CODEWITHSUNDEEP
keras
Konrad S
Konrad S

Reputation: 71

Multi-layer autoencoder using keras, specifying different optimizers

Currently I'm trying to implement a multi-layer autoencoder using Keras, working on the Mnist dataset (handwritten digits). My code is looking like this:

from keras.layers import Input, Dense, initializers
from keras.models import Model
import numpy as np
from Dataset import Dataset
import matplotlib.pyplot as plt
from keras import optimizers, losses
from keras import backend as K
import tensorflow as tf
from keras.callbacks import TensorBoard
from keras.layers import Dropout
from keras.models import Sequential
from keras import models
from keras import layers
import keras
from keras.optimizers import Adam


#global variables
d = Dataset()
num_features = d.X_train.shape[1]
low_dim = 32


def autoencoder(epochs):
    w = initializers.RandomNormal(mean=0.0, stddev=0.05, seed=None)
    model = Sequential()
    #First autoencoder
    model.add(Dense(400, activation='relu', kernel_initializer=w, input_dim=num_features, name='hidden'))
    model.add(Dropout(0.2))
    model.add(Dense(num_features, activation='sigmoid', input_dim = 400, name = 'output'))
    #Second autoencoder
    model.add(Dense(100, activation='relu', kernel_initializer=w, input_dim=num_features, name='hidden2'))
    model.add(Dropout(0.2))
    model.add(Dense(num_features, activation = 'sigmoid', input_dim = 100, name='output2'))
    #Third autoencoder
    model.add(Dense(50, activation='relu', kernel_initializer=w, input_dim=num_features, name='hidden3'))
    model.add(Dropout(0.2))
    model.add(Dense(num_features, activation='sigmoid', input_dim=10, name='output3'))
    model.compile(optimizer=Adam(lr=0.01), loss='binary_crossentropy', metrics=['accuracy'])
    history = model.fit(d.X_train, d.X_train,
                        epochs=epochs,
                        batch_size=64,
                        shuffle=True,
                        validation_data=(d.X_test, d.X_test))
    model.test_on_batch(d.X_test, d.X_test)
    print(history.history.keys())
    plt.plot(history.history['acc'])
    print(history.history['acc'])
    plt.show()
    return model

def finding_index():
    elements, index = np.unique(d.Y_test, return_index = True)
    return elements, index

def plotting():
    ae = autoencoder(2)
    elements, index = finding_index()
    y_proba = ae.predict(d.X_test)
    plt.figure(figsize=(20, 4))
    #size = 20
    for i in range(len(index)):
        ax = plt.subplot(2, len(index), i + 1)
        plt.imshow(d.X_test[index[i]].reshape(28, 28))
        plt.gray()
        ax.get_xaxis().set_visible(False)
        ax.get_yaxis().set_visible(False)

        ax = plt.subplot(2, len(index), i + 1 + len(index))
        plt.imshow(y_proba[index[i]].reshape(28, 28))
        plt.gray()
        ax.get_xaxis().set_visible(False)
        ax.get_yaxis().set_visible(False)
    plt.show()

plotting()

I have two questions, is it supposed to be like this when you stack autoencoders or should I let one layer reduce dimensions to let's say 400 and then the next to a 100 and so on, or the way I have done it? The second one is, can you different optimizers (in my case Adam) for different layers? I would like to use SGD (stochastic gradient descent) for the last layer. Thanks in advance!

Upvotes: 0

Views: 238

Answers (1)

Syrius
Syrius

Reputation: 941

You should not do it the way you've done it, but the way you described it in the question. Also you should go down first and then up again (e.g 400, 100, 50, 25, 10, 25, 50, 100, 400) in granular steps.

For the second question is the answer that it depends. You could train the model with Adam first and then freeze all but the last layer to train this further with SGD. But you can't tell Keras to use different classifiers for different layers.

Upvotes: 1

Related Questions