Transfer learning for DQN

Question

I research how to use transfer learning in deep reinforcement learning.

I want to use the pre-trained model (h5f. file) in my project by transfer learning. I have image input and scalar inputs. The image is the input of a convolutional neural network (CNN).

I have also tried to load the weights from the pre-trained model and I have tried to decide which layers can be trainable.

dqn.load_weights('checkpoint_reward_176.h5f')

for i in range(4):
    model.layers[1].trainable = False

for i in range(4,8):
    model.layers[i].trainable = True

To sum up, How can I transfer the layers to the layers which are not trained. Is it possible to use transfer learning in this case?

I appreciate all answers, Thank you very much.

Here is the DQN code.

import tensorflow as tf
from keras.backend.tensorflow_backend import set_session   
env = gym.make(args.env_name)
np.random.seed(123)
env.seed(123)
nb_actions = env.action_space.n

img_shape = env.simage.shape
vel_shape = env.svelocity.shape
dst_shape = env.sdistance.shape
geo_shape = env.sgeofence.shape


AE_shape = env.sAE.shape

img_kshape = (1,) + img_shape

#Sequential model for convolutional layers applied to image
image_model = Sequential()
image_model.add(Conv2D(32, (4, 4), strides=(4, 4) ,activation='relu', input_shape=img_kshape, data_format = "channels_first"))
image_model.add(Conv2D(64, (3, 3), strides=(2, 2),  activation='relu'))
image_model.add(Flatten())
print(image_model.summary())

#Input and output of the Sequential model
image_input = Input(img_kshape)
encoded_image = image_model(image_input)

#Inputs and reshaped tensors for concatenate after with the image
velocity_input = Input((1,) + vel_shape)
distance_input = Input((1,) + dst_shape)
geofence_input = Input((1,) + geo_shape)
vel = Reshape(vel_shape)(velocity_input)
dst = Reshape(dst_shape)(distance_input)
geo = Reshape(geo_shape)(geofence_input)


AE_input  = Input((1,) + AE_shape)
ae=Reshape(AE_shape)(AE_input)#Concatenation of image, position, distance and geofence values.
#3 dense layers of 256 units
denses = concatenate([encoded_image, vel, dst, geo, ae])
denses = Dense(256, activation='relu')(denses)
denses = Dense(256, activation='relu')(denses)
denses = Dense(256, activation='relu')(denses)

#Last dense layer with nb_actions for the output
predictions = Dense(nb_actions, kernel_initializer='zeros', activation='linear')(denses)

model = Model(
        inputs=[image_input, velocity_input, distance_input, geofence_input, AE_input],
        outputs=predictions
        )
print(model.summary())


train = True

memory = SequentialMemory(limit=100000, window_length=1)                        

processor = MultiInputProcessor(nb_inputs=5)

policy = LinearAnnealedPolicy(EpsGreedyQPolicy(), attr='eps', value_max=1., value_min=.1, value_test=0.0,
                              nb_steps=50000)

dqn = DQNAgent(model=model, processor=processor, nb_actions=nb_actions, memory=memory, nb_steps_warmup=50, 
               enable_double_dqn=True, 
               enable_dueling_network=False, dueling_type='avg', 
               target_model_update=1e-2, policy=policy, gamma=.99)

dqn.compile(Adam(lr=0.00025), metrics=['mae'])'

Updates in DQN code:

# Obtaining shapes from Gym environment
img_shape = env.simage.shape
vel_shape = env.svelocity.shape
dst_shape = env.sdistance.shape
geo_shape = env.sgeofence.shape

AE_shape = env.sAE.shape

# Keras-rl interprets an extra dimension at axis=0
# added on to our observations, so we need to take it into account
img_kshape = (1,) + img_shape

input_layer = Input(shape=img_kshape)
conv1 = Conv2D(32, (4, 4), strides=(4, 4), activation='relu', input_shape=img_kshape, name='conv1',
               data_format="channels_first")(input_layer)
conv2 = Conv2D(64, (3, 3), strides=(2, 2), activation='relu', name='conv2')(conv1)
flat1 = Flatten(name='flat1')(conv2)

auxiliary_input1 = Input(vel_shape, name='vel')
auxiliary_input2 = Input(dst_shape, name='dst')
auxiliary_input3 = Input(geo_shape, name='geo')
auxiliary_input4 = Input(AE_shape, name='ae')

denses = concatenate([flat1, auxiliary_input1, auxiliary_input2, auxiliary_input3, auxiliary_input4])
denses = Dense(256, activation='relu')(denses)
denses = Dense(256, activation='relu')(denses)
denses = Dense(256, activation='relu')(denses)

predictions = Dense(nb_actions, kernel_initializer='zeros', activation='linear')(denses)

model = Model(inputs=[input_layer, auxiliary_input1, auxiliary_input2, auxiliary_input3, auxiliary_input4],
              outputs=predictions)

print(model.summary())

Model summary

Transfer learning for DQN

Answers (1)

Related Questions