Merge different CNN models

Question

We are novice students in data science and we are trying to merge two different CNN models (one with 2 classes and the other with 3 classes). the codes of the models are:

Gender model

    #initialize the model along with the input shape
    model = Sequential()
    inputShape = (height, width, depth)
    chanDim = -1
   
    if K.image_data_format() == 'channels_first':
        inputShape = (depth, height, width)
        chanDim = 1
       
    # CONV -> RELU -> MAXPOOL
    model.add(Convolution2D(64, (3,3), padding='same', input_shape=inputShape))
    model.add(Activation('relu'))
    model.add(BatchNormalization(axis=chanDim))
    model.add(MaxPooling2D(pool_size=(3,3)))
    model.add(Dropout(0.25))
   
    # (CONV -> RELU)*2 -> AVGPOOL
    model.add(Convolution2D(128, (3,3), padding='same'))
    model.add(Activation('relu'))
    model.add(BatchNormalization(axis=chanDim))
    model.add(Convolution2D(128, (3,3), padding='same'))
    model.add(Activation('relu'))
    model.add(BatchNormalization(axis=chanDim))
    model.add(AveragePooling2D(pool_size=(3,3) ))
    model.add(Dropout(0.25))
   
    # CONV -> RELU -> MAXPOOL
    model.add(Convolution2D(256, (3,3), padding='same'))
    model.add(Activation('relu'))
    model.add(BatchNormalization(axis=chanDim))
    model.add(MaxPooling2D(pool_size=(3,3)))
    model.add(Dropout(0.25))
   
    # CONV -> RELU -> AVGPOOL
    model.add(Convolution2D(512, (3,3), padding='same'))
    model.add(Activation('relu'))
    model.add(BatchNormalization(axis=chanDim))
    model.add(AveragePooling2D(pool_size=(3,3)))
    model.add(Dropout(0.25))
   
    # DENSE -> RELU
    model.add(Flatten())
    model.add(Dense(1024))
    model.add(Activation('relu'))
    model.add(BatchNormalization())
    model.add(Dropout(0.25))
   
    # DENSE -> RELU
    model.add(Dense(512))
    model.add(Activation('relu'))
    model.add(BatchNormalization())
    model.add(Dropout(0.25))
   
    # sigmoid -> just to check the accuracy with this (softmax would work too)
    model.add(Dense(classes))
    model.add(Activation('sigmoid'))
   
    return model
model = build(img_size, img_size, 3, 2)
model.compile(loss='binary_crossentropy', optimizer='rmsprop', metrics=['accuracy'])

Ethnicity model:

    #initialize the model along with the input shape
    model = Sequential()
    inputShape = (height, width, depth)
    chanDim = -1
   
    if K.image_data_format() == 'channels_first':
        inputShape = (depth, height, width)
        chanDim = 1
       
    # CONV -> RELU -> MAXPOOL
    model.add(Convolution2D(64, (3,3), padding='same', input_shape=inputShape))
    model.add(Activation('relu'))
    model.add(BatchNormalization(axis=chanDim))
    model.add(MaxPooling2D(pool_size=(3,3)))
    model.add(Dropout(0.25))
   
    # (CONV -> RELU)*2 -> AVGPOOL
    model.add(Convolution2D(128, (3,3), padding='same'))
    model.add(Activation('relu'))
    model.add(BatchNormalization(axis=chanDim))
    model.add(Convolution2D(128, (3,3), padding='same'))
    model.add(Activation('relu'))
    model.add(BatchNormalization(axis=chanDim))
    model.add(AveragePooling2D(pool_size=(3,3) ))
    model.add(Dropout(0.25))
   
    # CONV -> RELU -> MAXPOOL
    model.add(Convolution2D(256, (3,3), padding='same'))
    model.add(Activation('relu'))
    model.add(BatchNormalization(axis=chanDim))
    model.add(MaxPooling2D(pool_size=(3,3)))
    model.add(Dropout(0.25))
   
    # CONV -> RELU -> AVGPOOL
    model.add(Convolution2D(512, (3,3), padding='same'))
    model.add(Activation('relu'))
    model.add(BatchNormalization(axis=chanDim))
    model.add(AveragePooling2D(pool_size=(3,3)))
    model.add(Dropout(0.25))
   
    # DENSE -> RELU
    model.add(Flatten())
    model.add(Dense(1024))
    model.add(Activation('relu'))
    model.add(BatchNormalization())
    model.add(Dropout(0.25))
   
    # DENSE -> RELU
    model.add(Dense(512))
    model.add(Activation('relu'))
    model.add(BatchNormalization())
    model.add(Dropout(0.25))
   
    # softmax
    model.add(Dense(classes))
    model.add(Activation('softmax'))
   
    return model
model = build(img_size, img_size, 3, 3)
model.compile(loss= 'categorical_crossentropy', optimizer='adam', metrics=['accuracy'])

We tried to merged the models with the concatenate keras function but failed to undesrtand how to merge two models with different numbers of classes. Our goal is: given a photo we want to predict the gender and the ethnicity at the same time Thanks for the attention.

Answer 1

Let's call the first model model_1 and the second model model_2. The first step you need to do is changing the inputs to the models to some common input.

inputs = keras.layers.Input(shape=inputShape)

outputs_1 = model_1(inputs)
outputs_2 = model_2(inputs

Next create a model with these inputs and outputs

new_model = keras.Model(inputs=inputs, outputs=[outputs_1, outputs_2])

Now the model has one input and two outputs. You can get two predictions from single inputs.

Fixing name conflict

In case when the models have the same name and/or the layers of the models have layers with the same name, use the following code to rename the models and layers of the models.

model_1._name = "model_1_"+model_1.name
model_2._name = "model_2_"+model_2.name

for layer in model_1.layers:
    layer._name = "model_1_layer_"+layer.name

for layer in model_2.layers:
    layer._name = "model_2_layer_"+layer.name