Concatenate two layers in keras, tensorflow

Question

I want to build the below architecture of neural network layers

I have a cnn layer:

cnn1 = keras.Sequential([
    layers.Input((32, 32, 3)),
    layers.Conv2D(32, (5, 5), activation='relu')
    ]
)

And module:

from tensorflow.keras.layers import Concatenate, Dense

'''Module 1'''
module1_left = keras.Sequential([
    layers.Input((28, 28, 32)),
    layers.Conv2D(32, (1, 1), activation='relu', padding='same')
    ]
)
module1_middle = keras.Sequential([
    layers.Input((28, 28, 32)),
    layers.Conv2D(32, (1, 1), activation='relu', padding='same'),
    layers.Conv2D(64, (3, 3), activation='relu', padding='same')
    ]
)
module1_right = keras.Sequential([
    layers.Input((28, 28, 32)),
    layers.MaxPooling2D((3, 3), padding='same', strides=(1, 1)),
    layers.Conv2D(32, (1, 1), activation='relu', padding='same')
    ]
)
module1 = keras.layers.concatenate([module1_left.outputs[0], module1_middle.outputs[0], module1_right.outputs[0]], axis=-1)

Then I try to combine cnn1 and module 1, by cnn1.add(module1), here's the error for the last line:

TypeError: The added layer must be an instance of class Layer. Found: Tensor("concatenate_27/Identity:0", shape=(None, 28, 28, 128), dtype=float32)

Then I try another approach to concatenate:

module1 = Concatenate([module1_left, module1_middle, module1_right])

Then I get the error:

ValueError: A Concatenate layer should be called on a list of at least 2 inputs

Please let me know what's wrong with those approaches. Thank you!

Answer 1

The best (most flexible,elegant) solution is to use the Functional API in Keras.

Here is a working solution. Notice I am using the Model() (Functional API) instantiation and not Sequential():

from tensorflow.keras import Model
image_input = keras.layers.Input((32, 32, 3))
output_cnn_1 = cnn1(image_input)
output_left = module1_left(output_cnn_1)
output_middle = module1_middle(output_cnn_1)
output_right = module1_right(output_cnn_1)
concatenated_output = keras.layers.Concatenate()([output_left,output_middle,output_right])
final_model = Model(inputs=image_input, outputs=concatenated_output)
final_model.summary()



Layer (type)                    Output Shape         Param #     Connected to                     
==================================================================================================
input_29 (InputLayer)           [(None, 32, 32, 3)]  0                                            
__________________________________________________________________________________________________
sequential_11 (Sequential)      (None, 28, 28, 32)   2432        input_29[0][0]                   
__________________________________________________________________________________________________
sequential_12 (Sequential)      (None, 28, 28, 32)   1056        sequential_11[8][0]              
__________________________________________________________________________________________________
sequential_13 (Sequential)      (None, 28, 28, 64)   19552       sequential_11[8][0]              
__________________________________________________________________________________________________
sequential_14 (Sequential)      (None, 28, 28, 32)   1056        sequential_11[8][0]              
__________________________________________________________________________________________________
concatenate_15 (Concatenate)    (None, 28, 28, 128)  0           sequential_12[8][0]              
                                                                 sequential_13[8][0]              
                                                                 sequential_14[8][0]              
==================================================================================================
Total params: 24,096
Trainable params: 24,096
Non-trainable params: 0
__________________________________________________________________________________________________
    

Where the definition has slightly changed (we do not declare the input for each of the modules, since we are using the output of the cnn1).

import tensorflow
from tensorflow import keras
from tensorflow.keras import layers

cnn1 = keras.Sequential([
    layers.Conv2D(32, (5, 5), activation='relu')
    ]
)

'''Module 1'''
module1_left = keras.Sequential([
    layers.Conv2D(32, (1, 1), activation='relu', padding='same')
    ]
)
module1_middle = keras.Sequential([
    layers.Conv2D(32, (1, 1), activation='relu', padding='same'),
    layers.Conv2D(64, (3, 3), activation='relu', padding='same')
    ]
)
module1_right = keras.Sequential([
    layers.MaxPooling2D((3, 3), padding='same', strides=(1, 1)),
    layers.Conv2D(32, (1, 1), activation='relu', padding='same')
    ]
)