combined multiple tensorflow models

Question

i am new to deep learning, and tensorflow. and it seems that all example online are very simple sequential model. but i have a lit bit complex model that i am trying to implement using tensorflow 2.1. so in nutshell, i have two models A, B that i am trying to combined its output and used as input for model C. please refer to diagram i attached to have more clear understanding of model architecture that i am trying to achieve

here is un-completed code so far i am still getting a lot of errors. any suggestions of how i can implement below network. thanks in advance.

def model_a():
    model_small = Sequential()
    model_small.add(Conv1D(filters=64, kernel_size=50, activation=None, input_shape=(3000, 1)))
    model_small.add(MaxPool1D(pool_size=8, strides=8))
    model_small.add(Dropout(0.5))
    model_small.add(Conv1D(filters=128, kernel_size=8, strides=1, activation=None))
    model_small.add(Conv1D(filters=128, kernel_size=8, strides=1, activation=None))
    model_small.add(Conv1D(filters=128, kernel_size=8, strides=1, activation=None))
    model_small.add(MaxPool1D(pool_size=4, strides=4))
    model_small.add(Flatten())
    return model_small
    #return model_small.add(Flatten())

def model_c():
    model = Sequential()
    model.add(Bidirectional(LSTM(512)))
    model.add(Dropout(0.5))
    model.add(Dense(4, activation='sigmoid'))

def model_b():
    model_large = Sequential()
    # the number of strides in this layer is too large at 50?
    model_large.add(Conv1D(filters=64, kernel_size=400, activation=None, input_shape=(3000, 1)))
    model_large.add(MaxPool1D(pool_size=4))
    model_large.add(Dropout(0.5))
    model_large.add(Conv1D(filters=128, kernel_size=6, activation=None))
    model_large.add(Conv1D(filters=128, kernel_size=6, activation=None))
    model_large.add(Conv1D(filters=128, kernel_size=6, strides=1, activation=None))
    model_large.add(MaxPool1D(pool_size=2))
    model_large.add(Flatten())
    return model_large
    #return model_large.add(Flatten())

def final_model():
    input1 = model_a()
    input2 = model_b()
    model_concat = concatenate([input1.output, input2.output], axis=1)
    model_concat = Dropout(0.5)(model_concat)
    # try to fix model_c here but i don't how 
    model= Model(inputs=[input1.input, input2.input], outputs=model_concat)

    return model

model_2 = final_model()
model_2.compile(
    loss=tf.keras.losses.sparse_categorical_crossentropy,
    optimizer=tf.keras.optimizers.SGD(learning_rate=0.01),
    metrics=['accuracy']  # can add more metrics
)


model_2.fit(x=INPUT, epochs=10)

Answer 1

You're on the right track. You can do it two ways: build one model split into functional parts, or build multiple models and link them together. Functionally they're the same thing, since a layer is a model; a model can be a layer.

I'm going to use some simple Dense layers to represent model_a through model_c. As a caveat, there's a bug that you might run into doing this my way if you use the sequential api, so I'll demonstrate with a functional api, but I assure you it's just as easy to use (the model is just defined after the layers, rather than before).

As one model split into functions:

import tensorflow as tf

def model_a(x):
  return tf.keras.layers.Dense(56,name='model_a_whatever')(x) # returning output of layer

def model_b(x):
  x = tf.keras.layers.Dense(56,name='model_b_whatever')(x)
  return tf.keras.layers.Dense(56,name='model_b_more_whatever')(x)

def model_c(x):
  x = tf.keras.layers.Dense(56,name='model_c_whatever')(x)
  x = tf.keras.layers.Dense(56,name='model_c_more_whatever')(x)
  return tf.keras.layers.Dense(56,name='model_c_even_more_whatever')(x)

# in through the input layer
main_input = tf.keras.layers.Input(shape=(12,34,56),name='INPUT')

# now through functions containing different models' layers
left = model_a(main_input)
right = model_b(main_input)

# concatenate their outputs
concatenated = tf.keras.layers.Concatenate(axis=-1)([left,right])

# now through function containing layers of model c
left = model_c(concatenated)

# and the juke right to a fully connected layer
right = tf.keras.layers.Dense(56,name='FC')(concatenated)

# then add the outputs and apply softmax activation
added = tf.keras.layers.Add(name='add')([left,right])
outputs = tf.keras.layers.Activation('softmax',name='Softmax')(added)

# now define the model
model = tf.keras.models.Model(main_input,outputs) # Model(input layer, final output))

print(model.summary())
tf.keras.utils.plot_model(model, to_file='just_a_model.png')

The diagram will look more cluttered than yours, since all layers will be visible:

As many models joined together:

# as separate models linked together

def build_model_a():
    a = tf.keras.layers.Input(shape=(12,34,56),name='model_a_input')
    b = tf.keras.layers.Dense(56,name='model_a_whatever')(a) # whatever layers
    return tf.keras.models.Model(a,b,name='MODEL_A') # returning model, not just layer output

def build_model_b():
  a = tf.keras.layers.Input(shape=(12,34,56),name='model_b_input')
  b = tf.keras.layers.Dense(56,name='model_b_whatever')(a)
  b = tf.keras.layers.Dense(56,name='model_b_more_whatever')(b)
  return tf.keras.models.Model(a,b,name='MODEL_B')

def build_model_c():
  a = tf.keras.layers.Input(shape=(12,34,112),name='model_c_input') # axis 2 is doubled because concatenation.
  b = tf.keras.layers.Dense(56,name='model_c_whatever')(a)
  b = tf.keras.layers.Dense(56,name='model_c_more_whatever')(b)
  b = tf.keras.layers.Dense(56,name='model_c_even_more_whatever')(b)
  return tf.keras.models.Model(a,b,name='MODEL_C')

# define the main input
main_input = tf.keras.layers.Input(shape=(12,34,56),name='INPUT')

# build the models
model_a = build_model_a()
model_b = build_model_b()
model_c = build_model_c()

# pass input through models a and b
a = model_a(main_input)
b = model_b(main_input)

# concatenate their outputs
ab = tf.keras.layers.Concatenate(axis=-1,name='Concatenate')([a,b])

# pass through model c and fully-connected layer
c = model_c(ab)
d = tf.keras.layers.Dense(56,name='FC')(ab)

# add their outputs and apply softmax activation
add = tf.keras.layers.Add(name="add")([c,d])
outputs = tf.keras.layers.Activation('softmax',name='Softmax')(add)

model = tf.keras.models.Model(main_input,outputs)

print(model.summary())
tf.keras.utils.plot_model(model, to_file='multi_model.png')

Although this is functionally the same network as in the first case, the diagram now matches yours:

Either method will work. As you can see, the first method is just a code-cleanup, really; putting separate data-pipelines into functions for clarity. If you want to get more complicated, like having different loss functions for sub-models and such, then the second method might simplify the process. The bug I mentioned only happens if you use the sequential api with the second method.