Only layers of same output shape can be merged using sum mode. Layer shapes

Question

My model is U-Net implementation -

from keras.layers import Input, merge, Convolution2D, MaxPooling2D, 

UpSampling2D
from keras.optimizers import Adam
from keras.callbacks import ModelCheckpoint, LearningRateScheduler
from keras import backend as K
from keras.models import Model

def seg_score(y_true, y_pred):
    smooth = 1.0
    y_true_f = K.flatten(y_true)
    y_pred_f = K.flatten(y_pred)
    intersection = K.sum(y_true_f * y_pred_f)
    true_sum = K.sum(y_true_f); pred_sum = K.sum(y_pred_f)
    if(true_sum > pred_sum):
        max_sum = true_sum
    else:
        max_sum = pred_sum
    return (intersection + smooth) / (max_sum + smooth)

def seg_score_loss(y_true, y_pred):
    return -seg_score(y_true, y_pred)

def dice_coef(y_true, y_pred):
    smooth = 1.
    y_true_f = K.flatten(y_true)
    y_pred_f = K.flatten(y_pred)
    intersection = K.sum(y_true_f * y_pred_f)
    return (2. * intersection + smooth) / (K.sum(y_true_f) + K.sum(y_pred_f) + smooth)

def dice_coef_loss(y_true, y_pred):
    return -dice_coef(y_true, y_pred)


def get_unet(num_color_component, dimension):

    img_rows = dimension; img_cols = dimension;
    inputs = Input((num_color_component, img_rows, img_cols))
    conv1 = Convolution2D(32, 3, 3, activation='relu', border_mode='same')(inputs)
    conv1 = Convolution2D(32, 3, 3, activation='relu', border_mode='same')(conv1)
    pool1 = MaxPooling2D(pool_size=(2, 2))(conv1)

    conv2 = Convolution2D(64, 3, 3, activation='relu', border_mode='same')(pool1)
    conv2 = Convolution2D(64, 3, 3, activation='relu', border_mode='same')(conv2)
    pool2 = MaxPooling2D(pool_size=(2, 2))(conv2)

    conv3 = Convolution2D(128, 3, 3, activation='relu', border_mode='same')(pool2)
    conv3 = Convolution2D(128, 3, 3, activation='relu', border_mode='same')(conv3)
    pool3 = MaxPooling2D(pool_size=(2, 2))(conv3)

    conv4 = Convolution2D(256, 3, 3, activation='relu', border_mode='same')(pool3)
    conv4 = Convolution2D(256, 3, 3, activation='relu', border_mode='same')(conv4)
    pool4 = MaxPooling2D(pool_size=(2, 2))(conv4)

    conv5 = Convolution2D(512, 3, 3, activation='relu', border_mode='same')(pool4)
    conv5 = Convolution2D(512, 3, 3, activation='relu', border_mode='same')(conv5)

    up6 = merge([UpSampling2D(size=(2, 2))(conv5), conv4], mode='concat', concat_axis=1)

    conv6 = Convolution2D(256, 3, 3, activation='relu', border_mode='same')(up6)
    conv6 = Convolution2D(256, 3, 3, activation='relu', border_mode='same')(conv6)

    up7 = merge([UpSampling2D(size=(2, 2))(conv6), conv3], mode='concat', concat_axis=1)
    conv7 = Convolution2D(128, 3, 3, activation='relu', border_mode='same')(up7)
    conv7 = Convolution2D(128, 3, 3, activation='relu', border_mode='same')(conv7)

    up8 = merge([UpSampling2D(size=(2, 2))(conv7), conv2], mode='concat', concat_axis=1)
    conv8 = Convolution2D(64, 3, 3, activation='relu', border_mode='same')(up8)
    conv8 = Convolution2D(64, 3, 3, activation='relu', border_mode='same')(conv8)

    up9 = merge([UpSampling2D(size=(2, 2))(conv8), conv1], mode='concat', concat_axis=1)
    conv9 = Convolution2D(32, 3, 3, activation='relu', border_mode='same')(up9)
    conv9 = Convolution2D(32, 3, 3, activation='relu', border_mode='same')(conv9)

    conv10 = Convolution2D(1, 1, 1, activation='sigmoid')(conv9)

    model = Model(input=inputs, output=conv10)

    #model.compile(optimizer=Adam(lr=1e-5), loss=seg_score_loss, metrics=[seg_score])
    model.compile(optimizer=Adam(lr=1e-5), loss=dice_coef_loss, metrics=[dice_coef])

    return model

I am getting error as follows-

Traceback (most recent call last): File "/home/zaverichintan/Chintan/PycharmProjects/CNN_wbc_identification/train.py", line 60, in model = mo.get_unet(num_color_component, filter_size); File "/home/zaverichintan/Chintan/PycharmProjects/CNN_wbc_identification/models.py", line 63, in get_unet up7 = merge([UpSampling2D(size=(2, 2))(conv6), conv3], mode='concat', concat_axis=1) File "/home/zaverichintan/anaconda2/lib/python2.7/site-packages/keras/legacy/layers.py", line 456, in merge name=name) File "/home/zaverichintan/anaconda2/lib/python2.7/site-packages/keras/legacy/layers.py", line 107, in init node_indices, tensor_indices) File "/home/zaverichintan/anaconda2/lib/python2.7/site-packages/keras/legacy/layers.py", line 187, in _arguments_validation 'Layer shapes: %s' % (input_shapes)) ValueError: "concat" mode can only merge layers with matching output shapes except for the concat axis. Layer shapes: [(None, 0, 16, 256), (None, 0, 16, 128)]

Changed Concat axis to 3 then I am getting this -

File "/home/zaverichintan/Chintan/PycharmProjects/CNN_wbc_identification/train.py", line 60, in model = mo.get_unet(num_color_component, filter_size); File "/home/zaverichintan/Chintan/PycharmProjects/CNN_wbc_identification/models.py", line 71, in get_unet up8 = keras.layers.merge([UpSampling2D(size=(2, 2))(conv7), conv2], mode='concat', concat_axis=1) File "/home/zaverichintan/anaconda2/lib/python2.7/site-packages/keras/legacy/layers.py", line 456, in merge name=name) File "/home/zaverichintan/anaconda2/lib/python2.7/site-packages/keras/legacy/layers.py", line 107, in init node_indices, tensor_indices) File "/home/zaverichintan/anaconda2/lib/python2.7/site-packages/keras/legacy/layers.py", line 187, in _arguments_validation 'Layer shapes: %s' % (input_shapes)) ValueError: "concat" mode can only merge layers with matching output shapes except for the concat axis. Layer shapes: [(None, 0, 32, 128), (None, 1, 32, 64)]

Answer 1

You have to set image_data_format": "channels_first" as described here https://keras.io/backend/ or change the input to

 inputs = Input((img_rows, img_cols, num_color_component))

Then the concat_axis has to correspond with data format.

Here is an example of how to implement a U-net in Keras: https://github.com/jocicmarko/ultrasound-nerve-segmentation/blob/master/train.py#L34

Answer 2

This is pretty straight forward :

ValueError: "concat" mode can only merge layers with matching output shapes except for the concat axis. Layer shapes: [(None, 0, 16, 256), (None, 0, 16, 128)]

you have :

up6 = merge([UpSampling2D(size=(2, 2))(conv5), conv4], mode='concat', concat_axis=1)

they explicitely say that the shapes should be the same except for the concat axis

the dimension where the shape is different is the 3rd dimension (one is 256, the other is 128). So you should set the concat axis to 3 not 1. As in :

up6 = merge([UpSampling2D(size=(2, 2))(conv5), conv4], mode='concat', concat_axis=3)

I hope this helps :)