Keras changes dimensions of input shape when checking shape compatibility

Question

I have the following keras model which takes in nonsequential and sequential inputs

# Model parameters
units = 100
batch_size = 64
epochs = 1

encoder_inputs = Input(shape=(None, 1), name='encoder')
# Allows handling of variable length inputs by applying a binary mask to the specified mask_value.
masker = Masking(mask_value=sys.float_info.max)
masker(encoder_inputs)


nonseq_inputs = np.array([
    tensors['product_popularity'],
    tensors['quarter_autocorr'],
    tensors['year_autocorr']
]).T

nonseq_dim = nonseq_inputs.shape[1]
nonseq_input = Input(shape=(nonseq_dim,), name='nonsequential_input')
hidden_dense = Dense(units)(nonseq_input)
zeros = Lambda(lambda x: K.zeros_like(x), output_shape=lambda s: s)(hidden_dense)

encoder = LSTM(units, return_state=True)
encoder_outputs, state_h, state_c = encoder(encoder_inputs, initial_state=[hidden_dense, zeros])

# Keep encoder states for decoder, discard outputs
encoder_states = [state_h, state_c]

# Set up the decoder taking the encoder_states to be the initial state vector of the decoder.
decoder_inputs = Input(shape=(None, 1), name='decoder')

# Full output sequences and internal states are returned.  Returned states are used in prediction / inference
masker(decoder_inputs)
decoder = LSTM(units, return_sequences=True, return_state=True)
decoder_outputs, _, _ = decoder(decoder_inputs, initial_state=encoder_states)

# Gives continuous output at each time step
decoder_dense = Dense(1)
decoder_outputs = decoder_dense(decoder_outputs)

# create model that takes encoder_input_data and decoder_input_data and creates decoder_target_data
model = Model([nonseq_input, encoder_inputs, decoder_inputs], decoder_outputs)

model.summary()

plot_model(model, 'model.png')

# Get encoder inputs and standardise
encoder_input = get_time_block_series(series_array, date_to_index, train_encoding_start, train_encoding_end)
encoder_input, encoder_series_mean = centre_data(encoder_input)

# Get targets for the decoder
decoder_targets = get_time_block_series(series_array, date_to_index, train_pred_start, train_pred_end)
decoder_targets, _ = centre_data(decoder_targets, means=encoder_series_mean)

# Lag the target series to apply teacher forcing to mitigate error propagtion
decoder_input = np.zeros_like(decoder_targets)
decoder_input[:, 1:, 0] = decoder_targets[:, :-1, 0]
decoder_input[:, 0, 0] = encoder_input[:, -1, 0]

model.compile(Adam(), loss='mean_absolute_error')

history = model.fit(
    [nonseq_inputs, encoder_input, decoder_input],
    decoder_targets,
    batch_size=batch_size,
    epochs=epochs,
    validation_split=0.2,
    shuffle=True
)

# Build a model to predict with
encoder_model = Model([nonseq_input, encoder_inputs], encoder_states)

decoder_state_input_h = Input(shape=(units,))
decoder_state_input_c = Input(shape=(units,))
decoder_initial_state = [decoder_state_input_h, decoder_state_input_c]

decoder_outputs, state_h, state_c = decoder(decoder_inputs, initial_state=decoder_initial_state)
decoder_states = [state_h, state_c]

decoder_model = Model([decoder_inputs] + decoder_initial_state, [decoder_outputs] + decoder_states)

# Predict
encoder_input_data = get_time_block_series(series_array, date_to_index, val_encoding_start, val_encoding_end)
encoder_input_data, encoder_series_mean = centre_data(encoder_input_data)

decoder_target_data = get_time_block_series(series_array, date_to_index, val_pred_start, val_pred_end)
decoder_target_data, _ = centre_data(decoder_target_data, encoder_series_mean)

series, y, yhat = predict(
    encoder_model,
    decoder_model,
    encoder_input_data,
    decoder_targets,
    encoder_series_mean,
    horizon,
    sp,
    nonseq_inputs
)

def predict(encoder_model, decoder_model, encoder_input, decoder_targets, means, horizon, sample_index, nonseq_inputs):
    encode_series = encoder_input[sample_index:sample_index + 1]
    nonseq_input = nonseq_inputs[sample_index, :]
    yhat = decode_sequence(encoder_model, decoder_model, encode_series, horizon, nonseq_input)

    encode_series = encode_series.flatten()
    yhat = yhat.flatten()
    y = decoder_targets[sample_index, :, :1].flatten()

    encode_series, yhat, y = invert_transform(encode_series, yhat, y, means[sample_index])

    return encode_series, y, yhat

def decode_sequence(encoder_model, decoder_model, input_sequence, output_length, nonseq_input=None):
    # Encode input as state vectors
    state_values = encoder_model.predict([nonseq_input, input_sequence], batch_size=1)

    # Generate empty target sequence of length 1
    target_sequence = np.zeros((1, 1, 1))

    # Populate the first target sequence with the end of the encoding series
    target_sequence[0, 0, 0] = input_sequence[0, -1, 0]

    # Sampling loop for a batch of sequences - we will fill decoded_sequence with predictions
    # (to simplify we assume a batch_size of 1)
    decoded_sequence = np.zeros((1, output_length, 1))

    for i in range(output_length):
        output, h, c = decoder_model.predict([target_sequence] + state_values)

        decoded_sequence[0, i, 0] = output[0, 0, 0]

        # Update the target sequence (of length 1)
        target_sequence = np.zeros((1, 1, 1))
        target_sequence[0, 0, 0] = output[0, 0, 0]

        # Update states
        state_values = [h, c]

    return decoded_sequence

Here is an image of the model:

When I call the predict function feeding in one set on nonsequential input and a set of sequential inputs I get the following error:

ValueError: Error when checking input: expected nonsequential_input to have shape (3,) but got array with shape (1,)

I can confirm that I do indeed pass in an array of shape (3,) as required in the list of model inputs (I printed it out for a sanity check). When I debug the code it takes me all the way to standardize_input_data in the training_utils.py module down to the shape checking compatibility:

# Check shapes compatibility.
    if shapes:
        for i in range(len(names)):
            if shapes[i] is not None and not K.is_tensor(data[i]):
                data_shape = data[i].shape
                shape = shapes[i]
                if data[i].ndim != len(shape):
                    raise ValueError(
                        'Error when checking ' + exception_prefix +
                        ': expected ' + names[i] + ' to have ' +
                        str(len(shape)) + ' dimensions, but got array '
                        'with shape ' + str(data_shape))
                if not check_batch_axis:
                    data_shape = data_shape[1:]
                    shape = shape[1:]
                for dim, ref_dim in zip(data_shape, shape):
                    if ref_dim != dim and ref_dim:
                        raise ValueError(
                            'Error when checking ' + exception_prefix +
                            ': expected ' + names[i] + ' to have shape ' +
                            str(shape) + ' but got array with shape ' +
                            str(data_shape))

When I step through this code, right up to the line 'if not check_batch_axis' the variable data_shape has the correct shape dimension (3 that is). This function however is always called with check_batch_axis=False which means the if statement always passes. In this part of the code the data_shape which is correctly set gets overwritten and set incorrectly to be 1.:

if not check_batch_axis:
    data_shape = data_shape[1:]
    shape = shape[1:]

I don't know why this is the case or if I am doing something else incorrect. All I can confirm is that the numpy arrays I pass to the predict function in a list do have the correct shape, but they get changed above in the aforementioned bit of code. Does anybody know why or what I am doing incorrectly?

The model is based on the code from this blog post: https://blog.keras.io/a-ten-minute-introduction-to-sequence-to-sequence-learning-in-keras.html

EDIT: Requested details underneath

Shape of the array that is passed into the fit function:

The arrays are passed in a list with following shapes:

[(478, 3), (478, 240), (478, 26)].

As background I have 478 unique series; of these there are three time invariant features which I pass in as the first input, the second input contains the actual sequences and the last element are the inputs to the decoder which is used to predict 26 points. I have updated the code above to show the line with the fit call.

EDIT 2: Added line to print out the shape outputs in the decode function:

def decode_sequence(encoder_model, decoder_model, input_sequence, output_length, nonseq_input=None):
    # Encode input as state vectors
    print('nonseq_input.shape: {}'.format(nonseq_input.shape))
    print('input_sequence.shape: {}'.format(input_sequence.shape))
    state_values = encoder_model.predict([nonseq_input, input_sequence], batch_size=1)

(the rest of the function stays the same as before, only added in the print statements). The output is below:

Train on 382 samples, validate on 96 samples
Epoch 1/1
2019-01-13 08:37:08.112955: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA
 64/382 [====>.........................] - ETA: 9s - loss: 2.7368
128/382 [=========>....................] - ETA: 4s - loss: 2.6203
192/382 [==============>...............] - ETA: 2s - loss: 2.4305
256/382 [===================>..........] - ETA: 1s - loss: 2.2558
320/382 [========================>.....] - ETA: 0s - loss: 2.2033
382/382 [==============================] - 4s 10ms/step - loss: 2.2386 - val_loss: 3.1458
nonseq_input.shape: (3,)
input_sequence.shape: (1, 240, 1)

Exception is the same as indicated in first part of the question.

Answer 1

The problem is that the input layer is expecting a batch of data, i.e. a two-dimensional array where the first axis is the batch dimension and the second axis is the data dimension but you pass in a single sample as one-dimensional array. While nonseq_inputs is two-dimensional and has shape (477, 3) because sp is an integer the new array nonseq_input = nonseq_inputs[sample_index, :] has shape (3,) and is one-dimensional. Instead you should be using

nonseq_input = nonseq_inputs[[sample_index], :]

to maintain a 2D array.