Index out of range when loading data into CNN LSTM Model keras

Question

I would like to implement a convolutional recurrent neural net with lstm in keras. I am a only a beginner in Machine Learning therefore I struggle understanding everything. Here is my code:

def initialize_model():
    'Function to initialize the Convolutional Neural Network'
    seed=123
    # define CNN model
    model = Sequential()
    model.add(TimeDistributed(Conv2D(input_shape=(None,1,2048,1),kernel_size=(256,1),strides=(32,1),filters=16
                         ,activation=LeakyReLU(0.01),data_format='channels_first')))
    #cnn.add(MaxPooling2D(...))
    model.add(TimeDistributed(Flatten()))
    # define LSTM model
    #model = Sequential()
    #model.add(TimeDistribu(cnn))
    model.add(LSTM(1024,recurrent_activation=LeakyReLU(0.01),recurrent_initializer=glorot_normal(seed)
                         ,recurrent_regularizer=regularizers.l2(0.005)))
    model.add(Dense(2048))
    return model

def compile_model(model,learning_rate,loss_function):
    #optimizer
    optimizer = Adam(lr=learning_rate,beta_1=0.9,beta_2=0.999,epsilon=1e-8)

    #Compile the model
    model.compile(optimizer=optimizer,loss=loss_function,metrics=["accuracy"]) 
    return model

# import and split data
data = get_data_from_files(10)
print(data.shape)
X_train, X_test, y_train, y_test = train_test_split(data[0],data[1], test_size=0.2, random_state=123)

print(X_train.shape) -> 160000,2048
print(y_train.shape) -> 160000,2048
print(X_test.shape) ->  40000,2048
print(y_test.shape) ->  40000,2048

X_tr = np.expand_dims(np.expand_dims(X_train,axis=2),axis=3)
X_te = np.expand_dims(np.expand_dims(X_test,axis=2),axis=3)
#X_tr shape -->(160000, 2048, 1, 1)
#X_te shape -->(40000, 2048, 1, 1)
model = initialize_model()
model =compile_model(model,0.001,"cosine_proximity")
hist = model.fit(X_tr,y_train
                 ,validation_data=[X_te,y_test]
                 ,epochs=5,batch_size=1000)

It returns the following error:

    ValueError                                Traceback (most recent call last)
~/PycharmProjects/Test/venv/lib/python3.6/site-packages/tensorflow/python/framework/tensor_shape.py in merge_with(self, other)
    666       try:
--> 667         self.assert_same_rank(other)
    668         new_dims = []

~/PycharmProjects/Test/venv/lib/python3.6/site-packages/tensorflow/python/framework/tensor_shape.py in assert_same_rank(self, other)
    711         raise ValueError("Shapes %s and %s must have the same rank" % (self,
--> 712                                                                        other))
    713 

ValueError: Shapes (256, 1, 2048, 16) and (?, ?, ?) must have the same rank

During handling of the above exception, another exception occurred:

ValueError                                Traceback (most recent call last)
~/PycharmProjects/Test/venv/lib/python3.6/site-packages/tensorflow/python/framework/tensor_shape.py in with_rank(self, rank)
    741     try:
--> 742       return self.merge_with(unknown_shape(ndims=rank))
    743     except ValueError:

~/PycharmProjects/Test/venv/lib/python3.6/site-packages/tensorflow/python/framework/tensor_shape.py in merge_with(self, other)
    672       except ValueError:
--> 673         raise ValueError("Shapes %s and %s are not compatible" % (self, other))
    674 

ValueError: Shapes (256, 1, 2048, 16) and (?, ?, ?) are not compatible

During handling of the above exception, another exception occurred:

ValueError                                Traceback (most recent call last)
<ipython-input-36-1633649265a0> in <module>()
      6 hist = model.fit(X_tr,y_train
      7                  ,validation_data=[X_te,y_test]
----> 8                  ,epochs=5,batch_size=1000)
      9 

~/PycharmProjects/Test/venv/lib/python3.6/site-packages/keras/engine/training.py in fit(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)
    953             sample_weight=sample_weight,
    954             class_weight=class_weight,
--> 955             batch_size=batch_size)
    956         # Prepare validation data.
    957         do_validation = False

~/PycharmProjects/Test/venv/lib/python3.6/site-packages/keras/engine/training.py in _standardize_user_data(self, x, y, sample_weight, class_weight, check_array_lengths, batch_size)
    674             # to match the value shapes.
    675             if not self.inputs:
--> 676                 self._set_inputs(x)
    677 
    678         if y is not None:

~/PycharmProjects/Test/venv/lib/python3.6/site-packages/keras/engine/training.py in _set_inputs(self, inputs, outputs, training)
    574                 assert len(inputs) == 1
    575                 inputs = inputs[0]
--> 576             self.build(input_shape=(None,) + inputs.shape[1:])
    577             return
    578 

~/PycharmProjects/Test/venv/lib/python3.6/site-packages/keras/engine/sequential.py in build(self, input_shape)
    225             self.inputs = [x]
    226             for layer in self._layers:
--> 227                 x = layer(x)
    228             self.outputs = [x]
    229 

~/PycharmProjects/Test/venv/lib/python3.6/site-packages/keras/engine/base_layer.py in __call__(self, inputs, **kwargs)
    458             # Actually call the layer,
    459             # collecting output(s), mask(s), and shape(s).
--> 460             output = self.call(inputs, **kwargs)
    461             output_mask = self.compute_mask(inputs, previous_mask)
    462 

~/PycharmProjects/Test/venv/lib/python3.6/site-packages/keras/layers/wrappers.py in call(self, inputs, training, mask)
    246                 inner_mask_shape = self._get_shape_tuple((-1,), mask, 2)
    247                 kwargs['mask'] = K.reshape(mask, inner_mask_shape)
--> 248             y = self.layer.call(inputs, **kwargs)
    249             if hasattr(y, '_uses_learning_phase'):
    250                 uses_learning_phase = y._uses_learning_phase

~/PycharmProjects/Test/venv/lib/python3.6/site-packages/keras/layers/convolutional.py in call(self, inputs)
    166                 padding=self.padding,
    167                 data_format=self.data_format,
--> 168                 dilation_rate=self.dilation_rate)
    169         if self.rank == 3:
    170             outputs = K.conv3d(

~/PycharmProjects/Test/venv/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py in conv2d(x, kernel, strides, padding, data_format, dilation_rate)
   3564         strides=strides,
   3565         padding=padding,
-> 3566         data_format=tf_data_format)
   3567 
   3568     if data_format == 'channels_first' and tf_data_format == 'NHWC':

~/PycharmProjects/Test/venv/lib/python3.6/site-packages/tensorflow/python/ops/nn_ops.py in convolution(input, filter, padding, strides, dilation_rate, name, data_format)
    777         dilation_rate=dilation_rate,
    778         name=name,
--> 779         data_format=data_format)
    780     return op(input, filter)
    781 

~/PycharmProjects/Test/venv/lib/python3.6/site-packages/tensorflow/python/ops/nn_ops.py in __init__(self, input_shape, filter_shape, padding, strides, dilation_rate, name, data_format)
    854         filter_shape=filter_shape,
    855         spatial_dims=spatial_dims,
--> 856         data_format=data_format)
    857 
    858   def _build_op(self, _, padding):

~/PycharmProjects/Test/venv/lib/python3.6/site-packages/tensorflow/python/ops/nn_ops.py in __init__(self, input_shape, dilation_rate, padding, build_op, filter_shape, spatial_dims, data_format)
    437         raise ValueError("dilation_rate must be positive")
    438       if np.all(const_rate == 1):
--> 439         self.call = build_op(num_spatial_dims, padding)
    440         return
    441 

~/PycharmProjects/Test/venv/lib/python3.6/site-packages/tensorflow/python/ops/nn_ops.py in _build_op(self, _, padding)
    863         data_format=self.data_format,
    864         strides=self.strides,
--> 865         name=self.name)
    866 
    867   def __call__(self, inp, filter):  # pylint: disable=redefined-builtin

~/PycharmProjects/Test/venv/lib/python3.6/site-packages/tensorflow/python/ops/nn_ops.py in __init__(self, input_shape, filter_shape, padding, data_format, strides, name)
    134       strides=None,
    135       name=None):
--> 136     filter_shape = filter_shape.with_rank(input_shape.ndims)
    137     self.padding = padding
    138     self.name = name

~/PycharmProjects/Test/venv/lib/python3.6/site-packages/tensorflow/python/framework/tensor_shape.py in with_rank(self, rank)
    742       return self.merge_with(unknown_shape(ndims=rank))
    743     except ValueError:
--> 744       raise ValueError("Shape %s must have rank %d" % (self, rank))
    745 
    746   def with_rank_at_least(self, rank):

ValueError: Shape (256, 1, 2048, 16) must have rank 3

I am completely stuck and have no idea how to resolve the problem.

EDIT: I updated my code with the new error which is now a error of shape.

Answer 1

Fo anyone with a similar problem, I changed the trained and test data shape as follow :

X_tr = np.expand_dims(np.expand_dims(np.expand_dims(X_train,axis=1),axis=3),axis=1)
X_te = np.expand_dims(np.expand_dims(np.expand_dims(X_test,axis=1),axis=3),axis=1)

and the first layer :

model.add(TimeDistributed(Conv2D(kernel_size=(256,1),strides=(32,1),filters=16,activation=LeakyReLU(0.01),data_format='channels_first'),input_shape=(None,1,2048,1)))

It know run very well.

Answer 2

I am fairly confident that I have found the solution so this answer is edited to reflect that. According to keras documentation, when ising TimeDistribution with conv2d the input_shape must be a touple of length 4

Below is a part of the keras documentation.

TimeDistributed can be used with arbitrary layers, not just Dense, for instance with a Conv2D layer:

model = Sequential()
model.add(TimeDistributed(Conv2D(64, (3, 3)),
                      input_shape=(10, 299, 299, 3)))