What could cause a type error during the build process of Keras Layer?

Question

I was creating a custom layer based off the NALU paper, but when testing my code in Google Colab (tensorflow version 1.10.0) I got a type error. This did not hapen in my local Jupyter notebook(tensorflow cpu version 1.8.0).

The type error appears to occurring when adding a weight in the build function of Layer. When adding the weight a type error occurs when code tries to divide the Dimension by a float. I thought this might be an error with how Keras handles arguments, so I dropped the initializer. Even without the initializer, it is not possible to add a weight to the layer. I got the structure of my code from tensorflow's keras tutorial and I am wondering if there is anything that I need to do in order to update my code to work with the most recent version of tensorflow.

TypeError                                 Traceback (most recent call last)
<ipython-input-7-55fb94a8f3b1> in <module>()
     82 y_test = x_test[:, 0] * x_test[:, 1]
     83 
---> 84 model = nalu_model()
     85 
     86 model.compile(optimizer='RMSProp',

<ipython-input-7-55fb94a8f3b1> in nalu_model()
     48 def nalu_model():
     49     inp = tf.keras.layers.Input(shape=(2,))
---> 50     out = NALU(1)(inp)
     51 
     52     model = tf.keras.models.Model(inputs=inp, outputs=out)

/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/base_layer.py in __call__(self, inputs, *args, **kwargs)
    726         if all(hasattr(x, 'shape') for x in input_list):
    727           input_shapes = nest.map_structure(lambda x: x.shape, inputs)
--> 728         self.build(input_shapes)
    729         self.built = True
    730 

<ipython-input-7-55fb94a8f3b1> in build(self, input_shape)
      9         shape = tf.TensorShape((input_shape[1], self.num_outputs))
     10         get = tf.keras.initializers.get
---> 11         self.W_ = self.add_variable("W_", shape=shape, initializer=get('glorot_uniform'))
     12         self.M_ = self.add_variable("M_", shape=shape, initializer=get('glorot_uniform'))
     13         self.GAM = self.add_variable("GAM", shape=shape, initializer=get('glorot_uniform')) # Gate add & multiply

/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/base_layer.py in add_variable(self, *args, **kwargs)
    459   def add_variable(self, *args, **kwargs):
    460     """Alias for `add_weight`."""
--> 461     return self.add_weight(*args, **kwargs)
    462 
    463   def add_weight(self,

/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/base_layer.py in add_weight(self, name, shape, dtype, initializer, regularizer, trainable, constraint, partitioner, use_resource, synchronization, aggregation, getter)
    563         use_resource=use_resource,
    564         synchronization=synchronization,
--> 565         aggregation=aggregation)
    566 
    567     if regularizer is not None:

/usr/local/lib/python3.6/dist-packages/tensorflow/python/training/checkpointable/base.py in _add_variable_with_custom_getter(self, name, shape, dtype, initializer, getter, overwrite, **kwargs_for_getter)
    533     new_variable = getter(
    534         name=name, shape=shape, dtype=dtype, initializer=initializer,
--> 535         **kwargs_for_getter)
    536 
    537     # If we set an initializer and the variable processed it, tracking will not

/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/base_layer.py in make_variable(name, shape, dtype, initializer, partition_info, trainable, caching_device, validate_shape, constraint, use_resource, synchronization, aggregation, partitioner)
   1916       use_resource=use_resource,
   1917       synchronization=synchronization,
-> 1918       aggregation=aggregation)
   1919   return v

/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/variable_scope.py in variable(initial_value, trainable, collections, validate_shape, caching_device, name, dtype, constraint, use_resource, synchronization, aggregation)
   2441       use_resource=use_resource,
   2442       synchronization=synchronization,
-> 2443       aggregation=aggregation)
   2444 
   2445 

/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/variable_scope.py in <lambda>(**kwargs)
   2423              synchronization=VariableSynchronization.AUTO,
   2424              aggregation=VariableAggregation.NONE):
-> 2425   previous_getter = lambda **kwargs: default_variable_creator(None, **kwargs)
   2426   for getter in ops.get_default_graph()._variable_creator_stack:  # pylint: disable=protected-access
   2427     previous_getter = _make_getter(getter, previous_getter)

/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/variable_scope.py in default_variable_creator(next_creator, **kwargs)
   2393         collections=collections, validate_shape=validate_shape,
   2394         caching_device=caching_device, name=name, dtype=dtype,
-> 2395         constraint=constraint)
   2396   elif not use_resource and context.executing_eagerly():
   2397     raise RuntimeError(

/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/resource_variable_ops.py in __init__(self, initial_value, trainable, collections, validate_shape, caching_device, name, dtype, variable_def, import_scope, constraint)
    310           name=name,
    311           dtype=dtype,
--> 312           constraint=constraint)
    313 
    314   # pylint: disable=unused-argument

/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/resource_variable_ops.py in _init_from_args(self, initial_value, trainable, collections, validate_shape, caching_device, name, dtype, constraint)
    415               with ops.name_scope("Initializer"), ops.device(None):
    416                 initial_value = ops.convert_to_tensor(
--> 417                     initial_value(), name="initial_value", dtype=dtype)
    418               self._handle = _eager_safe_variable_handle(
    419                   shape=initial_value.get_shape(),

/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/base_layer.py in <lambda>()
   1901         initializer = initializer(dtype=dtype)
   1902       init_val = lambda: initializer(  # pylint: disable=g-long-lambda
-> 1903           shape, dtype=dtype, partition_info=partition_info)
   1904       variable_dtype = dtype.base_dtype
   1905   if use_resource is None:

/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/init_ops.py in __call__(self, shape, dtype, partition_info)
    474       scale /= max(1., fan_out)
    475     else:
--> 476       scale /= max(1., (fan_in + fan_out) / 2.)
    477     if self.distribution == "normal" or self.distribution == "truncated_normal":
    478       # constant taken from scipy.stats.truncnorm.std(a=-2, b=2, loc=0., scale=1.)

TypeError: unsupported operand type(s) for /: 'Dimension' and 'float'

Code: https://gist.github.com/Carliceiv/0b68fe0d9f2a9fc9a1b901e8a722f6cd

Answer 1

The stacktrace indicates that somewhere in the code a Dimension value is given as n argument where you usually expect an integer/float.

I think that this might be caused by this line

shape = tf.TensorShape((input_shape[1], self.num_outputs))

please try to change it to

shape = (input_shape[1], self.num_outputs)

Edit: As pointed out in the comments, this is not working of input_shape is a Dimension object and not an int. To solve this, use this modification:

shape = tf.TensorShape((input_shape[1], self.num_outputs)).as_list()

This will at first create a new TensorShape which is able to deal with the Dimension data type and then convert this to a list of integers.