Pad data using tf.data.Dataset

Question

I have to use tf.data.Dataset for creating a input pipeline for an RNN model in tensorflow. I am providing a basic code, by which I need to pad the data in batch with a pad token and use it for further manipulation.

    import pandas as pd
    import numpy as np
    import tensorflow as tf
    import functools

    total_data_size = 10000
    embedding_dimension = 25
    max_len = 17


    varying_length = np.random.randint(max_len, size=(10000)) # varying length data
    X = np.array([np.random.randint(1000, size=(value)).tolist()for index, value in enumerate(varying_length)]) # data of arying length
    Y = np.random.randint(2, size=(total_data_size)).astype(np.int32) # target binary
    embedding = np.random.uniform(-1,1,(1000, embedding_dimension))   # word embedding



    def gen():

        for  index in range(len(X)):
            yield X[index] , Y[index]

    dataset = tf.data.Dataset.from_generator(gen,(tf.int32,tf.int32))
    dataset = dataset.batch(batch_size=25)
    padded_shapes = (tf.TensorShape([None])) # sentence of unknown size
    padding_values = (tf.constant(-111))   # the value with which pad index needs to be filled 

    dataset = (dataset
        .padded_batch(25, padded_shapes=padded_shapes, padding_values=padding_values)
    )

    iter2 = dataset.make_initializable_iterator()
    sess = tf.InteractiveSession()
    sess.run(tf.global_variables_initializer())
    sess.run(iter2.initializer)    
    print(sess.run(iter2.get_next()))

I hope the code is self explanatory with comments. But I am getting following error,

InvalidArgumentError (see above for traceback): Cannot batch tensors with different shapes in component 0. First element had shape [11] and element 1 had shape [12]. [[Node: IteratorGetNext = IteratorGetNext[output_shapes=[[?,?], [?]], output_types=[DT_INT32, DT_INT32], _device="/job:localhost/replica:0/task:0/device:CPU:0"](Iterator)]]

Answer 1

Finally got the answer. The issue was for the second padded shapes instead of Tensorshape([None]), we should provide [], because the second item returned by the generator is a scalar. If using Tensorshape([None]),, make sure we are returning a vector

    import pandas as pd
    import numpy as np
    import tensorflow as tf
    import functools

    total_data_size = 10000
    embedding_dimension = 25
    max_len = 17


    varying_length = np.random.randint(max_len, size=(10000)) # varying length data
    X = np.array([np.random.randint(1000, size=(value)).tolist()for index, value in enumerate(varying_length)]) # data of arying length
    Y = np.random.randint(2, size=(total_data_size)).astype(np.int32) # target binary
    embedding = np.random.uniform(-1,1,(1000, embedding_dimension))   # word embedding



    def gen():
        for index in range(len(X)):
            yield X[index] , Y[index]

    dataset = tf.data.Dataset.from_generator(gen, (tf.int32, tf.int32), (tf.TensorShape([None]), []))
    padded_shapes = (tf.TensorShape([None]), [])  # sentence of unknown size
    dataset = (dataset
               .padded_batch(25, padded_shapes=padded_shapes, padding_values=(-111, 0))
               )

    iter2 = dataset.make_initializable_iterator()
    sess = tf.InteractiveSession()
    sess.run(tf.global_variables_initializer())
    sess.run(iter2.initializer)

    sess.run(iter2.get_next())

Answer 2

I believe that since your generator yields two outputs, your padded_shapes and padded_values tuples must have a length of two. For me, this works:

dataset = tf.data.Dataset.from_generator(gen, (tf.int32, tf.int32))
dataset = dataset.batch(batch_size=25)
padded_shapes = (tf.TensorShape([None]), tf.TensorShape([None]))  # sentence of unknown size
padding_values = (tf.constant(-111), tf.constant(-111))  # the value with which pad index needs to be filled 

dataset = (dataset
           .padded_batch(25, padded_shapes=padded_shapes, padding_values=padding_values)
           )

iter2 = dataset.make_initializable_iterator()
sess = tf.InteractiveSession()
sess.run(tf.global_variables_initializer())
sess.run(iter2.initializer)