Tensor flow and RandomShuffleQueue "insufficient elements (requested 64, current size 0)"

Question

I currently playing around with tensorFlow, and even thoughthe tutorials were kinda simple to complete, the true work starts when we try to input our own datas.

I used a very basic dataset composer of animals and background.

I created 3 tfrecords (train/val/test). I then attempt to read them and to train a simple model (Alexnet here). I tried to use the "FLAGS.num_iter" to be sure i'm not out the iteration range.

This code processing get me a nice RandomShuffleQueue "insufficient elements (requested 64, current size 0)" error.

I tried to dig the web but i found no answer to my questions. Here they are: How do we fix this ? How can we check if our tfrecord contain any mistakes? Can we write any condition to ensure that we have enough elements? If You have any further question on my code, I stay around!

Best regards,

import tensorflow as tf
import os.path
from model import Model
from alexnet import Alexnet


FLAGS = tf.app.flags.FLAGS
NUM_LABELS = 2

IMAGE_WIDTH = 64
IMAGE_HEIGHT = 64
NUMBER_OF_CHANNELS = 3
#SOURCE_DIR = './data/'
#TRAINING_IMAGES_DIR = SOURCE_DIR + 'train/'
#LIST_FILE_NAME = 'list.txt'
BATCH_SIZE = 2
#TRAINING_SET_SIZE = 81112
TRAIN_FILE = '/home/sebv/SebV/datas/tfRecording/train.tfrecords'
VAL_FILE = '/home/sebv/SebV/datas/tfRecording/val.tfrecor'

def read_and_decode(filename_queue):
    reader = tf.TFRecordReader()
    _, serialized_example = reader.read(filename_queue)
    features = tf.parse_single_example(
        serialized_example,
        # Defaults are not specified since both keys are required.
        features={
          'image/encoded': tf.FixedLenFeature([], tf.string),
          'image/format': tf.FixedLenFeature([], tf.string),
          'image/class/label': tf.FixedLenFeature([], tf.int64),
          'image/height': tf.FixedLenFeature([], tf.int64),
          'image/width': tf.FixedLenFeature([], tf.int64),
        })

    # Convert from a scalar string tensor (whose single string has
    # length mnist.IMAGE_PIXELS) to a uint8 tensor with shape
    # [mnist.IMAGE_PIXELS].
    image = tf.image.decode_png(features['image/encoded'], 3, tf.uint8)

    # OPTIONAL: Could reshape into a 28x28 image and apply distortions
    # here.  Since we are not applying any distortions in this
    # example, and the next step expects the image to be flattened
    # into a vector, we don't bother.

    # Convert from [0, 255] -> [-0.5, 0.5] floats.
    image = tf.cast(image, tf.float32)# * (1. / 255) - 0.5
    image = tf.reshape(image, [IMAGE_WIDTH,IMAGE_HEIGHT,NUMBER_OF_CHANNELS])
    # Convert label from a scalar uint8 tensor to an int32 scalar.
    label = tf.cast(features['image/class/label'], tf.int64)

    return image, label


def inputs(train, filen, batch_size, num_epochs):
    """Reads input data num_epochs times.
    Args:
    train: Selects between the training (True) and validation (False) data.
    batch_size: Number of examples per returned batch.
    num_epochs: Number of times to read the input data, or 0/None to
    train forever.
    Returns:
    A tuple (images, labels), where:
    * images is a float tensor with shape [batch_size, mnist.IMAGE_PIXELS]
    in the range [-0.5, 0.5].
    * labels is an int32 tensor with shape [batch_size] with the true label,
    a number in the range [0, mnist.NUM_CLASSES).
    Note that an tf.train.QueueRunner is added to the graph, which
    must be run using e.g. tf.train.start_queue_runners().
    """
    if not num_epochs: num_epochs = None
    filename = filen
    filename_queue = tf.train.string_input_producer([filename], num_epochs=num_epochs)

    # Even when reading in multiple threads, share the filename
    # queue.
    image, label = read_and_decode(filename_queue)
    # Shuffle the examples and collect them into batch_size batches.
    # (Internally uses a RandomShuffleQueue.)
    # We run this in two threads to avoid being a bottleneck.
    images, sparse_labels = tf.train.shuffle_batch([image, label], batch_size=batch_size, num_threads=2,capacity=20000 + 3 * batch_size,min_after_dequeue=20000)
    sparse_labels = tf.reshape(sparse_labels, [batch_size])
    return images, sparse_labels


def train():
    model = Alexnet()
    with tf.Graph().as_default():

        x = tf.placeholder(tf.float32, [None, IMAGE_WIDTH,IMAGE_HEIGHT,NUMBER_OF_CHANNELS], name='x-input')
        y = tf.placeholder(tf.float32, [None], name='y-input')

        images, labels = inputs(train=True, filen=TRAIN_FILE, batch_size=FLAGS.batch_size,num_epochs=FLAGS.num_iter)

        images_val, labels_val = inputs(train=False, filen=VAL_FILE, batch_size=FLAGS.batch_size,num_epochs=1)

        keep_prob = tf.placeholder(tf.float32, name='dropout_prob')
        global_step = tf.contrib.framework.get_or_create_global_step()

        logits = model.inference(images, keep_prob=keep_prob)
        loss = model.loss(logits=logits, labels=labels)

        accuracy = model.accuracy(logits, labels)
        summary_op = tf.summary.merge_all()
        train_op = model.train(loss, global_step=global_step)

        saver = tf.train.Saver()

        with tf.Session(config=tf.ConfigProto(log_device_placement=True)) as sess:
            writer = tf.summary.FileWriter(FLAGS.summary_dir, sess.graph)
            sess.run(tf.global_variables_initializer())
            sess.run(tf.local_variables_initializer())
            coord = tf.train.Coordinator()
            threads = tf.train.start_queue_runners(sess=sess, coord=coord)
            for i in xrange(FLAGS.num_iter):
                _, cur_loss, summary = sess.run([train_op, loss, summary_op],
                                                feed_dict={keep_prob: 0.5})
                writer.add_summary(summary, i)

                if i % 10 == 0:

                    batch_x = sess.run(images_val)
                    batch_y = sess.run(labels_val)
                    validation_accuracy = accuracy.eval(feed_dict={x: batch_x, y: batch_y, keep_prob: 1.0})
                    print('Iter {} Accuracy: {}'.format(i, validation_accuracy))
                    saver.save(sess, FLAGS.checkpoint_file_path, global_step)
                if i == FLAGS.num_iter:
                    coord.request_stop()
                    coord.join(threads)



def main(argv=None):
    train()


if __name__ == '__main__':
    tf.app.flags.DEFINE_integer('batch_size', 64, 'size of training batches')
    tf.app.flags.DEFINE_integer('num_iter', 4001, 'number of training iterations') #10000
    tf.app.flags.DEFINE_string('checkpoint_file_path', 'checkpoints/model.ckpt-10000', 'path to checkpoint file')
    tf.app.flags.DEFINE_string('train_data', 'data', 'path to train and test data')
    tf.app.flags.DEFINE_string('summary_dir', 'graphs', 'path to directory for storing summaries')

    tf.app.run()

Answer 1

I actually figured out why i get this error. In fact, some of the images i was sending were not sized 64*64. therefore they couldn't be reshaped as 1 64 64 3. I don't know why there was no error here and I only get it on the shuffle image.

I've put an image resize before the reshape and it's all good now!

Also, thanks jsimsa i'll note that for later!

Answer 2

Using the queue runner API for I/O is deprecated. Instead, I recommend using tf.data API. Here is a detailed example of an input function for AlexNet data that can be used along with an Estimator:

def input_fn(params):
  """Passes data to the estimator as required."""

  batch_size = params["batch_size"]

  def parser(serialized_example):
    """Parses a single tf.Example into a 224x224 image and label tensors."""

    final_image = None
    final_label = None
    if FLAGS.preprocessed:
      features = tf.parse_single_example(
          serialized_example,
          features={
              "image": tf.FixedLenFeature([], tf.string),
              "label": tf.FixedLenFeature([], tf.int64),
          })
      image = tf.decode_raw(features["image"], tf.float32)
      image.set_shape([224 * 224 * 3])
      final_label = tf.cast(features["label"], tf.int32)
    else:
      features = tf.parse_single_example(
          serialized_example,
          features={
              "image/encoded": tf.FixedLenFeature([], tf.string),
              "image/class/label": tf.FixedLenFeature([], tf.int64),
          })
      image = tf.image.decode_jpeg(features["image/encoded"], channels=3)
      image = tf.image.resize_images(
          image,
          size=[224, 224])
      final_label = tf.cast(features["image/class/label"], tf.int32)

    final_image = (tf.cast(image, tf.float32) * (1. / 255)) - 0.5

    return final_image, final_label

  file_pattern = os.path.join(FLAGS.data_dir, "train-*")
  dataset = tf.data.Dataset.list_files(file_pattern)

  if FLAGS.filename_shuffle_buffer_size > 0:
    dataset = dataset.shuffle(buffer_size=FLAGS.filename_shuffle_buffer_size)
  dataset = dataset.repeat()

  def prefetch_map_fn(filename):
    dataset = tf.data.TFRecordDataset(
        filename, buffer_size=FLAGS.dataset_reader_buffer_size)
    if FLAGS.prefetch_size is None:
      dataset = dataset.prefetch(batch_size)
    else:
      if FLAGS.prefetch_size > 0:
        dataset = dataset.prefetch(FLAGS.prefetch_size)
    return dataset

  if FLAGS.use_sloppy_interleave:
    dataset = dataset.apply(
        tf.contrib.data.sloppy_interleave(
            prefetch_map_fn, cycle_length=FLAGS.cycle_length))
  else:
    dataset = dataset.interleave(
        prefetch_map_fn, cycle_length=FLAGS.cycle_length)

  if FLAGS.element_shuffle_buffer_size > 0:
    dataset = dataset.shuffle(buffer_size=FLAGS.element_shuffle_buffer_size)

  dataset = dataset.map(
      parser,
      num_parallel_calls=FLAGS.num_parallel_calls).prefetch(batch_size)

  dataset = dataset.batch(batch_size)    
  dataset = dataset.prefetch(1)
  images, labels = dataset.make_one_shot_iterator().get_next()
  return (
      tf.reshape(images, [batch_size, 224, 224, 3]),
      tf.reshape(labels, [batch_size])
  )

You can learn more about the tf.data API in this programmer's guide.