How do you run a validation loop during `slim.learning.train()`

Question

I'm looking at this answer for running evaluation metrics during training:

How to use evaluation_loop with train_loop in tf-slim

and it seems like overriding train_step_fn=train_step_fn is the reasonable approach. But I want to run a validation loop, not evaluation. My graph is something like this:

with tf.Graph().as_default():

    train_dataset = slim.dataset.Dataset(data_sources= "train_*.tfrecord")
    train_images, _, train_labels = load_batch(train_dataset, 
                batch_size=mini_batch_size,
                is_training=True)

    val_dataset = slim.dataset.Dataset(data_sources= "validation_*.tfrecord")
    val_images, _, val_labels = load_batch(val_dataset, 
                batch_size=mini_batch_size,
                is_training=False)


    with slim.arg_scope(vgg.vgg_arg_scope(weight_decay=0.0005)):
        net, end_points = vgg.vgg_16(train_images, 
                                      num_classes=10,
                                      is_training=is_training)
    predictions = tf.nn.softmax(net)
    labels = train_labels

    ...

    init_fn = slim.assign_from_checkpoint_fn(
        checkpoint_path,
        slim.get_variables_to_restore(exclude=['vgg_16/fc8']),
        ignore_missing_vars=True
        )     

    final_loss = slim.learning.train(train_op, TRAIN_LOG, 
                        train_step_fn=train_step_fn,
                        init_fn=init_fn,
                        global_step=global_step,
                        number_of_steps=steps,
                        save_summaries_secs=60,
                        save_interval_secs=600,
                        session_config=sess_config,
                      )

I want to add something like this to do a validation loop with a mini-batch against the current weights for the network

    def validate_on_checkpoint(sess, *args, **kwargs ):
        loss,mean,stddev = sess.run([val_loss, val_rms_mean, val_rms_stddev], 
                        feed_dict={images: val_images, 
                                   labels: val_labels, 
                                   is_training: is_training })
        validation_writer = tf.train.SummaryWriter(LOG_DIR + '/validation')                                              
        validation_writer.add_summary(loss, global_step)
        validation_writer.add_summary(mean, global_step)
        validation_writer.add_summary(stddev, global_step)


    def train_step_fn(sess, *args, **kwargs):
        total_loss, should_stop = train_step(sess, *args, **kwargs)

        if train_step_fn.step % FLAGS.validation_every_n_step == 0:
            validate_on_checkpoint(sess, *args, **kwargs )

        train_step_fn.step += 1
        return [total_loss, should_stop]   

but I got an error=Graph is finalized and cannot be modified.

Conceptually I'm not sure how I should add this. The training loop needs to needs gradients, dropouts, and weight updates for the net, but the validation loop skips all of that. I keep getting variations on Graph is finalized and cannot be modified. if I try to modify the Graph or XXX is not defined if I use an if is_training: else: approach

Answer 1

I figured out one way to make this work from a few other stackoverflow answers. Here are the basics:

1) get inputs and labels for both train and validation datasets

x_train, y_train = produce_batch(320)
x_validation, y_validation = produce_batch(320)

2) use reuse=True to reuse model weights between the train and validation loop. Here's one way:

  with tf.variable_scope("model") as scope:
    # Make the model, reuse weights for validation batches
    predictions, nodes = regression_model(inputs, is_training=True)
    scope.reuse_variables()
    val_predictions, _ = regression_model(val_inputs, is_training=False)

3) define your losses, put your validation losses in a different collection so it does not get added to train losses in tf.losses.get_losses()

  loss = tf.losses.mean_squared_error(labels=targets, predictions=predictions)
  total_loss = tf.losses.get_total_loss()

  val_loss = tf.losses.mean_squared_error(labels=val_targets, predictions=val_predictions,
                                          loss_collection="validation"
                                         )

4) define a train_step_fn() to trigger the validation loop, as necessary

VALIDATION_INTERVAL = 1000 . # validate every 1000 steps
# slim.learning.train(train_step_fn=train_step_fn)
def train_step_fn(sess, train_op, global_step, train_step_kwargs):
  """
  slim.learning.train_step():
    train_step_kwargs = {summary_writer:, should_log:, should_stop:}
  """
  train_step_fn.step += 1  # or use global_step.eval(session=sess)

  # calc training losses
  total_loss, should_stop = slim.learning.train_step(sess, train_op, global_step, train_step_kwargs)


  # validate on interval
  if train_step_fn.step % VALIDATION_INTERVAL == 0:
    validiate_loss, validation_delta = sess.run([val_loss, summary_validation_delta])
    print(">> global step {}:    train={}   validation={}  delta={}".format(train_step_fn.step, 
                        total_loss, validiate_loss, validiate_loss-total_loss))


  return [total_loss, should_stop]
train_step_fn.step = 0

5) add train_step_fn() to your training loop

  # Run the training inside a session.
  final_loss = slim.learning.train(
      train_op,
      train_step_fn=train_step_fn,
      ...
      )

see the complete results in this Colaboratory notebook