Evaluating pairwise distances between the output of a tf.keras.model

Question

I am trying to create a custom loss function in tensorflow. I am using tensorflow v2.0.rc0 for running the code. Following is the code and the function min_dist_loss computes the pairwise loss between the output of the neural network. Here's the code

def min_dist_loss(_, y_pred):
    distances = []
    for i in range(0, 16):
        for j in range(i + 1, 16):
            distances.append(tf.linalg.norm(y_pred[i] - y_pred[j]))
    return -tf.reduce_min(distances)

and the module is being initialized and compiled as follows

import tensorflow as tf
from tensorboard.plugins.hparams import api as hp

HP_NUM_UNITS = hp.HParam('num_units', hp.Discrete([6, 7]))
HP_OPTIMIZER = hp.HParam('optimizer', hp.Discrete(['adam', 'sgd']))

METRIC_ACCURACY = 'accuracy'

with tf.summary.create_file_writer('logs\hparam_tuning').as_default():
    hp.hparams_config(
        hparams=[HP_NUM_UNITS, HP_OPTIMIZER],
        metrics=[hp.Metric(METRIC_ACCURACY, display_name='Accuracy')]
    )

def train_test_model(logdir, hparams):
    weight1 = np.random.normal(loc=0.0, scale=0.01, size=[4, hparams[HP_NUM_UNITS]])
    init1 = tf.constant_initializer(weight1)
    weight2 = np.random.normal(loc=0.0, scale=0.01, size=[hparams[HP_NUM_UNITS], 7])
    init2 = tf.constant_initializer(weight2)
    model = tf.keras.models.Sequential([
        # tf.keras.layers.Flatten(),
        tf.keras.layers.Dense(hparams[HP_NUM_UNITS], activation=tf.nn.sigmoid, kernel_initializer=init1),
        tf.keras.layers.Dense(7, activation=tf.nn.sigmoid, kernel_initializer=init2) if hparams[HP_NUM_UNITS] == 6 else
        None,
    ])
    model.compile(
        optimizer=hparams[HP_OPTIMIZER],
        loss=min_dist_loss,
        # metrics=['accuracy'],
    )
    x_train = [list(k) for k in itertools.product([0, 1], repeat=4)]
    shuffle(x_train)
    x_train = 2 * np.array(x_train) - 1
    model.fit(
        x_train, epochs=1, batch_size=16,
        callbacks=[
            tf.keras.callbacks.TensorBoard(logdir),
            hp.KerasCallback(logdir, hparams)
        ],
    )

Now since the tensor object y_pred in the min_dist_loss is an object of shape [?, 7], indexing with i is throwing the following error:

Traceback (most recent call last):
  File "/home/pc/Documents/user/code/keras_tensorflow/src/try1.py", line 95, in <module>
    run('logs\hparam_tuning' + run_name, hparams)
  File "/home/pc/Documents/user/code/keras_tensorflow/src/try1.py", line 78, in run
    accuracy = train_test_model(run_dir, hparams)
  File "/home/pc/Documents/user/code/keras_tensorflow/src/try1.py", line 66, in train_test_model
    hp.KerasCallback(logdir, hparams)
  File "/home/pc/Documents/user/code/keras_tensorflow/venv/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/training.py", line 734, in fit
    use_multiprocessing=use_multiprocessing)
  File "/home/pc/Documents/user/code/keras_tensorflow/venv/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/training_v2.py", line 324, in fit
    total_epochs=epochs)
  File "/home/pc/Documents/user/code/keras_tensorflow/venv/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/training_v2.py", line 123, in run_one_epoch
    batch_outs = execution_function(iterator)
  File "/home/pc/Documents/user/code/keras_tensorflow/venv/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/training_v2_utils.py", line 86, in execution_function
    distributed_function(input_fn))
  File "/home/pc/Documents/user/code/keras_tensorflow/venv/lib/python3.6/site-packages/tensorflow_core/python/eager/def_function.py", line 427, in __call__
    self._initialize(args, kwds, add_initializers_to=initializer_map)
  File "/home/pc/Documents/user/code/keras_tensorflow/venv/lib/python3.6/site-packages/tensorflow_core/python/eager/def_function.py", line 370, in _initialize
    *args, **kwds))
  File "/home/pc/Documents/user/code/keras_tensorflow/venv/lib/python3.6/site-packages/tensorflow_core/python/eager/function.py", line 1847, in _get_concrete_function_internal_garbage_collected
    graph_function, _, _ = self._maybe_define_function(args, kwargs)
  File "/home/pc/Documents/user/code/keras_tensorflow/venv/lib/python3.6/site-packages/tensorflow_core/python/eager/function.py", line 2147, in _maybe_define_function
    graph_function = self._create_graph_function(args, kwargs)
  File "/home/pc/Documents/user/code/keras_tensorflow/venv/lib/python3.6/site-packages/tensorflow_core/python/eager/function.py", line 2038, in _create_graph_function
    capture_by_value=self._capture_by_value),
  File "/home/pc/Documents/user/code/keras_tensorflow/venv/lib/python3.6/site-packages/tensorflow_core/python/framework/func_graph.py", line 915, in func_graph_from_py_func
    func_outputs = python_func(*func_args, **func_kwargs)
  File "/home/pc/Documents/user/code/keras_tensorflow/venv/lib/python3.6/site-packages/tensorflow_core/python/eager/def_function.py", line 320, in wrapped_fn
    return weak_wrapped_fn().__wrapped__(*args, **kwds)
  File "/home/pc/Documents/user/code/keras_tensorflow/venv/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/training_v2_utils.py", line 73, in distributed_function
    per_replica_function, args=(model, x, y, sample_weights))
  File "/home/pc/Documents/user/code/keras_tensorflow/venv/lib/python3.6/site-packages/tensorflow_core/python/distribute/distribute_lib.py", line 760, in experimental_run_v2
    return self._extended.call_for_each_replica(fn, args=args, kwargs=kwargs)
  File "/home/pc/Documents/user/code/keras_tensorflow/venv/lib/python3.6/site-packages/tensorflow_core/python/distribute/distribute_lib.py", line 1787, in call_for_each_replica
    return self._call_for_each_replica(fn, args, kwargs)
  File "/home/pc/Documents/user/code/keras_tensorflow/venv/lib/python3.6/site-packages/tensorflow_core/python/distribute/distribute_lib.py", line 2132, in _call_for_each_replica
    return fn(*args, **kwargs)
  File "/home/pc/Documents/user/code/keras_tensorflow/venv/lib/python3.6/site-packages/tensorflow_core/python/autograph/impl/api.py", line 292, in wrapper
    return func(*args, **kwargs)
  File "/home/pc/Documents/user/code/keras_tensorflow/venv/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/training_v2_utils.py", line 264, in train_on_batch
    output_loss_metrics=model._output_loss_metrics)
  File "/home/pc/Documents/user/code/keras_tensorflow/venv/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/training_eager.py", line 311, in train_on_batch
    output_loss_metrics=output_loss_metrics))
  File "/home/pc/Documents/user/code/keras_tensorflow/venv/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/training_eager.py", line 252, in _process_single_batch
    training=training))
  File "/home/pc/Documents/user/code/keras_tensorflow/venv/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/training_eager.py", line 166, in _model_loss
    per_sample_losses = loss_fn.call(targets[i], outs[i])
IndexError: list index out of range

---

How do I compute the minimum distance in this setting? Any help is appreciated. Also, if there are any errors in other parts of the code, please feel free to point it out. I am new to using keras on tensorflow.

Answer 1

Keras is expecting you to provide the true labels as well. Since you're defining your own loss function and you're not using the true labels, you can pass some garbage labels. Eg: np.arange(16).

Change your model.fit as below and it should work

model.fit(
    x_train, np.arange(x_train.shape[0]), epochs=1, batch_size=16,
    callbacks=[
        tf.keras.callbacks.TensorBoard(logdir),
        hp.KerasCallback(logdir, hparams)
    ],
)