tf.nn.in_top_k(logits,y,1) out of range error but equal actually

Question

I'm doing my first Neural Network with a binary classification, but I got an error when I try to evaluate the model with:

correct = tf.nn.in_top_k(logits,y,1)

where

• logits tensor is : predictions : shape [batch_size = 52, num_classes = 1], type float32
• y tensor is : targets: shape [batch_size=52], type int32

I got this error :

targets[1] is out of range
     [[{{node in_top_k/InTopKV2}}]]

---

After some debugging time , I understood that the values of my tensor y must be <= to num_classes, so the first value of the tensor y equal to 1 is considered as out of range, even tough the parameter num_classes = 1.

How can I allow my tensor values to be equal to num_classes and only strictly inferior ? Or is there another way ?

In my opinion, num_classes should equal 1 because it's a binary classification so 1 neuron output is needed.

---

EDIT Here's my full code :

import tensorflow as tf
n_inputs = 28 
n_hidden1 = 15
n_hidden2 = 5
n_outputs = 1
reset_graph()
X = tf.placeholder(tf.float32, shape=(None, n_inputs), name="X") 
y = tf.placeholder(tf.int32, shape=(None), name="y")   #None => any
def neuron_layer(X, n_neurons, name, activation=None):
    with tf.name_scope(name):
        n_inputs = int(X.shape[1])
        stddev = 2 / np.sqrt(n_inputs) 
        init = tf.truncated_normal((n_inputs, n_neurons), stddev=stddev) #matrice n_inputs x n_neurons values proche de 0    
        W = tf.Variable(init,name="kernel")  #weights random
        b = tf.Variable(tf.zeros([n_neurons]), name="bias")
        Z = tf.matmul(X, W) + b
        tf.cast(Z,tf.int32)
        if activation is not None:
            return activation(Z)
        else:
            return Z
def to_one_hot(y):
    n_classes = y.max() + 1
    m = len(y)
    Y_one_hot = np.zeros((m, n_classes))
    Y_one_hot[np.arange(m), y] = 1
    return Y_one_hot
hidden1 = neuron_layer(X, n_hidden1, name="hidden1",
                           activation=tf.nn.relu)
hidden2 = neuron_layer(hidden1, n_hidden2, name="hidden2",
                           activation=tf.nn.relu)
logits = neuron_layer(hidden2, n_outputs, name="outputs")
xentropy = tf.keras.backend.binary_crossentropy(tf.to_float(y),logits) 
loss = tf.reduce_mean(xentropy)
learning_rate = 0.01
optimizer = tf.train.GradientDescentOptimizer(learning_rate)
training_op = optimizer.minimize(loss)
correct = tf.nn.in_top_k(logits,y,1)
labels_max = tf.reduce_max(y)
accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))

init = tf.global_variables_initializer()
saver = tf.train.Saver()
n_epochs = 40
batch_size = 50
def shuffle_batch(X, y, batch_size):  #Homogeneisation et decoupage en paquets(n_batches)
    rnd_idx = np.random.permutation(len(X))
    n_batches = len(X) // batch_size
    for batch_idx in np.array_split(rnd_idx, n_batches):
        X_batch, y_batch = X[batch_idx], y[batch_idx]
        yield X_batch, y_batch


with tf.Session() as sess:
    init.run()
    X_temp,Y_temp = X_batch,y_batch
    feed_dict={X: X_batch, y: y_batch}
    print("feed",feed_dict)
    print("\n y_batch :",y_batch,y_batch.dtype)
    print("\n X_batch :",X_batch,X_batch.dtype,X_batch.shape)

    for epoch in range(n_epochs):
        for X_batch, y_batch in shuffle_batch(X_train, Y_train, batch_size):
            y_batch=y_batch.astype(np.int32)
            X_batch=X_batch.astype(np.float32)
            sess.run(training_op,feed_dict={X: X_batch, y: y_batch})
        #acc_batch = accuracy.eval(feed_dict={X: X_batch, y: y_batch})
        #acc_val = accuracy.eval(feed_dict={X: X_valid, y: y_valid})
        #print(epoch, "Batch accuracy:", acc_batch, "Val accuracy:", acc_val)
    save_path = saver.save(sess, "./my_model_final.ckpt")
    #some tests
    print("y eval :",y.eval(feed_dict={X:X_temp,y:Y_temp}).shape)

    y_one_hot=to_one_hot(y.eval(feed_dict={X:X_temp,y:Y_temp}))
    print("y_one_hot :",y_one_hot.shape)

    print("logits eval : ",logits.eval(feed_dict={X:X_temp,y:Y_temp}))
    #print(correct.eval(feed_dict={X:X_temp,y:Y_temp}))
    print(labels_max.eval(feed_dict={X:X_temp,y:Y_temp}))

Answer 1

As per the documentation here, tf.nn.in_top_k(predictions, targets, k) has arguments:

• predictions: A Tensor of type float32. A batch_size x classes tensor.
• targets: A Tensor. Must be one of the following types: int32, int64. A batch_size vector of class ids.
• k: An int. Number of top elements to look at for computing precision.

As you are performing binary classification, i.e., has two classes, so the shape of logits tensor in your case should be (52, 2) while the shape of y should be (52,). Here, logits is basically one-hot encoded tensor. This is the reason why your are getting above error.

Consider the below example:

Example 1:

res = tf.nn.in_top_k([[0,1], [1,0], [0,1], [1, 0], [0, 1]], [0, 1, 1, 1, 1], 1)

Here, shape of logits is (5, 2) while y is (5,). If you will do tf.reduce_max(y), you will get 1, which is less than number of classes and hence okay.

This will work fine and output [False False True False True]

Example 2:

res = tf.nn.in_top_k([[0,1], [1,0], [0,1], [1, 0], [0, 1]], [0, 2, 1, 1, 1], 1)

If you will do tf.reduce_max(y), you will get 2, which is equal to the number of classes. This will raises an error: InvalidArgumentError: targets[1] is out of range

EDIT: In your above code, make following modifications:

• change n_outputs = 1 to n_outputs = 2
• change sess.run(training_op,feed_dict={X: X_batch, y: y_batch}) to _, cost, acc = sess.run([training_op, loss, accuracy], feed_dict={X: X_batch, y: to_one_hot(y_batch)})
• change correct = tf.nn.in_top_k(logits, y, 1) to correct = tf.nn.in_top_k(logits, tf.argmax(y, 1), 1)

Code(random data used):

n_inputs = 28 
n_hidden1 = 15
n_hidden2 = 5
n_outputs = 2

X = tf.placeholder(tf.float32, shape=(None, n_inputs), name="X") 
y = tf.placeholder(tf.int32, shape=(None, 2), name="y")   #None => any

def neuron_layer(X, n_neurons, name, activation=None):
    with tf.name_scope(name):
        n_inputs = int(X.shape[1])
        stddev = 2 / np.sqrt(n_inputs) 
        init = tf.truncated_normal((n_inputs, n_neurons), stddev=stddev) #matrice n_inputs x n_neurons values proche de 0    
        W = tf.Variable(init,name="kernel")  #weights random
        b = tf.Variable(tf.zeros([n_neurons]), name="bias")
        Z = tf.matmul(X, W) + b
        tf.cast(Z,tf.int32)
        if activation is not None:
            return activation(Z)
        else:
            return Z

def to_one_hot(y):
    n_classes = y.max() + 1
    m = len(y)
    Y_one_hot = np.zeros((m, n_classes))
    Y_one_hot[np.arange(m), y] = 1
    return Y_one_hot

hidden1 = neuron_layer(X, n_hidden1, name="hidden1",
                           activation=tf.nn.relu)
hidden2 = neuron_layer(hidden1, n_hidden2, name="hidden2",
                           activation=tf.nn.relu)
logits = neuron_layer(hidden2, n_outputs, name="outputs")
xentropy = tf.keras.backend.binary_crossentropy(tf.to_float(y),logits) 
loss = tf.reduce_mean(xentropy)
learning_rate = 0.01
optimizer = tf.train.GradientDescentOptimizer(learning_rate)
training_op = optimizer.minimize(loss)
correct = tf.nn.in_top_k(logits,tf.argmax(y, 1),1)
accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))

init = tf.global_variables_initializer()
saver = tf.train.Saver()
n_epochs = 1

X_train = np.random.rand(100, 28)
X_train = X_train.astype(np.float32)

Y_train = np.random.randint(low = 0, high = 2, size = 100, dtype=np.int32) 

with tf.Session() as sess:
    init.run()
    for epoch in range(n_epochs):
        _, cost, corr, acc = sess.run([training_op, loss, correct, accuracy], feed_dict={X: X_train, y: to_one_hot(Y_train)})
        print(corr)
        print('Loss: {} Accuracy: {}'.format(cost, acc))