tf.losses.log_loss for multiply classes

Question

I have tested tf.losses.log_loss with binary classification problem comparing with numpy implementation and sklearn implementation, it looks good:

# Array of samples
y_pred_arr = np.array([0.6,0.3,0.9], np.float32)
y_true_arr = np.array([1,0,1], np.float32)

print('y_pred_arr.shape', y_pred_arr.shape)
print('y_true_arr.shape', y_true_arr.shape)

# 1 : Implemented in numpy
def cross_entropy_loss(y_pred_arr, y_true_arr, eps=1e-6):
    p_arr = np.clip(y_pred_arr, eps, 1 - eps)
    n_samples = p_arr.shape[0]
    loss_arr = np.zeros((n_samples,), np.float32)
    for i in range(n_samples):        
        if y_true_arr[i] == 1:
            loss_arr[i] = -np.log(p_arr[i])
        else:
            loss_arr[i] = -np.log(1 - p_arr[i])
    loss = np.mean(loss_arr)
    return loss
loss = cross_entropy_loss(y_pred_arr, y_true_arr)
print(loss)

# 2 : Implemented in sklearn
from sklearn.metrics import log_loss
loss = log_loss(y_true_arr, y_pred_arr, labels=[0,1], eps=1e-6)
print(loss)

# 3 : Impemented in tensorflow using tf.losses.log_loss
y_pred_tf = tf.convert_to_tensor(y_pred_arr, np.float32)
y_true_tf = tf.convert_to_tensor(y_true_arr, np.float32)
loss_tf = tf.losses.log_loss(labels=y_true_tf, predictions=y_pred_tf, epsilon=1e-6)
with tf.Session() as sess:
    loss = sess.run(loss_tf)
    print(loss)

Output:

y_pred_arr.shape (3,)
y_true_arr.shape (3,)
0.32428703
0.3242870296041171
0.32428563

But for multiclass problem results are different:

# Array of samples
y_pred_arr = np.array([[0.4,0.3,0.3], [1.0,0.0,0.0], [0.0,0.0,1.0]], np.float32)
y_true_arr = np.array([[1,0,0], [1,0,0], [0,0,1]], np.float32)

print('y_pred_arr.shape', y_pred_arr.shape)
print('y_true_arr.shape', y_true_arr.shape)

# 1 : Implemented in numpy
def cross_entropy_loss(y_pred_arr, y_true_arr, eps=1e-6):
    p_arr = np.clip(y_pred_arr, eps, 1 - eps)
    n_samples = p_arr.shape[0]
    n_classes = p_arr.shape[1]
    loss_arr = np.zeros((n_samples,), np.float32)
    for i in range(n_samples):
        for c in range(n_classes):
            loss_arr[i] += -y_true_arr[i][c] * np.log(p_arr[i][c])
    loss = np.mean(loss_arr)
    return loss
loss = cross_entropy_loss(y_pred_arr, y_true_arr)
print(loss)

# 2 : Implemented in sklearn
from sklearn.metrics import log_loss
loss = log_loss(np.argmax(y_true_arr,axis=1), y_pred_arr, labels=[0,1,2], eps=1e-6)
print(loss)

# 3 : Impemented in tensorflow using tf.losses.log_loss
y_pred_tf = tf.convert_to_tensor(y_pred_arr, np.float32)
y_true_tf = tf.convert_to_tensor(y_true_arr, np.float32)
loss_tf = tf.losses.log_loss(labels=y_true_tf, predictions=y_pred_tf, epsilon=1e-6)
with tf.Session() as sess:
    loss = sess.run(loss_tf)
    print(loss)

Output:

y_pred_arr.shape (3, 3)
y_true_arr.shape (3, 3)
0.30543092
0.30543154478209544
0.18106996

Answer 1

Like @functor mentioned tf.losses.log_loss is for binary crossentropy, you can check source code here: https://github.com/tensorflow/tensorflow/blob/r1.1/tensorflow/python/ops/losses/losses_impl.py#L309

Also https://github.com/tensorflow/tensorflow/issues/2462

Here is example:

# Array of samples
y_pred_arr = np.array([[0.4,0.3,0.3], [1.0,0.0,0.0], [0.0,0.0,1.0]], np.float32)
y_true_arr = np.array([[1,0,0], [1,0,0], [0,0,1]], np.float32)

print('y_pred_arr.shape', y_pred_arr.shape)
print('y_true_arr.shape', y_true_arr.shape)

# 1 : Implemented in numpy
def cross_entropy_loss(y_pred_arr, y_true_arr, eps=1e-6):
    p_arr = np.clip(y_pred_arr, eps, 1 - eps)
    n_samples = p_arr.shape[0]
    n_classes = p_arr.shape[1]
    loss_arr = np.zeros((n_samples,), np.float32)
    for i in range(n_samples):
        for c in range(n_classes):
            loss_arr[i] += -y_true_arr[i][c] * np.log(p_arr[i][c])
    loss = np.mean(loss_arr)
    return loss
loss = cross_entropy_loss(y_pred_arr, y_true_arr)
print(loss)

# 2 : Implemented in sklearn
from sklearn.metrics import log_loss
loss = log_loss(np.argmax(y_true_arr,axis=1), y_pred_arr, labels=[0,1,2], eps=1e-6)
print(loss)

# 3 : Implemented in tensorflow
y_pred_tf = tf.convert_to_tensor(y_pred_arr, np.float32)
y_true_tf = tf.convert_to_tensor(y_true_arr, np.float32)
eps = 1e-6
cliped_y_pref_tf = tf.clip_by_value(y_pred_tf, eps, 1-eps)
loss_tf = tf.reduce_mean(-tf.reduce_sum(y_true_tf * tf.log(cliped_y_pref_tf), axis=1))
with tf.Session() as sess:
    loss = sess.run(loss_tf)
    print(loss)

Output:

y_pred_arr.shape (3, 3)
y_true_arr.shape (3, 3)
0.30543092
0.30543154478209544
0.30543092

Answer 2

tf.losses.log_loss only works for binary classes. For multiclass loss, you need to compute manually,

like:

cliped_y_pref_tf = tf.clip_by_value(y_pred_tf, eps, 1-eps)
loss_tf = tf.reduce_sum(tf.multiply(-y_true_tf,tf.log(cliped_y_pref_tf )))/len(y_pred_arr)