Custom f1_score metric in tensorflow

Question

I want to implement the f1_score metric for tf.keras.

from tensorflow.keras.models import Model, Sequential
from tensorflow.keras.layers import Dense
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.losses import binary_crossentropy
from tensorflow.keras.metrics import Accuracy, BinaryAccuracy
from sklearn.metrics import accuracy_score
import numpy as np
import tensorflow as tf

class F1_Score(tf.keras.metrics.Metric):

    def __init__(self, name='f1_score', **kwargs):
        super().__init__(name=name, **kwargs)
        self.f1 = self.add_weight(name='f1', initializer='zeros')

    def update_state(self, y_true, y_pred, sample_weight=None):
        p = Precision(thresholds=0.5)(y_true, y_pred)
        r = Recall(thresholds=0.5)(y_true, y_pred)
        self.f1 = 2 * ((p * r) / (p + r + 1e-6))

    def result(self):
        return self.f1

    def reset_states(self):
        self.f1.assign(0)
        
model = Sequential([
  Dense(64, activation='relu', input_shape=(784,)),
  Dense(64, activation='relu'),
  Dense(4, activation='sigmoid'),
])
x = np.random.normal(size=(10, 784))
y = np.random.choice(2, size=(10, 4))
model.compile(optimizer=Adam(0.001), loss='binary_crossentropy',
                  metrics=['accuracy', , F1_Score()])
model.fit(x[:1], y[:1], batch_size=1, epochs=1, verbose=1)

I got an error:

ValueError: tf.function-decorated function tried to create variables on non-first call.

Answer 1

You can use tensorflow-addons which has a built-in method for F1-Score. (don't forget to pip install tensorflow-addons)

Have a look below:

  model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=0.00001),
                  loss=tf.keras.losses.CategoricalCrossentropy(),
                  metrics=[tf.keras.metrics.CategoricalAccuracy(),
                           tfa.metrics.F1Score(num_classes=n_classes, average='macro'),
                           tfa.metrics.FBetaScore(beta=2.0, num_classes=n_classes, average='macro')])

If you do have a multi-label classification problem, you can change it to:

  model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=0.00001),
                      loss=tf.keras.losses.BinaryCrossentropy(),
                      metrics=[tf.keras.metrics.BinaryAccuracy(),
                               tfa.metrics.F1Score(num_classes=1, average='macro',threshold=0.5),
                               tfa.metrics.FBetaScore(beta=2.0, num_classes=1, average='macro',threshold=0.5)])

Answer 2

This is my code scoring f1 for Tensorflow 2.0:

class F1Score(tf.keras.metrics.Metric):
  def __init__(self, name='F1Score', **kwargs):
    super(F1Score, self).__init__(name=name, **kwargs)
    self.f1score = self.add_weight(name='F1Score', initializer='zeros')
    self.count = self.add_weight(name='F1ScoreCount', initializer='zeros')

  def update_state(self, y_true, y_pred, sample_weight=None):
    y_true = tf.cast(y_true, tf.bool)
    y_pred = tf.cast(y_pred, tf.bool)

    true_positives = tf.logical_and(tf.equal(y_true, True), tf.equal(y_pred, True))
    true_positives = tf.cast(true_positives, self.dtype)
    count_true_positives = tf.reduce_sum(true_positives)

    possible_positives = tf.cast(y_true, self.dtype)
    count_possible_positives = tf.reduce_sum(possible_positives)

    predicted_positives = tf.cast(y_pred, self.dtype)
    count_predicted_positives = tf.reduce_sum(predicted_positives)

    precision = count_true_positives / (count_predicted_positives + K.epsilon())
    recall = count_true_positives / (count_possible_positives + K.epsilon())
    f1_cal = 2*(precision*recall)/(precision + recall + K.epsilon())

    self.count.assign_add(1)
    a = 1.0 / self.count
    b = 1.0 - a
    self.f1score.assign(a*f1_cal+b*self.f1score)

  def result(self):
    return self.f1score

Answer 3

You get this error because you want to instantiate some tf.Variables during the update_state function. When instantiate object from the class Precision and Recall, you are creating some tf.Variables.

Instantiate the objects in the constructor, and call them in the update_state function:

class F1_Score(tf.keras.metrics.Metric):

    def __init__(self, name='f1_score', **kwargs):
        super().__init__(name=name, **kwargs)
        self.f1 = self.add_weight(name='f1', initializer='zeros')
        self.precision_fn = Precision(thresholds=0.5)
        self.recall_fn = Recall(thresholds=0.5)

    def update_state(self, y_true, y_pred, sample_weight=None):
        p = self.precision_fn(y_true, y_pred)
        r = self.recall_fn(y_true, y_pred)
        # since f1 is a variable, we use assign
        self.f1.assign(2 * ((p * r) / (p + r + 1e-6)))

    def result(self):
        return self.f1

    def reset_states(self):
        # we also need to reset the state of the precision and recall objects
        self.precision_fn.reset_states()
        self.recall_fn.reset_states()
        self.f1.assign(0)

---

Explanation of the behavior :

Tensorflow allow to create Variable only on the first call of a tf.function, see the documentation :

tf.function only allows creating new tf.Variable objects when it is called for the first time

Keras metrics are wrapped in a tf.function to allow compatibility with tensorflow v1. You can find this comment in the code

If update_state is not in eager/tf.function and it is not from a built-in metric, wrap it in tf.function. This is so that users writing custom metrics in v1 need not worry about control dependencies and return ops.

You also have another bug in your class, is that you override the f1 tf.Variable that you created with the calculation of your f1 score. To update the value of a variable, you need to use assign. And we must not forget to reset the states of the Precision and Recall Metrics objects in use!