ValueError: Classification metrics can't handle a mix of multilabel-indicator and binary targets

Question

I want to apply KerasCLassifier to solve multi-class classification problem. The value of y is one-hot-encoded, e.g.:

0 1 0
1 0 0
1 0 0

This is my code:

from keras.models import Sequential
from keras.layers import Dense
from keras.wrappers.scikit_learn import KerasClassifier

# Function to create model, required for KerasClassifier
def create_model(optimizer='rmsprop', init='glorot_uniform'):
    # create model
    model = Sequential()
    model.add(Dense(2048, input_dim=X_train.shape[1], kernel_initializer=init, activation='relu'))
    model.add(Dense(512, kernel_initializer=init, activation='relu'))
    model.add(Dense(y_train_onehot.shape[1], kernel_initializer=init, activation='softmax'))
    # Compile model
    model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['accuracy'])
    return model

# create model
model = KerasClassifier(build_fn=create_model, class_weight="balanced", verbose=2)

# grid search epochs, batch size and optimizer
optimizers = ['rmsprop', 'adam']
epochs = [10, 50]
batches = [5, 10, 20]
init = ['glorot_uniform', 'normal', 'uniform']

param_grid = dict(optimizer=optimizers, epochs=epochs, batch_size=batches, init=init)
grid = model_selection.GridSearchCV(estimator=model, param_grid=param_grid, scoring='accuracy')

grid_result = grid.fit(X_train], y_train_onehot)

When I run the last line of code, it throws the following error after 10 epochs:

/opt/conda/lib/python3.6/site-packages/sklearn/metrics/classification.py in accuracy_score(y_true, y_pred, normalize, sample_weight) 174 175 # Compute accuracy for each possible representation --> 176 y_type, y_true, y_pred = _check_targets(y_true, y_pred) 177 check_consistent_length(y_true, y_pred, sample_weight) 178 if y_type.startswith('multilabel'):

/opt/conda/lib/python3.6/site-packages/sklearn/metrics/classification.py in _check_targets(y_true, y_pred) 79 if len(y_type) > 1: 80 raise ValueError("Classification metrics can't handle a mix of {0} " ---> 81 "and {1} targets".format(type_true, type_pred)) 82 83 # We can't have more than one value on y_type => The set is no more needed

ValueError: Classification metrics can't handle a mix of multilabel-indicator and binary targets

When I write categorical_accuracy or balanced_accuracy instead of accuracy, I cannot compile a model.

Answer 1

Here is a working demo:

import numpy as np
from sklearn.model_selection import GridSearchCV
from keras.models import Sequential
from keras.layers import Dense
from keras.wrappers.scikit_learn import KerasClassifier

N = 100
X_train = np.random.rand(N, 4)
Y_train = np.random.choice([0,1,2], N, p=[.5, .3, .2])

# Function to create model, required for KerasClassifier
def create_model(optimizer='rmsprop', init='glorot_uniform'):
    # create model
    model = Sequential()
    model.add(Dense(2048, input_dim=X_train.shape[1], kernel_initializer=init, activation='relu'))
    model.add(Dense(512, kernel_initializer=init, activation='relu'))
    model.add(Dense(len(np.unique(Y_train)), kernel_initializer=init, activation='softmax'))
    # Compile model
    model.compile(loss='sparse_categorical_crossentropy', optimizer=optimizer, metrics=['sparse_categorical_accuracy'])
    return model

# create model
model = KerasClassifier(build_fn=create_model, class_weight="balanced", verbose=2)

# grid search epochs, batch size and optimizer
optimizers = ['rmsprop', 'adam']
epochs = [10, 50]
batches = [5, 10, 20]
init = ['glorot_uniform', 'normal', 'uniform']

param_grid = dict(optimizer=optimizers, epochs=epochs, batch_size=batches, init=init)
grid = GridSearchCV(estimator=model, param_grid=param_grid, scoring='accuracy')

grid_result = grid.fit(X_train, Y_train)

PS please pay attention at the usage of sparse_categorical_* loss function and metrics.