Error using GridSearchCV but not without GridSearchCV - Python 3.6.7

Question

I am experiencing a weird bug where my code fails when using GridSearchCV, but not when just running sklearnMLPRegressor on its own.

The following code:

from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
from sklearn.neural_network import MLPRegressor
from sklearn.model_selection import train_test_split
from sklearn.neural_network import MLPRegressor
from sklearn import preprocessing
import pandas as pd
import numpy as np

def str_to_num(arr):
    le = preprocessing.LabelEncoder()
    new_arr = le.fit_transform(arr)
    return new_arr

def compare_values(arr1, arr2):
    thediff = 0
    thediffs = []
    for thing1, thing2 in zip(arr1, arr2):
        thediff = abs(thing1 - thing2)
        thediffs.append(thediff)

    return thediffs

def print_to_file(filepath, arr):
    with open(filepath, 'w') as f:
        for item in arr:
            f.write("%s\n" % item)

data = pd.read_csv('data2.csv')

# create the labels, or field we are trying to estimate
label = data['TOTAL']
# remove the header
label = label[1:]

# create the data, or the data that is to be estimated
data = data.drop('TOTAL', axis=1)
data = data.drop('SERIALNUM', axis=1)
# remove the header
data = data[1:]

# # split into training and testing sets
X_train, X_test, y_train, y_test = train_test_split(data, label, test_size = 0.2)

mlp = MLPRegressor(activation = 'relu', solver = 'lbfgs', verbose=False)
mlp.fit(X_train, y_train)
mlp_predictions = mlp.predict(X_test)
mlp_differences = compare_values(y_test, mlp_predictions)
mlp_Avg = np.average(mlp_differences)
print(mlp_Avg)

Prints the following:

32.92041129078561 (Yes I know that average error is bad)

However, when trying to optimize parameters, this same parameter setting yields an error:

from sklearn.neural_network import MLPRegressor
from sklearn.model_selection import train_test_split, GridSearchCV
from sklearn.neural_network import MLPRegressor
from sklearn import preprocessing
import pandas as pd
import numpy as np


def str_to_num(arr):
    le = preprocessing.LabelEncoder()
    new_arr = le.fit_transform(arr)
    return new_arr

def compare_values(arr1, arr2):
    thediff = 0
    thediffs = []
    for thing1, thing2 in zip(arr1, arr2):
        thediff = abs(thing1 - thing2)
        thediffs.append(thediff)

    return thediffs

def print_to_file(filepath, arr):
    with open(filepath, 'w') as f:
        for item in arr:
            f.write("%s\n" % item)

data = pd.read_csv('data2.csv')

# create the labels, or field we are trying to estimate
label = data['TOTAL_DAYS_TO_COMPLETE']
# remove the header
label = label[1:]

# create the data, or the data that is to be estimated
data = data.drop('TOTAL_DAYS_TO_COMPLETE', axis=1)
data = data.drop('SERIALNUM', axis=1)
# remove the header
data = data[1:]

# # split into training and testing sets
X_train, X_test, y_train, y_test = train_test_split(data, label, test_size = 0.2)

param_grid = {
    #'hidden_layer_sizes': [(1,),(2,),(3,),(10,),(15,),(20,),(25,)],
    'activation': ['identity', 'logistic', 'relu'],
    #'activation': ['relu'],
    'solver': ['lbfgs', 'sgd', 'adam'],
    #'solver': ['adam']
    #'alpha': [0.0001, 0.0005, 0.0009],
    #'learning_rate': ['constant', 'invscaling', 'adaptive'],
    #'learning_rate_init': [0.001, 0.01, 0.99],
    #'warm_start': [True, False]
    #'momentum': [0.1, 0.9, 0.99]
    # Did not solver-specifics...yet
}# Create a based model

mlp = MLPRegressor()# Instantiate the grid search model
grid_search = GridSearchCV(estimator = mlp, param_grid = param_grid, 
                          cv = 3, n_jobs = -1, verbose = 2)
grid_search.fit(X_train, y_train)
print()
print(grid_search.best_params_)
print(grid_search.best_score_)
print()
print("Grid scores on development set: ")
print()
answers = grid_search.predict(X_test)
results = compare_values(answers, y_test)
print("Accuracy: ", np.average(results))
print()

Yields the following:

Fitting 3 folds for each of 9 candidates, totalling 27 fits [Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers. [CV] activation=identity, solver=lbfgs ............................... [CV] activation=identity, solver=lbfgs ............................... [CV] activation=identity, solver=sgd ................................. C:\Python367-64\lib\site-packages\sklearn\neural_network_base.py:195: RuntimeWarning: overflow encountered in square return ((y_true - y_pred) ** 2).mean() / 2 [CV] activation=identity, solver=adam ................................ [CV] activation=identity, solver=lbfgs ............................... [CV] activation=identity, solver=sgd ................................. [CV] activation=identity, solver=sgd .................................

< removed extra lines that were working fine >

!!! Here is where it starts to fail [CV] !!!!

.................... activation=relu, solver=lbfgs, total= 0.5s

joblib.externals.loky.process_executor._RemoteTraceback: """ Traceback (most recent call last): File "C:\Python367-64\lib\site-packages\joblib\externals\loky\process_executor.py", line 418, in _process_worker r = call_item() File "C:\Python367-64\lib\site-packages\joblib\externals\loky\process_executor.py", line 272, in call return self.fn(*self.args, **self.kwargs) File "C:\Python367-64\lib\site-packages\joblib_parallel_backends.py", line 567, in call return self.func(*args, **kwargs) File "C:\Python367-64\lib\site-packages\joblib\parallel.py", line 225, in call for func, args, kwargs in self.items] File "C:\Python367-64\lib\site-packages\joblib\parallel.py", line 225, in for func, args, kwargs in self.items] File "C:\Python367-64\lib\site-packages\sklearn\model_selection_validation.py", line 554, in _fit_and_score test_scores = _score(estimator, X_test, y_test, scorer, is_multimetric) File "C:\Python367-64\lib\site-packages\sklearn\model_selection_validation.py", line 597, in _score return _multimetric_score(estimator, X_test, y_test, scorer) File "C:\Python367-64\lib\site-packages\sklearn\model_selection_validation.py", line 627, in _multimetric_score score = scorer(estimator, X_test, y_test) File "C:\Python367-64\lib\site-packages\sklearn\metrics\scorer.py", line 240, in _passthrough_scorer return estimator.score(*args, **kwargs) File "C:\Python367-64\lib\site-packages\sklearn\base.py", line 410, in score y_type, _, _, _ = _check_reg_targets(y, y_pred, None) File "C:\Python367-64\lib\site-packages\sklearn\metrics\regression.py", line 79, in _check_reg_targets y_pred = check_array(y_pred, ensure_2d=False) File "C:\Python367-64\lib\site-packages\sklearn\utils\validation.py", line 542, in check_array allow_nan=force_all_finite == 'allow-nan') File "C:\Python367-64\lib\site-packages\sklearn\utils\validation.py", line 56, in _assert_all_finite raise ValueError(msg_err.format(type_err, X.dtype)) ValueError: Input contains NaN, infinity or a value too large for dtype('float64'). """

The above exception was the direct cause of the following exception:

Traceback (most recent call last): File "mlp_optimizer.py", line 93, in grid_search.fit(X_train, y_train) File "C:\Python367-64\lib\site-packages\sklearn\model_selection_search.py", line 687, in fit self._run_search(evaluate_candidates) File "C:\Python367-64\lib\site-packages\sklearn\model_selection_search.py", line 1148, in _run_search evaluate_candidates(ParameterGrid(self.param_grid)) File "C:\Python367-64\lib\site-packages\sklearn\model_selection_search.py", line 666, in evaluate_candidates cv.split(X, y, groups))) File "C:\Python367-64\lib\site-packages\joblib\parallel.py", line 934, in call self.retrieve() File "C:\Python367-64\lib\site-packages\joblib\parallel.py", line 833, in retrieve self._output.extend(job.get(timeout=self.timeout)) File "C:\Python367-64\lib\site-packages\joblib_parallel_backends.py", line 521, in wrap_future_result return future.result(timeout=timeout) File "C:\Python367-64\lib\concurrent\futures_base.py", line 432, in result return self.__get_result() File "C:\Python367-64\lib\concurrent\futures_base.py", line 384, in __get_result raise self._exception ValueError: Input contains NaN, infinity or a value too large for dtype('float64').

Why would it work when not using GridSearchCV, but using GridSearchCV causes it to fail?

Answer 1

The issue was related to this line:

'solver': ['lbfgs', 'sgd', 'adam'],

the sgd option requires certain parameters in a certain threshold per the documentation

Simply changing 'solver': ['lbfgs', 'sgd', 'adam'],

to

'solver': ['lbfgs', 'adam'],

fixed the problem