Invalid parameter for sklearn estimator MLP

Question

Using data from a regression created with make_regression from sklearn.datasets

from sklearn.datasets import make_regression
from sklearn.model_selection import train_test_split
X, y = make_regression(n_samples=200, random_state=1)
X_train, X_test, Y_train, Y_test = train_test_split(X, y, random_state=1)

I am trying to apply Bayesian optimization over hyperparameters in MLPRegressor from sklearn as shown:

from skopt import BayesSearchCV
from sklearn.neural_network import MLPRegressor

# Bayesian
n_iter = 100

# MLP regressor
mlpr = MLPRegressor()

param_grid = {
    "activation": ["logistic", "tanh", "relu"],
    "solver": ["lbfgs", "sgd", "adam"],
    "regressor__learning_rate": (0.0001, 0.001, 0.01),
}

reg_bay = BayesSearchCV(estimator=mlpr,
                        search_spaces=param_grid,
                        n_iter=n_iter,
                        cv=5,
                        n_jobs=8,
                        scoring='neg_mean_squared_error',
                        random_state=123)

model_bay = reg_bay.fit(X_train, Y_train)

And I got this:

ValueError: Invalid parameter regressor for estimator MLPRegressor(activation='tanh'). 
Check the list of available parameters with `estimator.get_params().keys()`.

It doesn't work even though 'tanh' is a function for hidden layer activation. I was wondering if anyone could help me solve this!

Versions:

sklearn==1.0.1

python==3.7.14

---

UPDATE:

from skopt import BayesSearchCV
from sklearn.neural_network import MLPRegressor
from sklearn.pipeline import Pipeline

# Bayesian
n_iter = 100

# MLP regressor
pipe = Pipeline(steps=[
    ('MLPRegressor', MLPRegressor())
])

param_grid = {
    "MLPRegressor__activation": ["logistic", "tanh", "relu"],
    "MLPRegressor__solver": ["lbfgs", "sgd", "adam"],
    "MLPRegressor__learning_rate": : ["constant", "invscaling", "adaptive"],
}

reg_bay = BayesSearchCV(estimator=pipe ,
                        search_spaces=param_grid,
                        n_iter=n_iter,
                        cv=5,
                        n_jobs=8,
                        scoring='neg_mean_squared_error',
                        random_state=123)

model_bay = reg_bay.fit(X_train, Y_train)

I got:

---> model_bay = reg_bay.fit(X_tr, y_tr)

ValueError: Not all points are within the bounds of the space.