How can I weigh the observations for lasso regression in following Python code?

Question

I wrote the following code for implementing lasso regression in Python. But I want to weigh the observations with given weight vector w. How can I change the code for this purpose?

import numpy as np
import matplotlib.pyplot as plt
from sklearn.linear_model import Lasso, LassoCV, LassoLarsCV

# dataset
data = [
    [0.067732, 3.176513], [0.427810, 3.816464], [0.995731, 4.550095], [0.738336, 4.256571], [0.981083, 4.560815],
    [0.526171, 3.929515], [0.378887, 3.526170], [0.033859, 3.156393], [0.132791, 3.110301], [0.138306, 3.149813],
    [0.247809, 3.476346], [0.648270, 4.119688], [0.731209, 4.282233], [0.236833, 3.486582], [0.969788, 4.655492],
    [0.607492, 3.965162], [0.358622, 3.514900], [0.147846, 3.125947], [0.637820, 4.094115], [0.230372, 3.476039],
    [0.070237, 3.210610], [0.067154, 3.190612], [0.925577, 4.631504], [0.717733, 4.295890], [0.015371, 3.085028],
    [0.335070, 3.448080], [0.040486, 3.167440], [0.212575, 3.364266], [0.617218, 3.993482], [0.541196, 3.891471]
]

dataMat = np.array(data)
X = dataMat[:, 0:1]
y = dataMat[:, 1]

model = Lasso(alpha=0.01)
# model = LassoCV()
# model = LassoLarsCV()
model.fit(X, y)
print('coefficients：\n', model.coef_)
print('The linear model is: \n', model)

predicted = model.predict(X)

plt.scatter(X, y, marker='x')
plt.plot(X, predicted,c='r')
plt.xlabel("x")
plt.ylabel("y")
plt.show()

Answer 1

Scikit-learn does not support weighted lasso.

We can easily bypass this because weighted linear regression corresponds to doing a regression on np.sqrt(w) * x or np.sqrt(w) * y.

This results in the following code-snippet:

# Create the weight vector
w = np.array([1,1,1,2,1,1,2, ...])

# Create a weight-matrix
W = np.diag(np.sqrt(w))

# Create an intercept column
n_rows, n_cols = np.shape(X)
X_intercept = np.append(X, np.ones([n_rows, 1]),axis=1)

# Transform the variables according to weights
X_trans = np.dot(W, X_intercept)
y_trans = np.dot(W, y)

# Fit the models
linear_model1 = LinearRegression(fit_intercept=True)
linear_model2 = LinearRegression(fit_intercept=False)
lasso_model = Lasso(fit_intercept=False, alpha=1)

weighted_linear1 = linear_model1.fit(X, y, w)
weighted_linear2 = linear_model2.fit(X_trans, y_trans)
weighted_lasso = lasso_model.fit(X_trans, y_trans)

# Check that weighted_linear1 and weighted_linear 2 are the same
print(f"intercept 1:\t {weighted_linear1.intercept_}")
print(f"intercept 2:\t {weighted_linear2.coef_[-1]}")

print(f"intercept 1:\t {weighted_linear1.coef_}")
print(f"intercept 2:\t {weighted_linear2.coef_[:-1]}")

# Proof that both methods for weighted linear regression (non-lasso) are the same
# Note, that for the second method the intercept appears as the last coefficient
# This happens because we created a column of ones
print(f"intercept 1:\t {weighted_linear1.intercept_}")
print(f"intercept 2:\t {weighted_linear2.coef_[-1]}")    

print(f"intercept 1:\t {weighted_linear1.coef_}")
print(f"intercept 2:\t {weighted_linear2.coef_[:-1]}")

This script will output:

intercept 1:     4.922057369248413
intercept 2:     4.922057369248415
intercept 1:     [ 0.0240568   0.01956514  0.00033999 -0.05395817  0.00779717]
intercept 2:     [ 0.0240568   0.01956514  0.00033999 -0.05395817  0.00779717]

An interesting exercise would be to create a WeightedLasso class as described here.

Answer 2

I have changed my original code following the suggestion from Erik. Here is the new code:

    import numpy as np
    import matplotlib.pyplot as plt
    from sklearn.linear_model import Lasso, LassoCV, LassoLarsCV

    # observations, the first column is x, the second is y
    data = [
        [0.067732, 3.176513], [0.427810, 3.816464], [0.995731, 4.550095], [0.738336, 4.256571], [0.981083, 4.560815],
        [0.526171, 3.929515], [0.378887, 3.526170], [0.033859, 3.156393], [0.132791, 3.110301], [0.138306, 3.149813],
        [0.247809, 3.476346], [0.648270, 4.119688], [0.731209, 4.282233], [0.236833, 3.486582], [0.969788, 4.655492],
        [0.607492, 3.965162], [0.358622, 3.514900], [0.147846, 3.125947], [0.637820, 4.094115], [0.230372, 3.476039],
        [0.070237, 3.210610], [0.067154, 3.190612], [0.925577, 4.631504], [0.717733, 4.295890], [0.015371, 3.085028],
        [0.335070, 3.448080], [0.040486, 3.167440], [0.212575, 3.364266], [0.617218, 3.993482], [0.541196, 3.891471]
    ]

    # Create X and y
    dataMat = np.array(data)
    X = dataMat[:, 0:1]
    y = dataMat[:, 1]

    # Create a weight-matrix
    w = np.array([1, 1, 1, 2, 2, 2, 1, 1, 1, 1, 1, 2, 2, 2, 1, 1, 1, 2, 1, 2, 1, 1, 2, 1, 1, 2, 2, 2, 2, 1])
    W = np.diag(np.sqrt(w))

    # Create an intercept column
    n_rows, n_cols = np.shape(X)
    X_intercept = np.append(X, np.ones([n_rows, 1]), axis=1)

    # Transform the variables according to weights
    X_trans = np.dot(W, X_intercept)
    y_trans = np.dot(W, y)

    # Lasso regression with observations weighted
    weighted_lasso = Lasso(fit_intercept=False, alpha=0.05)

    weighted_lasso.fit(X_trans, y_trans)
    print(weighted_lasso)

    # model = Lasso(alpha=0.01)
    # model = LassoCV()
    # model = LassoLarsCV()
    # model.fit(X, y)
    # print('系数矩阵：\n', weighted_lasso.coef_)
    # print('线性回归模型: \n', weighted_lasso)

    # predict with fitted model
    # predicted = weighted_lasso.predict(X_trans)
    predicted = weighted_lasso.coef_[0]*X + weighted_lasso.coef_[1]
    print(predicted)
    print(y)
    # predicted = model.predict(X)

    # plot
    plt.scatter(X, y, marker='x')
    plt.scatter(X, predicted, marker='o')

    # add x and y axis
    plt.xlabel("x")
    plt.ylabel("y")

    # show plot
    plt.show()

Is this code right? Thank you.