How to loop through various train and test splits

Question

I have various train and test splits that I create using TimeSeriesSplit(). My dataframe has 377 observations with 6 input variables and 1 target variable.

I split my dataframe into train and test using the following code:

#train set 
i=0
for X_train, X_test in tscv.split(data):
    i=i+1
    print ("No of observations under train%s=%s"%(i,len(X_train)))
    print ("No of observations under test%s=%s" % (i, len(X_test)))

X_train1, X_test1 = data[:67, :-1],  data[67:129,:-1]
X_train2, X_test2 = data[:129,:-1], data[129:191,:-1]
X_train3, X_test3 = data[:191,:-1], data[191:253,:-1]
X_train4, X_test4 = data[:253,:-1], data[253:315,:-1]
X_train5, X_test5 = data[:315,:-1], data[315:377,:-1]

#test set
i=0
for y_train, y_test in tscv.split(data):
    i=i+1
    print ("No of observations under train%s=%s"%(i,len(y_train)))
    print ("No of observations under test%s=%s" % (i, len(y_test)))

y_train1, y_test1 = data[:67, -1], data[67:129 ,-1]
y_train2, y_test2 = data[:129,-1], data[129:191,-1]
y_train3, y_test3 = data[:191,-1], data[191:253,-1]
y_train4, y_test4 = data[:253,-1], data[253:315,-1]
y_train5, y_test5 = data[:315,-1], data[315:377,-1]

So i have 5 splits in total. I want to train my lstm model looping through these splits but I am not sure how best I can do that. Here’s the code for my lstm:

# split into input and outputs
train_X, train_y = X_train, y_train
test_X, test_y = X_test, y_test

#reshape input to be 3D [samples, timesteps, features]
train_X = train_X.reshape((train_X.shape[0], 1, train_X.shape[1]))
test_X = test_X.reshape((test_X.shape[0], 1, test_X.shape[1]))

from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense,LSTM, Flatten
import matplotlib.pyplot as pyplot
# design network
model = Sequential()
model.add(LSTM(50, input_shape=(train_X.shape[1], train_X.shape[2])))
model.add(Dense(1))
model.compile(loss='mae', optimizer='adam')
history = model.fit(train_X, train_y, epochs=700
                    , batch_size=72, validation_data=(test_X, test_y), verbose=2, shuffle=False)

# plot history
pyplot.plot(history.history['loss'], label='train')
pyplot.plot(history.history['val_loss'], label='test')
pyplot.legend()
pyplot.show()

#predictions
y_lstm = model.predict(test_X)

#metrics for test set
mse_lstm = mean_squared_error(y_test, y_lstm)
rmse_lstm = np.sqrt(mse_lstm)
r2_lstm = r2_score(y_test, y_lstm)
mae_lstm = mean_absolute_error(y_test, y_lstm)

#train metics
train     = model.predict(X_t_reshaped)
msetrain  = mean_squared_error(y_train, train)
rmsetrain = np.sqrt(msetrain)
r2train   = r2_score(y_train, train)

What can I do to use the above code to loop through all my different splits and store the results in a list or dataframe?

I want to also plot the predicted results as shown below

This is the grapgh am getting based on @Ashraful answer

Answer 1

Replace your last Code block using this,

from sklearn.metrics import  mean_squared_error
from sklearn.metrics import *
import numpy as np
import csv  

Round = 3      # define the number of digits after decimal point you want 

fields = ['Fold_No', 'mse_lstm', 'rmse_lstm', 'r2_lstm','mae_lstm']  
csvfile = open('Summary.csv', 'w') 
csvwriter = csv.writer(csvfile)  
csvwriter.writerow(fields) 


for fold in range(1,6):
    print(f'Running fold {fold}')
    # split into input and outputs
    train_X, train_y = eval(f'X_train{fold}'),eval(f'y_train{fold}')
    test_X, test_y = eval(f'X_test{fold}'),eval(f'y_test{fold}')
    print(train_X.shape)



    #reshape input to be 3D [samples, timesteps, features]
    train_X = train_X.reshape((train_X.shape[0], 1, train_X.shape[1]))
    test_X = test_X.reshape((test_X.shape[0], 1, test_X.shape[1]))

    from tensorflow.keras.models import Sequential
    from tensorflow.keras.layers import Dense,LSTM, Flatten
    import matplotlib.pyplot as pyplot
    # design network
    model = Sequential()
    model.add(LSTM(50, input_shape=(train_X.shape[1], train_X.shape[2])))
    model.add(Dense(1))
    model.compile(loss='mae', optimizer='adam')
    history = model.fit(train_X, train_y, epochs=2
                        , batch_size=72, validation_data=(test_X, test_y), verbose=2, shuffle=False)

    # plot history
    pyplot.plot(history.history['loss'], label='train')
    pyplot.plot(history.history['val_loss'], label='test')
    pyplot.legend()
    pyplot.show()

    #predictions
    train_output =  model.predict(train_X)
    y_lstm = model.predict(test_X)

    pyplot.plot(train_output, label='Training output')
    pyplot.plot(train_y, label='Obesrved Training Target')
    # pyplot.plot(train_y, label='Training value')
    pyplot.plot(test_y, label='Obesrved Predic. Target')
    pyplot.plot(y_lstm, label='Predicted Output')
    pyplot.legend(loc='upper right')
    # pyplot.legend()
    pyplot.show()
    
    #metrics for test set
    mse_lstm = mean_squared_error(y_test1, y_lstm)
    rmse_lstm = np.sqrt(mse_lstm)
    r2_lstm = r2_score(y_test1, y_lstm)
    mae_lstm = mean_absolute_error(y_test1, y_lstm)

    csvwriter.writerow([f'Fold_{fold}',round(mse_lstm,Round), round(rmse_lstm,Round), round(r2_lstm,Round),round(mae_lstm,Round)]) 


csvfile.close()

#read stored CSV file
summary= pd.read_csv('Summary.csv')

print(summary)

Also, my implementatin in colab file you can find here.