LSTM taking more time to train

Question

I'm using following simple architecture to train my model, but my model is showing 2-3 hours as elapsed time per epoch when I'm also using masked input with my padded input, why it's happening like that.

please find the below code for my model

class lstm_raw(tf.keras.Model):
  def __init__(self,name='spectrogram'):
    super().__init__(name=name)
    self.lstm = tf.keras.layers.LSTM(32,activation="tanh",kernel_initializer=tf.keras.initializers.he_uniform(seed=45),kernel_regularizer=tf.keras.regularizers.l2())
    self.dense1 = tf.keras.layers.Dense(64,activation="relu",kernel_initializer=tf.keras.initializers.he_uniform(seed=45))
    self.dense2 = tf.keras.layers.Dense(10,kernel_initializer=tf.keras.initializers.he_uniform(seed=45))
  def call(self,X):
    lstm_output = self.lstm(X[0],mask=X[1])
    dense1 = self.dense1(lstm_output)
    dense2 = self.dense2(dense1)
    return dense2

with tf.device('/device:GPU:0'):
  model1.fit(x=[X_train_pad_seq_test,X_train_mask_test],y=y_train,epochs=20,batch_size=4,steps_per_epoch=len(X_train_pad_seq_test)//4)

my input shape as below

((1400, 17640, 1), (1400, 17640, 1))

Answer 1

Here's a generic example, which shouldn't take more than 1-2 minutes, max.

from pandas_datareader import data as wb
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from matplotlib.pylab import rcParams
from sklearn.preprocessing import MinMaxScaler

start = '2019-06-30'
end = '2020-06-30'

tickers = ['GOOG']

thelen = len(tickers)

price_data = []
for ticker in tickers:
    prices = wb.DataReader(ticker, start = start, end = end, data_source='yahoo')[['Open','Adj Close']]
    price_data.append(prices.assign(ticker=ticker)[['ticker', 'Open', 'Adj Close']])

#names = np.reshape(price_data, (len(price_data), 1))

df = pd.concat(price_data)
df.reset_index(inplace=True)

for col in df.columns: 
    print(col) 
    
#used for setting the output figure size
rcParams['figure.figsize'] = 20,10
#to normalize the given input data
scaler = MinMaxScaler(feature_range=(0, 1))
#to read input data set (place the file name inside  ' ') as shown below


df['Adj Close'].plot()
plt.legend(loc=2)
plt.xlabel('Date')
plt.ylabel('Price')
plt.show()

ntrain = 80
df_train = df.head(int(len(df)*(ntrain/100)))
ntest = -80
df_test = df.tail(int(len(df)*(ntest/100)))


#importing the packages 
from sklearn.preprocessing import MinMaxScaler
from keras.models import Sequential
from keras.layers import Dense, Dropout, LSTM

#dataframe creation
seriesdata = df.sort_index(ascending=True, axis=0)
new_seriesdata = pd.DataFrame(index=range(0,len(df)),columns=['Date','Adj Close'])
length_of_data=len(seriesdata)
for i in range(0,length_of_data):
    new_seriesdata['Date'][i] = seriesdata['Date'][i]
    new_seriesdata['Adj Close'][i] = seriesdata['Adj Close'][i]
#setting the index again
new_seriesdata.index = new_seriesdata.Date
new_seriesdata.drop('Date', axis=1, inplace=True)
#creating train and test sets this comprises the entire data’s present in the dataset
myseriesdataset = new_seriesdata.values
totrain = myseriesdataset[0:255,:]
tovalid = myseriesdataset[255:,:]
#converting dataset into x_train and y_train
scalerdata = MinMaxScaler(feature_range=(0, 1))
scale_data = scalerdata.fit_transform(myseriesdataset)
x_totrain, y_totrain = [], []
length_of_totrain=len(totrain)
for i in range(60,length_of_totrain):
    x_totrain.append(scale_data[i-60:i,0])
    y_totrain.append(scale_data[i,0])
x_totrain, y_totrain = np.array(x_totrain), np.array(y_totrain)
x_totrain = np.reshape(x_totrain, (x_totrain.shape[0],x_totrain.shape[1],1))


#LSTM neural network
lstm_model = Sequential()
lstm_model.add(LSTM(units=50, return_sequences=True, input_shape=(x_totrain.shape[1],1)))
lstm_model.add(LSTM(units=50))
lstm_model.add(Dense(1))
lstm_model.compile(loss='mean_squared_error', optimizer='adadelta')
lstm_model.fit(x_totrain, y_totrain, epochs=10, batch_size=1, verbose=2)
#predicting next data stock price
myinputs = new_seriesdata[len(new_seriesdata) - (len(tovalid)+1) - 60:].values
myinputs = myinputs.reshape(-1,1)
myinputs  = scalerdata.transform(myinputs)
tostore_test_result = []
for i in range(60,myinputs.shape[0]):
    tostore_test_result.append(myinputs[i-60:i,0])
tostore_test_result = np.array(tostore_test_result)
tostore_test_result = np.reshape(tostore_test_result,(tostore_test_result.shape[0],tostore_test_result.shape[1],1))
myclosing_priceresult = lstm_model.predict(tostore_test_result)
myclosing_priceresult = scalerdata.inverse_transform(myclosing_priceresult)
    
totrain = df_train
tovalid = df_test

#predicting next data stock price
myinputs = new_seriesdata[len(new_seriesdata) - (len(tovalid)+1) - 60:].values


#  Printing the next day’s predicted stock price. 
print(len(tostore_test_result));
print(myclosing_priceresult);

For reference:

https://github.com/ASH-WICUS/Notebooks/blob/master/Long%20Short%20Term%20Memory%20-%20Stock%20Price%20Prediction.ipynb

Answer 2

The culprit in your code is the activation="relu" in your LSTM layer.

Tensorflow accelerates LSTM cells using CuDNN if and only if the activation is set to tanh.

Replace relu with tanh, and see your model take off!