TensorFlow Linear Regression gives 'NaN' result

Question

I am currently running the TensorFlow model with Linear Regression. However, I don't understand why, even when I decrease the learning_rate from 0.01 to 0.001 and increase the training iterations from 1000 to 50000, I still obtain the 'nan' result for the cost function, as well as the two coefficients. Could anyone please help me detect the problem in the following code?

 from __future__ import print_function

    import tensorflow as tf
    import numpy
    import matplotlib.pyplot as plt
    import pandas as pd
    from sklearn.model_selection import train_test_split
    import random

    rng = numpy.random


    # Parameters
    learning_rate = 0.001 
    training_epochs = 20000 #number of iterations
    display_step = 400



    #read csv file
    datapath = [directory path]

    Ha_Noi = pd.read_csv(datapath+"HaNoi_1month_LW_WeatherTest.csv")
    #Add an additional column into the table
    sLength = len(Ha_Noi['accept_rate'])
    Ha_Noi['accept_rate_timeT'] = pd.Series(Ha_Noi['accept_rate'], index=Ha_Noi.index)
    #Shift the entries in the accept_rate column upward
    Ha_Noi.accept_rate = Ha_Noi.accept_rate.shift(-1)

    Ha_Noi = Ha_Noi.dropna(subset = ["longwait_percent4"])
    Ha_Noi = Ha_Noi.dropna(subset=["accept_rate"])
    Ha_Noi = Ha_Noi.dropna(subset = ["longwait_percent2"])
    df2 = pd.DataFrame(Ha_Noi)

    #split the dataset into training and testing sets
    train_set, test_set = train_test_split(Ha_Noi, test_size=0.2, random_state = random.randint(20, 200))
    Xtrain = train_set['longwait_percent2'].reshape(-1,1)
    Ytrain = train_set['accept_rate'].reshape(-1,1)

    Xtrain2 = train_set['Weather Weight_Longwait_percent2'].reshape(-1,1)
    Xtest2 = test_set['Weather Weight_Longwait_percent2'].reshape(-1,1)

    # Xtest = test_set['longwait_percent2'].reshape(-1,1)
    # Ytest = test_set['accept_rate'].reshape(-1,1)

    # Training Data
    train_X = Xtrain
    train_Y = Ytrain
    n_samples = train_X.shape[0]

    #Testing Data
    Xtest = np.asarray(test_set['longwait_percent2'])
    Ytest = np.asarray(test_set['accept_rate'])

    # tf Graph Input
    X = tf.placeholder("float")
    Y = tf.placeholder("float")

    # Set model weights
    W = tf.Variable(rng.randn(), name="weight")
    b = tf.Variable(rng.randn(), name="bias")

    # Construct a linear model
    pred = tf.add(tf.multiply(X, W), b)

    # Mean squared error
    cost = tf.sqrt(tf.reduce_sum(tf.pow(pred-Y, 2))/(n_samples))

    # Gradient descent method
    #  Note, minimize() knows to modify W and b because Variable objects are "trained" (trainable=True by default)
    optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(cost)

    # Initializing the variables
    init = tf.global_variables_initializer()
    saver = tf.train.Saver() #save all the initialized data

    # Launch the graph
    with tf.Session() as sess:
        sess.run(init)

        # Fit all training data
        for epoch in range(training_epochs):
            for (x, y) in zip(train_X, train_Y):
                sess.run(optimizer, feed_dict={X: x, Y: y})

            # Display logs per epoch step
            if (epoch+1) % display_step == 0: # checkpoint every 50 epochs
                c = sess.run(cost, feed_dict={X: train_X, Y:train_Y})
                print("Epoch:", '%04d' % (epoch+1), "cost=", "{:.9f}".format(c), \
                    "W=", sess.run(W), "b=", sess.run(b))

        print("Optimization Finished!")
        training_cost = sess.run(cost, feed_dict={X: train_X, Y: train_Y})
        print("Training cost=", training_cost, "W=", sess.run(W), "b=", sess.run(b), '\n')

        # Graphic display
        plt.plot(train_X, train_Y, 'ro', label='Original data')
        plt.plot(train_X, sess.run(W) * train_X + sess.run(b), label='Fitted line')
        plt.legend()
        plt.show()

        testing_cost = sess.run(
            tf.reduce_sum(tf.pow(pred - Y, 2)) / (Xtest.shape[0]),
            feed_dict={X: Xtest, Y: Ytest})  # square root of function cost above
        print("Root Mean Square Error =", tf.sqrt(testing_cost))
        print("Absolute mean square loss difference:", abs(
            training_cost - testing_cost))

        plt.plot(Xtest, Ytest, 'bo', label='Testing data')
        plt.plot(train_X, sess.run(W) * train_X + sess.run(b), label='Fitted line')
        plt.legend()
        plt.show()

Answer 1

Don't have your data, so it's hard to tell whether the problem is caused by data or by training problem. You can make learning rate and training iteration much smaller such 0.00005 and 100 to see is there still NaN.