How to increase Tensorflow Prediction accuracy

Question

I am working with tensorflow to create a model which can classify digits using the SVHN dataset provided by google.My accracy is really low(~25%) but i have seen a notebook which has 88% accuracy.Reference Notebook

I was wondering if anybody could give me some tips regarding how should i improve my accuracy in order to make my model better.

Here is my model code.

filename='extra.pickle'

with open(filename,'rb') as f:
    other=pickle.load(f)
    train_data=other['train_dataset']
    test_data=other['test_dataset']

    del other

train_dataset=train_data['X']
test_dataset=test_data['X']
train_labels=train_data['y']
test_labels=test_data['y']

print(len(test_dataset))
print(len(test_labels))
print(test_dataset.shape)
#print(train_dataset.length())

classes=10

batch_size=32

num_steps = 200000

graph=tf.Graph()

#Placeholder for the data
with graph.as_default():

    data_placeholder = tf.placeholder(tf.float32, shape=(batch_size,32,32,3))
    label_placeolder = tf.placeholder(tf.int64, shape=(batch_size, classes))

    tf_test_dataset = tf.placeholder(tf.float32, shape=(batch_size,32,32,3))

    tf_label_dataset = tf.placeholder(tf.float32, shape=(batch_size, classes))

    layer1_weights=tf.Variable(tf.truncated_normal([3,3,3,16]))
    layer1_biases=tf.Variable(tf.zeros([16]))

    layer2_weights=tf.Variable(tf.truncated_normal([3,3,16,32]))
    layer2_biases=tf.Variable(tf.zeros([32]))

    layer3_weights=tf.Variable(tf.truncated_normal([2,2,32,64]))
    layer3_biases=tf.Variable(tf.zeros([64]))

    layer4_weights=tf.Variable(tf.truncated_normal([1024,10]))
    layer4_biases=tf.Variable(tf.zeros([10]))

    layer5_weights=tf.Variable(tf.truncated_normal([10,classes]))
    layer5_biases=tf.Variable(tf.zeros([classes]))


    def layer_multiplication(data_input_given,dropping=False):

        #Convolutional Layer 1

        CNN1=tf.nn.relu(tf.nn.conv2d(data_input_given,layer1_weights,strides=[1,1,1,1],padding='SAME')+layer1_biases)

        print('CNN1 Done!!')

        #Pooling Layer

        Pool1=tf.nn.max_pool(CNN1,ksize=[1,2,2,1],strides=[1,2,2,1],padding='SAME')
        print('Pool1 DOne')

        #second Convolution layer

        CNN2=tf.nn.relu(tf.nn.conv2d(Pool1,layer2_weights,strides=[1,1,1,1],padding='SAME'))+layer2_biases
        print('CNN2 Done')
        #Second Pooling

        Pool2 = tf.nn.max_pool(CNN2, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')
        print('pool2 Done')
        #Third Convolutional Layer

        print(Pool2.shape)

        CNN3 = tf.nn.relu(tf.nn.conv2d(Pool2, layer3_weights, strides=[1, 1, 1, 1], padding='SAME')) + layer3_biases
        print('CNN3 Done')
        #Third Pooling Layer

        Pool3 = tf.nn.max_pool(CNN3, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')
        print('Pool3 DOne')
        #Fully Connected Layer

        #print(Pool3.shape)

        shape = Pool3.get_shape().as_list()

        # print(shape)

        reshape = tf.reshape(Pool3, [shape[0], shape[1] * shape[2] * shape[3]])

        #print(reshape.shape)

        FullyCon = tf.nn.relu(tf.matmul(reshape, layer4_weights) + layer4_biases)

        #print(FullyCon.shape)

        if dropping==False:
            print('Training')
            dropout = tf.nn.dropout(FullyCon, 0.6)
            z=tf.matmul(dropout,layer5_weights)+layer5_biases
            return z

        else:
            print('Testing')
            z = tf.matmul(FullyCon, layer5_weights) + layer5_biases
            return z


    gloabl_step = tf.Variable(0, trainable=False)

    decay_rate=tf.train.exponential_decay(1e-6,gloabl_step,4000,0.96,staircase=False,)

    train_input=layer_multiplication(data_placeholder,False)

    test_prediction = tf.nn.softmax(layer_multiplication(tf_test_dataset,True))

    loss=(tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=label_placeolder,logits=train_input))
                                   + 0.01 * tf.nn.l2_loss(layer1_weights)
                                   + 0.01 * tf.nn.l2_loss(layer2_weights)
                                   + 0.01 * tf.nn.l2_loss(layer3_weights)
                                   + 0.01 * tf.nn.l2_loss(layer4_weights)
                                   + 0.01 * tf.nn.l2_loss(layer5_weights)
                                   )

    optimizer = tf.train.GradientDescentOptimizer(name='Stochastic', learning_rate=decay_rate).minimize(loss,global_step=gloabl_step)


    def accuracy(predictions, labels):
        print(predictions.shape[0])
        return (100.0 * np.sum(np.argmax(predictions, 1) == np.argmax(labels, 1))
                / predictions.shape[0])

    config=tf.ConfigProto()
    config.gpu_options.allocator_type ='BFC'

    saver = tf.train.Saver()

    test_accuracy=[]

    with tf.Session(config=config) as session:
        tf.global_variables_initializer().run()
        print('Initialized')
        tf.train.write_graph(session.graph_def, '.', './SVHN.pbtxt')

        for step in range(num_steps):
            offset = (step * batch_size) % (train_labels.shape[0] - batch_size)
            batch_data = train_dataset[offset:(offset + batch_size), :, :]
            batch_labels = train_labels[offset:(offset + batch_size), :]

            batch_test_data =  test_dataset[offset:(offset + batch_size), :, :]
            batch_test_labels = test_labels[offset:(offset + batch_size),:]
            #print(batch_data)
            #print(batch_test.shape)

            feed_dict = {data_placeholder:batch_data, label_placeolder:batch_labels}
            _, l, predictions = session.run(
                [optimizer, loss, train_input], feed_dict=feed_dict)
            if (step % 500 == 0):
                #print(session.run(decay_rate))
                print('Minibatch loss at step %d: %f' % (step, l))
                print('Minibatch accuracy: %.1f%%' % accuracy(predictions, batch_labels))
                if(batch_test_data.shape!=(32,32,32,3)):
                    print('Skip')
                else:
                    correct_prediction = tf.equal(tf.argmax(test_prediction, 1), tf.argmax(tf_label_dataset, 1))
                    accuracy_for_test = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
                    print("Test Accuracy")

                    test_accuracy.append(accuracy_for_test.eval(feed_dict={tf_test_dataset:batch_test_data,
                                                tf_label_dataset:batch_test_labels}))

                    print(accuracy_for_test.eval(feed_dict={tf_test_dataset:batch_test_data,
                                                tf_label_dataset:batch_test_labels}))

        print(np.mean(test_accuracy))


        saver.save(sess=session, save_path='./SVHN.ckpt')

Ps-The code does work in My system.The issue seems to be with my architecture i believe.

Answer 1

First of all there is not single parameter which will increase your accuracy. Training in tensorflow depend on lot of parameter like initial weight, learning rate, number of step to train model. It is kind of try with different combination. Like learning rate should not be low as well not be very much other wise you will miss the minima.It is like our brain , some time we learn thing very fast when we have clear data about that thing.and some time we need more data to find the thing in better manner.

So do training on different configaration of all these parameter and then check your accuracy

Answer 2

Use a different weight initialization and your results should be way better. In your reference notebook they use stddev=0.1, but you can also take a look at glorot or he initialization, which should work even better.

Also your learning rate is really low and the decay makes the learning rate even lower, so the network will not learn much this way. By doing a better initialization of the network you can increase the learning rate and learn something useful from your data.