CNN with Tensorflow, low accuracy on CIFAR-10 and not improving

Question

On running the first training epoch of a 3-layer convnet on CIFAR-10, I am neither able to achieve a high enough validation accuracy nor minimize the objective function.

Specifically, the accuracy varies on the first iteration, and then settles at 8.7% for the following iterations. What's peculiar is that I've also trained a 2-layer, fully-connected network which does substantially better, consistently getting around 43% accuracy on the validation set.

NOTE: Bulk of the code is from a Jupyter notebook designed as an introduction to barebones Tensorflow (and Keras) provided as part of an assignment for Stanford's CS231n Convolutional Neural Networks for Visual Recognition and although I am neither a student of the course nor of the university, I am doing this purely for experiential purposes and out of my newborn interests in CV / deep learning. My contribution is only implementations for the forward pass and the initialization of the network's parameters.

The author of the notebook left a comment stating that when correctly implemented this model should achieve above 40% accuracy after the first epoch without any hyperparameter tuning.

Implementation Notes

• 49,000 / 1000 : train/validation split, batch size = 64

• Weights are initialized using Kaiming normalization, bias initialized with 0s

• learning rate = 3e-3

• Here are each of the layers of convnet in detail:

  1. Convolutional layer (with bias) with 32 5x5 filters, with zero-padding 2

  2. ReLU Convolutional layer (with bias) with 16 3x3 filters, with zero-padding 1

  3. ReLU Fully-connected layer (with bias) to compute scores for 10 classes

Code

( mine is written between the 'TODO' comment blocks )

import tensorflow as tf
import numpy as np


def load_cifar10(num_training=49000, num_validation=1000, num_test=10000):
    cifar10 = tf.keras.datasets.cifar10.load_data()
    (X_train, y_train), (X_test, y_test) = cifar10
    X_train = np.asarray(X_train, dtype=np.float32)
    y_train = np.asarray(y_train, dtype=np.int32).flatten()
    X_test = np.asarray(X_test, dtype=np.float32)
    y_test = np.asarray(y_test, dtype=np.int32).flatten()

    mask = range(num_training, num_training + num_validation)
    X_val = X_train[mask]
    y_val = y_train[mask]
    mask = range(num_training)
    X_train = X_train[mask]
    y_train = y_train[mask]
    mask = range(num_test)
    X_test = X_test[mask]
    y_test = y_test[mask]

    mean_pixel = X_train.mean(axis=(0, 1, 2), keepdims=True)
    std_pixel = X_train.std(axis=(0, 1, 2), keepdims=True)
    X_train = (X_train - mean_pixel) / std_pixel
    X_val = (X_val - mean_pixel) / std_pixel
    X_test = (X_test - mean_pixel) / std_pixel

    return X_train, y_train, X_val, y_val, X_test, y_test


class Dataset(object):
    def __init__(self, X, y, batch_size, shuffle=False):
        assert X.shape[0] == y.shape[0], 'Got different numbers of data and labels'
        self.X, self.y = X, y
        self.batch_size, self.shuffle = batch_size, shuffle

    def __iter__(self):
        N, B = self.X.shape[0], self.batch_size
        idxs = np.arange(N)
        if self.shuffle:
            np.random.shuffle(idxs)
        return iter((self.X[i:i+B], self.y[i:i+B]) for i in range(0, N, B))


def flatten(x):
    N = tf.shape(x)[0]
    return tf.reshape(x, (N, -1))


def three_layer_convnet(x, params):
    conv_w1, conv_b1, conv_w2, conv_b2, fc_w, fc_b = params
    scores = None
    ############################################################################
    # TODO: Implement the forward pass for the three-layer ConvNet.            #
    ############################################################################
    h1_conv = tf.nn.conv2d(x, 
                           conv_w1 + conv_b1, 
                           strides=[1, 1, 1, 1], 
                           padding='SAME'
    )
    h1 = tf.nn.relu(h1_conv)

    h2_conv = tf.nn.conv2d(h1, 
                           conv_w2 + conv_b2, 
                           strides=[1, 1, 1, 1], 
                           padding='SAME'
    )
    h2 = tf.nn.relu(h2_conv)

    fc_params = flatten(fc_w + fc_b)
    h2 = flatten(h2)
    scores = tf.matmul(h2, fc_params)
    ############################################################################
    #                              END OF YOUR CODE                            #
    ############################################################################
    return scores


def training_step(scores, y, params, learning_rate):
    losses = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=scores)
    loss = tf.reduce_mean(losses)
    grad_params = tf.gradients(loss, params)

    new_weights = []   
    for w, grad_w in zip(params, grad_params):
        new_w = tf.assign_sub(w, learning_rate * grad_w)
        new_weights.append(new_w)

    with tf.control_dependencies(new_weights):
        return tf.identity(loss)


def check_accuracy(sess, dset, x, scores, is_training=None):
    num_correct, num_samples = 0, 0
    for x_batch, y_batch in dset:
        feed_dict = {x: x_batch, is_training: 0}
        scores_np = sess.run(scores, feed_dict=feed_dict)
        y_pred = scores_np.argmax(axis=1)
        num_samples += x_batch.shape[0]
        num_correct += (y_pred == y_batch).sum()
    acc = float(num_correct) / num_samples
    print('Got %d / %d correct (%.2f%%)' % (num_correct, num_samples, 100 * acc))


def kaiming_normal(shape):
    if len(shape) == 2:
        fan_in, fan_out = shape[0], shape[1]
    elif len(shape) == 4:
        fan_in, fan_out = np.prod(shape[:3]), shape[3]
    return tf.random_normal(shape) * np.sqrt(2.0 / fan_in)


def three_layer_convnet_init():
    params = None
    ############################################################################
    # TODO: Initialize the parameters of the three-layer network.              #
    ############################################################################
    conv_w1 = tf.Variable(kaiming_normal((5, 5, 3, 32)))
    conv_b1 = tf.Variable(tf.zeros((32,)))
    conv_w2 = tf.Variable(kaiming_normal((3, 3, 32, 16)))
    conv_b2 = tf.Variable(tf.zeros((16,)))
    fc_w = tf.Variable(kaiming_normal((32 * 32 * 16, 10)))
    fc_b = tf.Variable(tf.zeros((10,)))
    params = [conv_w1, conv_b1, conv_w2, conv_b2, fc_w, fc_b]
    ############################################################################
    #                             END OF YOUR CODE                             #
    ############################################################################
    return params


def main():
    learning_rate = 3e-3
    tf.reset_default_graph()
    is_training = tf.placeholder(tf.bool, name='is_training')

    X_train, y_train, X_val, y_val, X_test, y_test = load_cifar10()
    train_dset = Dataset(X_train, y_train, batch_size=64, shuffle=True)
    test_dset = Dataset(X_test, y_test, batch_size=64)
    val_dset = Dataset(X_val, y_val, batch_size=64, shuffle=False)
    print('Train data shape: ', X_train.shape)
    print('Train labels shape: ', y_train.shape, y_train.dtype)
    print('Validation data shape: ', X_val.shape)
    print('Validation labels shape: ', y_val.shape)
    print('Test data shape: ', X_test.shape)
    print('Test labels shape: ', y_test.shape)

    device = '/cpu:0'

    with tf.device(device):
        x = tf.placeholder(tf.float32, [None, 32, 32, 3])
        y = tf.placeholder(tf.int32, [None])
        params = three_layer_convnet_init()
        scores = three_layer_convnet(x, params)
        loss = training_step(scores, y, params, learning_rate)

    with tf.Session() as sess:
        sess.run(tf.global_variables_initializer())
        for t, (x_np, y_np) in enumerate(train_dset):
            feed_dict = {x: x_np, y: y_np}
            loss_np = sess.run(loss, feed_dict=feed_dict)

            if t % 100 == 0:
                print('Iteration %d, loss = %.4f' % (t, loss_np))
                check_accuracy(sess, val_dset, x, scores, is_training)

if __name__=="__main__":
    main()

EDIT: removed unnecessary comments and code

Answer 1

the problem is here

h1_conv = tf.nn.conv2d(x, 
                       conv_w1 + conv_b1, 
                       strides=[1, 1, 1, 1], 
                       padding='SAME'
)

This is wrong as here you are adding bias values (conv_b1) to the filter conv_w1 but bias has to be added to the output of conv layer. The right way would be something like this

h1_conv = tf.nn.conv2d(x, 
                       conv_w1, 
                       strides=[1, 1, 1, 1], 
                       padding='SAME'

)
h1_bias = tf.nn.bias_add(h1_conv, conv_b1)
h1 = tf.nn.relu(h1_bias)

And Correct it for h2 too.