Classification using deep nets on CIFAR-10 dataset

Question

I am trying to build a classifier using deep learning techniques and used cifar-10 dataset to build a classifier.I have tried to build a classifier with 1024 hidden nodes. Each image is of 32*32*3(R-G-B) size.I have loaded data from only 3/5 files of dataset as my computer has low processing power.

from __future__ import print_function
import matplotlib.pyplot as plt
import numpy as np
import tensorflow as tf
import os
import sys
import tarfile
import random
from IPython.display import display, Image
from scipy import ndimage
from sklearn.linear_model import LogisticRegression
from six.moves.urllib.request import urlretrieve
from six.moves import cPickle as pickle
from sklearn.preprocessing import MultiLabelBinarizer

folder='/home/cifar-10-batches-py/'

training_data=np.ndarray((30000,3072),dtype=np.float32)
training_labels=np.ndarray(30000,dtype=np.int32)

testing_data=np.ndarray((10000,3072),dtype=np.float32)
testing_labels=np.ndarray(10000,dtype=np.int32)

no_of_files=3

begin=0
end=10000

for i in range(no_of_files):
    with open(folder+"data_batch_"+str(i+1),'rb') as f:
        s=pickle.load(f,encoding='bytes')
        training_data[begin:end]=s[b'data']
        training_labels[begin:end]=s[b'labels']
        begin=begin+10000
        end=end+10000

test_path='/home/cifar-10-batches-py/test_batch'
with open(test_path,'rb') as d:
    s9=pickle.load(d,encoding='bytes')
    tdata=s9[b'data']
    testing_data=tdata
    tlabels=s9[b'labels']
    testing_labels=tlabels
test_data=np.ndarray((5000,3072),dtype=np.float32)
test_labels=np.ndarray(5000,dtype=np.int32)
valid_data=np.ndarray((5000,3072),dtype=np.float32)
valid_labels=np.ndarray(5000,dtype=np.int32)

valid_data[:,:]=testing_data[:5000, :]
valid_labels[:]=testing_labels[:5000]
test_data[:,:]=testing_data[5000:, :]
test_labels[:]=testing_labels[5000:]
onehot_training_labels=np.eye(10)[training_labels.astype(int)]
onehot_test_labels=np.eye(10)[test_labels.astype(int)]
onehot_valid_labels=np.eye(10)[valid_labels.astype(int)]
image_size=32
num_labels=10
train_subset = 10000

def accuracy(predictions, labels):
  return (100.0 * np.sum(np.argmax(predictions, 1) == np.argmax(labels, 1))
          / predictions.shape[0])

batch_size = 128
relu_count = 1024 #hidden nodes count

graph = tf.Graph()
with graph.as_default():
    tf_train_dataset = tf.placeholder(tf.float32,
                                    shape=(batch_size, image_size * image_size*3))
    tf_train_labels = tf.placeholder(tf.float32, shape=(batch_size, num_labels))
    tf_valid_dataset = tf.constant(valid_data)
    tf_test_dataset = tf.constant(test_data)
    beta_regul = tf.placeholder(tf.float32)

    weights1 = tf.Variable(
    tf.truncated_normal([image_size * image_size*3, relu_count]))
    biases1 = tf.Variable(tf.zeros([relu_count]))
    weights2 = tf.Variable(
    tf.truncated_normal([relu_count, num_labels]))

    biases2 = tf.Variable(tf.zeros([num_labels]))

    preds = tf.matmul( tf.nn.relu(tf.matmul(tf_train_dataset, weights1) + biases1), weights2) + biases2


    loss = tf.reduce_mean(
    tf.nn.softmax_cross_entropy_with_logits(logits=preds, labels=tf_train_labels))+ \
      beta_regul * (tf.nn.l2_loss(weights1) + tf.nn.l2_loss(weights2))

    optimizer = tf.train.GradientDescentOptimizer(0.5).minimize(loss)
    train_prediction = tf.nn.softmax(preds)
    lay1_valid = tf.nn.relu(tf.matmul(tf_valid_dataset, weights1) + biases1)
    valid_prediction = tf.nn.softmax(tf.matmul(lay1_valid, weights2) + biases2)
    lay1_test = tf.nn.relu(tf.matmul(tf_test_dataset, weights1) + biases1)
    test_prediction = tf.nn.softmax(tf.matmul(lay1_test, weights2) + biases2)
num_steps = 5000

with tf.Session(graph=graph) as session:
    tf.initialize_all_variables().run()
    print("Initialized")
    for step in range(num_steps):
        offset = (step * batch_size) % (onehot_training_labels.shape[0] - batch_size)
        batch_data = training_data[offset:(offset + batch_size), :]
        batch_labels = onehot_training_labels[offset:(offset + batch_size), :]
        feed_dict = {tf_train_dataset : batch_data, tf_train_labels : batch_labels,beta_regul : 1e-3}
        _, l, predictions = session.run(
          [optimizer, loss, train_prediction], feed_dict=feed_dict)
        if (step % 500 == 0):
            print("Minibatch loss at step %d: %f" % (step, l))
            print("Minibatch accuracy: %.1f%%" % accuracy(predictions, batch_labels))
            print("Validation accuracy: %.1f%%" % accuracy(
            valid_prediction.eval(), onehot_valid_labels))
    print("Test accuracy: %.1f%%" % accuracy(test_prediction.eval(), onehot_test_labels))

the output for this code is:

WARNING:tensorflow:From /usr/local/lib/python3.5/dist-packages/tensorflow/python/util/tf_should_use.py:170: initialize_all_variables (from tensorflow.python.ops.variables) is deprecated and will be removed after 2017-03-02.
Instructions for updating:
Use `tf.global_variables_initializer` instead.
Initialized
Minibatch loss at step 0: 117783.914062
Minibatch accuracy: 14.8%
Validation accuracy: 10.2%
Minibatch loss at step 500: 3632989892247552.000000
Minibatch accuracy: 12.5%
Validation accuracy: 10.1%
Minibatch loss at step 1000: 2203224941527040.000000
Minibatch accuracy: 6.2%
Validation accuracy: 9.9%
Minibatch loss at step 1500: 1336172110413824.000000
Minibatch accuracy: 10.9%
Validation accuracy: 9.8%
Minibatch loss at step 2000: 810328996708352.000000
Minibatch accuracy: 8.6%
Validation accuracy: 10.1%
Minibatch loss at step 2500: 491423044468736.000000
Minibatch accuracy: 9.4%
Validation accuracy: 10.1%
Minibatch loss at step 3000: 298025566076928.000000
Minibatch accuracy: 12.5%
Validation accuracy: 9.8%
Minibatch loss at step 3500: 180741635833856.000000
Minibatch accuracy: 10.9%
Validation accuracy: 9.8%
Minibatch loss at step 4000: 109611013111808.000000
Minibatch accuracy: 15.6%
Validation accuracy: 10.1%
Minibatch loss at step 4500: 66473376612352.000000
Minibatch accuracy: 3.9%
Validation accuracy: 9.9%
Test accuracy: 10.2%

where am i doing it wrong? I see the accuracy is very low.

Answer 1

1. As far as I can see, you are building a simple 2-layer FNN with Tensorflow. While it is okay, you won't get a really high accuracy. But if you try, you need to carefully tune hyperparameters - learning rate, regularization strength, decay rate, number of neurons in hidden layers.

2. You are using not all data, so it will surely descrease the quality of predictions. It could still work, but you should check classes distribution in train, val and test sets. It is possible that some classes have too little values in one of datasets. You need at least to stratify your selection.

3. Are you sure you have a deep understanding of deep learning? It could be a good idea to try cs231n course.