How to perform iterative inference using Tensorflow Slim library

Question

There are many examples out there that show how to use the tf.contrib.slim library to classify a single image downloaded from the web. In fact the tensorflow github provides this. However, I am struggling to understand the best way to do this in a loop. Any application that uses Tensorflow for classification will have to classify more than one batch of images. The inference process involves building a graph, and loading the weights from a checkpoint. When iteratively running, it seems wasteful to repeat those steps again, and again. In fact, when I try that rudimentary method, I can see that the memory allocated to python continues to grow each iteration. Can someone please help suggest how to modify the basic examples to achieve repetitive/iterative inference? Here is my current method which works, but is clearly wasteful with memory resources (This code crashes a machine with limited memory, new images are periodically dumped in global frame): def classification():

def classification():
  global frame
  global count

  slim = tf.contrib.slim
  image_size = inception_v4.inception_v4.default_image_size
  names = imagenet.create_readable_names_for_imagenet_labels()
  checkpoints_dir = '../../checkpoints'

  # Don't classify the first few frames
  while count < 5:
    pass

  while True:
    start = count
    with tf.Graph().as_default():
      image = tf.convert_to_tensor(frame,dtype=tf.float32)
      processed_image = inception_preprocessing.preprocess_image(image, image_size, image_size, is_training=False)
      processed_images  = tf.expand_dims(processed_image, 0)
      # processed_images will be a 1x299x299x3 tensor of float32

    # Create the model, use the default arg scope to configure the batch norm parameters.
    with slim.arg_scope(inception_v4.inception_v4_arg_scope()):
      logits, _ = inception_v4.inception_v4(processed_images, num_classes=1001, is_training=False)
      probabilities = tf.nn.softmax(logits)

    init_fn = slim.assign_from_checkpoint_fn(
      os.path.join(checkpoints_dir, 'inception_v4.ckpt'),
      slim.get_model_variables('InceptionV4'))

    with tf.Session() as sess:
      init_fn(sess)
      np_image, probabilities = sess.run([image, probabilities])
      probabilities = probabilities[0, 0:]
      sorted_inds = [i[0] for i in sorted(enumerate(-probabilities), key=lambda x:x[1])]

    for i in range(5):
      index = sorted_inds[i]
      print('Probability %0.2f%% => [%s]' % (probabilities[index] * 100, names[index]))

    end = count
    print "Classification latency = %d frames" % (end-start)

Answer 1

One option can solve the problem by defining a class, and you load the model in the init method. Also, add a method called classify. So, you initiate the class first. Then, for every frame, you call method classify. Below you find how did I modify your code:

import os

import cv2
import matplotlib.pyplot as plt
import tensorflow as tf

from datasets import imagenet
from nets import inception_v4
from preprocessing import inception_preprocessing


def show_image(img_path):
    img = cv2.imread(img_path)
    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
    img_plot = plt.imshow(img)
    # Set up the plot and hide axes
    plt.title('test')
    img_plot.axes.get_yaxis().set_ticks([])
    img_plot.axes.get_xaxis().set_ticks([])
    plt.show()


def load_image(img_path):
    img = cv2.imread(img_path)
    return cv2.cvtColor(img, cv2.COLOR_BGR2RGB)


class ImageClassifier():
    def __init__(self):
        self.slim = tf.contrib.slim
        self.image_size = inception_v4.inception_v4.default_image_size
        self.checkpoints_dir = 'checkpoints'
        self.names = imagenet.create_readable_names_for_imagenet_labels()
        self.arg_scope = inception_v4.inception_v4_arg_scope()

        self.image = tf.placeholder(tf.uint8, [480, 640, 3])

        self.processed_image = inception_preprocessing.preprocess_image(self.image,
                                                                        self.image_size, self.image_size,
                                                                        is_training=False)
        self.processed_images = tf.expand_dims(self.processed_image, 0)

        # processed_images will be a 1x299x299x3 tensor of float32

        # Create the model, use the default arg scope to configure the batch norm parameters.
        with self.slim.arg_scope(self.arg_scope):
            self.logits, self.end_points = inception_v4.inception_v4(self.processed_images, num_classes=1001,
                                                                     is_training=False)
            self.probs = tf.nn.softmax(self.logits)

        self.init_fn = self.slim.assign_from_checkpoint_fn(
            os.path.join(self.checkpoints_dir, 'inception_v4.ckpt'),
            self.slim.get_model_variables('InceptionV4'))

        self.session = tf.Session()
        self.init_fn(self.session)

    def classify(self, img):
        height, width = img.shape[:2]

        feed_dict = {self.image: img}
        probabilities = self.session.run(self.probs, feed_dict=feed_dict)
        probabilities = probabilities[0, 0:]
        sorted_inds = [i[0] for i in sorted(enumerate(-probabilities), key=lambda x: x[1])]

        for i in range(5):
            index = sorted_inds[i]
            print('Probability %0.2f%% => [%s]' % (probabilities[index] * 100, self.names[index]))


def main():
    imgs_dir = "./imgs/wep"
    image_classifier = ImageClassifier()
    for img_name in os.listdir(imgs_dir):
        img = load_image(os.path.join(imgs_dir, img_name))
        img = cv2.resize(img, (640, 480))
        print(img_name)
        image_classifier.classify(img)

Answer 2

I got this to work, would still appreciate some wisdom from others. My solution was to build the graph with a placeholder as the input. Then the video frame can be fed into the session run method used feed_dict. This allows me to put the while loop around the call to session run. The latency using this method was 1/10th the original I shared, and the memory fingerprint is stable. Here is my full code used to classify video frames from a webcam. Note that there is an issue with it. I have no mechanism to exit the threads cleanly. Ctrl+C will not kill the script. Also, note that to run this, you would need to clone the github tensorflow models repo, and download and untar the pretrained weights at ../../checkpoints.

import cv2
import os
import time
import numpy as np
from threading import Thread
import tensorflow as tf
from datasets import imagenet
from nets import inception_v4
from preprocessing import inception_preprocessing
######################################################

# Global Variables Shared by threads
frame = None
count = 0

######################################################
def capture():
######################################################
  global frame
  global count

  video_capture = cv2.VideoCapture(0)

  while True:
    # Capture frame-by-frame
    ret, frame_bgr = video_capture.read()

    # Display the resulting frame
    cv2.imshow('Video', frame_bgr)

    # Convert to RGB format (Inception expects RGB not BGR color channels)
    frame = cv2.cvtColor(frame_bgr,cv2.COLOR_BGR2RGB)

    # Increment frame counter (Used only to calculate latency)
    count += 1

    # Kill loop when user hits q
    if cv2.waitKey(1) & 0xFF == ord('q'):
      break

  # When everything is done, release the capture
  video_capture.release()
  cv2.destroyAllWindows()
######################################################

######################################################
def classification():
######################################################
  global frame
  global count

  slim = tf.contrib.slim
  image_size = inception_v4.inception_v4.default_image_size
  names = imagenet.create_readable_names_for_imagenet_labels()
  checkpoints_dir = '../../checkpoints'

  # Don't classify the None Object
  time.sleep(5)

  with tf.Graph().as_default():
    image = tf.placeholder(tf.uint8,[480,640,3])
    processed_image = inception_preprocessing.preprocess_image(image, 
    image_size, image_size, is_training=False)
    processed_images  = tf.expand_dims(processed_image, 0)
    # processed_images will be a 1x299x299x3 tensor of float32

    # Create the model, use the default arg scope to configure the batch norm parameters.
    with slim.arg_scope(inception_v4.inception_v4_arg_scope()):
      logits, _ = inception_v4.inception_v4(processed_images, num_classes=1001, is_training=False)
      probs = tf.nn.softmax(logits)

    init_fn = slim.assign_from_checkpoint_fn(
      os.path.join(checkpoints_dir, 'inception_v4.ckpt'),
      slim.get_model_variables('InceptionV4'))

    with tf.Session() as sess:
      init_fn(sess)

      while True:
        start = count
        probabilities = sess.run(probs,feed_dict={image: frame})
        probabilities = probabilities[0, 0:]
        sorted_inds = [i[0] for i in sorted(enumerate(-probabilities), key=lambda x:x[1])]

        for i in range(5):
          index = sorted_inds[i]
          print('Probability %0.2f%% => [%s]' % (probabilities[index] * 100, names[index]))

        end = count
        print "Classification latency = %d frames" % (end-start)

    # How to end this thread cleanly?
######################################################

# Start the threads
capture_thread = Thread(target=capture)
classify_thread = Thread(target=classification)
capture_thread.start()
classify_thread.start()