How can I save the coordinates of bounding boxes of Tensorflow object detection in a csv file?

Question

I am currently working with the full course on Tensorflow object detection on youtube. Find the video here.

I am currently applying the code for "detection from the Webcam" to a video and now I am trying to sum up the detections in an excel or csv file. In detail, I need to have a file with every detection, its position as well as when it happened.

I have found some examples for how to print out coordinates of bounding boxes or using the detect.py function when working with YOLO, but still I did not find any ideas for my specific problem.

This is the code I am using for detection from a video or my webcam. In this example, I am detecting from the webcam:

cap = cv2.VideoCapture(0)
width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))

while cap.isOpened(): 
    ret, frame = cap.read()
    image_np = np.array(frame)

input_tensor = tf.convert_to_tensor(np.expand_dims(image_np, 0), dtype=tf.float32)
detections = detect_fn(input_tensor)

num_detections = int(detections.pop('num_detections'))
detections = {key: value[0, :num_detections].numpy()
              for key, value in detections.items()}
detections['num_detections'] = num_detections

# detection_classes should be ints.
detections['detection_classes'] = detections['detection_classes'].astype(np.int64)

label_id_offset = 1
image_np_with_detections = image_np.copy()

viz_utils.visualize_boxes_and_labels_on_image_array(
            image_np_with_detections,
            detections['detection_boxes'],
            detections['detection_classes']+label_id_offset,
            detections['detection_scores'],
            category_index,
            use_normalized_coordinates=True,
            max_boxes_to_draw=1,
            min_score_thresh=.75,
            agnostic_mode=False)

cv2.imshow('object detection', cv2.resize(image_np_with_detections, (800, 600)))



if cv2.waitKey(10) & 0xFF == ord('q'):
    cap.release()
    cv2.destroyAllWindows()
    break

Thank you so much for your help! <3

Answer 1

tf.image.draw_bounding_boxes( ) input is tf.constant( ) you can direct save to target file.

Sample: On the working function you remark the image display and save it with panda. Love concatenate, accumulate data create summaries.

"""""""""""""""""""""""""""""""""""""""""""""""""""""""""
ALE: 
EnvSpecEnvSpec(ChopperCommand-v4)
"""""""""""""""""""""""""""""""""""""""""""""""""""""""""

import os
from os.path import exists

import gym
import ale_py

import tensorflow as tf

import matplotlib.pyplot as plt
import matplotlib.animation as animation

import pandas as pd

"""""""""""""""""""""""""""""""""""""""""""""""""""""""""
[PhysicalDevice(name='/physical_device:GPU:0', device_type='GPU')]
None
"""""""""""""""""""""""""""""""""""""""""""""""""""""""""
physical_devices = tf.config.experimental.list_physical_devices('GPU')
assert len(physical_devices) > 0, "Not enough GPU hardware devices available"
config = tf.config.experimental.set_memory_growth(physical_devices[0], True)
print(physical_devices)
print(config)

"""""""""""""""""""""""""""""""""""""""""""""""""""""""""
Games Environments
"""""""""""""""""""""""""""""""""""""""""""""""""""""""""
env = gym.make("ALE/KungFuMaster-v5", render_mode='human')
n_outputs = env.action_space.n
obs = env.reset()
observation, reward, done, info, prob = env.step(1)

"""""""""""""""""""""""""""""""""""""""""""""""""""""""""
: Variables
"""""""""""""""""""""""""""""""""""""""""""""""""""""""""
CROP_SIZE = [ 210, 160 ]
IMAGE_SIZE = [ 210, 160, 1 ] 
BATCH_SIZE = 1
NUM_BOXES = 1
LONG_STEPS = 100000000000

global dataset
dataset = {
    "bounding_box" :[],
    "label" : []
}

fig = plt.figure()
image = plt.imread( "F:\\datasets\\downloads\\cats_name\\train\\Symbols\\01.jpg" )
im = plt.imshow(image)

"""""""""""""""""""""""""""""""""""""""""""""""""""""""""
: Functions
"""""""""""""""""""""""""""""""""""""""""""""""""""""""""
def crop_game_area( observation ):

    boxes = tf.constant([ 0.26, 0.05, 0.8, 1.0 ], shape=(1, 4))
    box_indices = tf.constant([ 0 ], shape=(1, ))
    image_array = tf.keras.preprocessing.image.img_to_array( observation )
    ###
    image_array = tf.image.rgb_to_grayscale( image_array ).numpy()
    ###
    image_cropped = tf.image.crop_and_resize( tf.expand_dims(image_array, axis=0), boxes, box_indices, CROP_SIZE )
    image_cropped = tf.reshape( image_cropped[0], IMAGE_SIZE )
    image_cropped = tf.keras.preprocessing.image.array_to_img( image_cropped )

    return image_cropped

def animate( i ):
    action = 1
    scores = 0
    observation, reward, done, info, prob = env.step(action)
    env.render()

    if done:
        observation, info = env.reset(return_info=True)
        action = 0
        scores = 0

    image_cropped = crop_game_area( observation )   
    image_cropped, object_count, prediction_scores, list_label, list_position_data = search_screen( image_cropped )

    im.set_array( image_cropped )
    plt.xlabel( str( object_count ) + " " + str( prediction_scores ), fontsize=22 )
    plt.ylabel( "scores: " + str( scores ) )
    plt.show()

    return im,
    
def search_screen( image_cropped ):
    global dataset
    
    image_cropped = tf.keras.preprocessing.image.img_to_array( image_cropped )
    image_cropped = tf.cast( image_cropped, dtype=tf.float32 )
    width = image_cropped.shape[1]
    height = image_cropped.shape[0]
    channels = image_cropped.shape[2]
    box_sizes = 10
    n_boxes = 10

    object_position = [ 0, 0, 0 ]
    object_properties = [ 0, 0, 0, 0, 0 ]
    object_count = [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ]

    list_input_data = tf.zeros([ 1, 21, 21, 3 ]).numpy()
    list_position_data = tf.zeros([ 1, 3 ]).numpy()
    list_label = tf.zeros([ 1, 1 ]).numpy()

    list_input_data = list_input_data[-100:,-2100:,-2100:,-300:]
    list_input_data = tf.cast( list_input_data, dtype=tf.float32 ).numpy()
    list_position_data = list_position_data[-100:,-300:]
    list_position_data = tf.cast( list_position_data, dtype=tf.float32 ).numpy()
    list_label = list_label[-100:,-100:]
    list_label = tf.cast( list_label, dtype=tf.float32 ).numpy()
    
    for i in range(n_boxes):
        for j in range(n_boxes):
            cropped_image_cell_search = tf.image.crop_to_bounding_box(image_cropped, int( CROP_SIZE[0] / 10 ) * i, 
            int( CROP_SIZE[1] / 10 ) * j, int( CROP_SIZE[0] / 10 ), int( CROP_SIZE[1] / 10 ) )
            input_data = tf.squeeze( cropped_image_cell_search )
            object_properties = tf.constant( object_properties, shape=( 1, 5 ), dtype=tf.float32 )
            

        colors = tf.constant([[0.5, 0.5, 0.5]])
        boxes_custom_input = tf.constant([ 0.26, 0.05, 0.8, 1.0 ], shape=(1, 1, 4))
        prediction_scores = 0
        
        #############################################
        ### 1. Save dataset using Panda
        new_dataset_folder = "F:\\temp\\Python\\excel"
        dataset["bounding_box"].append(boxes_custom_input.numpy())
        dataset["label"].append(prediction_scores)

        ### 
        df = pd.DataFrame(dataset)
        df.to_csv(os.path.join(new_dataset_folder, "boundary_box.csv"), index=False)
        #############################################

        image_cropped = tf.image.draw_bounding_boxes(tf.constant(image_cropped, shape=(1, IMAGE_SIZE[0], IMAGE_SIZE[1], IMAGE_SIZE[2]), dtype=tf.float32), boxes_custom_input, colors)
        image_cropped = tf.keras.preprocessing.image.img_to_array( tf.squeeze(image_cropped) *  255.0 )
        
    return image_cropped, object_count, prediction_scores, list_label, list_position_data

"""""""""""""""""""""""""""""""""""""""""""""""""""""""""
: Tasks
"""""""""""""""""""""""""""""""""""""""""""""""""""""""""
while LONG_STEPS > 0:
    ani = animation.FuncAnimation(fig, animate, interval=50, blit=True)
    plt.show()

Output:

bounding_box,label
[[[0.26 0.05 0.8  1.  ]]],0
[[[0.26 0.05 0.8  1.  ]]],0
[[[0.26 0.05 0.8  1.  ]]],0
[[[0.26 0.05 0.8  1.  ]]],0