How to use complex or irregular shape as my roi for image detection and tracking using mouse function?

Question

I am currently trying to create roi of complex/irregular shape for detection and tracking.But unable to create it.

I am able to draw a curve around my complex shape using mouse callbackfunction but i don't know how to use my closed curve for roi.

This is the frame from the video which I am working on.

Image 1

Image 2

The shape in the image2 is the area I want my roi to be for tracking.I don't want my roi to be rectangle in shape because it is not able to track my roi. The shape in the image2 is actually an object which I want to track in the video.

Below is the code for tracking with rectangle roi.

import cv2
import sys
from matplotlib import pyplot as plt

(major_ver, minor_ver, subminor_ver) = (cv2.__version__).split('.')



# Set up tracker.
# Instead of MIL, you can also use

tracker_types = ['BOOSTING', 'MIL','KCF', 'TLD', 'MEDIANFLOW', 'GOTURN', 'MOSSE', 'CSRT']
tracker_type = tracker_types[1]


if tracker_type == 'BOOSTING':
    tracker = cv2.TrackerBoosting_create()
elif tracker_type == 'MIL':
    tracker = cv2.TrackerMIL_create()
elif tracker_type == 'KCF':
    tracker = cv2.TrackerKCF_create()
elif tracker_type == 'TLD':
    tracker = cv2.TrackerTLD_create()
elif tracker_type == 'MEDIANFLOW':
    tracker = cv2.TrackerMedianFlow_create()
elif tracker_type == 'GOTURN':
    tracker = cv2.TrackerGOTURN_create()
elif tracker_type == 'MOSSE':
    tracker = cv2.TrackerMOSSE_create()
elif tracker_type == "CSRT":
    tracker = cv2.TrackerCSRT_create()

# Read video
video = cv2.VideoCapture("3umresize4.avi")


# Exit if video not opened.
if not video.isOpened():
    print "Could not open video"
    sys.exit()

# Read first frame.
ok, frame = video.read()
if not ok:
    print 'Cannot read video file'
    sys.exit()

# Define an initial bounding box
bbox = (287, 23, 86, 320)

# Uncomment the line below to select a different bounding box
bbox = cv2.selectROI(frame, False)

# Initialize tracker with first frame and bounding box
ok = tracker.init(frame, bbox)

while True:
    # Read a new frame
    ok, frame = video.read()
    if not ok:
        break

    # Start timer
    timer = cv2.getTickCount()

    # Update tracker
    ok, bbox = tracker.update(frame)

    # Calculate Frames per second (FPS)
    fps = cv2.getTickFrequency() / (cv2.getTickCount() - timer);

    # Draw bounding box
    if ok:
        # Tracking success
        p1 = (int(bbox[0]), int(bbox[1]))
        p2 = (int(bbox[0] + bbox[2]), int(bbox[1] + bbox[3]))
        cv2.rectangle(frame, p1, p2, (255,0,0), 2, 1)
    else :
        # Tracking failure
        cv2.putText(frame, "Tracking failure detected", (100,80), cv2.FONT_HERSHEY_SIMPLEX, 0.75,(0,0,255),2)

    # Display tracker type on frame
    cv2.putText(frame, tracker_type + " Tracker", (100,20), cv2.FONT_HERSHEY_SIMPLEX, 0.75, (50,170,50),2);

    # Display FPS on frame
    cv2.putText(frame, "FPS : " + str(int(fps)), (100,50), cv2.FONT_HERSHEY_SIMPLEX, 0.75, (50,170,50), 2);


    # Display result
    cv2.namedWindow("Tracking",cv2.WINDOW_NORMAL)
    cv2.imshow("Tracking", frame)

    # Exit if ESC pressed
    k = cv2.waitKey(1) & 0xff
    if k == 27 :

        break



cv2.destroyAllWindows()

Below is the mouse callback to create image2 in which I want the area under the contour to be the roi in the above code.

import cv2
import numpy as np 

drawing=False # true if mouse is pressed
mode=True # if True, draw rectangle. Press 'm' to toggle to curve

# mouse callback function
def drawcurve(event,former_x,former_y,flags,param):
    global current_former_x,current_former_y,drawing, mode


    if event==cv2.EVENT_LBUTTONDOWN:
        drawing=True
        current_former_x,current_former_y=former_x,former_y

    elif event==cv2.EVENT_MOUSEMOVE:
        if drawing==True:
            if mode==True:
                cv2.line(im,(current_former_x,current_former_y),(former_x,former_y),(0,0,255),5)
                current_former_x = former_x
                current_former_y = former_y
                #print former_x,former_y
    elif event==cv2.EVENT_LBUTTONUP:
        drawing=False
        if mode==True:
            cv2.line(im,(current_former_x,current_former_y),(former_x,former_y),(0,0,255),5)
            current_former_x = former_x
            current_former_y = former_y
    return former_x,former_y    

im = cv2.imread("sampleuct.bmp")
cv2.namedWindow("Bill BEGUERADJ OpenCV")
cv2.setMouseCallback('Bill BEGUERADJ OpenCV',drawcurve)
while(1):
    cv2.imshow('Bill BEGUERADJ OpenCV',im)
    k=cv2.waitKey(1)&0xFF
    if k==27:
        break
cv2.destroyAllWindows()

Can you help me out it integrating my roi with tracking. Or any other way I could track it.

Answer 1

OpenCV's functions will not directly get you what you want, all these trackers are designed to do specific things which in the end track a localised region which is a box.

Since you want to track the shape as it is you may consider Kalman Filter based tracking, where you essentially track points. You could pick points of interest and connnect them in every frame and track only these points. You may want to implement one yourself too. You can refer this link as a tutorial to directly use OpenCV's implementation of Kalman Filter tracking.

You could also consider a deep learning based approach by using Semantic Networks, where you segment regions of interest at every frame.

One dirty trick is to create your roi as it is and every time the box is detected, you could adjust your roi size to the box size as required and use it.