Integrating texture-feature to color-based object detection for Video frames

Question

I'm working on a project where i have to detect colored vehicles from bird's eye view. See image-frame below

Because i believe that for this case, deep learning is overkill, I decided for the classical detection pipeline by applying Histogram Backprojection method for detecting the red vehicle.

The code i use is shown below

#import packages
import cv2 
import numpy as np
import matplotlib.pyplot as plt

#defining disk for the convolution
disc = cv2.getStructuringElement(cv2.MORPH_RECT,(10,10))
#defining Kernel
kernel=cv2.getStructuringElement(cv2.MORPH_RECT, (3,3))
kernel1 = np.ones((5,5),np.uint8)



#preparing the object model by cropping the target object and reading it
Red_Model=cv2.imread(r'C:/Users/kjbaili/.spyder-py3/webcam_calib/red_new_uniform.png')

#convert object_model to hsv
hsv_red =  cv2.cvtColor(Red_Model, cv2.COLOR_BGR2HSV)
cv2.imshow('HSV_Model',hsv_red)

#calculating histogram of object_model
M = cv2.calcHist([hsv_red],[0,1],None,[180,256],[0,180,0,256])

#reading the image where we want to search for the target object
img=cv2.imread(r'C:/Users/kjbaili/.spyder-py3/Mean_Shift/MST_with_3D_projection/messigray1.png')

#converting the image to hsv
hsv_img =  cv2.cvtColor(img, cv2.COLOR_BGR2HSV)

#calculating histogram of image
I= cv2.calcHist([hsv_img],[0,1],None,[180,256],[0,180,0,256])

#calculating the ration histogram
R=M.copy() /I.copy()

#extracting the image channels
h,s,v = cv2.split(hsv_img.copy())


 
#backprojecting the R_red hist
B1 = R[h.ravel(),s.ravel()]

#reshaping the resulted vector into image and normalize the values between 0 and 1
B2=B1.reshape(hsv_img.copy().shape[:2])
B = np.minimum(B1.copy(),1)
B = B.reshape(hsv_img.copy().shape[:2])
 



#removing noises
open_B_red=cv2.morphologyEx(B.copy(),cv2.MORPH_OPEN,kernel)     
Filter_red=cv2.filter2D(open_B_red.copy(),-1,disc,open_B_red)

#displaying results
cv2.imshow('Filter',Filter_red)



cv2.waitKey(0)
cv2.destroyAllWindows()

The algorithm works for the illustrated seen (see result mask image)

However, when object’s colors are similar with background’s colors like the image below_ it also detects my T-shirt, it detects both of them and thus fails to only discriminate the target object

Therefore, I thought of using additional feature along with the color feature in order to enhance the detection, so that it still detects the target object, despite the small similarity in colors and came to the so called "Local Binary Pattern-LBP", which calculate the texture of the object. In fact, i could calculate the LBP for the object and image, however don't know to integrate it with the above method so that the red vehicle is being detected through color- texture detection.

So I would really appreciate an idea of how to improve the detection in such a case either using Texture features from LBP or even another additional features.

for info about LBP please see : https://towardsdatascience.com/face-recognition-how-lbph-works-90ec258c3d6b

for infor about how Histogram back projection works : https://theailearner.com/2019/04/18/histogram-backprojection/

Here is the original image: Here is the resulted Mask thanks in advance

Answer 1

Here is one way to do that in Python/OpenCV. First threshold on color. Then apply some morphology to clean extraneous regions. Then get contours and loop over each one. Compute the rotated rectangle for each and its area and aspect ratio (largest dimension / smallest dimension). Filter the contours on a range of both and draw a white contour and a white rotated rectangle on the input image.

Input:

import cv2
import numpy as np

image = cv2.imread("red_car.png")
hh, ww = image.shape[:2]

# convert to HSV
hsv = cv2.cvtColor(image, cv2.COLOR_BGR2HSV)

# create a binary thresholded image
lower = (80,100,180)
upper = (255,255,255)
thresh = cv2.inRange(hsv, lower, upper)


# apply morphology
kernel = np.ones((5,5), np.uint8)
clean = cv2.morphologyEx(thresh, cv2.MORPH_CLOSE, kernel)

# get external contours
contours = cv2.findContours(clean, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
contours = contours[0] if len(contours) == 2 else contours[1]

area_thresh1 = 500
area_thresh2 = 1000
aspect_thresh1 = 2
aspect_thresh2 = 4
result1 = image.copy()
result2 = image.copy()
for c in contours:
    
    # get rotated rectangle from contour
    # get its dimensions
    # get angle relative to horizontal from rotated rectangle
    rotrect = cv2.minAreaRect(c)
    box = cv2.boxPoints(rotrect)
    box = np.int0(box)
    (center), (dim1,dim2), angle = rotrect
    maxdim = max(dim1,dim2)
    mindim = min(dim1,dim2)
    area = dim1 * dim2
    if area > 0:
        aspect = maxdim / mindim
        #print(area, aspect)

    if area > area_thresh1 and area < area_thresh2 and aspect > aspect_thresh1 and aspect < aspect_thresh2:
        # draw contour on input
        cv2.drawContours(result1,[c],0,(255,255,255),1)
        # draw rectangle on input
        cv2.drawContours(result2,[box],0,(255,255,255),1)
        print(area, aspect)

# save result
cv2.imwrite("red_car_thresh.png",thresh)
cv2.imwrite("red_car_clean.png",clean)
cv2.imwrite("red_car_thresh_result1.png",result1)
cv2.imwrite("red_car_thresh_result2.png",result2)

# display result
cv2.imshow("thresh", thresh)
cv2.imshow("clean", clean)
cv2.imshow("result1", result1)
cv2.imshow("result2", result2)
cv2.waitKey(0)
cv2.destroyAllWindows()

Threshold image:

Morphology cleaned image:

Car contour:

Car rotated bounding box:

Answer 2

Template Matching with few control parameters as (Size ..etc) can work for your case :

import cv2
import numpy as np
from matplotlib import pyplot as plt

img = cv2.imread('Path_To_\\target.jpg',0)
img2 = img.copy()
imgDisp = cv2.cvtColor(img2 , cv2.COLOR_GRAY2RGB)
template = cv2.imread('Path_t0\\template.jpg',0)
w, h = template.shape[::-1]

# All the 6 methods for comparison in a list
methods = ['cv2.TM_CCOEFF', 'cv2.TM_CCOEFF_NORMED', 'cv2.TM_CCORR',
        'cv2.TM_CCORR_NORMED', 'cv2.TM_SQDIFF', 'cv2.TM_SQDIFF_NORMED']

for meth in methods:
    img = img2.copy()
    method = eval(meth)

    # Apply template Matching
    res = cv2.matchTemplate(img,template,method)
    min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(res)

    # If the method is TM_SQDIFF or TM_SQDIFF_NORMED, take minimum
    if method in [cv2.TM_SQDIFF, cv2.TM_SQDIFF_NORMED]:
       top_left = min_loc
    else:
       top_left = max_loc
    bottom_right = (top_left[0] + w, top_left[1] + h)

    cv2.rectangle(imgDisp,top_left, bottom_right, 255, 2)

    plt.subplot(121),plt.imshow(res,cmap = 'gray')
    plt.title('Matching Result'), plt.xticks([]), plt.yticks([])
    plt.subplot(122),plt.imshow(imgDisp,cmap = 'gray')
    plt.title('Detected Point'), plt.xticks([]), plt.yticks([])
    plt.suptitle(meth)

    plt.show()

Result :

You can fine the code HERE