OpenCV fails to match template with image(matchTemplate)

Question

So I have an image

and a template

and I want to find the template image inside the image but my code doesn't find anything. I tried down sizing but still no detection. Please help me with the example:

import cv2
import imutils
import glob, os
import numpy as np

image = cv2.imread("mainimage.png")
gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
h, w = gray.shape[:2]
for file in glob.glob("template.png"):
    template = cv2.imread(file)
    template = cv2.cvtColor(template, cv2.COLOR_BGR2GRAY)
    found = None
    (tH, tW) = template.shape[:2]
    cv2.imshow("Template", template)

    for scale in np.linspace(1, 0.2, 20)[::-1]:
        resized = imutils.resize(gray, width=int(gray.shape[1] * scale))
        r = gray.shape[1] / float(resized.shape[1])

        if resized.shape[0] < tH or resized.shape[1] < tW:
            break
        edged = cv2.Canny(resized, 50, 200)
        result = cv2.matchTemplate(edged, template, cv2.TM_CCOEFF)
        (_, maxVal, _, maxLoc) = cv2.minMaxLoc(result)

        if found is None or maxVal > found[0]:
            found = (maxVal, maxLoc, r)

    (_, maxLoc, r) = found
    (startX, startY) = (int(maxLoc[0] * r), int(maxLoc[1] * r))
    (endX, endY) = (int((maxLoc[0] + tW) * r), int((maxLoc[1] + tH) * r))
    cv2.rectangle(image, (startX, startY), (endX, endY), (0, 0, 255), 2)
    cv2.imshow("Image", image)
    cv2.waitKey(0)

Answer 1

Your code is mostly good. In your posted code, you were scaling the wrong way. You were shrinking the mainimage instead of growing it. Also, you need to do Canny on both template and image.

In your posted image, the template was larger (160x160) than the region in mainimage( 88x88). If you scale mainimage, then the scale factor should be 1.818. It would probably be much faster to scale the template.

import cv2
# import imutils
import glob, os
import numpy as np

image = cv2.imread("mainimage.png")
gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
h, w = gray.shape[:2]
for file in glob.glob("template.png"):
    template = cv2.imread(file)
    template = cv2.cvtColor(template, cv2.COLOR_BGR2GRAY)
    found = None
    (tH, tW) = template.shape[:2]
    # cv2.imshow("Template", template)

    tEdged = cv2.Canny(template, 50, 200)

    for scale in np.linspace(1, 2, 20):
        # resized = imutils.resize(gray, width=int(gray.shape[1] * scale))
        resized = cv2.resize(gray, dsize = (0,0), fx = scale, fy = scale)

        r = gray.shape[1] / float(resized.shape[1])

        if resized.shape[0] < tH or resized.shape[1] < tW:
            break
        edged = cv2.Canny(resized, 50, 200)
        result = cv2.matchTemplate(edged, tEdged, cv2.TM_CCOEFF)
        (_, maxVal, _, maxLoc) = cv2.minMaxLoc(result)

        if found is None or maxVal > found[0]:
            found = (maxVal, maxLoc, r)

    (_, maxLoc, r) = found
    (startX, startY) = (int(maxLoc[0] * r), int(maxLoc[1] * r))
    (endX, endY) = (int((maxLoc[0] + tW) * r), int((maxLoc[1] + tH) * r))
    cv2.rectangle(image, (startX, startY), (endX, endY), (0, 0, 255), 2)

    # cv2.imshow("Image", image)
    cv2.imwrite('output.jpg', image)
    # ~ cv2.waitKey(0)

On my computer, this code takes 6 seconds to run.

Keypoint matching + Homography

As an alternative, keypoint matching + homography is insensitive to scale. In the following code dst holds points containing the bounding box of the found template. For me, this following code executes in 0.06 seconds:

import cv2
# import imutils
import glob, os
import numpy as np
import time

image = cv2.imread("mainimage.png")
gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
h, w = gray.shape[:2]

MIN_MATCH_COUNT = 3

start_time = time.time()

for file in glob.glob("template.png"):
    template = cv2.imread(file, 0)

    patchSize = 16

    orb = cv2.ORB_create(edgeThreshold = patchSize, 
                            patchSize = patchSize)
    kp1, des1 = orb.detectAndCompute(template, None)
    kp2, des2 = orb.detectAndCompute(gray, None)

    FLANN_INDEX_LSH = 6
    index_params= dict(algorithm = FLANN_INDEX_LSH,
               table_number = 6,
               key_size = 12,    
               multi_probe_level = 1)
    search_params = dict(checks = 50)

    flann = cv2.FlannBasedMatcher(index_params, search_params)
    matches = flann.knnMatch(des1,des2,k=2)
    # store all the good matches as per Lowe's ratio test.
    good = []

    for pair in matches:
        if len(pair) == 2:
            if pair[0].distance < 0.7*pair[1].distance:
                good.append(pair[0])

    print('len(good) ', len(good))
    print('match %03d, min_match %03d, kp %03d' % (len(good), MIN_MATCH_COUNT, len(kp1)))
    if len(good)>MIN_MATCH_COUNT:
        src_pts = np.float32([ kp1[m.queryIdx].pt for m in good ]).reshape(-1,1,2)
        dst_pts = np.float32([ kp2[m.trainIdx].pt for m in good ]).reshape(-1,1,2)
        M, mask = cv2.findHomography(src_pts, dst_pts, cv2.RANSAC,5.0)
        matchesMask = mask.ravel().tolist()
        h,w = template.shape
        pts = np.float32([ [0,0],[0,h-1],[w-1,h-1],[w-1,0] ]).reshape(-1,1,2)
        dst = cv2.perspectiveTransform(pts,M)

        # dst contains points of bounding box of template in image. 
        # draw a close polyline around the found template:
        image = cv2.polylines(image,[np.int32(dst)], 
                              isClosed = True,
                              color = (0,255,0),
                              thickness = 3, 
                              linetype = cv2.LINE_AA)                    
    else:
        print( "Not enough matches are found - {}/{}".format(len(good), MIN_MATCH_COUNT) )
        matchesMask = None

    draw_params = dict(matchColor = (0,255,0), # draw matches in green color
               singlePointColor = None,
               matchesMask = matchesMask, # draw only inliers
               flags = 2)

    if len(good)>MIN_MATCH_COUNT:
        output2 = cv2.drawMatches(template,kp1,gray,kp2,good,None,**draw_params)

    print('elapsed time ', time.time()-start_time)
    # cv2.imshow("Image", image)
    cv2.imwrite('output_homography.jpg', image)
    cv2.imwrite('output2.jpg', output2)

output2 from cv2.drawMatches function

One of the important parameters for keypoint detection is the patchSize. In the code, we use patchSize = 16 for both image and template. As you make patchSize smaller, you'll get more keypoints. The smallest you can go is 2. But as you get too small, you start to get bad matches. I'm not sure how to find the sweet spot.