Why doesn't template_matching delete the previous ROI

Question

I am making a program that executes image-to-image correspondence between a PCB and its diagram. I have made a function template_matching that handles template matching and deleting visited areas so it can find all instances of the template above a certain threshold. The program should calculate the angle from the reference point (0,0) and see if the components match (I am doing this because i also want to check if their polarity is the same, otherwise I would have used BF Matching). The problem is that it still overlaps matches, meaning it didn't delete the previous matches from the image.

I have tried some logging prints and displaying the image at different points, but the problem seems to be how it deletes the zones. This is the code:

import cv2
import numpy as np

def template_matching(image, template, threshold=0.38):
    matches = []
    result = cv2.matchTemplate(image, template, cv2.TM_CCOEFF_NORMED)
    h, w = template.shape[:2]

    while True:
        min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(result)

        if max_val < threshold:
            break

        top_left = max_loc
        bottom_right = (top_left[0] + w, top_left[1] + h)

        # Append the match
        matches.append((top_left, bottom_right, max_val))

        # Zero out the region in the result matrix to avoid future detections in this area
        result[top_left[1]:bottom_right[1], top_left[0]:bottom_right[0]] = 0
        print(np.array(result))
        cv2.imshow('result', result)
        cv2.waitKey(0)
        cv2.destroyAllWindows(0)
        
    return matches

def calculate_angle(point1, point2):
    delta_y = point2[1] - point1[1]
    delta_x = point2[0] - point1[0]
    angle = np.arctan2(delta_y, delta_x) * 180 / np.pi
    return angle

def load_image(filepath):
    image = cv2.imread(filepath)
    if image is None:
        raise FileNotFoundError(f"Error: Unable to open or read image file at '{filepath}'. Please check the file path and file integrity.")
    return image

# File paths
pcb_photo_path = "C:\\Users\\q77597\\Desktop\\project\\templates\\template_5.jpeg"
diagram_path = "C:\\Users\\q77597\\Desktop\\project\\ceplm.png"
pcb_resistor_template_path = "C:\\Users\\q77597\\Desktop\\project\\templates\\pcb_resistor_template.png"
pcb_capacitor_template_path = "C:\\Users\\q77597\\Desktop\\project\\templates\\pcb_capacitor_template.png"
diagram_resistor_template_path = "C:\\Users\\q77597\\Desktop\\project\\templates\\d_resistor_template.png"
diagram_capacitor_template_path = "C:\\Users\\q77597\\Desktop\\project\\templates\\d_capacitor_template.png"

# Load images
try:
    pcb_photo = load_image(pcb_photo_path)
    diagram = load_image(diagram_path)
    pcb_resistor_template = load_image(pcb_resistor_template_path)
    pcb_capacitor_template = load_image(pcb_capacitor_template_path)
    diagram_resistor_template = load_image(diagram_resistor_template_path)
    diagram_capacitor_template = load_image(diagram_capacitor_template_path)

    pcb_copy = pcb_photo.copy()
    diagram_copy = diagram.copy()

except FileNotFoundError as e:
    print(e)
    exit(1)

# List of components and their respective templates
components = [
    ('resistor', pcb_resistor_template, diagram_resistor_template),
    ('capacitor', pcb_capacitor_template, diagram_capacitor_template)
]

# Reference points (these should be meaningful points in your specific use case)
point1_pcb = (0, 0)  # Reference point for the PCB
point1_diagram = (0, 0)  # Reference point for the diagram

for component_name, pcb_template, diagram_template in components:
    # Match components on the PCB photo
    pcb_matches = template_matching(pcb_copy, pcb_template)
    diagram_matches = template_matching(diagram_copy, diagram_template)

    for pcb_match, diagram_match in zip(pcb_matches, diagram_matches):
        top_left_pcb, bottom_right_pcb, match_score_pcb = pcb_match
        top_left_diagram, bottom_right_diagram, match_score_diagram = diagram_match

        print(f"PCB {component_name}: Top Left: {top_left_pcb}, Bottom Right: {bottom_right_pcb}, Match Score: {match_score_pcb}")
        print(f"Diagram {component_name}: Top Left: {top_left_diagram}, Bottom Right: {bottom_right_diagram}, Match Score: {match_score_diagram}")

        component_center_pcb = ((top_left_pcb[0] + bottom_right_pcb[0]) // 2, (top_left_pcb[1] + bottom_right_pcb[1]) // 2)
        component_center_diagram = ((top_left_diagram[0] + bottom_right_diagram[0]) // 2, (top_left_diagram[1] + bottom_right_diagram[1]) // 2)

        # Calculate angles
        angle_pcb = calculate_angle(point1_pcb, component_center_pcb)
        angle_diagram = calculate_angle(point1_diagram, component_center_diagram)

        # Check if the angles are approximately the same
        if np.isclose(angle_pcb, angle_diagram, atol=10):  # Adjust tolerance as needed
            print(f"{component_name} polarity is correct")
        else:
            print(f"{component_name} polarity is incorrect")

        # Visualize matches on PCB photo
        cv2.rectangle(pcb_copy, top_left_pcb, bottom_right_pcb, (0, 0, 255), 2)
        # Draw line to show angle on PCB
        cv2.line(pcb_copy, point1_pcb, component_center_pcb, (0, 0, 255), 2)
        cv2.putText(pcb_copy, f'Angle: {angle_pcb:.2f}', (component_center_pcb[0] + 10, component_center_pcb[1]), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 0, 0), 1)

        # Visualize matches on diagram
        cv2.rectangle(diagram_copy, top_left_diagram, bottom_right_diagram, (0, 0, 255), 2)
        # Draw line to show angle on diagram
        cv2.line(diagram_copy, point1_diagram, component_center_diagram, (0, 0, 255), 2)
        cv2.putText(diagram_copy, f'Angle: {angle_diagram:.2f}', (component_center_diagram[0] + 10, component_center_diagram[1]), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 0, 0), 1)

# Show the images
cv2.imshow('Matched components on PCB', pcb_copy)
cv2.imshow('Matched components on Diagram', diagram_copy)
cv2.waitKey(0)
cv2.destroyAllWindows()

These are the photos i used (the diagram and the pcb photo are 1:1, i overlapped them and drew some symbols):

Answer 1

Ok so I've taken it apart completely, and tried a different approach. I figured out working with two images wasn't necessary and I mainly worked on the diagram. Basically, what it does now is taking the coordinates of the ROI from the diagram and projecting them on the photo. To make an analogy, it's like putting two layers of dough one on top of the other and then using a cookie cutter:). I've made a little prototype of how it will work, the only thing that I need to do now is make it iterative, since I'll have to scan for more than one component. Here is the code and please let me know if you have any suggestions.

from TemplateMatching import load_image
import cv2

# File paths
pcb_photo_path = "C:\\Users\\q77597\\Desktop\\project\\templates\\template_5.jpeg"
pcb_capacitor_template_path = "C:\\Users\\q77597\\Desktop\\project\\templates\\pcb_capacitor_template.png"
diagram_path = "C:\\Users\\q77597\\Desktop\\project\\ceplm.png"
diagram_capacitor_template_path = "C:\\Users\\q77597\\Desktop\\project\\templates\\d_capacitor_template.png"

pcb = load_image(pcb_photo_path)
p_capacitor = load_image(pcb_capacitor_template_path)
diagram = load_image(diagram_path)
d_capacitor = load_image(diagram_capacitor_template_path)

def template_matching1(image, template):
    w, h = template.shape[1], template.shape[0]
    result = cv2.matchTemplate(image, template, cv2.TM_CCOEFF_NORMED)
    minVal, maxVal, minLoc, maxLoc = cv2.minMaxLoc(result)

    topLeft = maxLoc
    bottomRight = (maxLoc[0] + w, maxLoc[1] + h)

    cv2.rectangle(image, topLeft, bottomRight, (0, 0, 255), 2)
    cv2.imshow("test", image)
    cv2.waitKey(0)
    cv2.destroyAllWindows()
    
    return topLeft, bottomRight

def template_matching2(image, template, roi):
    topLeft_roi, bottomRight_roi = roi[0], roi[1]
    roi_img = image[topLeft_roi[1]:bottomRight_roi[0]+20, topLeft_roi[0]:bottomRight_roi[1]-20]
    w, h = template.shape[1], template.shape[0]
    result = cv2.matchTemplate(roi_img, template, cv2.TM_CCOEFF_NORMED)

    minVal, maxVal, minLoc, maxLoc = cv2.minMaxLoc(result)
    topLeft = maxLoc
    bottomRight = (maxLoc[0] + w, maxLoc[1] + h)

    cv2.rectangle(image, topLeft_roi, bottomRight_roi, (0, 0, 255), 2)

    cv2.imshow('test2', image)
    cv2.waitKey(0)
    cv2.destroyAllWindows()
    return 
    
    
  
result = template_matching1(diagram, d_capacitor)
template_matching2(pcb, p_capacitor, result)

(load_image is just a function I made in another file that reads the image)

Answer 2

In normalized template matching, the result matrix contains values between -1 to 1, where 1 represents a perfect match. Setting regions to 0 can still leave them in a detectable range if the threshold is not well below 0.

So instead of zeroing out the detected area you can try to set it as -1 like:

result[top_left[1]:bottom_right[1], top_left[0]:bottom_right[0]] = -1