Recognize number on 7 segment display with OpenCV

Question

I am trying to recognize the number on the 7 segment display.

I am using python on Jupyter notebook .

I have the 0~9 7 segment displayed number image, and each number with . is saved seperately. Below is the sample image of 3 ,3. ,2 , 2.

and I want to find these image on the target image.

I heard there are tools to find similar image on OpenCV.

I tried Brute-Force Matching with SIFT Descriptors and Ratio Test but the output does not seem accurate.

import numpy as np
import cv2 as cv
import matplotlib.pyplot as plt

img1 = cv2.imread('C:\\Users\\USER\\Desktop\\test\\deeplearningimage\\thermo\\3..png',cv2.IMREAD_GRAYSCALE) # trainImage
img2 = cv2.imread('C:\\Users\\USER\\Desktop\\test\\thermosample.jpg',cv2.IMREAD_GRAYSCALE)          # queryImage
# Initiate SIFT detector
sift = cv.SIFT_create()
# find the keypoints and descriptors with SIFT
kp1, des1 = sift.detectAndCompute(img1,None)
kp2, des2 = sift.detectAndCompute(img2,None)
# BFMatcher with default params
bf = cv2.BFMatcher()
matches = bf.knnMatch(des1,des2,k=2)
# Apply ratio test
good = []
for m,n in matches:
    if m.distance < 0.75*n.distance:
        good.append([m])
# cv.drawMatchesKnn expects list of lists as matches.
img3 = cv2.drawMatchesKnn(img1,kp1,img2,kp2,good,None,flags=cv.DrawMatchesFlags_NOT_DRAW_SINGLE_POINTS)
plt.imshow(img3),plt.show()'

here is the output of the code above

Not sure how to proceed with this. Any other opencv that would work for this problem?

Answer 1

You can use template matching after thresholding, and edge detection

import numpy as np
import matplotlib.pyplot as plt
import cv2

# Read Image
BGR = cv2.imread('input.jpg')
RGB = cv2.cvtColor(BGR, cv2.COLOR_BGR2RGB)

# Channels split
R = BGR[...,2]
G = BGR[...,1]
B = BGR[...,0]

# Threshold per channel
R[B>120] = 0
R[G>120] = 0
R[R<230] = 0

# Binarize
Binary = cv2.threshold(R, 127, 255, cv2.THRESH_BINARY)[1]
# Edge Detection
Edges = cv2.Canny(Binary, 50, 200)

# Read Template
templBGR = cv2.imread('templ.png')
templRGB =  cv2.cvtColor(templBGR, cv2.COLOR_BGR2RGB)
templateGray =  cv2.cvtColor(templBGR, cv2.COLOR_BGR2GRAY)
# Binarize Template
templateBinary = cv2.threshold(templateGray, 84, 255, cv2.THRESH_BINARY)[1]
# Denoise Template
templateFiltered = cv2.medianBlur(templateBinary,7)
# Resize Template
template = cv2.resize(templateFiltered, (templBGR.shape[1]//2, templBGR.shape[0]//2))
# Edge Detection Template
templateEdges = cv2.Canny(template, 50, 200)
# Extract Dimensions
h, w = template.shape

res = cv2.matchTemplate(Edges,templateEdges,cv2.TM_CCORR)

(_, _, _, maxLoc) = cv2.minMaxLoc(res)

img = RGB.copy()
cv2.rectangle(img, (maxLoc[0], maxLoc[1]), (maxLoc[0] + w, maxLoc[1] + h), (255,255,128), 2)

plt.subplot(221)
plt.imshow(RGB)
plt.title('Original')
plt.axis('off')

plt.subplot(222)
plt.imshow(Edges, cmap='gray')
plt.title('Segmented')
plt.axis('off')

plt.subplot(223)
plt.imshow(templRGB)
plt.title('Template')
plt.axis('off')

plt.subplot(224)
plt.imshow(img)
plt.title('Result')
plt.axis('off')

plt.show()

if you want to do multi-matching better to use a loop:

threshold = 0.8
Loc = np.where( res >= threshold)
for pt in zip(*Loc):
    cv2.rectangle(img, (Loc[0], Loc[1]), (Loc[0] + w, Loc[1] + h), (255,255,128), 2)