Pytesseract not giving expected output

Question

I'm new to pyhton and I'm making a number plate recognition system using haar cascade. My code works fine to detect the number plate and make contours but pytesseract ocr fails to recognise the characters and gives strange results. Please help.

The detected plate using haar cascades

contours made on the detected region

This is the output

import cv2
import numpy as np
import pytesseract

plate_cascade = cv2.CascadeClassifier('C:/Users/Jai/Desktop/Test/haarcascade_plate.xml')

img = cv2.imread('C:/Users/Jai/Desktop/Test/images/NumberPlates/IMG_20181029_194221.jpg', cv2.IMREAD_COLOR)

gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
gray = np.array(gray, dtype='uint8')
cv2.imshow('gray', gray)

plates = plate_cascade.detectMultiScale(gray, 1.3, 5)

for (x,y,w,h) in plates:
    cv2.rectangle(img, (x,y), (x+w,y+h),(255,0,0),5)
    roiGray = gray[y:y+h, x:x+w]
    roiImg = img[y:y+h, x:x+w]

roiUGray = cv2.bitwise_not(roiGray)
cv2.imshow('img', img)
cv2.imshow('roi_gray', roiUGray)

ret, thresh = cv2.threshold(roiUGray, 127, 255,0)
cv2.imshow("img", img)
height, width = thresh.shape
newImage = np.zeros((height, width, 3), np.uint8)
newImage[:, :] = (0,0,0)
im2, contours, hierarchy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_NONE)

num = 0
area = 0

for j in range(len(contours)):
    if hierarchy[0][j][3] != -1:
        area = area + cv2.contourArea(contours[j])
        num += 1

if num != 0:
    avgArea = float(area)/float(num)

for j in range(len(contours)):
    if hierarchy[0][j][3] != -1 and cv2.contourArea(contours[j]) > 0.2*avgArea:
        cv2.drawContours(newImage, contours[j], -1, (255,255,255), 1)

blur = cv2.GaussianBlur(newImage, (1, 1), 0)
cv2.imshow("roi", blur)

ocr_result = pytesseract.image_to_string(blur, lang='eng')
print(ocr_result)

cv2.waitKey(0)
cv2.destroyAllWindows()

Answer 1

This method uses contour detection to find the area of the number plate, and does a perspective transform on it. It then uses adaptive thresholding to detect the digits and does a medianBlur to get rid of noise that messes up the pytesseract operation.

The blue box is from the original image. The red box is from my number plate recognition haar cascade. The green is the contour detection of the number plate.

Here's the output of the perspective transform. I used the imutils module to do this.

This is the output of the adaptive thresholding and blurring, which I used the skimage module for.

Using this I got the output of:

\fUP1ADN7120

And removing all characters that aren't uppercase or a digit gives:

UP1ADN7120

Which is only one character wrong.

This method isn't too bad, but you could get even better using another method. If this didn't work for you, then you could create a CNN.

Here's the code:

import cv2
import numpy as np
import pytesseract
import imutils.perspective
from skimage.filters import threshold_local

plate_cascade = cv2.CascadeClassifier('/usr/local/lib/python3.5/dist-packages/cv2/data/haarcascade_russian_plate_number.xml')

img = cv2.imread('GNA0d.jpg', cv2.IMREAD_COLOR)

gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
gray = np.array(gray, dtype='uint8')

plates = plate_cascade.detectMultiScale(gray, 1.3, 5)

for (x,y,w,h) in plates:
    cv2.rectangle(img, (x,y), (x+w,y+h),(0,0,255),2)
    roiGray = gray[y:y+h, x:x+w]
    roiImg = img[y:y+h, x:x+w]

blur = cv2.GaussianBlur(roiGray, (5, 5), 0)
edges = cv2.Canny(blur, 75, 200)

contours = cv2.findContours(edges.copy(), cv2.RETR_LIST, cv2.CHAIN_APPROX_SIMPLE)[1]
contours = sorted(contours, key = cv2.contourArea, reverse = True)[:5]  #sort the contours, only getting the biggest to improve speed

for contour in contours:
    peri = cv2.arcLength(contour, True)
    approx = cv2.approxPolyDP(contour, 0.02 * peri, True)

    #find contours with 4 edges
    if len(approx) == 4:
        screenCnt = approx
        break

orig = roiImg.copy()
cv2.drawContours(roiImg, [screenCnt], -1, (0, 255, 0), 2)

#do a perspective transform
warped = imutils.perspective.four_point_transform(orig, screenCnt.reshape(4, 2))
graywarp = imutils.perspective.four_point_transform(roiGray, screenCnt.reshape(4, 2))

#threshold using adaptive thresholding
T = threshold_local(graywarp, 11, offset = 10, method = "gaussian")
graywarp = (graywarp > T).astype("uint8") * 255

#do a median blur to remove noise
graywarp = cv2.medianBlur(graywarp, 3)

text = pytesseract.image_to_string(graywarp)
print(text)
print("".join([c for c in text if c.isupper() or c.isdigit()]))


cv2.imshow("warped", warped)
cv2.imshow("graywarp", graywarp)
cv2.imshow('img', img)


cv2.waitKey(0)
cv2.destroyAllWindows()

Quite a bit was taken from pyimagesearch, who explains it better than I ever could.