Unable to segment handwritten characters

Question

I am trying to extract handwritten numbers and alphabet from an image, for that i followed this stackoverflow link. It is working fine for most of the images where letter is written using marker but when i am using image where data is written using Pen it is failing miserably. Need some help to fix this.

Below is my code:

import cv2
import imutils
from imutils import contours

# Load image, grayscale, Otsu's threshold
image = cv2.imread('xxx/pic_crop_7.png')
image = imutils.resize(image, width=350)
img=image.copy()

# Remove border
kernel_vertical = cv2.getStructuringElement(cv2.MORPH_RECT, (1,50))
temp1 = 255 - cv2.morphologyEx(image, cv2.MORPH_CLOSE, kernel_vertical)
horizontal_kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (50,1))
temp2 = 255 - cv2.morphologyEx(image, cv2.MORPH_CLOSE, horizontal_kernel)
temp3 = cv2.add(temp1, temp2)
result = cv2.add(temp3, image)

# Convert to grayscale and Otsu's threshold
gray = cv2.cvtColor(result, cv2.COLOR_BGR2GRAY)
gray = cv2.GaussianBlur(gray,(5,5),0)
_,thresh = cv2.threshold(gray, 0, 255, cv2.THRESH_OTSU | cv2.THRESH_BINARY_INV)
# thresh=cv2.dilate(thresh,None,iterations=1)

# Find contours and filter using contour area
cnts = cv2.findContours(thresh, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
cnts = cnts[0] if len(cnts) == 2 else cnts[0]

MIN_AREA=45

digit_contours = []
for c in cnts:
    if cv2.contourArea(c)>MIN_AREA:
        x,y,w,h = cv2.boundingRect(c)
        cv2.rectangle(img, (x, y), (x + w, y + h), (36,255,12), 2)
        digit_contours.append(c)
#         cv2.imwrite("C:/Samples/Dataset/ocr/segmented" + str(i) + ".png", image[y:y+h,x:x+w])


sorted_digit_contours = contours.sort_contours(digit_contours, method='left-to-right')[0]
contour_number = 0
for c in sorted_digit_contours:
    x,y,w,h = cv2.boundingRect(c)
    ROI = image[y:y+h, x:x+w]
    cv2.imwrite('xxx/segment_{}.png'.format(contour_number), ROI)
    contour_number += 1
    
    
cv2.imshow('thresh', thresh)
cv2.imshow('img', img)
cv2.waitKey()

It is correctly able to extract the numbers when written using marker.

Below is an example:

Original Image

Correctly extracting charachters

Image where it fails to read.

Original Image

Incorrectly Extracting

Answer 1

The solution that CodingPeter has given is perfectly fine, except that it may not be generic apropos the two test images you have posted. So, here's my take on it that might work on both of your test images, albeit with a little lesser accuracy.

import numpy as np
import cv2
import matplotlib.pyplot as plt

plt.rcParams['figure.figsize'] = (20, 20)
plt.rcParams["image.cmap"] = 'gray'

img_rgb = cv2.imread('path/to/your/image.jpg')
img = cv2.cvtColor(img_rgb, cv2.COLOR_BGR2GRAY)

th = cv2.adaptiveThreshold(img,255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C,cv2.THRESH_BINARY_INV,11,2)

kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (15,1))
horiz = cv2.morphologyEx(th, cv2.MORPH_OPEN, kernel, iterations=3)
ctrs, _ = cv2.findContours(horiz,cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)
for ctr in ctrs:
    x,y,w,h = cv2.boundingRect(ctr)
    if w < 20:
        cv2.drawContours(horiz, [ctr], 0, 0, cv2.FILLED)
kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (1,10))
vert = cv2.morphologyEx(th, cv2.MORPH_OPEN, kernel, iterations=3)
ctrs, _ = cv2.findContours(vert,cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)
for ctr in ctrs:
    x,y,w,h = cv2.boundingRect(ctr)
    if h < 25:
        cv2.drawContours(vert, [ctr], 0, 0, cv2.FILLED)

th = th - (horiz | vert)
ctrs, _ = cv2.findContours(th,cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)
min_ctr_area = 400 # Min character bounding box area

for ctr in ctrs:
    x, y, w, h = cv2.boundingRect(ctr)
    # Filter contours based on size
    if w * h > min_ctr_area and \
    w < 100 and h < 100:
        cv2.rectangle(img_rgb, (x, y), (x+w, y+h), (0, 255, 0), 1)
plt.imshow(img_rgb)

Of course some of the parameters here are hard-coded for filtering, which compare the contour height and width to ascertain whether it is a part of a line or maybe a character. With different images you may have to smartly change these values.

Answer 2

In this case, you only need to adjust your parameter. Because there is no vertical line in your handwritten characters' background, so I decided to delete them.

# Remove border
horizontal_kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (50,1))
temp2 = 255 - cv2.morphologyEx(image, cv2.MORPH_CLOSE, horizontal_kernel)
result = cv2.add(temp2, image)

And it works.