How to separate checkbox button and checkbox text from an image using Python?

Question

I have an image

consisting of 5 checkboxes arranged in a parallel manner.

I need to separate checkbox button

and checkbox text

and save them separately.

There can be any number of checkboxes in an image arranged in a parallel manner how can I split the checkbox button and checkbox text and save them separately.

I am new to computer vision. Please guide me how to achieve it.

Answer 1

I am a bit late with the answer but nevertheless maybe it will serve someone for an alternitive idea.

First you search your image for contours and create masks and final images to display the outcome.

For every contour you find the size of the contour so it will help you filter your boxes from letters (boxes are larger in size). For the checkmark I found the solution to filter it by size AND by distance from extreme point on the left and the right side of the contour (you could find a lot of different criterion to filter the checkpoints and probably a lot better ones - I focused only on the image you posted in your question).

To get to the area of interest you find x,y,h,w values with cv2.boundingRect()

Then you simply draw it on the images created in the begining (note that these outputs are only black and white). Maybe this approach will give you some idea. Cheers!

Example in code:

import cv2
import numpy as np

img = cv2.imread('checkbox.png')
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
ret, threshold = cv2.threshold(gray,170,255,cv2.THRESH_BINARY_INV)
im, contours, hierarchy = cv2.findContours(threshold,cv2.RETR_TREE,cv2.CHAIN_APPROX_NONE)
final = np.zeros(gray.shape,np.uint8)
mask = np.zeros(gray.shape,np.uint8)
final2 = np.zeros(gray.shape,np.uint8)
list1 = []


for i in range(0, len(contours)):
    cnt = contours[i]
    epsilon = 0.1*cv2.arcLength(cnt,True)
    approx = cv2.approxPolyDP(cnt,epsilon,True)
    size = cv2.contourArea(approx)
    extLeft = tuple(cnt[cnt[:, :, 0].argmin()][0])
    extRight = tuple(cnt[cnt[:, :, 0].argmax()][0])
    distance = np.sqrt((extLeft[0] - extRight[0])**2 + (extLeft[1] - extRight[1])**2)
    x,y,w,h = cv2.boundingRect(cnt)
    mask[x:x+h, y:y+w]=0
    if 700 > size > 220:
        cv2.drawContours(mask,contours,i,255,-1)
        cv2.drawContours(final,contours,i,255,2)
    elif 16 < distance < 17 and size > 60:
        list1.append(cnt)
    elif size < 250:
        cv2.drawContours(final2,contours,i,(255,255,255),1)

for i in list1:        
    cv2.drawContours(final, [i], -1, (255,255,255), -1)

cv2.bitwise_not(final,final)
cv2.bitwise_not(final2,final2)
cv2.imwrite('c_orig.png', img)
cv2.imwrite('c_boxes.png', final)
cv2.imwrite('c_text.png', final2)
cv2.imshow('img', img)
cv2.imshow('img2', final)
cv2.imshow('img3', final2)
cv2.waitKey(0)
cv2.destroyAllWindows()

Output:

Answer 2

I have split the images that are arranged in a parallel manner and saved them like the below images.

Here is my working code that splits the square checkbox and the text in this image.

# Import necessary libraries
from matplotlib import pyplot as plt
import cv2

# Read image
image = cv2.imread('data_2/5.png')

# Height and width of the image
height, width = image.shape[:2]
print("Dimensions of the image.")
print("Height:",height)
print("Width:",width)

gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
gray = cv2.GaussianBlur(gray, (5,5), 0)


# Finding Edges
edges = cv2.Canny(gray, 60, 255)


# contours -> an outline representing or bounding the shape.
_,cnts, hierarchy = cv2.findContours(edges.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
contours = sorted(cnts, key=cv2.contourArea, reverse=True)[:10]
count = 1
for contour in contours:
    if(count<=4):
        #print("Count:",count)
        count = count + 1
        area = cv2.contourArea(contour)
        if area > 100000 and area < 1000:
            contours.remove(contour)
        perimeter = cv2.arcLength(contour, True)
        approx = cv2.approxPolyDP(contour, 0.01*perimeter, True)
        if len(approx) == 4:


            cv2.circle(image, (720, 360), 5, (255,0,0), 5)
            cv2.drawContours(image, [approx], -1, (0, 255, 0), 2)   
            M = cv2.moments(approx)

            centers = []
            if M["m00"] != 0:
                cX = int(M["m10"] / M["m00"])
                cY = int(M["m01"] / M["m00"])
            else:
                cX, cY = 0, 0

            P1 = approx[0]
            P1x = P1[0][0]
            P1y = P1[0][1]

            P2 = approx[1]
            P2x = P2[0][0]
            P2y = P2[0][1]

            P3 = approx[2]
            P3x = P3[0][0]
            P3y = P3[0][1]

            P4 = approx[3]
            P4x = P4[0][0]
            P4y = P4[0][1]

plt.imshow(image)
plt.title('Detecting Square')
plt.show()

# Cropping the square_image using array slices -- it's a NumPy array
cropped_square = image[P1y:P3y, P2x:P3x]

# Cropping the text image
cropped_text = image[P1y:P3y,P3x+5:width]

# Displaying the cropped square and cropped text image.
plt.imshow(cropped_square)
plt.title('Cropped Square')
plt.show()

plt.imshow(cropped_text)
plt.title('Cropped Text')
plt.show()

# Now saving the cropped square and cropped text image
cv2.imwrite('results/square1.png',cropped_square)
cv2.imwrite('results/text1.png',cropped_text)

The Output of the above Program:

Answer 3

Assuming that irrespective of the number of check boxes, your images will follow a grid like pattern, a good choice would be to try template matching. MSTM is an example.

You should try finding checked or unchecked boxes in the image by using template matching and extract the region to the right side of it. Since you've mentioned that there can be upto 5 checkboxes in a row, you can check the width of the image, divide that by 5 and get an approximate size of the text region