Game board - square detection in image

Question

I am trying to detect slightly oval-shaped squares on the 15x15 game board area and the player's hand area in a screenshot from a game similar to scrabble.

Below is the original screenshot:

Here is the code i have got but it is missing some empty squares and most of the squares containing letters.

import cv2
import numpy as np

image = cv2.imread(r'image_path')

gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)

blurred = cv2.GaussianBlur(gray, (5, 5), 0)

edges = cv2.Canny(blurred, 50, 150)

contours, _ = cv2.findContours(edges, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)

output_image = image.copy()

for contour in contours:
    epsilon = 0.02 * cv2.arcLength(contour, True)
    approx = cv2.approxPolyDP(contour, epsilon, True)
    
    if len(approx) == 4 and cv2.isContourConvex(approx):
        (x, y, w, h) = cv2.boundingRect(approx)
        
        aspect_ratio = float(w) / h
        if aspect_ratio > 0.8 and aspect_ratio < 1.2:
            cv2.drawContours(output_image, [approx], -1, (0, 255, 0), 3)

cv2.imshow('Detected Squares', output_image)
cv2.imwrite('processed_cleaned_image.png', output_image)
cv2.waitKey(0)
cv2.destroyAllWindows()

Here is the output image:

Two letters in the player's hand and most of the squares on the board are missing.

Given the constraint that there will always be 15x15 squares on the board, and that there will be 7 squares in the player's hand, except for a few of the last rounds.

Also i tried the Square detection in image solution but it got worse.

Answer 1

I am referring to an example from Line detection in python with OpenCV | Houghline method and I hope the results are satisfactory.

The Hough Transform is being utilized to detect lines, particularly horizontal and vertical ones, and its results are then used to identify the locations of squares in the image. While this approach is effective, it may be resource-intensive. Below, I provide outputs for two different scenarios. If you disregard diagonal lines and remove duplicates, you should be able to locate the squares

# Python program to illustrate HoughLine
# method for line detection
import cv2
import numpy as np

# Reading the required image in
# which operations are to be done.
# Make sure that the image is in the same
# directory in which this python program is
img = cv2.imread('img.jpg')

# Convert the img to grayscale
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)

# Apply edge detection method on the image
edges = cv2.Canny(gray, 50, 200, apertureSize=3)

# This returns an array of r and theta values
lines = cv2.HoughLines(edges, 1, np.pi/180, 200)

# The below for loop runs till r and theta values
# are in the range of the 2d array
for r_theta in lines:
    arr = np.array(r_theta[0], dtype=np.float64)
    r, theta = arr
    # Stores the value of cos(theta) in a
    a = np.cos(theta)

    # Stores the value of sin(theta) in b
    b = np.sin(theta)

    # x0 stores the value rcos(theta)
    x0 = a*r

    # y0 stores the value rsin(theta)
    y0 = b*r

    # x1 stores the rounded off value of (rcos(theta)-1000sin(theta))
    x1 = int(x0 + 1000*(-b))

    # y1 stores the rounded off value of (rsin(theta)+1000cos(theta))
    y1 = int(y0 + 1000*(a))

    # x2 stores the rounded off value of (rcos(theta)+1000sin(theta))
    x2 = int(x0 - 1000*(-b))

    # y2 stores the rounded off value of (rsin(theta)-1000cos(theta))
    y2 = int(y0 - 1000*(a))

    # cv2.line draws a line in img from the point(x1,y1) to (x2,y2).
    # (0,0,255) denotes the colour of the line to be
    # drawn. In this case, it is red.
    cv2.line(img, (x1, y1), (x2, y2), (0, 0, 255), 2)

# All the changes made in the input image are finally
# written on a new image houghlines.jpg
cv2.imwrite('linesDetected.jpg', img)

output 1:

import cv2
import numpy as np

# Read image
image = cv2.imread('img.jpg')

# Convert image to grayscale
gray = cv2.cvtColor(image,cv2.COLOR_BGR2GRAY)

# Use canny edge detection
edges = cv2.Canny(gray,50,150,apertureSize=3)

# Apply HoughLinesP method to 
# to directly obtain line end points
lines_list =[]
lines = cv2.HoughLinesP(
            edges, # Input edge image
            1, # Distance resolution in pixels
            np.pi/180, # Angle resolution in radians
            threshold=100, # Min number of votes for valid line
            minLineLength=5, # Min allowed length of line
            maxLineGap=10 # Max allowed gap between line for joining them
            )

# Iterate over points
for points in lines:
      # Extracted points nested in the list
    x1,y1,x2,y2=points[0]
    # Draw the lines joing the points
    # On the original image
    cv2.line(image,(x1,y1),(x2,y2),(0,0, 255),2)
    # Maintain a simples lookup list for points
    lines_list.append([(x1,y1),(x2,y2)])
    
# Save the result image
cv2.imwrite('detectedLines.png',image)

output 2:

Additionally, you could omit the upper and lower side lines if the positions of those elements are consistent across each image.