Finding the corners of a rectangle

Question

I'm trying to get the corners of this rectangle:

.

I tried using cv2.cornerHarris(rectangle, 2, 3, 0.04), but the left edges are not showed due to image brightness, I guess. So I tried applying a threshold before using cornerHarris, but the image produced showed a lot of vertices along the edges, not being possible to filter the corners.

I know that I need to filter it before using cornerHarris, but I don't know how. Could someone help me with this problem?

Ps. I've already tried to use blur, but it also doesn't work.

import cv2
import numpy as np
import matplotlib.pyplot as plt

rectangle = cv2.imread('rectangle.png', cv2.IMREAD_GRAYSCALE)
rectangle = np.where(rectangle > np.mean(rectangle), 255, 0).astype(np.uint8)

dst_rectangle = cv2.cornerHarris(rectangle, 2, 3, 0.04)
dst_rectangle = cv2.dilate(dst_rectangle, None)

mask = np.where(dst_rectangle > 0.01*np.max(dst_rectangle), 255, 0).astype(np.uint8)
points = np.nonzero(mask)

plt.imshow(dst_rectangle, cmap='gray')
plt.plot(points[1], points[0], 'or')
plt.show()

Answer 1

I would approach it differently by getting the corners of the rotated bounding box of the contour after adaptive thresholding. Here is my code in Python/OpenCV.

Input:

import cv2
import numpy as np

# read image
img = cv2.imread("rectangle.png")

# convert img to grayscale
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
gray = 255-gray

# do adaptive threshold on gray image
thresh = cv2.adaptiveThreshold(gray, 255, cv2.ADAPTIVE_THRESH_MEAN_C, cv2.THRESH_BINARY, 17, 1)
thresh = 255-thresh

# apply morphology
kernel = np.ones((3,3), np.uint8)
morph = cv2.morphologyEx(thresh, cv2.MORPH_OPEN, kernel)
morph = cv2.morphologyEx(morph, cv2.MORPH_CLOSE, kernel)

# separate horizontal and vertical lines to filter out spots outside the rectangle
kernel = np.ones((7,3), np.uint8)
vert = cv2.morphologyEx(morph, cv2.MORPH_OPEN, kernel)
kernel = np.ones((3,7), np.uint8)
horiz = cv2.morphologyEx(morph, cv2.MORPH_OPEN, kernel)

# combine
rect = cv2.add(horiz,vert)

# thin
kernel = np.ones((3,3), np.uint8)
rect = cv2.morphologyEx(rect, cv2.MORPH_ERODE, kernel)

# get largest contour
contours = cv2.findContours(rect, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
contours = contours[0] if len(contours) == 2 else contours[1]
for c in contours:
    area_thresh = 0
    area = cv2.contourArea(c)
    if area > area_thresh:
        area = area_thresh
        big_contour = c

# get rotated rectangle from contour
rot_rect = cv2.minAreaRect(big_contour)
box = cv2.boxPoints(rot_rect)
box = np.int0(box)
print(box)

# draw rotated rectangle on copy of img
rot_bbox = img.copy()
cv2.drawContours(rot_bbox,[box],0,(0,0,255),2)

# write img with red rotated bounding box to disk
cv2.imwrite("rectangle_thresh.png", thresh)
cv2.imwrite("rectangle_outline.png", rect)
cv2.imwrite("rectangle_bounds.png", rot_bbox)

# display it
cv2.imshow("IMAGE", img)
cv2.imshow("THRESHOLD", thresh)
cv2.imshow("MORPH", morph)
cv2.imshow("VERT", vert)
cv2.imshow("HORIZ", horiz)
cv2.imshow("RECT", rect)
cv2.imshow("BBOX", rot_bbox)
cv2.waitKey(0)

Thresholded Image:

Rectangle Region Extracted:

Rotated Bounding Box on Image:

Rotated Bounding Box Corners:

[[446 335]
 [163 328]
 [168 117]
 [451 124]]

ADDITION:

Here is a slightly shorter version of the code, which is achievable by adding some gaussian blurring before thresholding.

import cv2
import numpy as np

# read image
img = cv2.imread("rectangle.png")

# convert img to grayscale
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
gray = 255-gray

# blur image
blur = cv2.GaussianBlur(gray, (3,3), 0)

# do adaptive threshold on gray image
thresh = cv2.adaptiveThreshold(blur, 255, cv2.ADAPTIVE_THRESH_MEAN_C, cv2.THRESH_BINARY, 75, 2)
thresh = 255-thresh

# apply morphology
kernel = np.ones((5,5), np.uint8)
rect = cv2.morphologyEx(thresh, cv2.MORPH_OPEN, kernel)
rect = cv2.morphologyEx(rect, cv2.MORPH_CLOSE, kernel)

# thin
kernel = np.ones((5,5), np.uint8)
rect = cv2.morphologyEx(rect, cv2.MORPH_ERODE, kernel)

# get largest contour
contours = cv2.findContours(rect, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
contours = contours[0] if len(contours) == 2 else contours[1]
for c in contours:
    area_thresh = 0
    area = cv2.contourArea(c)
    if area > area_thresh:
        area = area_thresh
        big_contour = c

# get rotated rectangle from contour
rot_rect = cv2.minAreaRect(big_contour)
box = cv2.boxPoints(rot_rect)
box = np.int0(box)
for p in box:
    pt = (p[0],p[1])
    print(pt)

# draw rotated rectangle on copy of img
rot_bbox = img.copy()
cv2.drawContours(rot_bbox,[box],0,(0,0,255),2)

# write img with red rotated bounding box to disk
cv2.imwrite("rectangle_thresh.png", thresh)
cv2.imwrite("rectangle_outline.png", rect)
cv2.imwrite("rectangle_bounds.png", rot_bbox)

# display it
cv2.imshow("IMAGE", img)
cv2.imshow("THRESHOLD", thresh)
cv2.imshow("RECT", rect)
cv2.imshow("BBOX", rot_bbox)
cv2.waitKey(0)

Thresholded Image:

Rectangle Region Extracted:

Rotated Bounding Box on Image:

Rotated Bounding Box Corners:

(444, 335)
(167, 330)
(170, 120)
(448, 125)

Answer 2

Here's a simple approach:

• Obtain binary image. We load the image, grayscale, Gaussian blur, then adaptive threshold.

• Morphological operations. We create a rectangular kernel and morph open to remove the small noise

• Find distorted rectangle contour and draw onto a mask. Find contours, determine rotated bounding box, and draw onto a blank mask

• Find corners. We use the Shi-Tomasi Corner Detector already implemented as cv2.goodFeaturesToTrack which is supposedly shows better results compared to the Harris Corner Detector

---

Here's a visualization of each step:

Binary image

Morph open

Find rotated rectangle contour and draw/fill onto a blank mask

Draw rotated rectangle and corners to get result

Corner coordinates

(448.0, 337.0)
(164.0, 332.0)
(452.0, 123.0)
(168.0, 118.0)

Code

import cv2
import numpy as np

# Load image, grayscale, Gaussian blur, adaptive threshold
image = cv2.imread("1.png")
mask = np.zeros(image.shape, dtype=np.uint8)
gray = 255 - cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
blur = cv2.GaussianBlur(gray, (3,3), 0)
thresh = cv2.adaptiveThreshold(blur, 255, cv2.ADAPTIVE_THRESH_MEAN_C, cv2.THRESH_BINARY_INV, 51, 3)

# Morph open
kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (3,3))
opening = cv2.morphologyEx(thresh, cv2.MORPH_OPEN, kernel, iterations=1)

# Find distorted rectangle contour and draw onto a mask
cnts = cv2.findContours(opening, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
cnts = cnts[0] if len(cnts) == 2 else cnts[1]
rect = cv2.minAreaRect(cnts[0])
box = cv2.boxPoints(rect)
box = np.int0(box)
cv2.drawContours(image,[box],0,(36,255,12),2)
cv2.fillPoly(mask, [box], (255,255,255))

# Find corners on the mask
mask = cv2.cvtColor(mask, cv2.COLOR_BGR2GRAY)
corners = cv2.goodFeaturesToTrack(mask, maxCorners=4, qualityLevel=0.5, minDistance=150)

for corner in corners:
    x,y = corner.ravel()
    cv2.circle(image,(x,y),8,(255,120,255),-1)
    print("({}, {})".format(x,y))

cv2.imshow("thresh", thresh)
cv2.imshow("opening", opening)
cv2.imshow("mask", mask)
cv2.imshow("image", image)
cv2.waitKey(0)

Answer 3

I was able to locate 3 out of the 4 points, the 4th point can be found easily given the other three points since it's rectangle. Here is my solution:

import cv2
import numpy as np

img = cv2.imread('6dUIr.png',1)
gray = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)

#smooth the image
kernel = np.ones((5,5),np.float32)/25
gray = cv2.filter2D(gray,-1,kernel)

#histogram equalization
clahe = cv2.createCLAHE(clipLimit=1.45, tileGridSize=(4,4))
cl1 = clahe.apply(gray)

#find edges
edges = cv2.Canny(cl1,4,100)

#find corners
dst = cv2.cornerHarris(edges,2,3,0.04)
#result is dilated for marking the corners, not important
dst = cv2.dilate(dst,None)
# Threshold for an optimal value, it may vary depending on the image.
img[dst>0.25*dst.max()]=[0,0,255]

cv2.imshow('edges', edges)
cv2.imshow('output', img)
# cv2.imshow('Histogram equalized', img_output)

cv2.waitKey(0)

The code has many hard coded thresholds but it's a good start.

Answer 4

You can try with an adaptive threshold. Then you may either use cornerHarris if you only need corners, or depending on what you need to do next, you could also find useful findContours, which returns a list of bounding boxes