Detect largest rectangle from an image using OpenCV

Question

I asked a previous question here and following the advice from the answer I built the below program which I thought would detect large rectangle but it doesn't detect the rectangle at all. It does work on this image though.

Original Image

Desired Image

I want the solution to work on not only this image but different images of this kind. Major part of the code below is from different answers on SO

My full program:

#include <cv.h>
#include <highgui.h>
using namespace cv;
using namespace std;

double angle(  Point pt1,  Point pt2,  Point pt0 ) {
    double dx1 = pt1.x - pt0.x;
    double dy1 = pt1.y - pt0.y;
    double dx2 = pt2.x - pt0.x;
    double dy2 = pt2.y - pt0.y;
    return (dx1*dx2 + dy1*dy2)/sqrt((dx1*dx1 + dy1*dy1)*(dx2*dx2 + dy2*dy2) + 1e-10);
}

void find_squares( Mat& image,  vector< vector< Point> >& squares)
{
    // blur will enhance edge detection
    Mat blurred(image);
    medianBlur(image, blurred, 9);

     Mat gray0(blurred.size(), CV_8U), gray;
     vector< vector< Point> > contours;

    // find squares in every color plane of the image
    for (int c = 0; c < 3; c++)
    {
        int ch[] = {c, 0};
         mixChannels(&blurred, 1, &gray0, 1, ch, 1);

        // try several threshold levels
        const int threshold_level = 2;
        for (int l = 0; l < threshold_level; l++)
        {
            // Use Canny instead of zero threshold level!
            // Canny helps to catch squares with gradient shading
            if (l == 0)
            {
                 Canny(gray0, gray, 10, 20, 3); // 

                // Dilate helps to remove potential holes between edge segments
                 dilate(gray, gray,  Mat(),  Point(-1,-1));
            }
            else
            {
                    gray = gray0 >= (l+1) * 255 / threshold_level;
            }

            // Find contours and store them in a list
             findContours(gray, contours, CV_RETR_LIST, CV_CHAIN_APPROX_SIMPLE);

            // Test contours
             vector< Point> approx;
            for (size_t i = 0; i < contours.size(); i++)
            {
                    // approximate contour with accuracy proportional
                    // to the contour perimeter
                     approxPolyDP( Mat(contours[i]), approx,  arcLength( Mat(contours[i]), true)*0.02, true);

                    // Note: absolute value of an area is used because
                    // area may be positive or negative - in accordance with the
                    // contour orientation
                    if (approx.size() == 4 &&
                            fabs(contourArea( Mat(approx))) > 1000 &&
                            isContourConvex( Mat(approx)))
                    {
                            double maxCosine = 0;

                            for (int j = 2; j < 5; j++)
                            {
                                    double cosine = fabs(angle(approx[j%4], approx[j-2], approx[j-1]));
                                    maxCosine = MAX(maxCosine, cosine);
                            }

                            if (maxCosine < 0.3)
                                    squares.push_back(approx);
                    }
            }
        }
    }
}

void find_largest_square(const vector<vector <Point> >& squares, vector<Point>& biggest_square) {
    if (!squares.size()) {
        return;
    }

    int max_width = 0;
    int max_height = 0;
    int max_square_idx = 0;
    const int n_points = 4;

    for (size_t i = 0; i < squares.size(); i++) {
        Rect rectangle = boundingRect(Mat(squares[i]));
        if ((rectangle.width >= max_width) && (rectangle.height >= max_height)) {
            max_width = rectangle.width;
            max_height = rectangle.height;
            max_square_idx = i;
        }
    }
    biggest_square = squares[max_square_idx];

}

int main(int argc, char* argv[])
{
    Mat img =  imread(argv[1]);
    if (img.empty())
    {
         cout << "!!! imread() failed to open target image" <<  endl;
        return -1;        
    }
    vector< vector< Point> > squares;
    find_squares(img, squares);
    vector<Point> largest_square;
    find_largest_square(squares, largest_square);
    for (int i = 0; i < 4; ++i) {
        line(img, largest_square[i], largest_square[(i+1)%4], Scalar(0, 255, 0), 1, CV_AA);
    }
    imwrite("squares.png", img);
    imshow("squares", img);
    waitKey(0);
    return 0;
}

Answer 1

I think you can do it easily using findContours function - http://docs.opencv.org/doc/tutorials/imgproc/shapedescriptors/find_contours/find_contours.html The biggest contour (or eventually second biggest) should be contour of black rectangle. Then just find the smallest rectangle which will surround this contour (just find points with the biggest/smallest x/y coordinates).