Opencv: find box within a box

Question

LOGISTICS

opencv 2.4 and Python 2.7

The image I'm working with:

THE PROBLEM

What I'm interested in is isolating the outline that forms the box around the 9 vertical and horizontal lines. I'm just not sure on how to go about this. I've looked at various tutorials, such as those done on Sudoku puzzles and those simply assume that the largest box is the one that you're looking for (since a sudoku puzzle doesn't have boxes within boxes, minus the actual grid). I've tried using the findContour function and filtering contours by size but with no luck. I end up with inconsistent results that sometimes finds the right contour but other times finds contours that are completely wrong. Can anyone point me in the right direction? Thanks.

Original Image:

Answer 1

It's better to put the original image, but I tried to interpret from your contours images

I did the following steps

1. you may need some noise removal (erosion)
2. calculate the horizontal and vertical projection from these contours
3. Plot the projection on the image to be able analyzing the bounds of the projections with regard to check sheet.
4. note that the vertical projection (green) how it gives you an indication to the left and right bounds, and so does the horizontal (blue) for the bottom and top of sheet.

You just need to smooth the projection and refine your search about the exact outlines.

if you think it's useful i can share the code implemented by opencv c++ (not python)

EDIT:

This is the code that i used to make horizontal and vertical projections, you may need to optimize it.

 void HVprojection(Mat image)
{
    // find the vertical projection
    Mat smothedRes = image.clone();
    vector<double> v_vl_proj; // holds the column sum values
    double max_vl_proj_h = 0,max_vl_proj_v=0; // holds the maximum value
    double average_v=0;
    for( int i=0;i<image.cols;++i )
    {
        Mat col;
        Scalar col_sum;
        // get individual columns
        col= image.col(i);
        col_sum = sum( col ); // find the sum of ith column
        v_vl_proj.push_back( col_sum.val[0] ); // push back to vector
        if( col_sum.val[0]>max_vl_proj_v )    max_vl_proj_v = col_sum.val[0];
        average_v+= col_sum.val[0];
    }
    average_v = average_v/image.cols;

    // find the horizontal projection
    vector<double> h_vl_proj; // holds the row sum values

    double average_h=0;
    for( int i=0;i<image.rows;++i )
    {
        Mat row;
        Scalar row_sum;
        // get individual columns
        row= image.row(i);
        row_sum = sum(row); // find the sum of ith row
        h_vl_proj.push_back(row_sum.val[0]); // push back to vector
        if( row_sum.val[0]>max_vl_proj_h )    max_vl_proj_h = row_sum.val[0];
        average_h+= row_sum.val[0];
    }
    average_h = average_h/image.rows;


    //******************Plotting vertical projection*******************
    for(int j =1;j<image.cols;j++)
    {
        int y1 = int(image.rows*v_vl_proj[j-1]/max_vl_proj_v);
        int y2 = int(image.rows*v_vl_proj[j]/max_vl_proj_v);
          line(image,Point(j-1,y1),Point(j,y2),Scalar(255,255,255),1,8);
        }
    int average_y = int(image.rows*average_v/max_vl_proj_v);  // zero level
    line(image,Point(0,average_y),Point(image.cols,average_y),Scalar(255,255,255),1,8);




    //***************Plotting horizontal projection**************
    for(int j =1;j<image.rows;j++)
    {
        int x1 = int(0.25*image.cols*h_vl_proj[j-1]/max_vl_proj_h);
        int x2 = int(0.25*image.cols*h_vl_proj[j]/max_vl_proj_h);
          line(image,Point(x1,j-1),Point(x2,j),Scalar(255,0,0),1,8);
        }
    int average_x = int(0.25*image.cols*average_h/max_vl_proj_h);
    line(image,Point(average_x,0),Point(average_x,image.rows),Scalar(255,0,0),1,8);

    imshow("horizontal_projection",image);
    imwrite("h_p.jpg",image);



// if you want to smooth the signal of projection in case of noisu signal
    v_vl_proj = smoothing(v_vl_proj);

    for(int j =1;j<image.cols;j++)
        {
            int y1 = int(image.rows*v_vl_proj[j-1]/max_vl_proj_v);
            int y2 = int(image.rows*v_vl_proj[j]/max_vl_proj_v);
              line(smothedRes,Point(j-1,y1),Point(j,y2),Scalar(0,255,0),1,8);
            }
        int average_y1 = int(smothedRes.rows*average_v/max_vl_proj_v);  // zero level
        line(smothedRes,Point(0,average_y1),Point(smothedRes.cols,average_y1),Scalar(0,255,0),1,8);

        imshow("SMoothed",smothedRes);
        imwrite("Vertical_projection.jpg",smothedRes);
        waitKey(0);
}

To smooth the projection signal:

vector<double> smoothing(vector<double> a)
 {
     //How many neighbors to smooth
     int NO_OF_NEIGHBOURS=5;
     vector<double> tmp=a;
     vector<double> res=a;

     for(int i=0;i<a.size();i++)
     {

         if(i+NO_OF_NEIGHBOURS+1<a.size())
         {
             for(int j=1;j<NO_OF_NEIGHBOURS;j++)
             {
                 res.at(i)+=res.at(i+j+1);
             }
             res.at(i)/=NO_OF_NEIGHBOURS;
         }
         else
         {
             for(int j=1;j<NO_OF_NEIGHBOURS;j++)
             {
                 res.at(i)+=tmp.at(i-j);
             }
             res.at(i)/=NO_OF_NEIGHBOURS;

         }


     }
     return res;

 }

Answer 2

Inspired by @dervish's answer, I have some idea.

1. Use cv::HoughLines() to get the axis directions.
2. Estimate the perspective transform matrix (M) to align image w.r.t. axis direction.
3. Use cv::warpPerspective() to warp the image.
4. Use @dervish's answer to get grid line candicates.
5. Filter the line candicates by color information (blue and white checkerboard) and line distance.
6. Do reverse transform on final grid with M to get grid in original image space.

Or find the final grid position directly in step 2 to get the Nth longest lines. And filter result by step 4.

python code:

import cv2
import numpy as np


def main():
    im = cv2.imread('image.png')
    #edge = cv2.imread('edge.png', 0)
    edge = cv2.Canny(im, 100, 200, apertureSize=3)
    lines = cv2.HoughLines(edge, 1, np.pi/180, 140)
    for rho, theta in lines[0]:
        a = np.cos(theta)
        b = np.sin(theta)
        x0 = a*rho
        y0 = b*rho
        x1 = int(x0 + 1000*(-b))
        y1 = int(y0 + 1000*(a))
        x2 = int(x0 - 1000*(-b))
        y2 = int(y0 - 1000*(a))
        cv2.line(im, (x1, y1), (x2, y2), (0, 0, 255), 2)
        # TODO: filter the lines by color and line distance

    cv2.imshow('image', im)
    cv2.imshow('edge', edge)
    cv2.waitKey(0)
    cv2.destroyAllWindows()

if __name__ == '__main__':
    main()