How to choose the right rectangles in an image?

Question

I want to detect the colors of a Rubiks Cube. That is what I want: Link
I'm able to recognize the 9 colored fields with the findContours function of Open CV.
Here is my code:

Mat input = new Mat(); //The image
Mat blur = new Mat();
Mat canny = new Mat();

Imgproc.GaussianBlur(input, blur, new Size(3,3), 1.5); //GaussianBlur to reduce noise

Imgproc.Canny(blur, canny, 60, 70); //Canny to detect the edges
Imgproc.GaussianBlur(canny, canny, new Size(3,3), 1.5); //Again GaussianBlur to reduce noise

List<MatOfPoint> contours = new ArrayList<>();
Mat hierachy = new Mat();

Imgproc.findContours(canny, contours, hierachy, Imgproc.RETR_LIST, Imgproc.CHAIN_APPROX_SIMPLE); //Find contours

List<MatOfPoint2f> approxedShapes = new ArrayList<>();
for(MatOfPoint point : contours){
    double area = Imgproc.contourArea(point);
    if(area > 1000){
        MatOfPoint2f shape = new MatOfPoint2f(point.toArray());
        MatOfPoint2f approxedShape = new MatOfPoint2f();

        double epsilon = Imgproc.arcLength(shape, true) / 10;

        Imgproc.approxPolyDP(shape, approxedShape, epsilon, true); //"Smooth" the edges with approxPolyDP
        approxedShapes.add(approxedShape);
    }
}

//Visualisation
for(MatOfPoint2f point : approxedShapes){
    RotatedRect rect = Imgproc.minAreaRect(new MatOfPoint2f(point.toArray()));
    Imgproc.circle(input, rect.center, 5, new Scalar(0, 0, 255));

    for(Point p : point.toArray()){
        Imgproc.circle(input, p, 5, new Scalar(0,255,0));
    }
}

This is the "raw" source image:

Image

It produces this output (Green circles: corners; Blue circles: center of the rectangle):

As you can see, there are more detected rectangles than 9. I want to get the nine midpoints in an array of Points.
How can I choose the right ones?
Hopefully, you can understand what I mean

Answer 1

I have written code to do this in OpenCV.

The basic process is as yours, find contours but then weed out small and non-convex contours.

After this you can iterate over your contours, for each one do the following:

1. Sort the border pixels in ascending order using Y then X coords
2. Iterate across the points. For each Y, add all points between each X and the next X to a vector. You now have a vector of all points contained within the vector. You can also use this to calculate the centroid and to calculate the mean RGB colour as below:

Some sample code is below although note that it's not complete, it should give you a ought idea.

void meanColourOfContour( const Mat& frame, vector<Point> contour, Vec3b& colour, vector<Point>& pointsInContour ) {
    sort(contour.begin(), contour.end(), pointSorter);
    
    
    //
    // Mean RGB values
    //
    int rsum = 0;
    int gsum = 0;
    int bsum = 0;
    
    int index = 0;
    Point lastP = contour[index++];
    pointsInContour.push_back(lastP);
    
    Vec3b rgbValue = frame.at<Vec3b>(lastP);
    rsum += rgbValue[0];
    gsum += rgbValue[1];
    bsum += rgbValue[2];
    
    int currentRow = lastP.y;
    int lastX = lastP.x;
    
    // For all remaining points in contour
    while( index < contour.size() ) {
        Point nextP = contour[index];
        
        // Save it
        pointsInContour.push_back(nextP);
        
        // If we're on the same row, add in values of intervening points
        if( nextP.y == currentRow ) {
            for( int x = lastX; x < nextP.x; x++ ) {
                Point p(x, currentRow);
                pointsInContour.push_back(p);
                rgbValue = frame.at<Vec3b>(p);
                rsum += rgbValue[0];
                gsum += rgbValue[1];
                bsum += rgbValue[2];
            }
        }
        // Add nextP
        rgbValue = frame.at<Vec3b>(nextP);
        rsum += rgbValue[0];
        gsum += rgbValue[1];
        bsum += rgbValue[2];
        
        lastX = nextP.x;
        currentRow = nextP.y;
        index++;
    }
    
    // Calculate mean
    size_t pointCount = pointsInContour.size();
    colour =Vec3b( rsum/pointCount, gsum/pointCount, bsum/pointCount);
}


void extractFacelets( const Mat& frame, vector<tFacelet>& facelets) {
    // Convert to Grey
    Mat greyFrame;
    cvtColor(frame, greyFrame, CV_BGR2GRAY);
    blur( greyFrame, greyFrame, Size(3,3));
    
    // Canny and find contours
    Mat cannyOut;
    Canny(greyFrame, cannyOut, 100, 200);
    
    vector<vector<Point>> contours;
    vector<Vec4i> hierarchy;
    findContours(cannyOut, contours, hierarchy, CV_RETR_TREE, CV_CHAIN_APPROX_NONE);
    
    // Filter out non convex contours
    for( int i=contours.size()-1; i>=0; i-- ) {
        if( contourArea(contours[i]) < 400 ) {
            contours.erase(contours.begin()+i);
        }
    }
    
    // For each contour, calculate mean RGB and plot in output
    int cindex = 0;
    for( auto iter = contours.begin(); iter != contours.end(); iter ++ ) {
        
        // Sort points in contour on ascending Y then X coord
        vector<Point> contour = (vector<Point>)*iter;
        vector<Vec3b> meanColours;
        
        Vec3b meanColour;
        vector<Point> pointsInContour;
        meanColourOfContour(frame, contour, meanColour, pointsInContour);
        
        meanColours.push_back(meanColour);
        
        long x=0; long y=0;
        for( auto iter=pointsInContour.begin(); iter != pointsInContour.end(); iter++ ) {
            Point p = (Point) *iter;
            x += p.x;
            y += p.y;
        }
        
        tFacelet f;
        f.centroid.x = (int) (x / pointsInContour.size());
        f.centroid.y = (int) (y / pointsInContour.size());
        f.colour = meanColour;
        f.visible = true;
        facelets.push_back(f);
    }
    
}