Basic Shape recognition (openCV C++)

Question

I have a project that I need to make for classes and I chose task that is a bit out of my skills. Target is to count result of dice rolls. For now, I'am trying to make it work on a sample pic:

sample pic of dices

and my current code is added below:

#include "opencv2/core/core.hpp"
#include "opencv2/highgui/highgui.hpp"
#include "opencv2/imgproc/imgproc.hpp"
#include "iostream"

using namespace cv;
using namespace std;

Mat KostkaFunkcja(Mat image, Mat in, Scalar low, Scalar high);
int getMaxAreaContourId(vector <vector<cv::Point>> contours);
vector<Point> contoursConvexHull(vector<vector<Point> > contours, int index);
Mat ZnakiFunkcja(Mat image, Mat in, Scalar low, Scalar high);


int main(int argc, char** argv)
{
    Mat image;
    image = imread("kostki.jpg", CV_LOAD_IMAGE_COLOR);

    if (!image.data)
    {
        cout << "Could not open or find the image" << std::endl;
        return -1;
    }

    Mat imgHSV;
    Mat workimage = image;

    cvtColor(workimage, imgHSV, COLOR_BGR2HSV); //Convert the captured frame from BGR to HSV
    //red dice
    workimage = KostkaFunkcja(workimage, imgHSV, Scalar(146, 0, 31), Scalar(179, 255, 255));
    //green dice
    workimage = KostkaFunkcja(workimage, imgHSV, Scalar(25, 147, 0), Scalar(98, 255, 154));
    //yellow dice
    workimage = KostkaFunkcja(workimage, imgHSV, Scalar(22, 45, 161), Scalar(91, 255, 255));
    //black dice
    workimage = KostkaFunkcja(workimage, imgHSV, Scalar(98, 0, 0), Scalar(179, 232, 107));
    //white symbols
    workimage = ZnakiFunkcja(workimage, imgHSV, Scalar(58, 0, 183), Scalar(179, 145, 255));
    namedWindow("Kostki_kontur", CV_WINDOW_AUTOSIZE);
    imshow("Kostki_kontur", workimage);
    waitKey(0);

    return 0;

}

Mat KostkaFunkcja(Mat image, Mat in, Scalar low, Scalar high)
{
    Mat temp;
    inRange(in, low, high, temp);
    erode(temp, temp, getStructuringElement(MORPH_ELLIPSE, Size(5, 5)));
    dilate(temp, temp, getStructuringElement(MORPH_ELLIPSE, Size(5, 5)));
    dilate(temp, temp, getStructuringElement(MORPH_ELLIPSE, Size(5, 5)));
    erode(temp, temp, getStructuringElement(MORPH_ELLIPSE, Size(5, 5)));

    Mat srcBlur, srcCanny;
    blur(temp, srcBlur, Size(3, 3));

    Canny(srcBlur, srcCanny, 0, 100, 3, true);
    vector<vector<Point> > contours;
    findContours(srcCanny, contours, CV_RETR_EXTERNAL, CV_CHAIN_APPROX_SIMPLE);

    int largest_contour_index = getMaxAreaContourId(contours);

    Mat drawing = Mat::zeros(srcCanny.size(), CV_8UC3);

    for (int i = 0; i< contours.size(); i++)
    {
        Scalar color = Scalar(255, 255, 255);
        drawContours(drawing, contours, i, color, 2);
    }

    vector<Point> ConvexHullPoints = contoursConvexHull(contours, largest_contour_index);
    polylines(image, ConvexHullPoints, true, Scalar(0, 0, 255), 2);
    return image;
}

vector<Point> contoursConvexHull(vector<vector<Point> > contours, int index)
{
    vector<Point> result;
    vector<Point> pts;
    for (size_t j = 0; j< contours[index].size(); j++)
        pts.push_back(contours[index][j]);
    convexHull(pts, result);
    return result;
}

int getMaxAreaContourId(vector <vector<cv::Point>> contours)
{
    double maxArea = 0;
    int maxAreaContourId = -1;
    for (int j = 0; j < contours.size(); j++) {
        double newArea = cv::contourArea(contours.at(j));
        if (newArea > maxArea) {
            maxArea = newArea;
            maxAreaContourId = j;
        }
        return maxAreaContourId;
    }
}
Mat ZnakiFunkcja(Mat image, Mat in, Scalar low, Scalar high)
{
    Mat temp;
    inRange(in, low, high, temp);
    erode(temp, temp, getStructuringElement(MORPH_ELLIPSE, Size(5, 5)));
    dilate(temp, temp, getStructuringElement(MORPH_ELLIPSE, Size(5, 5)));
    dilate(temp, temp, getStructuringElement(MORPH_ELLIPSE, Size(5, 5)));
    erode(temp, temp, getStructuringElement(MORPH_ELLIPSE, Size(5, 5)));

    Mat srcBlur, srcCanny;
    blur(temp, srcBlur, Size(3, 3));

    Canny(srcBlur, srcCanny, 0, 100, 3, true);
    vector<vector<Point> > contours;
    findContours(srcCanny, contours, CV_RETR_EXTERNAL, CV_CHAIN_APPROX_SIMPLE);
    Mat drawing = Mat::zeros(srcCanny.size(), CV_8UC3);
    for (int i = 0; i< contours.size(); i++)
    {
        Scalar color = Scalar(255, 255, 255);
        drawContours(drawing, contours, i, color, 2);
        polylines(image, contours, true, Scalar(0, 0, 255), 2);
        return image;
    }

}

Yet I have no idea how to count different shapes (hearts, lightnings, shields, numbers).

I will be greatfull if anybody would give me a tip or solution of how to do the job.

1) sorry for bad english 2) we had no openCV in classes [only basic c++] 3) tryed to found anything usefull on internet, but even if I found anything, I could't understand what was happening in the code

Answer 1

Your project can be splited in three steps:

1. find the dices.
2. extract the shapes from the visible face of the dices.
3. count the faces.

For the first step among all the possible approaches I think saliency map approaches can help. Saliency map are a family of segmentation algorithm that aim to detect the parts in the image which are more likely to attract visual attention.

OpenCV have a saliency API that already implement several saliency algorithm and for each of them you can get an segmentation map.

It is highlikely considering the example image you gave the saliency will be focus on the dices.

From this you can so extract the dices as rois from the original image.

For the step 2) saliency algorithms may also fit... or not that depend a lot of the statistical criterions that are used by the algorithm. However the previously extracted rois should only contain the face of the dice that does contain the shapes you want to count in step 3) so approaches based on contours detection may give quite good result.

Once you get the shapes among the way to count each shape you can use templateMatching (that is also already implement in OpenCV),a clustering approach based on the a shape sensitive metric (Hausdorff, Dice, ...), or many other.

Here is a code that can help you to deal with the two first step.

#ifndef _DEBUG
#define _DEBUG
#endif

#include <iostream>

#include <opencv2/core.hpp>
#include <opencv2/imgproc.hpp>
#include <opencv2/saliency.hpp>
#include <opencv2/highgui.hpp>

#include <list>

CV_EXPORTS_W void get_regions_of_interest(cv::InputArray _src, cv::OutputArrayOfArrays mv, cv::OutputArrayOfArrays mv2 = cv::noArray());

int main()
{
    cv::Mat tmp = cv::imread("C:\Desktop\dices.jpg");

    if(!tmp.empty())
    {
        cv::imshow("source",tmp);
        cv::waitKey(-1);
    }

    std::vector<cv::Mat> rois;

    get_regions_of_interest(tmp,rois);

    std::cout << "Hello World!" << std::endl;
    return 0;
}

void get_regions_of_interest(cv::InputArray _src, cv::OutputArrayOfArrays _rois, cv::OutputArrayOfArrays _contours)
{

    // Check that the first argument is an image and the second a vector of images.
    CV_Assert(_src.isMat() && !_src.depth() && (_src.channels() == 1 || _src.channels() == 3) && _rois.isMatVector() && (!_contours.needed() || (_contours.needed() && _contours.isMatVector()) ) );

    static cv::Ptr<cv::saliency::StaticSaliencySpectralResidual> saliency;

    if(!saliency)
        saliency = cv::saliency::StaticSaliencySpectralResidual::create();

    cv::Mat src = _src.getMat();
    cv::Mat gray;

    if(src.depth() == src.type())
        gray = src;
    else
        cv::cvtColor(src,gray,cv::COLOR_BGR2GRAY);

    bool is_ctr_needed = _contours.needed();
    std::list<cv::Mat> final_ctrs;

    // Step 1) Process the saliency in order to segment the dices.

    cv::Mat saliency_map;
    cv::Mat binary_map;

    saliency->computeSaliency(src,saliency_map);
    saliency->computeBinaryMap(saliency_map,binary_map);


    saliency_map.release();

    // Step 2) From the binary map get the regions of interest.

    cv::Mat1i stats;
    std::vector<cv::Mat> rois;

    cv::Mat labels;
    cv::Mat centroids;

    cv::connectedComponentsWithStats(binary_map, labels, stats, centroids);


    labels.release();
    centroids.release();

    // prepare the memory
    rois.reserve(stats.rows-1);

// Sort the stats in order to remove the background.

    stats = stats.colRange(0,stats.cols-1);

    // Extract the rois.

    for(int i=0;i<stats.rows;i++)
    {
        cv::Rect roi = *reinterpret_cast<cv::Rect*>(stats.ptr<int>(i));

        if(static_cast<std::size_t>(roi.area()) == gray.total())
            continue;

        rois.push_back(gray(roi));
#ifdef _DEBUG
        cv::imshow("roi_"+std::to_string(i),gray(roi));
#endif
    }



    // Step 3) Refine.

    // Because the final number of shape cannot be determine in advance it is better to use a linked list than a vector.
    // In practice except if there is a huge number of elements to work with the performance will be almost the same.
    std::list<cv::Mat> shapes;

    int cnt=0;
    for(const cv::Mat& roi : rois)
    {

        cv::Mat tmp = roi.clone();

        // Slightly sharpen the regions contours
        cv::morphologyEx(tmp,tmp, cv::MORPH_CLOSE, cv::noArray());
        // Reduce the influence of local unhomogeneous illumination.
        cv::GaussianBlur(tmp,tmp,cv::Size(31,31), 5);

        cv::Mat thresh;
        // Binarize the image.
        cv::threshold(roi,thresh,0.,255.,cv::THRESH_BINARY | cv::THRESH_OTSU);
#ifdef _DEBUG
        cv::imshow("thresh"+std::to_string(cnt++),thresh);
#endif
        // Find the contours of each sub region on interest
        std::vector<cv::Mat> contours;

        cv::findContours(thresh, contours, cv::RETR_TREE, cv::CHAIN_APPROX_SIMPLE);

        cv::Mat dc;

        cv::merge(std::vector<cv::Mat>(3,thresh),dc);

//        cv::drawContours(dc, contours,-1,cv::Scalar(0.,0.,255),2);
//        cv::imshow("ctrs"+std::to_string(cnt),dc);

        // Extract the sub-regions

        if(is_ctr_needed)
        {
            for(const cv::Mat& ctrs: contours)
            {

                cv::Rect croi = cv::boundingRect(ctrs);

                // If the sub region is to big or to small it is depreate
                if(static_cast<std::size_t>(croi.area()) == roi.total() || croi.area()<50)
                    continue;

                final_ctrs.push_back(ctrs);

                shapes.push_back(roi(croi));

#ifdef _DEBUG
                cv::rectangle(dc,croi,cv::Scalar(0.,0.,255.));

                cv::imshow("sub_roi_"+std::to_string(cnt++),roi(croi));
#endif
            }
        }
        else
        {
            for(const cv::Mat& ctrs: contours)
            {

                cv::Rect croi = cv::boundingRect(ctrs);

                // If the sub region is to big or to small it is depreate
                if(static_cast<std::size_t>(croi.area()) == roi.total() || croi.area()<50)
                    continue;

                shapes.push_back(roi(croi));

#ifdef _DEBUG
                cv::rectangle(dc,croi,cv::Scalar(0.,0.,255.));

                cv::imshow("sub_roi_"+std::to_string(cnt++),roi(croi));
#endif

            }
        }

    }
#ifdef _DEBUG
    cv::waitKey(-1);
#endif

    // Final Step: set the output

    _rois.create(shapes.size(),1,CV_8U);
    _rois.assign(std::vector<cv::Mat>(shapes.begin(),shapes.end()));

    if(is_ctr_needed)
    {
        _contours.create(final_ctrs.size(),1,CV_32SC2);
        _contours.assign(std::vector<cv::Mat>(final_ctrs.begin(), final_ctrs.end()));
    }

}