OpenCV detect which pins are bent

Question

I have this image of a pin header, and I need to detect if there are bent pins in the header using OpenCV.

UPDATE, solved

Thanks to Nick, I have made something that works pretty good, not prefect but oke!

I use the findContours function to find all contours. Then I loop over all the items and find the minAreaRect, and draw a box of the given size. When a box has a width greaten then a set threshold the pin is too bend (out of spec).

for (const auto &entry: fs::directory_iterator(SAMPLES)) {
    try {
        // Load src image
        src = imread(entry.path(), IMREAD_COLOR);
        // Blur
        medianBlur(src, blurred, 3);
        // Set threshold
        threshold(blurred, blurred, 100, 255, cv::THRESH_BINARY);

        // Edge detection
        Canny(blurred, detected_edges, thres1, thres2, 3);
//            imshow("blurred", blurred);

        vector<vector<Point> > contours;
        // Find all the contours in the image
        findContours(detected_edges, contours, RETR_TREE, CHAIN_APPROX_SIMPLE);

        vector<RotatedRect> minRect(contours.size());
        vector<vector<Point> > contours_poly(contours.size());
        vector<Rect> boundRect(contours.size());

        int bendPins = 0;

        for (size_t i = 0; i < contours.size(); i++) {
            // bind all shapes to the vectors
            minRect[i] = minAreaRect(contours[i]);
            approxPolyDP(contours[i], contours_poly[i], 3, true);
            boundRect[i] = boundingRect(contours_poly[i]);

            // Draw the min area rect need to fill the contour
            Rect rect(boundRect[i].tl(), boundRect[i].br());
            rectangle(src, rect, Scalar(0, 0, 255), 2);

            // When a pin it to bend
            if (rect.width > threshold_bend) {
                Point centerRect = (boundRect[i].br() + boundRect[i].tl()) * 0.5;
                circle(src, centerRect, 20, Scalar(255, 0, 255), 2);
                bendPins++;
            }

            // Draw a rect around the pin, bent or not
            Point2f rect_points[4];
            minRect[i].points(rect_points);
            for (int j = 0; j < 4; j++) {
                line(src, rect_points[j], rect_points[(j + 1) % 4], Scalar(0, 255, 255), 1);
            }
        }

        char buffer[100];
        snprintf(buffer, 100, "Found bend pin(s) : %d", bendPins);
        putText(src, buffer, Point(10, 25), FONT_HERSHEY_SIMPLEX, 0.75, Scalar(255, 255, 255), 2);

        imshow(entry.path().filename(), src);

        waitKey();

    } catch (const std::exception &e) {
        cout << e.what() << endl;
    }
}

The result:

Answer 1

Here is my approach with code and results.

Steps to produce:

(preprocessing)

1. Apply median to decrease noise in the image
2. Apply threshold to get a clear image

Here is the result of these 2 steps:

3. Check each row and get the sticks according to the thickness threshold.

Result image of this step:

4. Get mid point's y axis values of each stick and hold in an array.
5. Calculate the standard deviation of each array and choose the ones which are higher.

Here is the code:

#include <string>
#include <opencv2/opencv.hpp>
#include <opencv2/dnn_superres.hpp>
#include <numeric>
#include <boost/accumulators/accumulators.hpp>
#include <boost/accumulators/statistics/stats.hpp>
#include <boost/accumulators/statistics/mean.hpp>
#include <boost/accumulators/statistics/moment.hpp>
using namespace boost::accumulators;
using namespace std;
using namespace boost;

double stddev(std::vector<int> const & func)
{
    double mean = std::accumulate(func.begin(), func.end(), 0.0) / func.size();
    double sq_sum = std::inner_product(func.begin(), func.end(), func.begin(), 0.0,
        [](double const & x, double const & y) { return x + y; },
        [mean](double const & x, double const & y) { return (x - mean)*(y - mean); });
    return std::sqrt(sq_sum / func.size());
}


int main(){

    cv::Mat img = cv::imread("/home/yns/Downloads/aaa.jpg",cv::IMREAD_GRAYSCALE);

    cv::namedWindow("input",0);
    cv::namedWindow("output",0);
    cv::namedWindow("output2",0);

    cv::imshow("input",img);


    cv::Mat out;

    cv::medianBlur(img,img,3);

    cv::threshold(img,out,80,255,cv::THRESH_BINARY);

    cv::Mat out2;

    cv::cvtColor(out,out2,cv::COLOR_GRAY2BGR);


    cv::imshow("output",out);

    int start = 0;
    int cnt = 0;
    int refX = 0;
    int thresholdThickness = 5;

    int orderNum = 1;

    std::vector<int> yAxis_1;
    std::vector<int> yAxis_2;
    std::vector<int> yAxis_3;
    std::vector<int> yAxis_4;
    std::vector<int> yAxis_5;
    std::vector<int> yAxis_6;
    std::vector<int> yAxis_7;
    std::vector<int> yAxis_8;
    std::vector<int> yAxis_9;
    std::vector<int> yAxis_10;
    std::vector<int> yAxis_11;
    std::vector<int> yAxis_12;


    int annen = 0;

    for(int i=0; i<out.rows; i++)
    {
        orderNum = 1;
        annen = 0;
        for(int j=0; j<out.cols; j++)
        {

            if(out.at<uchar>(cv::Point(j,i))==255 && start != 1)
            {
                start = 1;
                refX = j;
                cnt = 0;
            }
            else if (out.at<uchar>(cv::Point(j,i))==255)
            {
                cnt++;
            }
            else if (out.at<uchar>(cv::Point(j,i))==0 && start == 1 && cnt>thresholdThickness) {
                cv::circle(out2,cv::Point((j+refX)/2,i),1,cv::Scalar(0,0,255),cv::FILLED);
                start = 0;
                annen++;
                if(orderNum == 1)
                    yAxis_1.push_back(j);
                if(orderNum == 2)
                    yAxis_2.push_back(j);
                if(orderNum == 3)
                    yAxis_3.push_back(j);
                if(orderNum == 4)
                    yAxis_4.push_back(j);
                if(orderNum == 5)
                    yAxis_5.push_back(j);
                if(orderNum == 6)
                    yAxis_6.push_back(j);
                if(orderNum == 7)
                    yAxis_7.push_back(j);
                if(orderNum == 8)
                    yAxis_8.push_back(j);
                if(orderNum == 9)
                    yAxis_9.push_back(j);
                if(orderNum == 10)
                    yAxis_10.push_back(j);
                if(orderNum == 11)
                    yAxis_11.push_back(j);
                if(orderNum == 12)
                    yAxis_12.push_back(j);

                orderNum++;

            }
            else if (out.at<uchar>(cv::Point(j,i))==0 && start == 1)
            {
                start = 0;
            }
        }

    }

    std::cout<<stddev(yAxis_1)<<std::endl;
    std::cout<<stddev(yAxis_2)<<std::endl;
    std::cout<<stddev(yAxis_3)<<std::endl;
    std::cout<<stddev(yAxis_4)<<std::endl;
    std::cout<<stddev(yAxis_5)<<std::endl;
    std::cout<<stddev(yAxis_6)<<std::endl;
    std::cout<<stddev(yAxis_7)<<std::endl;
    std::cout<<stddev(yAxis_8)<<std::endl;
    std::cout<<stddev(yAxis_9)<<std::endl;
    std::cout<<stddev(yAxis_10)<<std::endl;
    std::cout<<stddev(yAxis_11)<<std::endl;
    std::cout<<stddev(yAxis_12)<<std::endl;


    float average = accumulate( yAxis_6.begin(), yAxis_6.end(), 0.0)/yAxis_6.size();
    float average2 = accumulate( yAxis_7.begin(), yAxis_7.end(), 0.0)/yAxis_7.size();

    cv::circle(out2,cv::Point(average,out2.rows/2),25,cv::Scalar(0,255,255),5);
    cv::circle(out2,cv::Point(average2,out2.rows/2),25,cv::Scalar(0,255,255),5);




    cv::imshow("output2",out2);

    cv::waitKey(0);

  return 0;

}

Answer 2

Have a look at cv::findContours.

You should be able to extract the pins with that, maybe binarize first with cv::threshold(). Then using center-of-mass and the moment-of-area for the contours found, you can describe the position and angle of the pins. Or just using the bounding rectangle might even be enough.