Arrows segmentations in an image

Question

I need a way to segment each arrow alone. I tried OpenCv findContours but it broke it or add it to multiple shapes and arrows as the share the boundaries of shapes. I tried OpenCV connected components but this arrows almost in some graph connected all of it. Plus having trouble as the boundaries almost have the same color as the arrow. And in these kind of images each arrow contains different colors. Any opinion about this problem.

This is a sample diagram. I have to deal with harder diagrams like this.

Answer 1

Ok, work with new picture. 1. Binarization the arrows (and shapes):

cv::Mat imgCl = cv::imread("62uoU.jpg", cv::IMREAD_COLOR);
cv::Mat img;
cv::cvtColor(imgCl, img, cv::COLOR_BGR2GRAY);

cv::Mat mask1;
cv::threshold(img, mask1, 30, 255, cv::THRESH_BINARY_INV);

cv::Mat mask2;
cv::threshold(img, mask2, 120, 255, cv::THRESH_BINARY_INV);

cv::Mat diff;
cv::absdiff(mask1, mask2, diff);

cv::imshow("diff1", diff);

Result 1:

2. Remove rectangle shapes:

   cv::Rect objRect(0, 0, diff.cols, diff.rows); cv::Size minSize(objRect.width / 100, objRect.height / 100);

   cv::Mat bin = cv::Mat(diff, objRect).clone();
   
   for (;;)
   {
       cv::Rect cutRect;
       if (!PosRefinement(bin, cutRect, 0.9f, minSize))
       {
           break;
       }
       cv::rectangle(bin, cutRect, cv::Scalar(0, 0, 0), cv::FILLED);
       cv::rectangle(diff, cutRect, cv::Scalar(0, 0, 0), cv::FILLED);
   
       objRect.x += cutRect.x;
       objRect.y += cutRect.y;
       objRect.width = cutRect.width;
       objRect.height = cutRect.height;
   }
   
   cv::imshow("diff", diff);
   

Result 2:

3. Find lines:

   std::vector<cv::Vec4i> linesP;
   cv::HoughLinesP(diff, linesP, 1, CV_PI / 180, 20, 10, 5);
   for (size_t i = 0; i < linesP.size(); i++)
   {
       cv::Vec4i l = linesP[i];
       cv::line(imgCl, cv::Point(l[0], l[1]), cv::Point(l[2], l[3]), cv::Scalar(0, 0, 255), 3, cv::LINE_AA);
   }
   cv::imshow("img", imgCl);
   

Result 3:

Black arrows was founded. It can to improve this solution: find and delete text areas from image (tesseract or cv::text::ERFilter). And add a little morphology for draw arrow tips with Hough lines.

P.S. Utility function:

bool PosRefinement(
    cv::Mat bin,
    cv::Rect& cutRect,
    double kThreshold,
    cv::Size minSize
    )
{
    const double areaThreshold = 100;
    const int radius = 5;
    const int maxIters = 100;

    std::vector<std::vector<cv::Point>> contours;
    std::vector<cv::Vec4i> hierarchy;
    cv::findContours(bin, contours, hierarchy, cv::RETR_EXTERNAL, cv::CHAIN_APPROX_SIMPLE, cv::Point());

    size_t bestCont = contours.size();
    double maxArea = 0;
    for (size_t i = 0; i < contours.size(); i++)
    {
        double area = cv::contourArea(contours[i]);
        if (area > maxArea)
        {
            maxArea = area;
            bestCont = i;
        }
    }
    if (maxArea < areaThreshold)
    {
        return false;
    }

    cv::Moments m = cv::moments(contours[bestCont]);
    cv::Point mc(cvRound(m.m10 / m.m00), cvRound(m.m01 / m.m00));

    cv::Rect currRect(mc.x - radius / 2, mc.y - radius / 2, radius, radius);

    auto Clamp = [](int v, int hi) -> bool
    {
        if (v < 0)
        {
            v = 0;
            return true;
        }
        else if (hi && v > hi - 1)
        {
            v = hi - 1;
            return true;
        }
        return false;
    };
    auto RectClamp = [&](cv::Rect& r, int w, int h) -> bool
    {
        return Clamp(r.x, w) || Clamp(r.x + r.width, w) || Clamp(r.y, h) || Clamp(r.y + r.height, h);
    };

    int stepL = radius / 2;
    int stepR = radius / 2;
    int stepT = radius / 2;
    int stepB = radius / 2;

    double k = 0;

    struct State
    {
        double k = 0;
        int stepL = 0;
        int stepR = 0;
        int stepT = 0;
        int stepB = 0;
        cv::Rect currRect;

        State() = default;
        State(double k_, int stepL_, int stepR_, int stepT_, int stepB_, cv::Rect currRect_)
            :
              k(k_),
              stepL(stepL_),
              stepR(stepR_),
              stepT(stepT_),
              stepB(stepB_),
              currRect(currRect_)
        {
        }
        bool operator==(const State& st) const
        {
            return (st.k == k) && (st.stepL == stepL) && (st.stepR == stepR) && (st.stepT == stepT) && (st.stepB == stepB) && (st.currRect == currRect);
        }
    };
    const size_t statesCount = 2;
    State prevStates[statesCount];
    size_t stateInd = 0;

    for (int it = 0; it < maxIters; ++it)
    {
        cv::Rect rleft(currRect.x - stepL, currRect.y, currRect.width + stepL, currRect.height);
        cv::Rect rright(currRect.x, currRect.y, currRect.width + stepR, currRect.height);
        cv::Rect rtop(currRect.x, currRect.y - stepT, currRect.width, currRect.height + stepT);
        cv::Rect rbottom(currRect.x, currRect.y, currRect.width, currRect.height + stepB);

        double kleft = 0;
        double kright = 0;
        double ktop = 0;
        double kbottom = 0;

        if (!RectClamp(rleft, bin.cols, bin.rows))
        {
            cv::Rect rstep(currRect.x - stepL, currRect.y, stepL, currRect.height);
            if (cv::sum(bin(rstep))[0] / (255.0 * rstep.area()) > kThreshold / 2)
            {
                kleft = cv::sum(bin(rleft))[0] / (255.0 * rleft.area());
            }
        }
        if (!RectClamp(rright, bin.cols, bin.rows))
        {
            cv::Rect rstep(currRect.x + currRect.width, currRect.y, stepR, currRect.height);
            if (cv::sum(bin(rstep))[0] / (255.0 * rstep.area()) > kThreshold / 2)
            {
                kright = cv::sum(bin(rright))[0] / (255.0 * rright.area());
            }
        }
        if (!RectClamp(rtop, bin.cols, bin.rows))
        {
            cv::Rect rstep(currRect.x, currRect.y - stepT, currRect.width, stepT);
            if (cv::sum(bin(rstep))[0] / (255.0 * rstep.area()) > kThreshold / 2)
            {
                ktop = cv::sum(bin(rtop))[0] / (255.0 * rtop.area());
            }
        }
        if (!RectClamp(rbottom, bin.cols, bin.rows))
        {
            cv::Rect rstep(currRect.x, currRect.y + currRect.height, currRect.width, stepB);
            if (cv::sum(bin(rstep))[0] / (255.0 * rstep.area()) > kThreshold / 2)
            {
                kbottom = cv::sum(bin(rbottom))[0] / (255.0 * rbottom.area());
            }
        }

        bool wasEnlargeX = false;
        if (kleft > kThreshold)
        {
            currRect.x -= stepL;
            currRect.width += stepL;
            wasEnlargeX = true;

            if (kleft > k)
            {
                ++stepL;
            }
        }
        else
        {
            if (stepL > 1)
            {
                --stepL;
            }
            currRect.x += 1;
            currRect.width -= 1;
        }
        if (kright > kThreshold)
        {
            currRect.width += stepR;
            wasEnlargeX = true;

            if (kright > k)
            {
                ++stepR;
            }
        }
        else
        {
            if (stepR > 1)
            {
                --stepR;
            }
            currRect.width -= 1;
        }

        bool wasEnlargeY = false;
        if (ktop > kThreshold)
        {
            currRect.y -= stepT;
            currRect.height += stepT;
            wasEnlargeY = true;

            if (ktop > k)
            {
                ++stepT;
            }
        }
        else
        {
            if (stepT > 1)
            {
                --stepT;
            }
            currRect.y += 1;
            currRect.height -= 1;
        }
        if (kbottom > kThreshold)
        {
            currRect.height += stepB;
            wasEnlargeY = true;

            if (kbottom > k)
            {
                ++stepB;
            }
        }
        else
        {
            if (stepB > 1)
            {
                --stepB;
            }
            currRect.height -= 1;
        }

        k = cv::sum(bin(currRect))[0] / (255.0 * currRect.area());

        State currState(k, stepL, stepR, stepT, stepB, currRect);

        bool repState = false;
        for (size_t i = 0; i < statesCount; ++i)
        {
            if (prevStates[i] == currState)
            {
                repState = true;
                break;
            }
        }
        if (repState)
        {
            break;
        }
        else
        {
            prevStates[stateInd] = currState;
            stateInd = (stateInd + 1 < statesCount) ? (stateInd + 1) : 0;
        }

        if (k < kThreshold && (stepL + stepR + stepT + stepB == 4) && !wasEnlargeX && !wasEnlargeY)
        {
            break;
        }
    }

    cutRect.x = std::max(0, currRect.x - 1);
    cutRect.width = currRect.width + 2;
    cutRect.y = std::max(0, currRect.y - 1);
    cutRect.height = currRect.height + 2;

    return (cutRect.width >= minSize.width) && (cutRect.height >= minSize.height);
}

Answer 2

For your example it might be simple. The picture (png) has 4 channels and 4th channel is transparent mask. It can work only with transparent channel and filter arrows with moments:

cv::Mat img = cv::imread("voXFs.png", cv::IMREAD_UNCHANGED);
std::cout << "imsize = " << img.size() << ", chans = " << img.channels() << std::endl;
cv::imshow("img", img);

std::vector<cv::Mat> chans;
cv::split(img, chans);

cv::imshow("transp", chans.back());

cv::Mat mask;
cv::threshold(chans.back(), mask, 50, 255, cv::THRESH_BINARY | cv::THRESH_OTSU);

std::vector<std::vector<cv::Point> > contours;
cv::findContours(mask, contours, cv::RETR_EXTERNAL, cv::CHAIN_APPROX_SIMPLE);

cv::Mat draw;
cv::cvtColor(mask, draw, cv::COLOR_GRAY2BGR);

for (size_t i = 0; i < contours.size(); ++i)
{
    double area = cv::contourArea(contours[i]);
    double len = cv::arcLength(contours[i], false);
    double k = len / area;

    if (area > 10 && len > 60 && k > 2)
    {
        std::cout << "area = " << area << ", len = " << len << ", k = " << k << std::endl;
        cv::drawContours(draw, contours, i, cv::Scalar(255, 0, 0), 1);
    }
}

cv::imshow("mask", mask);
cv::imshow("draw", draw);
cv::waitKey(0);

But for more robust result:

1. Find and delete text areas from image (tesseract or cv::text::ERFilter).

2. Erode mask, find all shapes by contours, draw and dilate they. Bitwise and operation for mask and result.

3. The end!