Extracting blocks, inputs, and outputs from a scanned Simulink diagram

Question

I am working on a recognition software that takes a scanned Simulink diagram (in .png/.jpeg format) as input and extracts structured information about blocks, their inputs, and outputs. The goal is to generate an Excel spreadsheet that will be used by an in-house auto C code generator.

The recognition software is supposed to:

1. Identify basic blocks from the scanned image and assign unique names to blocks of the same type, with a running number suffix.
   • Example: The first BBANG block → BBANG1, the second BBANG block → BBANG2, etc.

Challenge

• How can I extract and recognize blocks from a scanned Simulink diagram?
• Any recommendations for approaches (e.g., edge detection, template matching) to accurately detect and classify blocks? Tried using template matching...not working as of now

What would be the best approach to achieve this in C++?

Code I have tried till now:

    #include <opencv2/opencv.hpp>
    #include <iostream>
    #include <fstream>
    #include <filesystem>
    #include <map>

    namespace fs = std::filesystem;
    using namespace cv;
    using namespace std;

    // Non-Maximum Suppression (NMS)
    void nonMaxSuppression(vector<Rect>& detectedRegions, double overlapThreshold) {
        vector<int> indices;
        sort(detectedRegions.begin(), detectedRegions.end(), [](const Rect& a, const Rect& b) {
            return a.area() > b.area();
        });

        for (size_t i = 0; i < detectedRegions.size(); ++i) {
            bool keep = true;
            for (size_t j = 0; j < i; ++j) {
                float intersectionArea = (detectedRegions[i] & detectedRegions[j]).area();
                float unionArea = detectedRegions[i].area() + detectedRegions[j].area() - intersectionArea;
                if ((intersectionArea / unionArea) > overlapThreshold) {
                    keep = false;
                    break;
                }
            }
            if (keep) {
                indices.push_back(i);
            }
        }

        vector<Rect> filteredDetections;
        for (int index : indices) {
            filteredDetections.push_back(detectedRegions[index]);
        }

        detectedRegions = filteredDetections;
    }

    // template matching
    void detectBlocks(const string& diagramPath, const string& templateFolder, ofstream& outputFile) {
        Mat inputImage = imread(diagramPath, IMREAD_GRAYSCALE);
        if (inputImage.empty()) {
            cout << "ERROR: Could not load input image: " << diagramPath << endl;
            return;
        }

        // Scale up input image if it's too small
        if (inputImage.cols < 500 || inputImage.rows < 500) {
            resize(inputImage, inputImage, Size(), 2.0, 2.0);  // Scale up by 2x
            cout << "Input image scaled up for better matching!" << endl;
        }

        map<string, int> blockCount;

        for (const auto& entry : fs::directory_iterator(templateFolder)) {
            string templatePath = entry.path().string();
            Mat templateImage = imread(templatePath, IMREAD_GRAYSCALE);
            if (templateImage.empty()) {
                cout << "Skipping: " << templatePath << " (Failed to load)" << endl;
                continue;
            }

            string blockName = entry.path().stem().string();
            vector<Rect> detectedRegions;

            // Scale down template 
            double maxScale = 1.0;
            if (templateImage.cols > inputImage.cols || templateImage.rows > inputImage.rows) {
                maxScale = min((double)inputImage.cols / templateImage.cols, (double)inputImage.rows / templateImage.rows);
            }

            for (double scale = maxScale; scale >= 0.3; scale -= 0.1) { // Reduce template size if needed
                Mat resizedTemplate;
                resize(templateImage, resizedTemplate, Size(), scale, scale);

                
                if (resizedTemplate.cols > inputImage.cols || resizedTemplate.rows > inputImage.rows) {
                    continue;
                }


                imshow("Resized Template", resizedTemplate);
                waitKey(200);

                Mat result;
                matchTemplate(inputImage, resizedTemplate, result, TM_CCOEFF_NORMED);

                double threshold = 0.6;
                for (int i = 0; i < result.rows; i++) {
                    for (int j = 0; j < result.cols; j++) {
                        if (result.at<float>(i, j) >= threshold) {
                            detectedRegions.emplace_back(j, i, resizedTemplate.cols, resizedTemplate.rows);
                        }
                    }
                }
            }


            nonMaxSuppression(detectedRegions, 0.3);

            for (const auto& rect : detectedRegions) {
                blockCount[blockName]++;
                string blockLabel = blockName + to_string(blockCount[blockName]);
                cout << "Detected: " << blockLabel << " at (" << rect.x << ", " << rect.y << ")" << endl;
                outputFile << diagramPath << "," << blockLabel << "," << rect.x << "," << rect.y << endl;
            }
        }
    }

    int main() {
        string diagramFolder = "./simulink_diagrams";
        string templateFolder = "./blocks";
        string outputFileName = "block_detection_results.csv";

        if (!fs::exists(diagramFolder) || !fs::exists(templateFolder)) {
            cout << "ERROR: One or both required folders are missing!" << endl;
            return -1;
        }

        ofstream outputFile(outputFileName);
        outputFile << "Diagram,Block Name,X,Y\n";

        for (const auto& entry : fs::directory_iterator(diagramFolder)) {
            string diagramPath = entry.path().string();
            cout << "Processing: " << diagramPath << endl;
            detectBlocks(diagramPath, templateFolder, outputFile);
        }

        outputFile.close();
        cout << "Detection complete! Results saved to " << outputFileName << endl;

        return 0;
    }