Multithreaded opencv video processing Qt/C++

Question

Hope your are doing good ! I'm strugguling from yesterday on how to implement a multithreaded video processing program with opencv.

I understand how threads actually work, how to use a simple mutex etc...But when it comes to implementation, i'm completely lost.

My goal is to create an interface with Qt and display two labels, one showing original video feed and the other showing the processed one, each one handeled by a thread.

For now i'm just trying to get on thread to display images, but i really struggle with it.

Here is what i've done so far :

CaptureThread a class inheriting from QThread : It is supposed to handle the start of cam capture etc...

#ifndef CAPTURETHREAD_H
#define CAPTURETHREAD_H

#include <QThread>

#include <opencv2/opencv.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/core/core.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include "opencv2/videoio.hpp"

#include <QTimer>

class CaptureThread : public QThread
{
    Q_OBJECT

public:
    explicit CaptureThread(QObject *parent);


protected:
    void run();

signals:
    void frameCaptured(cv::Mat);

public slots:
    void captureFrame();

private:
    QTimer* tmrTimer;
    cv::VideoCapture capWebam;
    cv::Mat capturedFrame;

};

#endif // CAPTURETHREAD_H

And this is its implementation :

#include "capturethread.h"
#include <QDebug>

CaptureThread::CaptureThread(QObject* parent):QThread(parent)
{
    capWebam.open(0);
}

void CaptureThread::run()
{
    tmrTimer = new QTimer(this);
    QObject::connect(tmrTimer, SIGNAL(timeout()), this, SLOT(captureFrame()));

    tmrTimer->start(10);
    exec();
}

void CaptureThread::captureFrame()
{
    if(capWebam.isOpened()){
        capWebam.read(capturedFrame);

        emit frameCaptured(capturedFrame);
    }

}

MainWindow use to display the camera feed etc...

#ifndef MAINWINDOW_H
#define MAINWINDOW_H

#include <QMainWindow>
#include <opencv2/opencv.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/core/core.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include "opencv2/videoio.hpp"
#include <capturethread.h>

#include <QTimer>

namespace Ui {
class MainWindow;
}

class MainWindow : public QMainWindow
{
    Q_OBJECT

public:
    explicit MainWindow(QWidget *parent = 0);
    ~MainWindow();

public slots:
    void pricessFrameAndUpdateGUI(cv::Mat matOriginal);

private slots:
    void on_button_clicked();

private:
    Ui::MainWindow *ui;
    QImage toGrayscale(QImage image);

    cv::VideoCapture capWebcam;
    cv::Mat matOriginal;
    cv::Mat matProcessed;

    QImage qimgOriginal;
    QImage qimgProcessed;

    QTimer* tmrTimer;

    CaptureThread* cpThread;
};

#endif // MAINWINDOW_H

And its implentation :

#include "mainwindow.h"
#include "ui_mainwindow.h"
#include <QtCore>
#include <cv.h>
#include <QColor>
#include <opencv/highgui.h>

MainWindow::MainWindow(QWidget *parent) :QMainWindow(parent),ui(new Ui::MainWindow)
{
    ui->setupUi(this);

    cpThread = new CaptureThread(this);
    QObject::connect(cpThread, SIGNAL(frameCaptured(cv::Mat)),this, SLOT(pricessFrameAndUpdateGUI(cv::Mat)));
    cpThread->start();

}

MainWindow::~MainWindow()
{
    delete ui;
}

void MainWindow::pricessFrameAndUpdateGUI(cv::Mat matOriginal)
{


    cv::Canny(matOriginal, matProcessed, 100, 300);

    cv::cvtColor(matOriginal, matOriginal, CV_BGR2RGB);
    QImage qimgOriginal((uchar*)matOriginal.data, matOriginal.cols, matOriginal.rows, matOriginal.step, QImage::Format_RGB888);
    QImage qimgProcessed((uchar*)matProcessed.data, matProcessed.cols, matProcessed.rows, matProcessed.step,QImage::Format_Indexed8);

    ui->original->setPixmap(QPixmap::fromImage(qimgOriginal));
    ui->modified->setPixmap(QPixmap::fromImage(qimgProcessed));

}

After compiling and executing the program i get this error :

Starting /media/wassim/BLAZER/Workspace/CPP/build-firstCV-Desktop_Qt_5_3_GCC_64bit-Debug/firstCV...

VIDEOIO ERROR: V4L/V4L2: VIDIOC_S_CROP

QObject: Cannot create children for a parent that is in a different thread.

(Parent is CaptureThread(0xc02be0), parent's thread is QThread(0xae82c0), current thread is CaptureThread(0xc02be0)

The program has unexpectedly finished.

/media/wassim/BLAZER/Workspace/CPP/build-firstCV-Desktop_Qt_5_3_GCC_64bit-Debug/firstCV crashed

Thanks you guys for your help !

Answer 1

The error you see is caused by the QTimer being passed a parent that lives in another thread. CaptureThread lives in the UI thread. The QTimer is created in another thread (it is in the run() method).

Simplest solution: move the instantiation of the QTimer (and the start call) into the ctor:

tmrTimer = new QTimer(this);
QObject::connect(tmrTimer, SIGNAL(timeout()),
                 this, SLOT(captureFrame()));
tmrTimer->start(40);

This should work. But it won't work as it should. The timeout() signal will queue a message in the thread in which CaptureThread lives, which is the UI thread. So everything will be done in the UI thread, not only the post-processing. Quickest solution:

CaptureThread::CaptureThread() : QObject()
{
    capWebam.open(0);

    QThread* t = new QThread();
    moveToThread(t);
    t->start();

    tmrTimer = new QTimer;
    QObject::connect(tmrTimer, SIGNAL(timeout()),
                     this, SLOT(captureFrame()));
    tmrTimer->start(40);
}

CaptureThread is moved to a new QThread (it is not a subclass of QThread). Also, move the post-processing to this thread. This is just the concept, then you'll have to handle cleanup, register the meta-type etc...

EDIT: Ok, just a quick test code (tested on Mac OS), may be broken and need optimization, I didn't check memory cleanup etc... (also pay attention to how you're hiding matOriginal in your code, you're not passing the class member to the QImage, but the local instance it seems):

main.cpp

#include <QApplication>
#include <QLabel>
#include <QTimer>
#include <QThread>

#include <opencv2/imgproc/imgproc.hpp>
#include <opencv2/highgui/highgui.hpp>

#include "src.h"

int main(int argc, char* argv[])
{
   QApplication a(argc, argv);

   cv::VideoCapture vc;
   if (!vc.open(0))
      return 1;

   QTimer t;
   QThread th;
   CaptureHandler handler(&vc);
   handler.moveToThread(&th);
   th.start();

   MainWidget w1;
   w1.resize(100, 100);
   w1.show();

   MainWidget w2;
   w2.resize(100, 100);
   w2.show();

   QObject::connect(&t, SIGNAL(timeout()),
                    &handler, SLOT(handleFrame()));
   QObject::connect(&handler, SIGNAL(frameReady(QImage)),
                    &w1, SLOT(onFrame(QImage)));
   QObject::connect(&handler, SIGNAL(framePpReady(QImage)),
                    &w2, SLOT(onFrame(QImage)));
   t.start(20);

   return a.exec();
}

src.h

#ifndef SRC_H
#define SRC_H

#include <QObject>
#include <QImage>
#include <QLabel>

#include <opencv2/opencv.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/core/core.hpp>

class CaptureHandler : public QObject
{
   Q_OBJECT
public:
   CaptureHandler(cv::VideoCapture* vc);
signals:
   void frameReady(QImage frame);
   void framePpReady(QImage frame);
public slots:
   void handleFrame();
private:
   cv::VideoCapture* vc;
};

class MainWidget : public QLabel
{
   Q_OBJECT
public slots:
   void onFrame(QImage frame);
};

#endif // SRC_H

src.cpp

#include <QObject>
#include <QImage>
#include <QLabel>

#include "src.h"

void cleanup_mat(void* info)
{
   delete (cv::Mat*)info;
}

CaptureHandler::CaptureHandler(cv::VideoCapture* vc) : QObject(), vc(vc) {}

void CaptureHandler::handleFrame() {
   cv::Mat* original = new cv::Mat;
   if (!vc->read(*original))
      return;

   cv::Mat* processed = new cv::Mat;
   cv::Canny(*original, *processed, 100, 300);
   cv::cvtColor(*original, *original, CV_BGR2RGB);
   QImage qimgOriginal((uchar*)original->data,
                       original->cols,
                       original->rows,
                       original->step,
                       QImage::Format_RGB888, cleanup_mat, original);
   QImage qimgProcessed((uchar*)processed->data,
                        processed->cols,
                        processed->rows,
                        processed->step,
                        QImage::Format_Indexed8, cleanup_mat, processed);

   emit frameReady(qimgOriginal);
   emit framePpReady(qimgProcessed);
}

void MainWidget::onFrame(QImage frame) {
   setPixmap(QPixmap::fromImage(frame));
}

Shot :-)