Painting a QVideoFrame on a QVideoWidget with QT6

Question

I'd like to process the stream of my webcam frame by frame with QT6. I've checked the internet but since QTMultimedia was heavily reworked with QT6, and since QT6 is pretty new, all the documentation/questions available are outdated.

So, In order to achieve my goal, I'm using a QMediaCaptureSession with a camera set on QMediaDevices::defaultVideoInput(). I checked that this was working by setting the video output of the QMediaCaptureSession to a QVideoWidget with m_session.setVideoOutput(ui->videowidget);, and it's working fine, except that I can't process the frames (basically, it's rendering my webcam on the QVideoWidget).

Now, to process the frames, I have to use a QVideoSink as far as I understand the documentation here and there. So I replaced m_session.setVideoOutput(ui->videowidget); with m_session.setVideoSink(&mysink);, where mysink is a QVideoSink.

Then, since I want to process the frames, I'm connecting the videoFrameChanged signal of mysink to a function processVideoFrame where I want to do 2 things :

1. process the current frame
2. render the result on the UI, ideally on ui->videowidget

This is the point where I'm struggling. I do not understand how to use the paint function of the class QVideoFrame to render the processed frame on the QVideoWidget. More precisely :

• I do not understand how I'm supposed to instantiate the QPainter. I tried a straightforward new QPainter(ui->videowidget) but it ends up in a QWidget::paintEngine: Should no longer be called exception and nothing is rendered
• I do not understand what is actually representing the second parameter rect of QVideoFrame::paint?

I made a MWE, code is below.

mwe_videosinkpainting.h

#ifndef MWE_VIDEOSINKPAINTING_H
#define MWE_VIDEOSINKPAINTING_H

#include <QMainWindow>
#include <QMediaCaptureSession>
#include <QMediaDevices>
#include <QCamera>
#include <QVideoSink>
#include <QPainter>

QT_BEGIN_NAMESPACE
namespace Ui { class MWE_VideoSinkPainting; }
QT_END_NAMESPACE

class MWE_VideoSinkPainting : public QMainWindow
{
    Q_OBJECT

public:
    MWE_VideoSinkPainting(QWidget *parent = nullptr);
    ~MWE_VideoSinkPainting();

private slots:
    void processVideoFrame();


private:
    Ui::MWE_VideoSinkPainting *ui;

    QVideoSink mysink;
    QMediaCaptureSession m_session;
    QScopedPointer<QCamera> m_camera;
};
#endif // MWE_VIDEOSINKPAINTING_H

mwe_videosinking.cpp

#include "mwe_videosinkpainting.h"
#include "ui_mwe_videosinkpainting.h"

MWE_VideoSinkPainting::MWE_VideoSinkPainting(QWidget *parent)
    : QMainWindow(parent)
    , ui(new Ui::MWE_VideoSinkPainting)
{
    ui->setupUi(this);

    m_camera.reset(new QCamera(QMediaDevices::defaultVideoInput()));
    m_session.setCamera(m_camera.data());
    //m_session.setVideoOutput(ui->videowidget);

    connect(&mysink, &QVideoSink::videoFrameChanged, this, &MWE_VideoSinkPainting::processVideoFrame);

    m_session.setVideoSink(&mysink);
    m_camera->start();
}

MWE_VideoSinkPainting::~MWE_VideoSinkPainting()
{
    delete ui;
}

void MWE_VideoSinkPainting::processVideoFrame()
{
    QVideoFrame videoframe = mysink.videoFrame();
    if(videoframe.map(QVideoFrame::ReadOnly))
    {
        //This is the part I'm struggling to understand and achieve
        videoframe.paint(new QPainter(ui->videowidget), QRectF(0.0f,0.0f,100.0f,100.0f), QVideoFrame::PaintOptions());
        videoframe.unmap();
    }
}

main.cpp

#include "mwe_videosinkpainting.h"

#include <QApplication>

int main(int argc, char *argv[])
{
    QApplication a(argc, argv);
    MWE_VideoSinkPainting w;
    w.show();
    return a.exec();
}

ui_mwe_videosinkpainting.h (just so that you have the whole code, it has no value for the question)

#ifndef UI_MWE_VIDEOSINKPAINTING_H
#define UI_MWE_VIDEOSINKPAINTING_H

#include <QtCore/QVariant>
#include <QtMultimediaWidgets/QVideoWidget>
#include <QtWidgets/QApplication>
#include <QtWidgets/QGridLayout>
#include <QtWidgets/QHBoxLayout>
#include <QtWidgets/QMainWindow>
#include <QtWidgets/QMenuBar>
#include <QtWidgets/QStatusBar>
#include <QtWidgets/QWidget>

QT_BEGIN_NAMESPACE

class Ui_MWE_VideoSinkPainting
{
public:
    QWidget *centralwidget;
    QGridLayout *gridLayout;
    QVideoWidget *videowidget;
    QHBoxLayout *horizontalLayout;
    QMenuBar *menubar;
    QStatusBar *statusbar;

    void setupUi(QMainWindow *MWE_VideoSinkPainting)
    {
        if (MWE_VideoSinkPainting->objectName().isEmpty())
            MWE_VideoSinkPainting->setObjectName(QString::fromUtf8("MWE_VideoSinkPainting"));
        MWE_VideoSinkPainting->resize(800, 600);
        centralwidget = new QWidget(MWE_VideoSinkPainting);
        centralwidget->setObjectName(QString::fromUtf8("centralwidget"));
        gridLayout = new QGridLayout(centralwidget);
        gridLayout->setObjectName(QString::fromUtf8("gridLayout"));
        videowidget = new QVideoWidget(centralwidget);
        videowidget->setObjectName(QString::fromUtf8("videowidget"));
        horizontalLayout = new QHBoxLayout(videowidget);
        horizontalLayout->setObjectName(QString::fromUtf8("horizontalLayout"));

        gridLayout->addWidget(videowidget, 0, 0, 1, 1);

        MWE_VideoSinkPainting->setCentralWidget(centralwidget);
        menubar = new QMenuBar(MWE_VideoSinkPainting);
        menubar->setObjectName(QString::fromUtf8("menubar"));
        menubar->setGeometry(QRect(0, 0, 800, 21));
        MWE_VideoSinkPainting->setMenuBar(menubar);
        statusbar = new QStatusBar(MWE_VideoSinkPainting);
        statusbar->setObjectName(QString::fromUtf8("statusbar"));
        MWE_VideoSinkPainting->setStatusBar(statusbar);

        retranslateUi(MWE_VideoSinkPainting);

        QMetaObject::connectSlotsByName(MWE_VideoSinkPainting);
    } // setupUi

    void retranslateUi(QMainWindow *MWE_VideoSinkPainting)
    {
        MWE_VideoSinkPainting->setWindowTitle(QCoreApplication::translate("MWE_VideoSinkPainting", "MWE_VideoSinkPainting", nullptr));
    } // retranslateUi

};

namespace Ui {
    class MWE_VideoSinkPainting: public Ui_MWE_VideoSinkPainting {};
} // namespace Ui

QT_END_NAMESPACE

#endif // UI_MWE_VIDEOSINKPAINTING_H

Answer 1

Since I cannot format code in a comment...

So, you mean change this...

    ui->setupUi(this);
    m_camera.reset(new QCamera(QMediaDevices::defaultVideoInput()));
    m_session.setCamera(m_camera.data());
    //m_session.setVideoOutput(ui->videowidget);
    connect(&mysink, &QVideoSink::videoFrameChanged, this, &MWE_VideoSinkPainting::processVideoFrame);
    m_session.setVideoSink(&mysink);
    m_camera->start();

...to this?

    ui->setupUi(this);
    m_camera.reset(new QCamera(QMediaDevices::defaultVideoInput()));
    m_session.setCamera(m_camera.data());
    m_session.setVideoSink(&mysink);
    m_session.setVideoOutput(ui->videowidget);
    connect(&mysink, &QVideoSink::videoFrameChanged, this, &MWE_VideoSinkPainting::processVideoFrame);
    m_camera->start();

I did this and I am still only getting black in my videoWidget.

Answer 2

The answer is quite straightforward : you can use setVideoSink AND setVideoOutput.

The code I gave in OP is good, you just have to uncomment setVideoOutput(ui->videowidget); of mwe_videosinking.cpp and to call setVideoSink BEFORE calling setVideoOutput