Display cv::Mat as QVideoFrame in a QML VideoOutput

Question

I have an OpenCV backend which retrieves the video frame from a camera device through cv::VideoCapture, does some processing and then passes the cv::Mat frame to a Qt5 application for display in a QML VideoOutput.

The problem is the frames drawn are empty/white.

The CameraService class is receiving a cv::Mat from a backend object which runs on its own thread using a Qt::QueuedConnection signal. Then I convert to it to a QImage, which I use to initialize a QVideoFrame and pass it to a QAbstractVideoSurface received from a QML VideoOutput, after setting a pixel format on it.

I have checked whether the cv::Mat has valid content before conversion to QVideoFrame, so this is not the case.

Or am I doing it completely wrong and should instead draw an image?

Relevant code:

CameraService.cpp

CameraService::CameraService(Video::Backend *backend) 
  : QObject(), 
  surface(nullptr),
  isFormatSet(false) {

  this->backend = backend;

  connect(
    backend, &Video::Backend::onFrameReady,
    this, &CameraService::onVideoFrameReady, 
    Qt::QueuedConnection);
}

CameraService::~CameraService() {

  backend->deleteLater();
}

QAbstractVideoSurface *CameraService::getVideoSurface() const {

  return surface;
}

void CameraService::setVideoSurface(QAbstractVideoSurface *surface) {

  if (!this->surface && surface)
    backend->start();

  if (this->surface && this->surface != surface && this->surface->isActive())      
    this->surface->stop();

  this->surface = surface;

  if (this->surface && format.isValid()) {

    format = this->surface->nearestFormat(format);
    this->surface->start(format);
  }
}

void CameraService::setFormat(
  int width, 
  int height, 
  QVideoFrame::PixelFormat frameFormat
){
    QSize size(width, height);
    QVideoSurfaceFormat format(size, frameFormat);

    this->format = format;

    if (surface) {

        if (surface->isActive())
            surface->stop();

        this->format = surface->nearestFormat(this->format);

        surface->start(this->format);
    }
}

void CameraService::onVideoFrameReady(cv::Mat currentFrame) {

  if (!surface || currentFrame.empty())
    return;

  cv::Mat continuousFrame;

  if (!currentFrame.isContinuous())
    continuousFrame = currentFrame.clone();
  else
    continuousFrame = currentFrame;

  if (!isFormatSet) {

    setFormat(
      continuousFrame.cols, 
      continuousFrame.rows, 
      QVideoFrame::PixelFormat::Format_BGR32);
    isFormatSet = true;
  }

  frame = QImage(
    (uchar *)continuousFrame.data,
    continuousFrame.cols,
    continuousFrame.rows,
    continuousFrame.step,
    QVideoFrame::imageFormatFromPixelFormat(
      QVideoFrame::PixelFormat::Format_BGR32));

  surface->present(QVideoFrame(frame));
}

QML object:

    VideoOutput {
        objectName: "videoOutput";
        anchors.fill: parent;
        fillMode: VideoOutput.PreserveAspectCrop;
        source: CameraService;
    }

The CameraService object is made available as a singleton to QML using this statement:

qmlRegisterSingletonInstance<Application::CameraService>("Application.CameraService", 1, 0, "CameraService", service);

Answer 1

Analyzing the code I have noticed that the conversion is not supported (I recommend you check if the format is valid or). For this I have made some changes:...

#ifndef CAMERASERVICE_H
#define CAMERASERVICE_H

#include "backend.h"

#include <QObject>
#include <QPointer>
#include <QVideoFrame>
#include <QVideoSurfaceFormat>
#include <opencv2/core/mat.hpp>

class QAbstractVideoSurface;

class CameraService : public QObject
{
    Q_OBJECT
    Q_PROPERTY(QAbstractVideoSurface* videoSurface READ videoSurface WRITE setVideoSurface NOTIFY surfaceChanged)
public:
    explicit CameraService(Backend *backend, QObject *parent = nullptr);
    QAbstractVideoSurface* videoSurface() const;

public Q_SLOTS:
    void setVideoSurface(QAbstractVideoSurface* surface);
Q_SIGNALS:
    void surfaceChanged(QAbstractVideoSurface* surface);
private Q_SLOTS:
    void onVideoFrameReady(cv::Mat currentFrame);
private:
    void setFormat(int width, int height, QVideoFrame::PixelFormat frameFormat);

    QPointer<QAbstractVideoSurface> m_surface;
    QScopedPointer<Backend> m_backend;
    QVideoSurfaceFormat m_format;
    bool m_isFormatSet;
    QImage m_image;
};

#endif // CAMERASERVICE_H

#include "backend.h"
#include "cameraservice.h"

#include <QAbstractVideoSurface>
#include <iostream>

CameraService::CameraService(Backend *backend, QObject *parent)
    : QObject(parent), m_backend(backend), m_isFormatSet(false)
{
    connect(m_backend.data(), &Backend::frameReady, this, &CameraService::onVideoFrameReady);
}

QAbstractVideoSurface *CameraService::videoSurface() const
{
    return m_surface;
}

void CameraService::setVideoSurface(QAbstractVideoSurface *surface){
    if (m_surface == surface)
        return;
    if(m_surface && m_surface != surface && m_surface->isActive())
        m_surface->stop();
    m_surface = surface;
    Q_EMIT surfaceChanged(m_surface);
    m_backend->start();
    if (m_surface && m_format.isValid()) {
        m_format = m_surface->nearestFormat(m_format);
        m_surface->start(m_format);
    }
}

void CameraService::setFormat(
        int width,
        int height,
        QVideoFrame::PixelFormat frameFormat
        ){
    QSize size(width, height);
    QVideoSurfaceFormat format(size, frameFormat);
    m_format = format;
    if (m_surface) {
        if (m_surface->isActive())
            m_surface->stop();
        m_format = m_surface->nearestFormat(m_format);
        m_surface->start(m_format);
    }
}

void CameraService::onVideoFrameReady(cv::Mat currentFrame){
    if (!m_surface || currentFrame.empty())
        return;
    cv::Mat continuousFrame;
    if (!currentFrame.isContinuous())
        continuousFrame = currentFrame.clone();
    else
        continuousFrame = currentFrame;
    if (!m_isFormatSet) {
        setFormat(continuousFrame.cols,
                  continuousFrame.rows,
                  QVideoFrame::Format_RGB32);
        m_isFormatSet = true;
    }
    m_image = QImage(continuousFrame.data,
                     continuousFrame.cols,
                     continuousFrame.rows,
                     continuousFrame.step,
                     QImage::Format_RGB888);
    m_image = m_image.rgbSwapped();
    m_image.convertTo(QVideoFrame::imageFormatFromPixelFormat(QVideoFrame::Format_RGB32));
    m_surface->present(QVideoFrame(m_image));
}

_{You can find the complete example here.}