Gstreamer appsrc to file is empty

Question

I'm working with Xilinx Petalinux and Vivado 2018.2 tools targeting a Zynqmp device with a (video codec unit) VCU.

I'm developing a gstreamer based application in Vivado SDK where the goal is to construct the next pipeline:

1. Capture RAW video frames from a USB3 camera (can't use v4l2, it uses its own API to capture the frames). Wrap frames to GstBuffer and push it to appsrc pipeline element.
2. Compress the video using the hardware VCU (H.264/H.265). (omxh264enc)
3. Save it to a file. (filesink)

At the moment I'm able to interface the camera, get the frames and wrap them in a GstBuffer type.

The problem is that the generated "output.h264" file is empty.

The relevant part of the code is:

    /* Create pipeline */
pipeline = gst_parse_launch("appsrc is-live=TRUE name=xsource  caps= video/x-raw,format=Y800,width=1280,height=1024 ! omxh264enc ! filesink location= media/test/output.h264", NULL);
if(!pipeline)
    goto finish;

/* we add a message handler */
bus = gst_pipeline_get_bus (GST_PIPELINE (pipeline));
bus_watch_id = gst_bus_add_watch (bus, bus_call, NULL);
gst_object_unref (bus);

appsrc=gst_bin_get_by_name(GST_BIN(pipeline), "xsource");

gst_element_set_state(pipeline, GST_STATE_PLAYING);

if(xiGetImage(xiH, 5000, &image) == XI_OK) //Get just one frame
{

    unsigned long buffer_size = image.width*image.height;

    buffer = gst_buffer_new();
    gst_buffer_insert_memory(buffer, -1, gst_memory_new_wrapped(GST_MEMORY_FLAG_READONLY, (guint8*)image.bp, buffer_size, 0, buffer_size, NULL, NULL));
    ret = gst_app_src_push_buffer(GST_APP_SRC(appsrc), buffer);
    if(ret != GST_FLOW_OK){
        break;
    }

}

gst_app_src_end_of_stream(GST_APP_SRC(appsrc));
gst_element_set_state(pipeline, GST_STATE_NULL);
gst_object_unref(GST_OBJECT(pipeline));

I checked (on SDK debug mode) the memory and buffers are not empty, so the camera interface and buffer push method to appsrc seems to be working fine. I suspect that the issue might be in the pipeline chain definition but I tried lot of configurations without success...

Any ideas/clues would be appreciated.

EDIT:

As suggested, I tried to wait for EOS confirmation and an error message check at the end of the code:

gst_app_src_end_of_stream(GST_APP_SRC(appsrc));

  /* Wait until error or EOS */
bus = gst_element_get_bus (pipeline);
msg =
gst_bus_timed_pop_filtered (bus, GST_CLOCK_TIME_NONE,
      GST_MESSAGE_ERROR | GST_MESSAGE_EOS);

gst_element_set_state(pipeline, GST_STATE_NULL);
gst_object_unref(GST_OBJECT(pipeline));

Also I tried to load more frames to see if that helps, I tried to load 500 like so:

    while(xiGetImage(xiH, 5000, &image) == XI_OK)
{
    unsigned long buffer_size = image.width*image.height;

    buffer = gst_buffer_new();
    gst_buffer_insert_memory(buffer, -1, gst_memory_new_wrapped(GST_MEMORY_FLAG_READONLY, (guint8*)image.bp, buffer_size, 0, buffer_size, NULL, NULL));

    ret = gst_app_src_push_buffer(GST_APP_SRC(appsrc), buffer);
    if(ret != GST_FLOW_OK){
        break;
    }

    if(frames > 500)
    {
         break;
    }else{
        frames++;
    }

}

But unfortunately it didn't help, still having empty file and no errors.

Any more ideas/clues?

Thanks.

Answer 1

I had similar problems. Every example that I've seen working uses the need-data callback in a pull fashion, rather than pushing frames as in your code.

• When I'm done streaming, I do a g_signal_emit_by_name (appsrc, "end-of-stream", &ret);
• Keep running until I see GST_MESSAGE_EOS on the pipeline bus.
• Then call gst_app_src_end_of_stream(GST_APP_SRC(appsrc));
• Then set gst_element_set_state (pipeline, GST_STATE_NULL);

Full source:

// based on https://gstreamer.freedesktop.org/documentation/application-development/advanced/pipeline-manipulation.html?gi-language=c
// and https://gist.github.com/floe/e35100f091315b86a5bf
// compile with:
// g++ -Wall $(pkg-config --cflags gstreamer-1.0) -o gst gst.cpp $(pkg-config --libs gstreamer-1.0) -lgstapp-1.0

#include <gst/gst.h>
#include <gst/app/gstappsrc.h>

#include <stdint.h>
#include <iostream>
using namespace std;

bool run = true;

const int WIDTH = 1280;
const int HEIGHT = 720;
const int FRAME_RATE = 10;

uint16_t b_white[WIDTH*HEIGHT];
uint16_t b_black[WIDTH*HEIGHT];

static void prepare_buffer(GstAppSrc* appsrc) {

    static gboolean white = FALSE;
    static GstClockTime timestamp = 0;
    GstBuffer *buffer;
    guint size;
    GstFlowReturn ret;
    double now = ((double)timestamp / (double)GST_SECOND);
    cout << now << endl;

    if(now > 3.0)
    {
        /* we are EOS, send end-of-stream and remove the source */
        g_signal_emit_by_name (appsrc, "end-of-stream", &ret);
        return;
    }

    size = WIDTH * HEIGHT * 2;
    for (int i = 0; i < WIDTH*HEIGHT; i++) { b_white[i] =  0xFFFF; }

    buffer = gst_buffer_new_wrapped_full( (GstMemoryFlags)0, (gpointer)(white?b_white:b_black), size, 0, size, NULL, NULL );

    white = !white;

    GST_BUFFER_PTS (buffer) = timestamp;
    GST_BUFFER_DURATION (buffer) = gst_util_uint64_scale_int (1, GST_SECOND, FRAME_RATE);

    timestamp += GST_BUFFER_DURATION (buffer);

    //This can also trigger callbacks including cb_need_data!
    ret = gst_app_src_push_buffer(appsrc, buffer);

    if (ret != GST_FLOW_OK) {
        run = false;
    }

}

static void cb_need_data (GstElement *appsrc, guint unused_size, gpointer user_data) {
    prepare_buffer((GstAppSrc*)appsrc);
}

static gboolean on_pipeline_message (GstBus * bus, GstMessage * message, GMainLoop *loop)
{
    cout << GST_MESSAGE_TYPE_NAME(message) << endl;

    switch (GST_MESSAGE_TYPE (message)) {
        case GST_MESSAGE_EOS:
            g_print ("Received End of Stream message\n");
            //g_main_loop_quit (loop);
            run = false;
            break;
    }
    return TRUE;
}

gint main (gint argc, gchar *argv[]) {

    GstElement *pipeline, *appsrc, *conv;

    for (int i = 0; i < WIDTH*HEIGHT; i++) { b_black[i] = 0; b_white[i] = 0xFFFF; }

    /* init GStreamer */
    gst_init (&argc, &argv);

    /* setup pipeline */
    pipeline = gst_pipeline_new ("pipeline");
    appsrc = gst_element_factory_make ("appsrc", "source");
    conv = gst_element_factory_make ("videoconvert", "conv");
    GstElement *enc = gst_element_factory_make ("x264enc", "enc");
    GstElement *mux = gst_element_factory_make ("matroskamux", "mux");
    GstElement *outFile = gst_element_factory_make ("filesink", "outFile");

    /* setup */
    g_object_set (G_OBJECT (appsrc), "caps",
            gst_caps_new_simple ("video/x-raw",
                         "format", G_TYPE_STRING, "RGB16",
                         "width", G_TYPE_INT, WIDTH,
                         "height", G_TYPE_INT, HEIGHT,
                         "framerate", GST_TYPE_FRACTION, FRAME_RATE, 1,
                         NULL), NULL);
    gst_bin_add_many (GST_BIN (pipeline), appsrc, conv, enc, mux, outFile, NULL);
    gst_element_link_many (appsrc, conv, enc, mux, outFile, NULL);

    /* setup appsrc */
    g_object_set (G_OBJECT (appsrc),
        "stream-type", 0, // GST_APP_STREAM_TYPE_STREAM
        "format", GST_FORMAT_TIME,
        "is-live", false,
        NULL);
    g_signal_connect (appsrc, "need-data", G_CALLBACK (cb_need_data), NULL);

    g_object_set (G_OBJECT (enc),
        "qp-min", 18,
        NULL);

    g_object_set (G_OBJECT (outFile),
        "location", "test.mkv",
        NULL);

    // play
    gst_element_set_state (pipeline, GST_STATE_PLAYING);

    auto bus = gst_element_get_bus (pipeline);

    //Wait for bus events
    while (run) {

        //This can trigger callbacks including cb_need_data
        GstClockTime timeout = 0 * GST_MSECOND;
        GstMessage *msg = gst_bus_timed_pop_filtered (bus, timeout, (GstMessageType)
            (GST_MESSAGE_STATE_CHANGED | GST_MESSAGE_ERROR | GST_MESSAGE_EOS));
        if(msg == nullptr) continue;

        on_pipeline_message (bus, msg, nullptr);
    }

    gst_object_unref (bus);

    /* clean up */
    gst_app_src_end_of_stream(GST_APP_SRC(appsrc));
    gst_element_set_state (pipeline, GST_STATE_NULL);
    gst_object_unref (GST_OBJECT (pipeline));

    return 0;
}

Answer 2

Most video encoders do not produce any buffers just by giving them one frame. They want to look ahead.. or have some latency in their internal pipelines.

So perhaps feeding more buffers into it helps.

Else, the video encoder probably will give you the frame when you properly tell it that no more frames will be coming. That means: send and EOS to the pipeline and wait for the EOS event to arrive on the bus. The encoder should hopefully push that one frame to your filesink by then.

EDIT: I noticed you already send EOS. You probably want to wait for the EOS on the bus before setting the pipeline's state to NULL.