How to extract frames at 30 fps using FFMPEG APIs on Android?

Question

We are working on a project that consumes FFMPEG library for video frame extraction on Android platform.

On Windows, we have observed:

• Using CLI, ffmpeg is capable of extracting frames at 30 fps using command ffmpeg -i input.flv -vf fps=1 out%d.png.
• Using Xuggler, we are able to extract frames at 30 fps.
• Using FFMPEG APIs directly in code, we are getting frames at 30 fps.

But when we use FFMPEG APIs directly on Android (See Hardware Details), we are getting following results:

• 720p video (1280 x 720) - 16 fps (approx. 60 ms/frame)
• 1080p video (1920 x 1080) - 7 fps (approx. 140 ms/frame)

We haven't tested Xuggler/CLI on Android yet.

Ideally, we should be able to get the data in constant time (approx. 30 ms/frame).

How can we get 30 fps on Android?

Code being used on Android:

if (avformat_open_input(&pFormatCtx, pcVideoFile, NULL, NULL)) {
    iError = -1;  //Couldn't open file
}

if (!iError) {
    //Retrieve stream information
    if (avformat_find_stream_info(pFormatCtx, NULL) < 0)
        iError = -2; //Couldn't find stream information
}

//Find the first video stream
if (!iError) {

    for (i = 0; i < pFormatCtx->nb_streams; i++) {
        if (AVMEDIA_TYPE_VIDEO
                == pFormatCtx->streams[i]->codec->codec_type) {
            iFramesInVideo = pFormatCtx->streams[i]->nb_index_entries;
            duration = pFormatCtx->streams[i]->duration;
            begin = pFormatCtx->streams[i]->start_time;
            time_base = (pFormatCtx->streams[i]->time_base.num * 1.0f)
                    / pFormatCtx->streams[i]->time_base.den;

            pCodecCtx = avcodec_alloc_context3(NULL);
            if (!pCodecCtx) {
                iError = -6;
                break;
            }

            AVCodecParameters params = { 0 };
            iReturn = avcodec_parameters_from_context(&params,
                    pFormatCtx->streams[i]->codec);
            if (iReturn < 0) {
                iError = -7;
                break;
            }

            iReturn = avcodec_parameters_to_context(pCodecCtx, &params);
            if (iReturn < 0) {
                iError = -7;
                break;
            }

            //pCodecCtx = pFormatCtx->streams[i]->codec;

            iVideoStreamIndex = i;
            break;
        }
    }
}

if (!iError) {
    if (iVideoStreamIndex == -1) {
        iError = -3; // Didn't find a video stream
    }
}

if (!iError) {
    // Find the decoder for the video stream
    pCodec = avcodec_find_decoder(pCodecCtx->codec_id);
    if (pCodec == NULL) {
        iError = -4;
    }
}

if (!iError) {
    // Open codec
    if (avcodec_open2(pCodecCtx, pCodec, NULL) < 0)
        iError = -5;
}

if (!iError) {
    iNumBytes = av_image_get_buffer_size(AV_PIX_FMT_RGB24, pCodecCtx->width,
            pCodecCtx->height, 1);

    // initialize SWS context for software scaling
    sws_ctx = sws_getContext(pCodecCtx->width, pCodecCtx->height,
            pCodecCtx->pix_fmt, pCodecCtx->width, pCodecCtx->height,
            AV_PIX_FMT_RGB24,
            SWS_BILINEAR,
            NULL,
            NULL,
            NULL);
    if (!sws_ctx) {
        iError = -7;
    }
}
clock_gettime(CLOCK_MONOTONIC_RAW, &end);
delta_us = (end.tv_sec - start.tv_sec) * 1000000
        + (end.tv_nsec - start.tv_nsec) / 1000;
start = end;
//LOGI("Starting_Frame_Extraction: %lld", delta_us);
if (!iError) {
    while (av_read_frame(pFormatCtx, &packet) == 0) {
        // Is this a packet from the video stream?
        if (packet.stream_index == iVideoStreamIndex) {
            pFrame = av_frame_alloc();
            if (NULL == pFrame) {
                iError = -8;
                break;
            }

            // Decode video frame
            avcodec_decode_video2(pCodecCtx, pFrame, &iFrameFinished,
                    &packet);
            if (iFrameFinished) {
                //OUR CODE
            }
            av_frame_free(&pFrame);
            pFrame = NULL;
        }
        av_packet_unref(&packet);
    }
}

Answer 1

Please take a look at https://gitter.im/mobile-ffmpeg/Lobby?at=5c5bb384f04ef00644f1bb4e A few lines below, they mention options to accelerate the process, such as ... -preset ultrafast, -threads 10, -tune zerolatency, -x264-params sliced-threads=1

Answer 2

You need some structures and functions from libavfilter.

The vf option means "video filter". The command line ffmpeg -i input -vf fps=30 out%d.png will output video_length_in_seconds * 30 regardless the original video fps. That means if the video is of 25 fps, you'll get some duplicate frames. While if the video is more than 30 fps, you'll lose some frames.

To achieve this, you have to init some filter context. See filtering_video.c example from ffmpeg source.

AVFilter* buffersrc  = avfilter_get_by_name("buffer");
AVFilter* buffersink = avfilter_get_by_name("buffersink");
AVFilterInOut* outputs = avfilter_inout_alloc();
AVFilterInOut* inputs  = avfilter_inout_alloc();
AVRational time_base = p_format_ctx->streams[video_stream]->time_base;
enum AVPixelFormat pix_fmts[] = { p_codec_ctx->pix_fmt, AV_PIX_FMT_NONE };

filter_graph = avfilter_graph_alloc();
if (!outputs || !inputs || !filter_graph) {
    // failed, goto cleanup
}

char args[512];
snprintf(args, sizeof(args),
         "video_size=%dx%d:pix_fmt=%d:time_base=%d/%d:pixel_aspect=%d/%d",
         p_codec_ctx->width, p_codec_ctx->height, p_codec_ctx->pix_fmt,
         time_base.num, time_base.den,
         p_codec_ctx->sample_aspect_ratio.num, p_codec_ctx->sample_aspect_ratio.den);

int ret = avfilter_graph_create_filter(&buffersrc_ctx, buffersrc, "in",
                                       args, NULL, filter_graph);

if (ret < 0) {
    LOG(ERROR) << "Cannot create buffer source";
    avfilter_inout_free(&inputs);
    avfilter_inout_free(&outputs);
    return false;
}

ret = avfilter_graph_create_filter(&buffersink_ctx, buffersink, "out",
                                   NULL, NULL, filter_graph);
if (ret < 0) {
    // failed... blabla
}

ret = av_opt_set_int_list(buffersink_ctx, "pix_fmts", pix_fmts,
                          AV_PIX_FMT_NONE, AV_OPT_SEARCH_CHILDREN);
if (ret < 0) {
    // failed... blabla
}

outputs->name       = av_strdup("in");
outputs->filter_ctx = buffersrc_ctx;
outputs->pad_idx    = 0;
outputs->next       = NULL;

inputs->name        = av_strdup("out");
inputs->filter_ctx  = buffersink_ctx;
inputs->pad_idx     = 0;
inputs->next        = NULL;

const char* filter_description[256] = "fps=fps=30";

if ((ret = avfilter_graph_parse_ptr(filter_graph, filters_descr.c_str(),
                                    &inputs, &outputs, NULL)) < 0) {
    // failed...
}

if ((ret = avfilter_graph_config(filter_graph, NULL)) < 0) {
    // failed...
}

Ok, this is all initialization needed.

And adding some codes to decoding part:

avcodec_decode_video2(p_codec_ctx, p_frame, &got_frame, &packet);
if (*got_frame) {
    p_frame->pts = av_frame_get_best_effort_timestamp(p_frame);
    if (av_buffersrc_add_frame_flags(buffersrc_ctx, p_frame, AV_BUFFERSRC_FLAG_KEEP_REF) < 0) {
        // failed... blabla
    }
    while (1) {
        int ret = av_buffersink_get_frame(buffersink_ctx, p_frame_stage);  
        // p_frame_stage is a AVFrame struct. Same size as p_frame. Need to allocated before.
        if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF)
            break;
        if (ret < 0) {
            // something wrong. filter failed.            
        }
        // Do something with p_frame_stage here.
    }
}