How can I time the presentation and extraction of frames from a video file?

Question

The goal is to control the frames extraction speed in the BackGroundWorker's DoWork event.

I tried Thread.Sleep(), but it throws an exception.

This is what I want to do. It is described it above and in the bottom.

using Accord.Video;
using Accord.Video.FFMPEG;
using System.ComponentModel;
using System.Drawing;
using System.IO;
using System.Windows.Forms;

public partial class Form1 : Form
{
    public Form1()
    {
        InitializeComponent();

        using (var vFReader = new VideoFileReader())
        {
            vFReader.Open(@"C:\Users\Chocolade 1972\Downloads\MyVid.mp4");
            trackBar1.Maximum = (int)vFReader.FrameCount;
        }
    }

    private void button1_Click(object sender, EventArgs e)
    {
        backgroundWorker1.RunWorkerAsync();
    }

    private void backgroundWorker1_DoWork(object sender, DoWorkEventArgs e)
    {
        using (var vFReader = new VideoFileReader())
        {
            vFReader.Open(@"C:\Users\Chocolade 1972\Downloads\MyVid.mp4");
            for (var i = 0; i < vFReader.FrameCount; i++)
            {
                backgroundWorker1.ReportProgress(0, vFReader.ReadVideoFrame());
            }

            // Not sure that this would be required as it might happen implicitly at the end of the 'using' block.
            vFReader.Close();
        }
    }

    private void backgroundWorker1_ProgressChanged(object sender, ProgressChangedEventArgs e)
    {
        pictureBox1.Image?.Dispose();
        pictureBox1.Image = (Image)e.UserState;
    }

    private void Form1_Resize(object sender, EventArgs e)
    {
        label1.Text = this.Size.ToString();
    }
}

It's working fine, but too fast. How can I use a Timer, or something that allows me to control the speed of the extraction of the frames?

Answer 1

I suggest some changes (quite a lot actually :) to the current code.

Main points:

1. Create an async method to perform the video playback. The VideoFileReader works on a ThreadPool Thread (2, actually), it won't cause the Form to freeze
2. Use an IProgress<T> delegate (of Type Progress<Bitmap>, named videoProgress here) that marshals to the UI Thread the new data, used to update a PictureBox control. The delegate method is named Updater
3. Use a single Bitmap object, set to the Image property of a PictureBox
4. Use a Graphics object derived from this Bitmap to draw the video frames. This allows to contain the resources used. The PictureBox is simply invalidated, to show the current content of the Bitmap
5. Allow the video playback method to accept a frame-rate value, here set to 25 frames per second. Of course, it can be adapted to slow down or speed up the playback (note that, setting more than 32~35 frames per second, you begin to lose some frames)
6. Use a CancellationTokenSource to signal the video playback method to stop the playback and terminate, either when a Stop Button is pressed or the Form is closed while the playback is active

Important notes:

• The Bitmap that VideoFileReader returns must be disposed. If you don't, you're going to see an increase in consumption of graphic resources, which won't stop
• Using a single Bitmap and drawing each new frame with a derived Graphics object, allows to preserve graphic resources. If you keep the Diagnostic Tool pane open while the video is playing, you'll notice that you're not leaking any resource and the memory usage remains constant.
  There's of course a slight increase when you open this Form and the container Bitmap is created, but when the Form is closed, that small amount of resources is reclaimed
• This also allows smoother transitions and some more speed in the rendering (move the Form around while the video is playing). Also, try to anchor / dock the PictureBox, set SizeMode = Zoom and maximize the Form (setting the Zoom mode of the PictureBox is a performance hit, you should resize the Bitmap instead)

---

buttonStart_Click, buttonStop_Click and buttonPause_Click are the Click handlers of the Buttons used to start, stop and pause the playback.
The syncRoot object is not strictly required here, but keep it there, it may become useful at some point

using System.Drawing;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using System.Windows.Forms;
using Accord.Video.FFMPEG;

public partial class Form1 : Form
{
    Bitmap frame = null;
    Graphics frameGraphics = null;
    bool isVideoRunning = false;
    IProgress<Bitmap> videoProgress = null;
    private CancellationTokenSource cts = null;
    private readonly object syncRoot = new object();
    private static long pause = 0;

    public Form1() => InitializeComponent(); 

    private async void buttonStart_Click(object sender, EventArgs e) {
        string fileName = "[The Video File Path]";

        if (isVideoRunning) return;
        isVideoRunning = true;

        using (var videoReader = new VideoFileReader()) {
            videoReader.Open(fileName);
            // Adds two pixels, to see the Frame's boundaries in the container
            frame = new Bitmap(videoReader.Width + 2, videoReader.Height + 2);
            trackBar1.Maximum = (int)videoReader.FrameCount;
        }

        videoProgress = new Progress<Bitmap>(Updater);
        cts = new CancellationTokenSource();
        pictureBox1.Image = frame;
        try {
            frameGraphics = Graphics.FromImage(frame);
            // Set the frame rate to 25 frames per second
            int frameRate = 1000 / 25;
            await GetVideoFramesAsync(videoProgress, fileName, frameRate, cts.Token);
        }
        finally {
            StopPlayback(false);
            frameGraphics?.Dispose();
            pictureBox1.Image?.Dispose();
            pictureBox1.Image = null;
            buttonPause.Text = "Pause";
            pause = 0;
            isVideoRunning = false;
        }
    }

    private void buttonStop_Click(object sender, EventArgs e) => StopPlayback(true);

    private void buttonPause_Click(object sender, EventArgs e)
    {
        if (pause == 0) {
            buttonPause.Text = "Resume";
            Interlocked.Increment(ref pause);
        }
        else {
            Interlocked.Decrement(ref pause);
            buttonPause.Text = "Pause";
        }
    }

    private void StopPlayback(bool cancel) {
        lock (syncRoot) {
            if (cancel) cts?.Cancel();
            cts?.Dispose();
            cts = null;
        }
    }

    private void Updater(Bitmap videoFrame) {
        using (videoFrame) frameGraphics.DrawImage(videoFrame, Point.Empty);
        pictureBox1.Invalidate();
    }

    private async Task GetVideoFramesAsync(IProgress<Bitmap> updater, string fileName, int intervalMs, CancellationToken token = default) {
        using (var videoReader = new VideoFileReader()) {
            if (token.IsCancellationRequested) return;
            videoReader.Open(fileName);

            while (!token.IsCancellationRequested) {
                // Resumes on a ThreadPool Thread
                await Task.Delay(intervalMs, token).ConfigureAwait(false);

                if (Interlocked.Read(ref pause) == 0) {
                    var frame = videoReader.ReadVideoFrame();
                    if (frame is null) break;
                    updater.Report(frame);
                }
            }
        }
    }

    protected override void OnFormClosing(FormClosingEventArgs e) {
        if (isVideoRunning) StopPlayback(true);
        base.OnFormClosing(e);
    }
}