Reputation: 465

How to avoid delta frames when going to previous frames using WebCodecs VideoDecoder?

I have created a custom video player using WebCodecs VideoDecoder and mp4box.js for mp4 parsing. I have also implemented frame-by-frame control, which works as expected. However, due to the limitation of VideoDecoder, when I go to previous frames, I must first process all of the frames since the last key frame until the target frame. As a result, all of these frames are rendered to the target canvas which doesn't look very good.

How can I prevent rendering intermediate frames when going to a previous frame and only display the target frame?

Here's my code:

<!DOCTYPE html>
<html lang="en">

<head>
  <meta charset="UTF-8">
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  <title>Custom Video Player</title>
</head>

<body>
  <canvas id="videoCanvas" width="640" height="360"></canvas>
  <br>
  <input type="file" id="fileInput" accept="video/mp4">
  <button id="play">Play</button>
  <button id="pause">Pause</button>
  <button id="nextFrame">Next frame</button>
  <button id="prevFrame">Previous frame</button>

  <script src="mp4box.all.min.js"></script>
  <script>
    const fileInput = document.getElementById('fileInput');
    const playButton = document.getElementById('play');
    const pauseButton = document.getElementById('pause');
    const nextFrameButton = document.getElementById('nextFrame');
    const prevFrameButton = document.getElementById('prevFrame');
    const canvas = document.getElementById('videoCanvas');
    const ctx = canvas.getContext('2d');

    let mp4boxFile;
    let videoDecoder;
    let playing = false;
    let frameDuration = 1000 / 50; // 50 fps
    let currentFrame = 0;
    let frames = [];
    let shouldRenderFrame = true;


    function findPreviousKeyFrame(frameIndex) {
      for (let i = frameIndex - 1; i >= 0; i--) {
        if (frames[i].type === 'key') {
          return i;
        }
      }
      return -1;
    }

    async function displayFramesInRange(start, end) {
      shouldRenderFrame = false;
      for (let i = start; i < end; i++) {
        if (i == end - 1) {
          shouldRenderFrame = true;
          console.log("end");
        }
        await videoDecoder.decode(frames[i]);
      }
    }

    function shouldRenderNextFrame() {
      return shouldRenderFrame;
    }

    async function prevFrame() {
      if (playing || currentFrame <= 1) return;

      // Find the previous keyframe.
      const keyFrameIndex = findPreviousKeyFrame(currentFrame - 1);

      // If no keyframe found, we can't go back.
      if (keyFrameIndex === -1) return;

      // Display frames from the previous keyframe up to the desired frame.
      await displayFramesInRange(keyFrameIndex, currentFrame - 1);
      currentFrame--;
    }

    async function initVideoDecoder() {
      videoDecoder = new VideoDecoder({
        output: displayFrame,
        error: e => console.error(e),
      });
    }

    function displayFrame(frame) {
      if (shouldRenderNextFrame()) {
        ctx.drawImage(frame, 0, 0);
      }
      frame.close();
    }

    function playVideo() {
      if (playing) return;
      console.log('Playing video');
      playing = true;
      (async () => {
        for (let i = currentFrame; i < frames.length && playing; i++) {
          await videoDecoder.decode(frames[i]);
          currentFrame = i + 1;
          await new Promise(r => setTimeout(r, frameDuration));
        }
        playing = false;
      })();
    }

    function getDescription(trak) {
      for (const entry of trak.mdia.minf.stbl.stsd.entries) {
        if (entry.avcC || entry.hvcC) {
          const stream = new DataStream(undefined, 0, DataStream.BIG_ENDIAN);
          if (entry.avcC) {
            entry.avcC.write(stream);
          } else {
            entry.hvcC.write(stream);
          }
          return new Uint8Array(stream.buffer, 8);  // Remove the box header.
        }
      }
      throw "avcC or hvcC not found";
    }

    function pauseVideo() {
      playing = false;
    }

    function nextFrame() {
      if (playing || currentFrame >= frames.length) return;
      videoDecoder.decode(frames[currentFrame]);
      currentFrame++;
    }

    fileInput.addEventListener('change', () => {
      if (!fileInput.files[0]) return;
      const fileReader = new FileReader();
      fileReader.onload = e => {
        mp4boxFile = MP4Box.createFile();
        mp4boxFile.onReady = info => {
          const videoTrack = info.tracks.find(track => track.type === 'video');
          const trak = mp4boxFile.getTrackById(videoTrack.id);
          videoDecoder.configure({
            codec: videoTrack.codec,
            codedHeight: videoTrack.video.height,
            codedWidth: videoTrack.video.width,
            description: this.getDescription(trak)
          });
          mp4boxFile.setExtractionOptions(videoTrack.id);
          mp4boxFile.start()
          mp4boxFile.onSamples = (id, user, samples) => {
            frames.push(...samples.map(sample => new EncodedVideoChunk({
              type: sample.is_sync
                ? 'key' : 'delta',
              timestamp: sample.dts,
              data: sample.data.buffer,
            })));
          };
          mp4boxFile.flush();
        };
        e.target.result.fileStart = 0;
        mp4boxFile.appendBuffer(e.target.result);
      };
      fileReader.readAsArrayBuffer(fileInput.files[0]);
    });

    playButton.addEventListener('click', playVideo);
    pauseButton.addEventListener('click', pauseVideo);
    nextFrameButton.addEventListener('click', nextFrame);
    prevFrameButton.addEventListener('click', prevFrame);

    initVideoDecoder();

  </script>
</body>

</html>

Upvotes: 0

Answers (1)

Daniel Robinson

Reputation: 465

I was able to resolve this by comparing the timestamp of the frame to the target frame

function displayFrame(frame) {
  if(frame.timestamp == frames[currentFrame - 1].timestamp){
    ctx.drawImage(frame, 0, 0);
  }
  frame.close();
}

Upvotes: 1

How to avoid delta frames when going to previous frames using WebCodecs VideoDecoder?

Answers (1)

Related Questions