Chrome stops playing video when I add small pieces to the MSE buffer

Question

I'm experimenting with the Media Source Extension, based on https://developers.google.com/web/updates/2016/03/mse-sourcebuffer

1. Download video (link) via XMLHttpRequest
2. I break it into 20 pieces via Blob & FileReader (1 piece about 3 sec.)
3. Create MediaSource and add buffer to it
4. I add these pieces to the buffer

I add the pieces not all at once, but one after the other, namely when the buffer is empty in the video, that is, the waiting event occurs.

1. After downloading the file and splitting, I immediately add the first piece to initialize and set the 2. duration
2. The video starts playing, everything is fine
3. Then, when there is no new data in the player to play (after the waiting event), I add a new piece
4. The video starts playing again, comes canplay, playing, canplaythrough, timeupdate, timeupdate ...
5. The cycle repeats 3 times well
6. When the timeline of the player reaches 10.61 seconds, the video freezes

The video freezes after adding another piece for 10.61 seconds, although there is new data in the buffer (2.5 seconds), but for some reason the canplay event does not occur after adding data. Chrome is expecting something, although it would seem there are new data - keep playing them. No errors, no video, no buffer, no MSE

What prevents chrome from playing this next piece is not clear, it just stops and the standard wheel turns and waits for data (?)

There are three ways to make a video start playing again:

• video.currentTime = video.currentTime // stupidly set the current position to the same
• Manually pull the timeline, no matter where a little back or forward
• Add a new piece

Then the video will continue to play again, but it stops already for 31.9 seconds (already longer)

When the video doesn't stop:

• If you break into fewer pieces, for example 5
• If you add a new chunk right after the onupdateend event

I can't understand what magic timestamps the video stops on: 10.61 and 31.9 seconds.

In Firefox, everything is played to the end, such a bug only in chrome-like browsers (checked in chrome and opera)

Here is a sample code, you can see there for 10 seconds. that there is loaded data on the scale, but it is not playing https://jsfiddle.net/1w4hyrke/1/

function playVideo(video) {
  var NUM_CHUNKS = 20;
  var sourceBuffer;
  var segments = [];
  var mimeCodec = 'video/mp4; codecs="avc1.4d401f,mp4a.40.2"';
  var mediaSource = new MediaSource();


  video.src = URL.createObjectURL(mediaSource);
  video.play();

  function addBuffer() {

    if (!segments.length) return console.error("segments empty");
    if (sourceBuffer.updating) return console.error("buffer updating");

    var segment = segments.shift();
    console.info("appendBuffer", segments.length, mediaSource.duration);
    sourceBuffer.appendBuffer(segment);
  }

  mediaSource.addEventListener('sourceopen', () => {
    sourceBuffer = mediaSource.addSourceBuffer(mimeCodec);
    //sourceBuffer.mode = 'segments';

    sourceBuffer.addEventListener('error', (e) => {
      console.error("sourceBuffer error", segments);
    });

    var xhr = new XMLHttpRequest();
    xhr.responseType = 'arraybuffer';

    sourceBuffer.onupdateend = () => {
      console.info("onupdateend");
      //addBuffer();
    };

    $(video).on("waiting abort canplay canplaythrough durationchange emptied encrypted  ended error interruptbegin interruptend loadeddata loadedmetadata loadstart mozaudioavailable pause play playing progress ratechange seeked seeking stalled suspend timeupdate volumechange", (e) => {

      var vbuf = video.buffered;
      var vstart = vbuf && vbuf.length > 0 ? vbuf.start(0) : "";
      var vend = vbuf && vbuf.length > 0 ? vbuf.end(0) : "";
      var videoDelta = vend ? (vend - video.currentTime).toFixed(2) : "-";
      var videoLen = vend ? (vend - vstart).toFixed(2) : "-";
      var quality = video.getVideoPlaybackQuality();
      let warn = ["timeupdate", "progress"].indexOf(e.type) === -1;

      console[!warn ? "log" : "error"].call(console, "VIDEO EVENT", e.type,
                                            `currentTime: ${video.currentTime}, videoDelta: ${videoDelta}, videoBuffLen: ${videoLen}, mediaSource: ${mediaSource.duration}`,
                                           );

      if (e.type === "waiting") {
        addBuffer();
      }

    });
    
    xhr.onload = () => {
      let buf = xhr.response;
      var uInt8Array = new Uint8Array(buf);
      var file = new Blob([uInt8Array], {
        type: 'video/mp4'
      });

      var i = 0;

      function readChunk() {
        var reader = new FileReader();

        reader.onload = function(e) {
          console.log('Appending chunk: ' + i);

          segments.push(new Uint8Array(e.target.result));

          if (i === 0) addBuffer(); //init first chunk          

          if (i === NUM_CHUNKS - 1) {

            sourceBuffer.addEventListener('updateend', function() {
              if (!sourceBuffer.updating && mediaSource.readyState === 'open') {
                //mediaSource.endOfStream();
              }
            });

          } else {
            readChunk(++i);
          }
        };

        var chunkSize = Math.ceil(file.size / NUM_CHUNKS);
        var startByte = chunkSize * i;
        var chunk = file.slice(startByte, startByte + chunkSize);
        reader.readAsArrayBuffer(chunk);
      }

      readChunk();
    };

    xhr.open("GET", "https://nickdesaulniers.github.io/netfix/demo/frag_bunny.mp4");
    xhr.send();

  });

  mediaSource.addEventListener('sourceended', () => {
    console.log("mediaSource ended");
  });
  mediaSource.addEventListener('sourceclose', () => {
    console.log("mediaSource closed");
  });

  mediaSource.addEventListener('error', () => {
    console.error("mediaSource error");
  });

  mediaSource.addEventListener('abort', () => {
    console.error("mediaSource abort");
  });

}

$(function() {
  var video = $('#video')[0];
  playVideo(video);

})

Answer 1

Based on my experience, most of the errors with the video stopping playing are caused by the video segments coming out of order. I think MediaSource expects the segments to be in monotonically increasing order. So, I changed my code from Async sending/receiving of the video buffers to Sync.

Answer 2

Your error is to listen to the event "waiting" from video. The definition of the event is :

Fired when playback has stopped because of a temporary lack of data.

It's a bad idea to use it since it means the player could pause at any moment (it seems that in Chrome it can happen after the event "waiting"), and this also cause stuterring.

A solution is to do the addBuffer() in the onupdateend event, I see you commented this line, maybe you did find an issue with that? If you don't want to push packets in the SourceBuffer when the buffer is ready you could also use setTimeout or setInterval to make asynchronous calls.

Here is a sample of code working on both Firefox and Chrome : https://jsfiddle.net/thomasjammet/bvm8ueoj/

I removed the FileReader part which is not necessary (you can push directly UInt8Array packets to the SourceBuffer instance).