CODEWITHSUNDEEP
javascriptfirefoxvideohtml5-videowebrtc
Rob Hague
Rob Hague

Reputation: 1429

Firefox's MediaRecorder interface only providing new video data every two seconds

I'm trying to send a video stream obtained via WebRTC's getUserMedia() method to the server for additional processing. Latency is important, as I wish to detect changes in the video stream and update the client immediately. For this particular use, a Firefox-only solution is acceptable, and so I'm investigating the MediaRecorder interface.

I've put together a simple test case, included below. There are no errors, and the ondataavailable callback is called every 500ms, as expected. However, for three out of four of these calls, the size of the data provided is zero. This suggests to me that the data is being grouped into chunks of about two seconds (possibly due to constraints of the video encoding being used).

Is it possible to get MediaRecorder to provide data at a finer granularity? If not, what is the best way to get video data from the userMedia stream to the server with low latency? An interface specific to Chrome or Firefox would be fine, but one that worked in both would be even better.

<html>
  <body>
    <h1>MediaRecorder Test</h1>
    <video id="video" width="640" style="border: 1px solid black"></video>
  </body>
</html>

<script>
 // The variable that holds the video stream
 var mediastream = null;

 // Start video capture (and provide a way to stop it)
 navigator.mozGetUserMedia ( { video: true, audio: false },
   function(stream_arg) {
     mediastream = stream_arg;
     var vendorURL = window.URL || window.webkitURL;
     video.src = vendorURL.createObjectURL(mediastream);
     video.play();
     recordStream();
   },
   function(err) { console.log("Error starting video stream: " + err); }
 );

 // Record the stream
 var recorder = null;
 function recordStream() {
   recorder = new MediaRecorder(mediastream);
   recorder.ondataavailable = function(ev) {
     console.log("Got: "+ev.data.size);
   };
   recorder.start(500);
 }
</script>

Upvotes: 4

Views: 1873

Answers (2)

Oleg
Oleg

Reputation: 23297

You may use another approach: each N milliseconds send video to (optionally hidden) canvas which allows to to get base64 representation of an image. Thus, you will get an array of base64 frames. Now you have 2 options:

  • send each frame to server in base64 format. Since base64 is a regular string, this is the easiest way;
  • transform each base64 frame to Blob and send it to server via FormData. In my case uploading this way is twice as fast as first one.

Below you can see my example (performing the second option). This example is quite big but every part of it is important.

index.html:

<!DOCTYPE html>
<html>
<head>
<script src="record-test.js"></script>
</head>
<body>

    <video id="video"></video>
    <canvas id="canvas" style="display:none;"></canvas>
    <input type="button" id="stopRecordBtn" value="Stop recording">

</body>
</html>

record-test.js:

(function() {
    'use strict';

    //you can play with these settings
    var FRAME_INTERVAL_MS = 500;  //take snapshot each 500 ms
    var FRAME_WIDTH = 320;    //width and
    var FRAME_HEIGHT = 240;   //height of resulting frame

    navigator.getUserMedia = navigator.getUserMedia || navigator.webkitGetUserMedia || navigator.mozGetUserMedia;
    window.URL = window.URL || window.webkitURL;

    var video, canvas, ctx;
    var mediaStream;
    var videoRecordItv;
    var base64Frames = [];

    var init = function() {
        video = document.getElementById('video');

        canvas = document.getElementById('canvas'); //use canvas to capture a frame and convert it to base64 data
        canvas.width = FRAME_WIDTH;
        canvas.height = FRAME_HEIGHT;
        ctx = canvas.getContext('2d');

        var stopBtn = document.getElementById('stopRecordBtn');
        stopBtn.addEventListener('click', stopRecording);

        navigator.getUserMedia({video: true}, onGotStream, function(e) {console.log(e);});
    }

    var onGotStream = function(stream) {
        mediaStream = stream;
        video.src = URL.createObjectURL(mediaStream);
        video.play();

        videoRecordItv = setInterval(function() {  //capture a frame each FRAME_INTERVAL_MS milliseconds
            var frame = getBase64FrameFromVideo();
            base64Frames.push(frame);
        }, FRAME_INTERVAL_MS);
    }

    var getBase64FrameFromVideo = function() {
        ctx.drawImage(video, 0, 0, FRAME_WIDTH, FRAME_HEIGHT);
        //a canvas snapshot looks like data:image/jpeg;base64,ACTUAL_DATA_HERE
        //we need to cut out first 22 characters:
        var base64PrefixLength = 'data:image/jpeg;base64,'.length;
        return canvas.toDataURL('image/jpeg').slice(base64PrefixLength);
    }

    var stopRecording = function() {
        mediaStream && mediaStream.stop && mediaStream.stop();
        mediaStream = null;
        clearInterval(videoRecordItv);  //stop capturing video

        uploadFramesToServer();
    }

    var uploadFramesToServer = function() {
        var sid = Math.random(); //generate unique id
        var curFrameIdx = 0;  //current frame index
        (function postFrame() {
            console.log('post frame #' + curFrameIdx);

            var base64Frame = base64Frames[curFrameIdx];
            var blobFrame = base64ToBlob(base64Frame, 'image/jpeg');
            var formData = new FormData;
            formData.append('frame', blobFrame, 'upload.jpg');
            formData.append('sid', sid);
            var xhr = new XMLHttpRequest();
            //post a single frame to /postFrame url with multipart/form-data enctype
            //on the server you get "sid" param and "frame" file as you would post a file with regular html form
            xhr.open('POST', '/postFrame', true);
            xhr.onload = function(e) {
                console.log(this.response);
                if (base64Frames[++curFrameIdx]) {
                    postFrame(); //post next frame
                } else {
                    //DONE!
                    console.log('finish post frames');
                }
            };
            xhr.send(formData);
        })();
    }

    var base64ToBlob = function(base64Data, contentType, sliceSize) {
        contentType = contentType || '';
        sliceSize = sliceSize || 512;

        var byteCharacters = atob(base64Data);
        var byteArrays = [];

        for (var offset = 0; offset < byteCharacters.length; offset += sliceSize) {
            var slice = byteCharacters.slice(offset, offset + sliceSize);

            var byteNumbers = new Array(slice.length);
            for (var i = 0; i < slice.length; i++) {
                byteNumbers[i] = slice.charCodeAt(i);
            }

            var byteArray = new Uint8Array(byteNumbers);

            byteArrays.push(byteArray);
        }

        return new Blob(byteArrays, {type: contentType});
    }

    document.addEventListener('DOMContentLoaded', init);
})();

On the server-side you still have to perform some actions, for example, create video from these frames with FFmpeg.

This approach works in both Chrome and Firefox.

Hope this helps. Sorry for my English and good luck!

Upvotes: 0

Brad
Brad

Reputation: 163468

The 500ms interval you're passing to MediaRecorder is advisory. The codec used may require larger chunks of data/time to work with. It's probably giving you data as fast as it can.

If you need low latency, MediaRecorder is the wrong tool for the job. A regular WebRTC call will use codec settings that optimize for latency over quality. I've heard of folks recording WebRTC server-side, but I do not know of anything open source to do this off the top of my head.

Perhaps some day in the future, the MediaRecorder API will allow us to choose codec parameters and this won't be an issue.

Upvotes: 1

Related Questions