celiofagundes
celiofagundes

Reputation: 3

Get a <video/> element audio without having to play the video

I'm developing a feature where i need to export a canvas (created with fabricjs) as a video file. Exporting the canvas as a video is already working by using MediaRecorderApi, the problem is that if the canvas has some video elements, i need to play all videos with sound so the exported canvas video has all the video audios. I want to know if there's actually a way to get the video elements audio and combine them with the record without having play them. Also they need to be playing and not muted. I want that to improve user experience when the video is being created, so he doesn't need to listen and see all the videos playing while waiting.

My current code looks like this


// Ffmpeg used to convert the record webm format to mp4
const ffmpegRef = useRef(new FFmpeg());

const load = async () => {
   const baseURL = 'https://unpkg.com/@ffmpeg/[email protected]/dist/umd';
   const ffmpeg = ffmpegRef.current;

   // toBlobURL is used to bypass CORS issue, urls with the same
   // domain can be used directly.
   await ffmpeg.load({
     coreURL: await toBlobURL(`${baseURL}/ffmpeg-core.js`, 'text/javascript'),
     wasmURL: await toBlobURL(`${baseURL}/ffmpeg-core.wasm`, 'application/wasm'),
   });
   ffmpeg.on('log', ({ message }) => {
     console.log('[ffmpeg]', message);
   });
 };
const transcode = async (file: Blob | File) => {
    setDownloadingVideoMessage('Otimizando vídeo');
    const ffmpeg = ffmpegRef.current;
    await ffmpeg.writeFile('input.webm', await fetchFile(file));
    await ffmpeg.exec([
      '-i',
      'input.webm',
      '-crf',
      '23',
      '-preset',
      'veryfast',
      '-movflags',
      '+faststart',
      '-r',
      '30',
      '-vcodec',
      'libx264',
      '-acodec',
      'aac',
      'output.mp4',
    ]);
    const data = await ffmpeg.readFile('output.mp4');
    var videoURL = URL.createObjectURL(new Blob([data], { type: 'video/mp4' }));
    var a = document.createElement('a');
    a.href = videoURL;
    a.download = 'video-canvas.mp4';
    a.click();
  };

const download = async() =>{
    // Load ffmpeg
    await load();
    
    // Get the canvas element and create a stream
    const canvasElement = document.getElementById('canvas');
    const canvasStream = canvasElement.captureStream(60);

    // Trigger a canvas render, mediarecorder for some reason only record canvas when something is draw on it when recording
    canvas.renderAll();

    
    // Get canvas objects so i cant iterate them
    const canvasToJson = canvas.toJSON(['height', 'width', 'data', '_element']); 
    const objs = canvasToJson.objects;
    
    // Create an audio context
    const audioContext = new AudioContext();
    const destination = audioContext.createMediaStreamDestination();
    
    let hasAnyVideo = false;
    
    // Iterate through objects, if they are a video create a node and connect to the audio context. also reset video time to beginning and play them.
    for (let i = 0; i < objs.length; i += 1) {
            const obj = objs[i];
            if (obj.data?.isVideo) {
              const videoElement = obj._element;
              videoElement.currentTime = 0.1;
              videoElement.loop = true;
    
              videoElement.play();
              const sourceNode = audioContext.createMediaElementSource(videoElement);
              sourceNode.connect(destination);
              sourceNode.connect(audioContext.destination);
              hasAnyVideo = true;
            }
          }
    const audioTrack = destination.stream.getAudioTracks()[0];
    
    // Combine canvasStream and audioTracks, i need to check if there's any video because otherwise the audio track will break the stream.
    
    const combinedStream = hasAnyVideo
            ? new MediaStream([...canvasStream.getVideoTracks(), audioTrack])
            : new MediaStream([...canvasStream.getVideoTracks()]);
    
    // Create the recorder
    const recorder = new MediaRecorder(combinedStream);
    let recordedBlobs: Blob[] = [];
    
    // Trigger a canvas render, mediarecorder for some reason only record canvas when something is draw on it when recording
    
    const animationLoop = () => {
        ctx.fillRect(0, 0, 1, 1);
        if (recorder.state !== 'inactive') {
          requestAnimationFrame(animationLoop);
        }
      };
    recorder.onstart = animationLoop;
    
    recorder.onstop = async (e) => {
            canvas.renderAll();
            // pauseall videos when recording ends
            if (hasAnyVideo) {
              for (let i = 0; i < objs.length; i += 1) {
                const obj = objs[i];
                if (obj.data?.isVideo) {
                  const videoElement = obj._element;
                  videoElement.currentTime = 0.1;
                  videoElement.loop = true;
                  videoElement.pause();
                }
              }   
            }
            const blob = new Blob(recordedBlobs, { type: 'video/webm' });
            recordedBlobs = [];
            
            try {
              // I need to convert the record which comes as a webm file to mp4 using ffmpeg, but i don't think it's relevant for this problem 
              await transcode(blob);
            } catch (err) {
              console.log(err);
            }
          };
    // Start recording, end recording after 20 seconds
    recorder.start();
    setTimeout(function () {
            recorder.stop();
    }, 20000);
}

Upvotes: 0

Views: 59

Answers (0)

Related Questions