high quality media recorder from canvas 30 fps at 1080p

Question

I have a canvas app that currently captures images of the canvas and compiles a video that is sent to ffmpeg which then outputs the video format of their choice. The problem is its super slow! Not on the video conversion but on the compiling of the actual frames, you see I have to pause the video and the animation and take a screenshot of the canvas. So rather than taking screenshots I was thinking about using MediaRecorder and canvas.captureStream. I am able to get video output but the quality is really low and the video keeps droping frames. I need to have the frame rate be at least 30 fps or higher and the quality be high. Heres my record function

async [RECORD] ({state}) {
    state.videoOutputURL = null;
    state.outputVideo = document.createElement("video");
    const videoStream = state.canvas.captureStream(30);
    const mediaRecorder = new MediaRecorder(videoStream);
    mediaRecorder.ondataavailable = function(e) {
      state.captures.push(e.data);
    };
    
    mediaRecorder.onstop = function(e) {
      const blob = new Blob(state.captures);
      state.captures = [];
      const videoURL = URL.createObjectURL(blob);
      state.outputVideo.src = videoURL;
      state.outputVideo.width = 1280;
      state.outputVideo.height = 720;
      document.body.append(state.outputVideo);
    }; 
    mediaRecorder.start();
    
    state.anim.start();
    state.video.play();
    lottie.play();
    
    state.video.addEventListener("ended", async () => {
      mediaRecorder.stop();
    });
  }

Answer 1

The best way I found to do this was to actually pause the video on a canvas and use canvas.toDataURL to take screenshots. I compile the screenshots into a video with a library called Whammy and send that over to FFmpeg to rip the final content. The following code should give a pretty good idea

async [TAKE_SCREENSHOT]({ state, dispatch }) {
    let seekResolve;
    if (!state.ended && state.video) {
      state.video.addEventListener("seeked", async () => {
        if (seekResolve) seekResolve();
      });
      await new Promise(async (resolve, reject) => {
        if (state.animations.length) {
          dispatch(PAUSE_LOTTIES);
        }
        dispatch(PAUSE_VIDEO);
        await new Promise(r => (seekResolve = r));
        if (state.layer) {
          state.layer.draw();
        }
        if (state.canvas) {
          state.captures.push(state.canvas.toDataURL("image/webp"));
        }
        resolve();
        dispatch(TAKE_SCREENSHOT);
      });
    }
  },
  async [PAUSE_VIDEO]({ state, dispatch, commit }) {
    state.video.pause();
    const oneFrame = 1 / 30;
    if (state.video.currentTime + oneFrame < state.video.duration) {
      state.video.currentTime += oneFrame;
      const percent = `${Math.round(
        (state.video.currentTime / state.video.duration) * 100
      )}%`;
      commit(SET_MODAL_STATUS, percent);
    } else {
      commit(SET_MODAL_STATUS, "Uploading your video");
      state.video.play();
      state.ended = true;
      await dispatch(GENERATE_VIDEO);
    }
  },
  async [PAUSE_LOTTIES]({ state }) {
    for (let i = 0; i < state.animations.length; i++) {
      let step = 0;
      let animation = state.animations[i].lottie;
      if (animation.currentFrame <= animation.totalFrames) {
        step = animation.currentFrame + 1;
      }
      await lottie.goToAndStop(step, true, animation.name);
    }
  },
  async [GENERATE_VIDEO]({ state, rootState, dispatch, commit }) {
    let status;
    state.editingZoom = null;
    const username =
      rootState.user.currentUser.username;
    const email = rootState.user.currentUser.email || rootState.user.guestEmail;
    const name = rootState.user.currentUser.firstName || "guest";
    const s3Id = rootState.templates.currentVideo.stock_s3_id || state.s3Id;
    const type = rootState.dataClay.fileFormat || state.type;
    const vid = new Whammy.fromImageArray(state.captures, 30);
    vid.lastModifiedDate = new Date();
    vid.name = "canvasVideo.webm";
    const data = new FormData();
    const id = `${username}_${new Date().getTime()}`;
    data.append("id", id);
    data.append("upload", vid);
    let projectId,
      fileName,
      matrix = null;
    if (!state.editorMode) {
      projectId = await dispatch(INSERT_PROJECT);
      fileName = `${rootState.dataClay.projectName}.${type}`;
      matrix = rootState.dataClay.matrix[0];
    } else {
      matrix = rootState.canvasSidebarMenu.selectedDisplay;
      projectId = id;
      fileName = `${id}.${type}`;
    }
    if (projectId || state.editorMode) {
      await dispatch(UPLOAD_TEMP_FILE, data);
      const key = await dispatch(CONVERT_FILE_TYPE, {
        id,
        username,
        type,
        projectId,
        matrix,
        name,
        email,
        editorMode: state.editorMode
      });
      const role = rootState.user.currentUser.role;
      state.file = `/api/files/${key}`;
      let message;
      let title = "Your video is ready";
      status = "rendered";
      if (!key) {
        status = "failed";
        message =
          "<p class='error'>Error processing video! If error continues please contact Creative Group. We are sorry for any inconvenience.</p>";
        title = "Error!";
      } else if (!rootState.user.currentUser.id) {
        message = `<p>Your video is ready. Signup for more great content!</p> <a href="${
          state.file
        }" download="${fileName}" class="btn btn-primary btn-block">Download</a>`;
      } else if (role != "banner") {
        message = `<p>Your video is ready.</p> <a href="${
          state.file
        }" download="${fileName}" class="btn btn-primary btn-block">Download</a>`;
      } else {
        message = `<p>Your video is ready. You may download your file from your banner account</p>`;
        await dispatch(EXPORT_TO_BANNER, {
          s3Id,
          fileUrl: key,
          extension: `.${type}`,
          resolution: matrix
        });
      }
      if (state.editorMode) {
        await dispatch(SAVE_CANVAS, { status, fileId: projectId });
      }
      state.video.loop = "loop";
      state.anim.stop();
      state.video.pause();
      lottie.unfreeze();
      await dispatch(DELETE_PROJECT_IN_PROGRESS);
      commit(RESET_PROJECT_IN_PROGRESS);
      commit(RESET_CANVAS);
      if (rootState.user.currentUser.id) {
        router.push("/account/projects");
      } else {
        router.push("/pricing");
      }
      dispatch(SHOW_MODAL, {
        name: "message",
        title,
        message
      });
    } else {
      await dispatch(FETCH_ALL_PUBLISHED_TEMPLATES);
      await dispatch(DELETE_PROJECT_IN_PROGRESS);
      commit(RESET_PROJECT_IN_PROGRESS);
      commit(RESET_CANVAS);
    }
  },