Node.js async-functions executes on parallel?

Question

The code below is made to:

1. CONVERT some videos and store them on the folder of converted.
2. CONCATENATE the converted videos.
3. CLEAR the converted folder.

BUT currently, when executing the code, it concatenates while converting, and eventually throw an error. I need to execute each function with priority in his respective order.

const glob = require('glob');
const exec = require('child_process').exec;
const { unlink } = require("fs").promises;

function start(index) {


  const path = `./files/${index}`;
  const ConvertedVideos=path+"/converted/*.mp4";
  const videos = glob.sync(path+"/videos/*.mp4");
  const outputnoaudio = path+"/outputnoaudio.mp4";




  //THIS IS THE ORDER OF EXECUTION

  await convertVideos()
  await withFfmpegConcatVideo()
  await clearConverted()



  //HERE THE DEFINITION OF EACH FUNCTION

  async function convertVideos() {
    return new Promise((resolve) => {

      console.log('>>>Starting conversion!!!');
      const promises = videos.map(
        (video, index) => {
          var command = `ffmpeg -i ${video} -c:v libx264 -vf scale=1920:1080 -r 60 -c:a aac -ar 48000 -b:a 160k -strict experimental -f mp4 ./converted/${index}.mp4`;

          return new Promise((resolve) => {
            exec(command, (error, stdout, stderr) => {
              if (error) {
                console.warn(error);
              }
              resolve(stdout ? stdout : stderr);
              console.log('Converted video', index);
            });
          });
        }
      )
      Promise.all(promises);
      resolve();
    })
  }

  async function withFfmpegConcatVideo() {
    return new Promise((resolve) => {

      console.log('Starting concatenation...');

      var converted = glob.sync(ConvertedVideos);

      console.log('There is ', converted.length, 'converted videos');

      if (converted.length > 0) {

        (async () =>
          await concat({
            output: outputnoaudio,
            videos: converted,
            transition: {
              name: 'fade',
              duration: 200
            }
          }))()
      }
      resolve(console.log('Conversion finished'));
    })
  }


  async function clearConverted() {

    return new Promise((resolve) => {

      const converted =
        glob.sync(ConvertedVideos)

      if (converted.length === 0)
        return Promise.resolve([])

      const promises =
        converted.map(v => unlink(v).then(_ => {
          v
        }))

      Promise.all(promises).then('Clean done.')
      resolve();
    })
  }
}

start(1);

I want to keep clean and reusable code. May you help me?

Answer 1

I just fixed the code thanks to your comments and found this solution.

const glob = require('glob');
const concat = require('ffmpeg-concat');
const {
  unlink
} =
require("fs").promises;
const exec = require("child_process").exec;

async function start(index){



  const path = `./files/${index}`;
  const pathToConverted = `${path}/converted`;
  const videos = glob.sync(`${path}/[0-99]/videoKeywords/[0-99]/*output.mp4`);
  const output = `${path}/output.mp4`;
  const outputmuted = `${path}/outputmuted.mp4`;




   async function clearConverted() {

    await new Promise(async (resolve) => {
      console.log('Starting cleaning...');
      const converted = glob.sync(`${pathToConverted}/*.mp4`)

      if (converted.length === 0)
        return Promise.resolve([])

      const promises =
        converted.map(v => unlink(v).then(_ => {
          console.log("   File Deleted");
          v
        }))

      await Promise.all(promises).then(async () => await console.log('Clean done.'))

      resolve()
    })
  }




async function convertVideos() {
    await new Promise(async (resolve) => {

      console.log('>>>Starting conversion of', videos.length, 'files!!!');


      for (let i = 0; i < videos.length; i++) {

        let video = videos[i];

        var command = `ffmpeg -i ${video} -c:v libx264 -vf scale=1920:1080 -r 60 -c:a aac -ar 48000 -b:a 160k -strict experimental -f mp4 ${pathToConverted}/${i}.mp4`;

        await new Promise((resolve) => {
          exec(command, (error, stdout, stderr) => {
            if (error) {
              console.warn(error);
            }
            resolve(stdout ? stdout : stderr);
            console.log('  Converted video', i);
          });
        });
      }
      resolve();
    }).then(async () => await console.log('Conversion finished!!!'))
  }




 async function clearConverted() {

    await new Promise(async (resolve) => {
      console.log('Starting cleaning...');
      const converted = glob.sync(`${pathToConverted}/*.mp4`)

      if (converted.length === 0)
        return Promise.resolve([])

      const promises =
        converted.map(v => unlink(v).then(_ => {
          console.log("   File Deleted");
          v
        }))

      await Promise.all(promises).then(async () => await console.log('Clean done.'))

      resolve()
    })
  }
}
start(0);

Notice that this implementation of function convertVideos() execute conversion sequentially asynchronous which means that it converts one file by step. On the previous implementation of this function, the execution is in parallel, which is optimal for a few videos to convert if you have several cores.

If you have any code suggestions to process a higher volume of videos at the same time, please feel free to support.

Answer 2

Your start function doesn't look correct because you are missing the async operator.

For clarity, I suggest treating this more like a C file and create an async function main() {} where you call and await your three functions.

In your convertVideos function, you are calling Promise.all(), but you aren't awaiting that. Note that Promise.all() actually returns a promise too, and you must await that, but in order for you to await it, the surrounding function must also have the async keyword. I see the same problem in the clearConverted function.

Try awaiting your Promise.all calls, and see if that helps fix your problem.

In fact, your convertVideos function seems to be wrapping everything up in a Promise, and I don't think you need that. Try this:

function convertVideos() {
    console.log('>>>Starting conversion!!!');
    const promises = videos.map(
        (video, index) => {
            var command = `ffmpeg -i ${video} -c:v libx264 -vf scale=1920:1080 -r 60 -c:a aac -ar 48000 -b:a 160k -strict experimental -f mp4 ./converted/${index}.mp4`;

            return new Promise((resolve) => {
                exec(command, (error, stdout, stderr) => {
                    if (error) {
                       console.warn(error);
                    } 
                resolve(stdout ? stdout : stderr);
                console.log('Converted video', index);
            });
        });
      });


    // Notice that you can return this promise from Promise.all.
    // The caller can then await this promise that will let you know when
    // all the videos are converted.
    return Promise.all(promises);
}

Answer 3

withFfmpegConcatVideo().then(() => clearConverted());

Should do the trick