AWS S3 / Javascript callback issue

Question

So, I'm having a problem with JavaScript asynchronous execution when making an API call to AWS S3.

I have a sequence of nested callbacks that are working fine up until a specific S3 call that my code is not waiting for. Here's my code:

getThumbUrls(contentIndex, function(data) {
  console.log('Returning from getThumbUrls');
  // let's just display thumbUrls[0] for now...
  console.log('The thumbUrls are ' + data[0]);
});

getThumbUrls() looks like this:

function getThumbUrls(contentIndex, callback) {
  console.log('Entering getThumbUrls');

  var thumbUrls = [];

  JSON.parse(contentIndex).forEach(videoKey => {
    // get the thumbnail: bucket-name/thumbnails/
    console.log('videoKey = ' + videoKey);

    getThumbFileName(videoKey, function(thumbFileName) {
      console.log('Returning from getThumbFileName');
      console.log('Returned thumb filename is ' + thumbFileName);

      thumbUrls.push(CLOUDFRONT_URL + videoKey + '/thumbnails/' + thumbFileName);

    });

  });

  callback(thumbUrls);
}

And getThumbFileName() looks like this:

function getThumbFileName(videoKey, callback) {
  console.log('Entering getThumbFileName...');

  const s3 = new AWS.S3({
    apiVersion: '2006-03-01',
    params: {
      Bucket: 'my-bucket-name'
    }
  });

  // Get the name of the file.
  params = {
    Bucket: 'my-bucket-name',
    Delimiter: '/',
    Prefix: videoKey + '/' + THUMBS_FOLDER,
    MaxKeys: 1
  };

  var urlKey;
  //console.log('listObjects params = ' + JSON.stringify(params, null, 4));
  s3.listObjectsV2(params, (err, data) => {
    if (err) {
      console.log(err, err.stack);
      callback(err);
      return;
    }

    var thumbsKey = data.Contents;
    // MaxKeys was 1 bc first thumbnail key is good enough for now. Therefore, only one iteration.
    thumbsKey.forEach(function (keys) {
      console.log('thumbKey = ' + keys.Key);
      urlKey = keys.Key;
    });

  });

  callback(urlKey);
  //callback('20161111-TheWind.jpg');
}

Obviously, what's happening is that execution doesn't wait for the s3.listObjectsV2 call to finish. I've verified that the entire flow works properly when all getThumbFileName() does is callback with the filename.

Would someone kindly show me how to force execution to wait for s3.listObjectsV2 to complete before calling back with undefined?

guijob · Accepted Answer

As discussed, you should avoid callbacks approach when dealing with asynchronous operations over iterations, due their difficulty.

(You can skip this section if you don't want to know motivation behind promises approach).

Just to mention, in a callback approach, you must have to wait for all callbacks to complete in your getThumbUrls(), using a if which will check if all callbacks has been called, then just call callback(thumbUrls); with all responses pushed into your thumbUrls array:

function getThumbUrls(contentIndex, callback) {
  const thumbUrls = [];

  // counter which will increment by one for every callback
  let counter = 0;
  JSON.parse(contentIndex).forEach(videoKey => {
    getThumbFileName(videoKey, function (thumbFileName) {
      thumbUrls.push(CLOUDFRONT_URL + videoKey + '/thumbnails/' + thumbFileName);

      // for each callback response you must add 1 to a counter and then
      counter++;
      // check if all callbacks already has been called
      if (counter === JSON.parse(contentIndex).length) {
        // right here, thumbsUrls are filled with all responses
        callback(thumbUrls);
      }
    });
  });
}

So, you can make use of Promises, and a Promise.all will be enough for you to handle all responses from api. You can study over internet and check your code below, which is using a promise approach. I've added some comments to help you understanding what is happening.

// when using promises, no callbacks is needed
getThumbUrls(contentIndex)
  .then(function (data) {
    console.log('Returning from getThumbUrls');
    // let's just display thumbUrls[0] for now...
    console.log('The thumbUrls are ' + data[0]);

  })

// when using promises, no callbacks is needed
function getThumbUrls(contentIndex) {
  console.log('Entering getThumbUrls');

  // not needed anymore, Promise.all will return all values
  // var thumbUrls = [];

  // Promise.all receives an array of promises and returns to next .then() all results
  // changing forEach to map to return promises to my Promise.all
  return Promise.all(JSON.parse(contentIndex).map(videoKey => {
    console.log('videoKey = ' + videoKey);

    // returning a promise
    return getThumbFileName(videoKey)
      .then(function (thumbFileName) {
        console.log('Returning from getThumbFileName');
        console.log('Returned thumb filename is ' + thumbFileName);

        return CLOUDFRONT_URL + videoKey + '/thumbnails/' + thumbFileName;
      });
  }))
}

// when using promises, no callbacks is needed
function getThumbFileName(videoKey) {
  console.log('Entering getThumbFileName...');

  const s3 = new AWS.S3({
    apiVersion: '2006-03-01',
    params: {
      Bucket: 'my-bucket-name'
    }
  });

  // Get the name of the file.
  params = {
    Bucket: 'my-bucket-name',
    Delimiter: '/',
    Prefix: videoKey + '/' + THUMBS_FOLDER,
    MaxKeys: 1
  };

  // urlKey not need anymore
  // var urlKey;

  // most of AWS functions has a .promise() method which returns a promise instead calling callback funcions
  return s3.listObjectsV2(params).promise()
    .then(function (data) {
      var thumbsKey = data.Contents;
      //if you want to return only first one thumbsKey:
      return thumbsKey[0];
    })
    .catch(function (err) {
      console.log(err, err.stack);
      callback(err);
      return;
    })
}

Hope this helps you out in your study.

AWS S3 / Javascript callback issue

Answers (2)

Related Questions