Adding delay or something between hundred of promises

Question

I'm using node for requesting app details from another website but the problem i'm facing is, it sends hundreds ( or maybe thousands ) of request, and then I get error and I receive nothing.

Read comments in code for info...

// I'm reading links from another file using 'fs'
fs.readFile('./google_apps/GAME_ACTION.json', 'utf8', function (err, data) {
    if (err) throw err;

    obj = JSON.parse(data);
    // creating a empty array
    var promiseStack = [];

    for (var index in obj['GAME_ACTION']) {
        var linksArray = obj['GAME_ACTION'][index];
        linksArray.forEach( function(link, index) {
            var appID = link.match(/id=.*/g)
            var instr = appID.toString();

            var appIDSliced = instr.slice(3)
            // Here appIDSliced is ID, which is sliced from a link
            // there are thousands on link in this file which I opened

            console.log('sending') // for testing purpose

            // here i'm pushing promises into that empty array
            // store.app({id: appIDSliced}) is a promise I guess because I can use .then() on it...
            // and store.app sends request to another website from which it receives an object in return. 
            // (Variable store is from another file 'require()' above in my node app fie )
            promiseStack.push( store.app({id: appIDSliced}))

        });
    }


    // After pushing all promises into array, now i'm trying to resolve them using Promise.all
    Promise.all(promiseStack).then((responses) => {
        console.log("Dealing with responses")

        // Dealing with response (which is an object coming)
        responses.map(response => {
            var title = response.title
            var package_name = response.appId
            var appCategory = response.primaryGenre
            var appSize = parseFloat((response.size/1024)/1024).toFixed(2)
            var developerName = response.developer
            var developerWebsite = response.developerWebsite
            if (typeof developerWebsite == 'undefined') {
                developerWebsite = "N/A"
            }
            var appPrice = response.price
            var lastUpdated = response.updated
            var contentRating = response.contentRating
            if (typeof contentRating == 'undefined') {
                contentRating = "N/A"
            }
            var userRating = response.score
            if (typeof userRating == 'undefined') {
                userRating = "N/A"
            }
            var dataRow = [appID, title, package_name, appCategory, appSize, developerName, developerWebsite, appPrice, lastUpdated, contentRating, userRating]
            var dataToAdd = [dataRow];
            console.log("Appending now")

            // here i'm using google API to append that into my sheet on google
            authentication.authenticate().then((auth)=>{
                appendData(auth, dataToAdd);
            });
        })
    })
})

See the image below ... Those are the Errors I receive on my console It keep logging 'sending' like for 80 secs and then I get error "UnhandledPromiseRejectionWarning" where I just get stuck for 2 mins and I keep pressing 'CTRL+C'.

Thanks.

Answer 1

In for loop when you are pushing store.app({id: appIDSliced}) & I guess it starts requesting the server thus in a burst mode. Promise.all is resolving requests which is getting stuck.

Why not run n numbers at a time and make them follow till all server requests get completed. This way there will be delay between most of the requests as at any point of time there will be max 5 request will be running instead of burst execution.

var functions = [];
var m;
for (var i = 0; i < 100; i++) {
  (function(j) {
    functions.push(function(callback) {
      var random = Math.floor(Math.random() * 10000) + 1;
      var s = document.getElementById('here');
      var d = document.createElement('div');
      d.innerHTML = 'function : ' + j + ' executed after :' + random;
      s.appendChild(d);
      setTimeout(callback, random);
      // the above code executes something like xhr. remove it
      // here you can write your own statements
      // store.app({id: appIDSliced}, (err, response) => {
      //     responses.map(response => {
      //         var title = response.title
      //         {...}
      //         here i'm using google API to append that into my sheet on google
      //         authentication.authenticate().then((auth)=>{
      //            appendData(auth, dataToAdd);
      //         });
      //     });
      //     callback(); // you can add callback here after all your statements execution
      // });
    });
  })(i);
}

function exec(method) {
  // here method will call the below function which is in your case promise.
  // function(callback) {
  //     console.log(j);
  //     callback();
  // }
  method(function() {
    if (functions.length > m) {
      exec(functions[m]);
      m++;
    }
  });
}

// lets start only 5 at a time.
// first to complete next to start
// function number:    1      2     3     4     5
// completes :        2nd    3rd   1st   5th   4th
// start new :         7      8     6     10    9
setTimeout(function() {
  for (m = 0; m < 5; m++) {
    (function(m) {
      exec(functions[m]); // this will execute 1,2,3,4,5 function
      m++;
    })(m);
  }
}, 0);

<div id="here"></div>

Answer 2

async.each may be more appropriate in your case, check the async.each

Promise.all() doesnt seem right in this case, because method returns a single Promise that resolves when all of the promises in the iterable argument have resolved or when the iterable argument contains no promises. It rejects with the reason of the first promise that rejects. promise.all

I have tried to rewrite your code with async, there few assumptions I have made, this is the general example you probably may tune it for you logic.

var async = require('async');
fs.readFile('./google_apps/GAME_ACTION.json', 'utf8', function (err, data) {
    if (err) throw err;

    var obj = JSON.parse(data);
    async.each(obj['GAME_ACTION'], function(linksArray, callback){
        linksArray.forEach( function(link,) {
            var appID = link.match(/id=.*/g);
            var instr = appID.toString();

            var appIDSliced = instr.slice(3);

            console.log('sending') // for testing purpose
            // I am assuming that store.app() is asyncronous and get a callback
            store.app({id: appIDSliced}, function(err, response){
                if (err){
                    console.log('something bad happend');
                    callback();
                }
                else{
                        var title = response.title;
                        var package_name = response.appId;
                        var appCategory = response.primaryGenre;
                        var appSize = parseFloat((response.size/1024)/1024).toFixed(2);
                        var developerName = response.developer;
                        var developerWebsite = response.developerWebsite;
                        if (typeof developerWebsite == 'undefined') {
                            developerWebsite = "N/A"
                        }
                        var appPrice = response.price;
                        var lastUpdated = response.updated;
                        var contentRating = response.contentRating;
                        if (typeof contentRating == 'undefined') {
                            contentRating = "N/A"
                        }
                        var userRating = response.score;
                        if (typeof userRating == 'undefined') {
                            userRating = "N/A"
                        }
                        var dataRow = [appID, title, package_name, appCategory, appSize, developerName, developerWebsite, appPrice, lastUpdated, contentRating, userRating]
                        var dataToAdd = [dataRow];
                        console.log("Appending now");

                        authentication.authenticate().then((auth)=>{
                            appendData(auth, dataToAdd);
                        });
                    callback();
                }
            });
        });
    }, function (err) {

    });

});

Answer 3

var Promise = require('bluebird');

function fetchData(objekt){
  return new Promise((resolve, reject) => {
      // api call and then 
      resolve(data);
  })
}

function doRequest(){

 Promise.coroutine(function *(){
        yield Promise.map(arr, fetchData);
 })()
 .catch(err => {
    console.error(err);
  })
}

A simple and shorter code with error handling.

Answer 4

You could use a pseudo recursive approach to slowly iterate one request after another:

var finished=new Promise(function(resolve){
   (function iterate( index=0) {
        var link=linksArray[index];
        var appID = link.match(/id=.*/g)
        var instr = appID.toString();
        var appIDSliced = instr.slice(3)
        var promise=store.app({id: appIDSliced});            
        promise.then(function(value){
             promiseStack.push(value);
             if(index<linksArray.length) return iterate(index+1);
             resolve(promiseStack);
        });
   })();
});

finished.then(function(responses){
 //your promise all code
});