Optimal way to chain Promise.all() with many calls many times

Question

I'm developing a chain of API calls that work by making calls to an api with an array of id's, getting a return of data, and making the next call to a different endpoint with some data retrieved.

Now this is totally the naïve solution - i'm hoping to get some input into how to best optimise this. I may have some 500 data points to go through in each of these.

What returns from apiHelper.apiCall(data, endpoint) is an array of promises that are resolved and managed in the Promise.all(); each time - meaning

Currently it looks something like 6 nested Promise.all() inside each others .then():

Promise.all(function(){})
.then(function(){
    Promise.all(function(){})
    .then(function(){
        Promise.all(function(){})
        .then(function(){
            Promise.all(function(){})
            .then(function(){
            ...and so on...
            });
        });
    });
});

ie in semi real code.

 var datapoints = req.body.toString().split("\n");
    var startingCount = datapoints.length;

    var preFailed = [];

    //initial naïve solution with nested Promise.all();
    var promises = apiHelper.apiCall(datapoints, endpoint + '?referenceId=');
    console.log('#INITIAL COUNT', promises.length);
    Promise
        .all(promises)
        .then(function (orderItems) {
            var itemHrefs = [];
            orderItems.forEach(function (oi) {
                if (oi.data.items && oi.data.items.length > 0) {
                    itemHrefs.push(oi.data.items[0].href); //get item href (has full order ref)
                }
            })

            //Find order items
            promises = apiHelper.apiCall(itemHrefs, '');
            Promise
                .all(promises)
                .then(function (retrievedOrders) {
                    var consumerOrderHrefs = [];
                    retrievedOrders.forEach(function (ro) {
                        if (ro.data.consumers) {
                            consumerOrderHrefs.push(ro.data.consumers.href); //get item href (has full order ref)
                        }
                    })


                    promises = apiHelper.apiCall(consumerOrderHrefs, '');
                    console.log('#STEP 3 FIND CONSUMERORDER COUNT', promises.length);
                    Promise
                        .all(promises)
                        .then(function (retrievedConsumers) {
                            var consumerHrefs = [];
                            retrievedConsumers.forEach(function (c) {
                                if (c.data.items[0]) {
                                    consumerHrefs.push(c.data.items[0].href); //get item href (has full order ref)
                                }
                            })

                            promises = apiHelper.apiCall(consumerHrefs, '');
                            console.log('#STEP 4 FIND CONSUMER COUNT', promises.length);
                            Promise
                                .all(promises)
                                .then(function (consumer) {
                                    var guestHrefs = [];
                                    consumer.forEach(function (c) {
                                        if (c.data.guest.href) {
                                            guestHrefs.push(c.data.guest.href); //get item href (has full order ref)
                                        }
                                    })

                                    promises = apiHelper.apiCall(guestHrefs, '');
                                    console.log('#STEP 5 GUEST COUNT', promises.length);
                                    Promise
                                        .all(promises)
                                        .then(function (guests) {
                                            var guest = [];
                                            guests.forEach(function (g) {
                                                if (g.data) {
                                                    guest.push(g.data); //get item href (has full order ref)
                                                }
                                            })
                                            res.status(200).json({guest});
                                        })
                                        .catch(function (err) {
                                            console.log('#ERR', err);
                                            res.status(500).send(err);
                                        });


                                })
                                .catch(function (err) {
                                    console.log('#ERR', err);
                                    res.status(500).send(err);
                                });
                        })
                        .catch(function (err) {
                            console.log('#ERR', err);
                            res.status(500).send(err);
                        });
                })
                .catch(function (err) {
                    console.log('#ERR', err);
                    res.status(500).send(err);
                });
        })
        .catch(function (err) {
            console.log('#ERR', err);
            res.status(500).send(err);
        });

Answer 1

The principle you should apply here is the same as you would for any other promises - return a promise within a then and handle its result in the next then:

Promise.all(apiCall()) 
    .then(function (results) {
         var nextItems = // do something with results 

         return Promise.all(apiHelper.apiCall(nextItems, ''));
    })
    .then(function (results) {
         var nextItems = // do something with results 

         return Promise.all(apiHelper.apiCall(nextItems, ''));
    })
    .then(function (results) {
        // ...
    });

Answer 2

Here's another way:

var promise = Promise.all(...);
promise = promise.then(() => Promise.all(...))
promise = promise.then(() => Promise.all(...))
promise = promise.then(() => Promise.all(...))
promise = promise.then(() => Promise.all(...))
promise.then(() => {
  console.log('all done!')
})

Answer 3

Look into async/await:

const allPromises = async () => {
    await Promise.all(...);
    await Promise.all(...);
    //Etc.
}