sequential and parallel processing of promises

Question

The idea is to sequentially iterate the array but parallel process each item in the subarray.

1. Once record #1 is processed in parallel then it moves to the record #2 and parallel process it's items and so on. So basically it's a combination of sequentiality and parallelism.

2. Concat all results in a single dimension array and display. (pending)

If input contains an array of arrays.

var items = [
    ["item1", "item2"],
    ["item3", "item4"],
    ["item5", "item6"],
    ["item7", "item8"],
    ["item9", "item10"]
]

And an action that processes these items.

function action(item) {
    return new Promise(function(resolve, reject){
        setTimeout(function(){
            resolve(item + ":processed");
        }, 100)
    });
}

Attempt

describe("", function(){
    this.timeout(0);

    it("should", function(done){
        items.reduce(function(accumulator, currentValue, currentIndex, array){
            return accumulator.then(function(result){
                return new Promise(function(resolve, reject){
                    Promise.all(currentValue.map(action))
                        .then(resolve, reject);
                });
            });
        }, Promise.resolve())

    });

});

Expectations:

Ideally a clean minimalistic and a functional approach (no state) to return the results to the caller.

---

Attempt 2

var chain = items.reduce(function(accumulator, currentValue, currentIndex, array){
    return accumulator.then(function(result){
        return new Promise(function(resolve, reject){
            Promise.all(currentValue.map(action))
                .then(resolve, reject);
        });
    });
}, Promise.resolve());

chain.then(console.log, console.error); // I need all results here

displays last result only. [ 'item9:processed', 'item10:processed' ]

---

Edit final solution based on the answer.

var chain = items.reduce(function(accumulator, currentValue, currentIndex, array){
    return accumulator.then(function(result){
        return new Promise(function(resolve, reject){

            Promise.all(currentValue.map(action))
                .then(function(data){
                    resolve(result.concat(data)) // new array
                }, reject);
        });

    });
}, Promise.resolve([]));

chain.then(console.log, console.error);

Answer 1

A simple functional way of doing this would be like

• map sub array with parallel data by promise returning async functions
• Promise.all() the resulting promises array
• sequence the sub arrays at the .then() stage of Promise.all() in a recursive fashion

ES6; tools like spread syntax / rest parameters and array destructuring are very handy for this job.

var items = [["item1", "item2"],
             ["item3", "item4"],
             ["item5", "item6"],
             ["item7", "item8"],
             ["item9", "item10"]],
    act   = i => new Promise(v => setTimeout(v, 1000, `${i}: processed`)),
    seqps = ([is,...iss]) => is && Promise.all(is.map(i => act(i)))
                                          .then(([p,q]) => (console.log(`${p} and ${q}`),
                                                            seqps(iss)));
seqps(items);

Answer 2

One way to do this:

var items = [
    ["item1", "item2"],
    ["item3", "item4"],
    ["item5", "item6"],
    ["item7", "item8"],
    ["item9", "item10"]
]

function action(item) {
    return new Promise(function(resolve, reject){
        setTimeout(function(){
            resolve(item + ":processed");
        }, 100)
    });
}

function process(items) {
  return items.reduce((m, d) => {
    const promises = d.map(i => action(i));
    let oldData;
    return m.then((data) => {
        oldData = data;
        return Promise.all(promises);
      })
      .then(values => {
        //oldData.push(...values);
        oldData.push.apply(oldData, values);
        return Promise.resolve(oldData);
      })
  }, Promise.resolve([]))
}

process(items).then(d => console.log(d))

//Prints:

// ["item1:processed","item2:processed","item3:processed","item4:processed","item5:processed","item6:processed","item7:processed","item8:processed","item9:processed","item10:processed"]