Run Promise all in an await for loop

Question

I have a nested array of Promise function. Eg:

let callSet = [
 [ func1 , func2 , func3],
 [ func4 , func5 , func6],
 [ func7 , func8 , func9],
]

the response of await func1() will be in below structure:

{
 data : [ {id: 1} , {id:2}],
 status: 400
}

I want to run it in a for loop so that they run in batch sequentially and incrementally load the data into array as they come in. Tried below code but I am lost on how to do it:

  const finalData = [];
  private asyncForEach(PromiseGroups): Promise<any> {
    return PromiseGroups.reduce(async (acc, cItem) => {
      const results = await acc;
      const res = await Promise.all(cItem) as any;
      finalData  = [...finalData  , ...[].concat(...res.map(r => r.data))];
      return results
    }, Promise.resolve([]))
  }

I would like to load it in as:

[ {id: 1}, {id:2} , {id: 3} ..... ]   

and this should get updated as the Promise all is getting resolved

I want to wait till func1 , func2 , func3 is resolved and then move to func4 , func5 , func6 . and once I get data of func4 , func5 , func6 , I want to push it with the data of func1 , func2 , func3

Answer 1

Try the below code:

  private init(){
   let callSet = [
     [ func1 , func2 , func3],
     [ func4 , func5 , func6],
     [ func7 , func8 , func9],
   ];
   this.asyncForEach(callSet,this.fetchInBatch.bind(this) )
  }

  private asyncForEach(funcGrpList, execFunc) {
    return funcGrpList.reduce((p,funcList) => {
        return p.then(() => execFunc(funcList));
    }, Promise.resolve());
  }

  private fetchInBatch(pageGroupList) {
    return Promise.all(pageGroupList).then((res: any) => {
      this.finalData = [...this.finalData  , ...[].concat(...res.map(r => r.data))];
    }) 
  }

This should work as expected

Answer 2

If you want to load the data in chunks but produce flat array of results, your easiest option is to use async/await syntax:

interface Data {
  id: number
}
interface DataResponse {
  data: Data[];
  status: number;
}
type AsyncCall = () => Promise<DataResponse>;

/* ... */

const result: Data[] = [];
for(const chunk of callSet) {
  const chunkResult = await Promise.all(chunk.map(f => f()));
  result.push(...chunkResult.flatMap(x => x.data));
}

Playground Link

JavaScript demo:

/* mock section */
const fakeFunc = (id1, id2) => ()=>
  Promise.resolve({
    data : [{id: id1} , {id: id2}],
    status: 400
  });

const func1 = fakeFunc(1, 2),
      func2 = fakeFunc(3, 4),
      func3 = fakeFunc(5, 6),
      func4 = fakeFunc(7, 8),
      func5 = fakeFunc(9, 10),
      func6 = fakeFunc(11, 12),
      func7 = fakeFunc(13, 14),
      func8 = fakeFunc(15, 16),
      func9 = fakeFunc(17, 18)
      ;
      
/* /mock section */

async function main() {
  let callSet = [
    [ func1 , func2 , func3],
    [ func4 , func5 , func6],
    [ func7 , func8 , func9],
  ];

  const result = [];
  for(const chunk of callSet) {
    const chunkResult = await Promise.all(chunk.map(f => f()));
    result.push(...chunkResult.flatMap(x => x.data));
  }
  
  return result;
}

main()
  .then(r => console.log(r));

I you prefer to use the Promise API only, instead of async/await, then you can reduce into a promise like this:

interface Data {
  id: number
}
interface DataResponse {
  data: Data[];
  status: number;
}
type AsyncCall = () => Promise<DataResponse>;

/* ... */

const result = callSet.reduce((p: Promise<Data[]>, chunk: AsyncCall[]) =>
   p.then(acc =>  
      Promise.all(chunk.map(f => f()))
        .then(chunkResult => acc.concat(chunkResult.flatMap(x => x.data))))
  , Promise.resolve([]));

Playground Link

JavaScript demo:

/* mock section */
const fakeFunc = (id1, id2) => () =>
  Promise.resolve({
    data : [{id: id1} , {id: id2}],
    status: 400
  });

const func1 = fakeFunc(1, 2),
      func2 = fakeFunc(3, 4),
      func3 = fakeFunc(5, 6),
      func4 = fakeFunc(7, 8),
      func5 = fakeFunc(9, 10),
      func6 = fakeFunc(11, 12),
      func7 = fakeFunc(13, 14),
      func8 = fakeFunc(15, 16),
      func9 = fakeFunc(17, 18)
      ;
      
/* /mock section */

function main() {
  let callSet = [
    [ func1 , func2 , func3],
    [ func4 , func5 , func6],
    [ func7 , func8 , func9],
  ];

  const result = callSet.reduce((p, chunk) =>
    p.then(acc =>  
      Promise.all(chunk.map(f => f()))
        .then(chunkResult => acc.concat(chunkResult.flatMap(x => x.data))))
    , Promise.resolve([]));

  return result;
}

main()
  .then(r => console.log(r));

However, it is a bit ugly. It can be improved by extracting some functions:

interface Data {
  id: number
}
interface DataResponse {
  data: Data[];
  status: number;
}
type AsyncCall = () => Promise<DataResponse>;

/* ... */

const combineWith = (acc: Data[]) => (chunkResult: DataResponse[]) =>
  acc.concat(chunkResult.flatMap(x => x.data));

const process = (chunk: AsyncCall[]) => (acc: Data[]) =>  
      Promise.all(chunk.map(f => f()))
        .then(combineWith(acc));

/* ... */

const result = callSet.reduce((p: Promise<Data[]>, chunk: AsyncCall[]) =>
    p.then(process(chunk))
  , Promise.resolve([]))

Playground Link

JavaScript demo:

/* mock section */
const fakeFunc = (id1, id2) => () =>
  Promise.resolve({
    data : [{id: id1} , {id: id2}],
    status: 400
  });

const func1 = fakeFunc(1, 2),
      func2 = fakeFunc(3, 4),
      func3 = fakeFunc(5, 6),
      func4 = fakeFunc(7, 8),
      func5 = fakeFunc(9, 10),
      func6 = fakeFunc(11, 12),
      func7 = fakeFunc(13, 14),
      func8 = fakeFunc(15, 16),
      func9 = fakeFunc(17, 18)
      ;
      
/* /mock section */

const combineWith = (acc) => (chunkResult) =>
  acc.concat(chunkResult.flatMap(x => x.data));

const process = (chunk) => (acc) =>  
      Promise.all(chunk.map(f => f()))
        .then(combineWith(acc));

function main() {
  let callSet = [
    [ func1 , func2 , func3],
    [ func4 , func5 , func6],
    [ func7 , func8 , func9],
  ];

  const result = callSet.reduce((p, chunk) =>
    p.then(process(chunk))
  , Promise.resolve([]));

  return result;
}

main()
  .then(r => console.log(r));

Answer 3

Edit: Assuming the last array result is return directly because there is no need to wait for next loop to be finished.

async function run(callSet) {
  const output = [];
  let prev = [];
  const len = callSet.length;

  for (let i = 0; i < len; i++) {
    const array = await Promise.all(callSet[i].map(func => func()));
    const data = array.map(item => item.data);

    if (i === 0) {
      // no need to append item to output
      // just append item to previous array for next loop to use.
      prev.push(...data);
    } else if (i < len) {
      // append item to output from previous result.
      output.push(...prev);
      prev = [];
      // append data to previous result for next loop.
      prev.push(...data);
    } else {
      //last loop, just append data from previous result and current result
      output.push(...prev);
      output.push(...data);
    }
  }
  console.log(output);
}

Answer 4

This will call the execution sets in the requested order and timing and add the return data as soon as the group of promises returns

const finalData = [];

async function execute() {
  for (const set of callSet) {
    const resp = await Promise.all(set.map(f => f()));
    finalData.push(...resp.map(r => r.data).flat());
  }
}

Once execute() is called finalData will be updated asynchronously once for each 'row' of functions.

For future readers

resp.map(r => r.data).flat() is due to the specified promises payload. If someone needs just to pack the results together the code would be:

for (const set of callSet) {
  const resp = await Promise.all(set);
  finalData.push(...resp);
}