Async/Await for recursive API Calls

Question

I am trying to collect all the ids from a paginated endpoint. I want to wait for all the calls to finish but it seems to only finish the first call and return the data and a promise to the next call.

How can I wait for all the calls to return.

const getSelectAllIds = async (url) => {
  let response = await axios.post(url);
  if (response.status === 200 && response.data.next != null) {
    console.log(`calling: ${response.config.url}`);
    return [
      ...response.data.results,
      await getSelectAllIds(response.data.next)
    ];
  }
};

const someFunction = async (url, data) => {
  const selectedIds = await getSelectAllIds(url);
  return selectedIds;
};

Answer 1

async generators

I'm going to suggest a different approach, using async generators. It's my hope that the benefits of this approach are self-evident -

async function post (url)
{ const r = await axios.post(url)
  if (r.status == 200)
    return r.data   // <- resolve data
  else
    throw r         // <- reject
}

async function* getSelectAllIds (url)
{ if (url == null) return
  let { config, results, next } = await post(url) // <- get
  console.log(`calling: ${config.url}`)           // <- debug output
  yield *results                                  // <- yield each
  for await (const v of getSelectAllIds(next))    // <- recur
    yield v
}

async function someFunction (url)
{ const r = []                                    // <- empty result
  for await (const v of getSelectAllIds(url))     // <- simple iteration
    r.push(v)                                     // <- collect results
  return r                                        // <- return
}

// run and catch errors
someFunction("/my/url").then(console.log, console.error)

demo

I want to demonstrate this in a way that you can verify the results in your own browser. To do that, we'll make a fake DB -

const DB =
  { "/1":
      { config: { url: "/1" }
      , results: [ "foo", "bar" ]
      , next: "/2"
      }
  , "/2":
      { config: { url: "/2" }
      , results: [ "cat", "dog" ]
      , next: "/3"
      }
  , "/3":
      { config: { url: "/3" }
      , results: [ "pea", "yam" ]
      , next: null
      }
  }

Next we need to make FAKE which is a stand-in for axios.post -

async function FAKE (url)      // <- fake axios.post
{ await sleep (1000)           
  if (url in DB)
    return { status: 200, data: DB[url] } // <- found resource
  else
    return { status: 404, data: null }    // <- notfound resource
}

Which depends on a little sleep function, to simulate delay -

const sleep = ms =>
  new Promise(r => setTimeout(r, ms))

To run the demo, we simply replace axios.post with our FAKE -

async function post (url)
{ const r = await FAKE(url)    // <- fake axios.post
  if (r.status == 200)
    return r.data
  else
    throw r
}

someFunction("/1").then(console.log, console.error)

Expand the snippet below to verify the results in your browser -

const sleep = ms =>            // <- sleep for demo
  new Promise(r => setTimeout(r, ms))

async function FAKE (url)
{ await sleep (1000)           // <- simulated delay
  if (url in DB)
    return { status: 200, data: DB[url] }
  else
    return { status: 404, data: null }
}

async function post (url)
{ const r = await FAKE(url)    // <- fake axios.post
  if (r.status == 200)
    return r.data
  else
    throw r
}

async function* getSelectAllIds (url)
{ if (url == null) return
  let { config, results, next } = await post(url)
  console.log(`calling: ${config.url}`)
  yield *results
  for await (const v of getSelectAllIds(next))
    yield v
}

async function someFunction (url)
{ const r = []
  for await (const v of getSelectAllIds(url))
    r.push(v)
  return r
}

const DB =
  {"/1":{config:{url:"/1"},results:[ "foo","bar" ],next:"/2"},"/2":{config:{url:"/2"},results:[ "cat","dog" ],next:"/3"},"/3":{config:{url:"/3"},results:[ "pea","yam" ],next:null}}

someFunction("/1").then(console.log, console.error)

calling: /1
calling: /2
calling: /3
=> [ "foo", "bar", "cat", "dog", "pea", "yam" ]

without generators

Should you choose not to use async generators, you can write a simple async function. One advantage to this approach is it skips the need for someFunction entirely, however it has the distinct disadvantage that it waits for all results before you can start using them -

async function getSelectAllIds (url)
{ if (url == null) return []                            // <-
  let { config, results, next } = await post(url)
  console.log(`calling: ${config.url}`)
  return [ ...results, ...await getSelectAllIds(next) ] // <-
}

getSelectAllIds("/1").then(console.log, console.error)

calling: /1
calling: /2
calling: /3
=> [ "foo", "bar", "cat", "dog", "pea", "yam" ]

Compare this to the result of using async generators where we can use the results as they arrive asynchronously -

async function someFunction (url)
{ for await (const v of getSelectAllIds(url)) // <- async generator
    console.log(v)
  return "done"
}

someFunction("/1").then(console.log, console.error)

calling: /1
foo
bar
calling: /2
cat
dog
calling: /3
pea
yam
done

Answer 2

All async functions return a promise, always. So both getSelectAllIds() and someFunction() will always return a promise. That caller will have to use .then() or await to get the value from the promise. You can't make them synchronously return the asynchronously retrieved value. Javascript does not work that way.

That's how asynchronous coding works in nodejs. You can't ever turn an asynchronous value into a synchronous value. You have to use the asynchronous value using asynchronous mechanisms. For promises, that means the caller has to use await of .then() to get the value.

getSelectAllIds(someUrl).then(allIDs => {
     console.log(allIDs);
}).catch(err => {
     console.log(err);
});

Note, it's not clear what you expect your code to do if response.status is not 200. And, it also looks like you're not collecting the data from the last page because you don't add the data into the array if there's no data.next.

Answer 3

I think you want this:

const getSelectAllIds = async (url) => {
  const response = await axios.post(url);
  if(!response.ok) throw new Error('HTTP error'); // lets not ignore these.
  return [
    ...response.data.results,
    ...(response.data.next ? await getSelectAllIds(response.data.next) : [])
  ];
}