async Iterate over API results

Question

I am calling an API which returns results in pages and I am trying to find an 'elegant' way of retrieving them.

Ideally I want to consume them like this:

let results = api.get();

for await (const page of results) {
    // do stuff with page
}

I can active close to this using a generator function like this:

class Results {
    constructor(url, token) {
        this.url = url;
        this.token = token;
    }

    async *page() {
        let url = this.url;

        while (true) {
            const response = await fetch(url, {
                headers: { 'Authorization': 'Bearer ' + this.token }
            });

            const data = await response.json();

            yield data.values;

            if (!data.next) return;
            url = data.next;
       }
    }
}

And calling it like:

for await (const page of results.page()) {
    // do stuff with page
}

I have tried to do it with a [Symbol.iterator] like this, but cannot get it to work:

[Symbol.iterator]() {
    let that = this;

    return {
        next: async function() {

            if (!that.page) {
                that.page = that.url;
                return {done: true};
            }

            const response = await fetch(that.page, {
                headers: { 'Authorization': 'Bearer ' + that.token }
            });

            const data = await response.json();

            that.page = data.data.next;

            return {
                value: data,
                done: false
            }
        }
    }
} 

This issue is I need to get the link to the next page from the current page to determine if there is a next page, but as its a promise i cannot access it in the function.

Any ideas how to get the iterator working?

Answer 1

Following advice here is a working function. [Symbol.asyncIterator] made all the difference. Thanks:

[Symbol.asyncIterator]() {
    let that = this;

    return {
      page: that.url,
      token: that.token,
      next: async function() {

        if (!this.page) {
            this.page = that.url;
            return {done: true};
        }

        const response = await fetch(this.page, {
            headers: { 'Authorization': 'Bearer ' + this.token }
        });

        const data = await response.json();

        this.page = data.next;

        return {
          value: data,
          done: false
        }
    }
}

Now that its working ideally I just want to be able to iterate through all results and not know about pages so here is a working solution to this for info:

[Symbol.asyncIterator]() {
    let that = this;

    return {
      page: that.url,
      token: that.token,
      values: [],
      next: async function() {

        if (!this.page && this.values.length === 0) {
            this.page = that.url;
            return {done: true};
        }

        if (this.values.length > 0) {
            return {
                value: this.values.pop(),
                done: false
            }
        }

        const response = await fetch(this.page, {
            headers: { 'Authorization': 'Bearer ' + this.token }
        });

        const data = await response.json();
        this.page = data.next;
        this.values = data.values;

        if (this.values.length === 0) {
            return { done: true }
        }

        return {
          value: this.values.pop(),
          done: false
        }
    }
}

This code can be simplified by using an async generator function like so:

async *[Symbol.asyncIterator]() {
    let url = this.url;

    const getPage = url =>
       fetch(url, this.header)
           .then(response => response.json())
           .then(data => ({
               next: data.next,
               values: data.values
           }));

    while(url) {
        const page = await getPage(url);

        for (const value of page.values) {
            yield value;
        }

        url = page.next;
    }
}

So the full class looks like this:

class Response {
    constructor(url, token) {
        this.url = url;
        this.header = {
            headers: {
                Authorization: `Bearer ${token}`
            }
        };
    }

    async* [Symbol.asyncIterator]() {
        let url = this.url;

        const getPage = url =>
            fetch(url, this.header)
                .then(response => response.json())
                .then(data => ({
                    next: data.next,
                    values: data.values
                }));

        while (url) {
            const page = await getPage(url);

            for (const value of page.values) {
                yield value;
            }

            url = page.next;
        }
    }
}

Which allows you to easily loop through paged API results like this:

for await (const item of response) {
    ...
}