Unknown number of promises dependent on result of previous promise?

Question

I'm trying to write a function that retrieves all of a user's data from an API. Unfortunately, the API only returns 50 pieces of data per request. In order to retrieve the next 'page' of results, another GET request needs to be made, with an extra path indicating the page of results.

(In my case, the API is Imgur, and the pieces of data are a user's albums.)

I'm trying to do this with Promises. The function populateAlbumList successfully returns only the first page of results.

I've attempted to modify it to get more pages of results in the function populateAlbumList2, but it's not working correctly.

How do I get these conditionally nested promises working? (I'd prefer not to use a library like bluebird/q, because I want to understand the concept and pattern itself.)

/**
 * Performs an AJAX get request to the Imgur API, retrieving all the albums owned by the user. When the albums are
 * populated, they are logged to the extension settings page's console.
 * @returns {Promise<void>}
 */
async function populateAlbumList() {
    const username = await getItemFromStorage(STORAGE_USERNAME, ERROR_STORAGE_USERNAME_NOT_FOUND);
    const ALBUMS_URL = `https://api.imgur.com/3/account/${username}/albums`;

    // Fetch the albums for the currently logged in user
    return fetch(ALBUMS_URL, {
        method: "GET",
        headers: {
            "Authorization": "Bearer " + CLIENT_ID,
            "Content-type": "application/json; charset=UTF-8"
        }
    })
        .then(response => response.json())
        .then(json => json.data)
        .then(albums => albums.forEach(album => addAlbumToPage(album)));
}

/**
 * Performs an AJAX get request to the Imgur API, retrieving all the albums owned by the user. When the albums are
 * populated, they are logged to the extension settings page's console.
 * @returns {Promise<Array>}
 */
async function populateAlbumList2() {
    const username = await getItemFromStorage(STORAGE_USERNAME, ERROR_STORAGE_USERNAME_NOT_FOUND);
    let ALBUMS_URL = `https://api.imgur.com/3/account/${username}/albums`;
    const allAlbums = [];
    let page = 0;
    const promises = [];

    await getAlbumsFromImgur()
        .then(() => console.log(allAlbums));

    function getAlbumsFromImgur() {
        if (page > 0) {
            ALBUMS_URL = `https://api.imgur.com/3/account/${username}/albums` + page;
        }

        promises.push(
            fetch(ALBUMS_URL, {
            method: "GET",
            headers: {
                "Authorization": "Bearer " + CLIENT_ID,
                "Content-type": "application/json; charset=UTF-8"
            }
        })
            .then(response => response.json())
            .then(json => json.data)
            .then(albums => {
                allAlbums.push(albums);

                if (albums.length >= 50) {
                    page++;
                    promises.push(getAlbumsFromImgur());
                }
            })
        );
    }
}

Answer 1

Since you're using an async function, you don't need to deal with the promises directly, just use await and write your logical flow. First, let's apply that to just getting the first page so we can see how it simplifies the function; see the *** comments:

async function populateAlbumList() {
    const username = await getItemFromStorage(STORAGE_USERNAME, ERROR_STORAGE_USERNAME_NOT_FOUND);
    const ALBUMS_URL = `https://api.imgur.com/3/account/${username}/albums`;

    // Fetch the albums for the currently logged in user
    // *** Use await to consume the promise
    const response = await fetch(ALBUMS_URL, {
        method: "GET",
        headers: {
            "Authorization": "Bearer " + CLIENT_ID,
            "Content-type": "application/json; charset=UTF-8"
        }
    });
    // Note: You have to check for errors
    if (!response.ok) {
        throw new Error("HTTP error " + response.status);
    }
    // Read and parse the JSON, get the `data` property from it using destructuring
    // *** Use await to consume the promise
    let { data: albums } = await response.json();
    // Add the albums; no need for `forEach` when we have `for-of` available to us
    for (const album of albums) {
        addAlbumToPage(album);
    }
}

Now let's extend that function so it keeps doing requests for subsequent pages until it gets back less than 50 results:

async function populateAlbumList() {
    const username = await getItemFromStorage(STORAGE_USERNAME, ERROR_STORAGE_USERNAME_NOT_FOUND);
    const ALBUMS_URL = `https://api.imgur.com/3/account/${username}/albums`;

    // Start on page 0
    let page = 0;
    let albums; // We'll need this in our while condition below
    do {
        // Fetch the albums for the currently logged in user,
        // adding in the page if page > 0
        const response = await fetch(
            page === 0 ? ALBUMS_URL : ALBUMS_URL + page, // Seems odd there's no `/` or `?page=` or similar
            {
                method: "GET",
                headers: {
                    "Authorization": "Bearer " + CLIENT_ID,
                    "Content-type": "application/json; charset=UTF-8"
                }
            }
        );
        if (!response.ok) {
            throw new Error("HTTP error " + response.status);
        }
        // Read and parse the JSON, get the `data` from it
        albums = (await response.json()).data;
        // Add the albums
        for (const album of albums) {
            addAlbumToPage(album);
        }
        ++page;

        // Keep going until we get fewer than 50 back
    } while (albums.length >= 50);
}

Note that I also added a check to see if the fetch worked, which was missing from your original code. It's not just you, most people forget to include that check (so much so that I wrote it up on my anemic little blog).