promise.all does not wait for firestore query to loop

Question

I am converting this code from the Realtime Database to Firestore.

In order to create some jobs to be handled later the code loops through each User (doc) in Firestore and then through each document of 2 nested Subcollections inside each User.

I want the function to wait for each query to finish before finishing. Promise.all() always fires after 3 promises have been added, of which the first one is undefined.

I have tried to use async/await, but thats not the goal anyway. I tried creating a separate promises array just for the most nested logic (writeJobToBackLog()). Both without success.

After hours of playing around I still don't even understand what is happening and my logging skills are probably what keep me from getting a clearer picture.

I'm no pro with promises, but I have done some work with them, mostly with the Realtime Database.

var database = admin.firestore();

// prepare()

test();

function test() {
  console.log("prepare() called ...");

  let promises = [];

    database
      .collection("users")
      .get()
      .then((snapshot) => {
        snapshot.forEach((user) => {
          user = user.data();
          const userId = user.userId;

            database
              .collection("users")
              .doc(userId)
              .collection("projects")
              .get()
              .then((snapshot) => {
                snapshot.forEach((project) => {
                  project = project.data();
                  const projectUrl = project.projectUrl;
                  const projectId = project.projectId;

                    database
                      .collection("users")
                      .doc(userId)
                      .collection("projects")
                      .doc(projectId)
                      .collection("subProjects")
                      .get()
                      .then((snapshot) => {
                        snapshot.forEach((subProject) => {

                          subProject.keywords.map(async (keyword) => {
                            let unreadyJob = {
                              keyword: keyword,
                            };

                            // returns a promise
                            let write = writeJobsToBackLog(unreadyJob);
                            writePromises.push(write);
                            return null;
                          });
                          return;
                        });
                        return;
                      })
                      .catch((error) => {
                        console.log(error);
                      })
                  return;
                });
                return;
              })
              .catch((error) => {
                console.log(error);
              })
        });
        return;
      })
      .catch((error) => {
        console.log(error);
      })
  Promise.all(promises)
    .then(() => {
      console.log("prepare() finished successfully..." +
          promises.map((promise) => {
            console.log(promise);
            return null;
          }));
      return null;
    })
    .catch((error) => {
      console.log("prepare() finished with error: " + error + "...");
      return null;
    });
}

function writeJobsToBackLog(unreadyJob) {
  console.log("writing job to Backlog ...");
  return database
    .collection("backLog")
    .doc()
    .set(unreadyJob);
}

Here is what is printed to the console:

prepare() called ...
prepare() finished successfully...
writing job to Backlog ...
writing job to Backlog ...
writing job to Backlog ...
writing job to Backlog ...
(... more of those ...)

Everything works as expected but the Promise.all logic. I expect it to fill the promises array with one returned promise for each 'write' and then wait for all writes to have succeeded.

No promises are added to the array at all.

Thank for any help!

---

So I changed the code:

async function test() {
  console.log("prepare() called ...");

  const users = await database.collection("users").get();
  users.forEach(async (user) => {
    const userId = user.data().userId;
    const projects = await database
      .collection("users")
      .doc(userId)
      .collection("projects")
      .get();

    projects.forEach(async (project) => {
      const projectUrl = project.data().projectUrl;
      const projectId = project.data().projectId;
      const subProjects = await database
        .collection("users")
        .doc(userId)
        .collection("projects")
        .doc(projectId)
        .collection("subProjects")
        .get();

      subProjects.forEach(async (subProject) => {

        subProject.data().keywords.map(async (keyword) => {
          let unreadyJob = {
            keyword: keyword,
          };
          await writeJobsToBackLog(unreadyJob);
        });
      });
    });
  });
  console.log("finished");
}

function writeJobsToBackLog(unreadyJob) {
  console.log("writing job to Backlog ...");
  return database
    .collection("backLog")
    .doc()
    .set(unreadyJob);
}

It is producing the same result:

prepare() called ...
finished
writing job to Backlog ...
writing job to Backlog ...
writing job to Backlog ...
...

What am I doing wrong. Thank you!

Answer 1

You can try this. I removed nested promise and it now uses promise chaining.

You have to add the error handling code yourself.

let users = await database.collection("users").get();

let userPromises = [];
users.forEach((userDoc) => {
    let userDocData = userDoc.data();
    let userId = userDocData.userId;

    // Create promises for each user to retrieve sub projects and do further operation on them.
    let perUserPromise = database.collection("users").doc(userId).collection("projects").get().then((projects) => {

        // For every project, get the project Id and use it to retrieve the sub project.
        let getSubProjectsPromises = [];
        projects.forEach((projDoc) => {
            const projectId = projDoc.data().projectId;
            getSubProjectsPromises.push(database.collection("users").doc(userId).collection("projects").doc(projectId).collection("subProjects").get());
        });

        // Resolve and pass result to the following then()
        return Promise.all(getSubProjectsPromises);

    }).then((subProjectSnapshots) => {

        let subProjectPromises = [];
        subProjectSnapshots.forEach((subProjSnapshot) => {
            subProjSnapshot.forEach((subProjDoc) => {

                // For every sub project, retrieve "keywords" field and write each keyword to backlog.
                const subProjData = subProjDoc.data();
                subProjectPromises.push(subProjData.keywords.map((keyword) => {
                    let unreadyJob = {
                        keyword: keyword,
                    };
                    return writeJobsToBackLog(unreadyJob);
                }));
            });
        });

        return Promise.all(subProjectPromises);
    });

    userPromises.push(perUserPromise);
});

// Start the operation and wait for results
await Promise.all(userPromises);

}

Answer 2

I would split the logic into several smaller functions (way easier to follow, test and debug - BTW, what is the projectUrl for?), and write something like this:

async function getUsers() {
   const users = await database.collection("users").get();
   const userIds = users.map(user => user.data().userId);
   const projectPromises = userIds.map(getUserProjects);

   const projects = await Promise.all(projectPromises);
   return projects;
}

async function getUserProjects(userId) {
   const projects = await database
      .collection("users")
      .doc(userId)
      .collection("projects")
      .get()
      .map(getUrlAndId);

   const subprojectPromises = projects.map(({
      projectId
   }) => getUserSubprojects(userId, projectId));

   const subprojects = await subprojectPromises;
   return subprojects;
}

function getUserSubprojects(userId, projectId) {
   const subProjects = await database
      .collection("users")
      .doc(userId)
      .collection("projects")
      .doc(projectId)
      .collection("subProjects")
      .get();

   const keywordJobPromises = subprojects.map(keywordJobs);
   return Promise.all(keywordJobPromises);
}

function keywordJobs = (subproject) {
   const keywordPromises = subproject.keywords.map((keyword) => {
      const unreadyJob = {
         keyword
      };
      return writeJobsToBackLog(unreadyJob);
   });

   // Running out of good variable names here...
   const keyword = await Promise.all(keywordPromises); 
   return keyword;
}

function getUrlAndId(project) {
   const data = project.data();
   return {
      projectUrl: data.projectUrl,
      projectId: data.projectId
   };
}

Answer 3

As in ECMAScript8 use await to get result from Promise

const users = await database.collection("users").get();
users.forEach(async (user) => {
    const userId = user.data().userId;
    const projects = await database.collection("users").doc(userId).collection("projects").get();
    ....
});