Adding items to a global Array from inside a callback function and then accessing it globally

Question

Here's the problem.

I have a global variable(Array type) named folders

let folders = [];

I modify it inside a Callback function inside yet another callback function. Here's how.

app.get("/", (req, res) => {
// TODO: Proceed only if the path is correct and it is a directory
fs.readdir(dir, (err, files) => {
    console.log("READING:");

    if (err) throw err;

    files.forEach(file => {
        const add = folder => folders.push(folder);
        fs.lstat(path.join(dir, file), (err, stats) => {
            if (err) throw err;

            if (stats.isDirectory()) {
                add(file);
            }
            console.log("INSIDE: " + folders);
        });
        console.log("OUTSITE: " + folders);
    });
});

res.send(folders.length.toString());
});

Now the problem is, that when I read it on this line:

res.send(folders.length.toString());

The length is always 0. And it is also 0 on console log line where I Print it with OUTSITE but it reads fine when I print it on the line where I mention it with INSIDE.

I know the problem after some search. It happens because the callback sets the variable on a later time in the event loop.(If it makes any sense, but you get the point).

I know the problem but I don't have any idea of how to solve it. I have tried various implementations including adding a global function that pushes to the array and calling it frpm inside the callback but the results are same.

Here's the full code.

const express = require("express");
const fs = require("fs");
const path = require("path");
const os = require("os");

// Initialize Express
const app = express();

// PORT on which the app process should be started
const PORT = process.env.PORT || 5100;

// Setting Up the path to Projects folder dynamically
// !Currently only works(tested) on the Darwin(MacOS) systems PS. I don't own a Windows
// TODO: Test on Windowsn and Linux
const homedir = os.homedir();
const dir = `${homedir}/Projects/`;

// TODO: Re-Write using Async/Await as it is not fully suppported as of Node version 10.0

let folders = [];

// Home Route
app.get("/", (req, res) => {
    // TODO: Proceed only if the path is correct and it is a directory
    fs.readdir(dir, (err, files) => {
        console.log("READING:");

        if (err) throw err;

        files.forEach(file => {
            const add = folder => folders.push(folder);
            fs.lstat(path.join(dir, file), (err, stats) => {
                if (err) throw err;

                if (stats.isDirectory()) {
                    add(file);
                }
                console.log("INSIDE: " + folders);
            });
            console.log("OUTSITE: " + folders);
        });
    });

    res.send(folders.length.toString());
});

// Start the express server
app.listen(PORT, err => {
    if (err) throw err;
    console.log(`Project Lister Running On PORT: ${PORT}`);
});

Any solutions?

Answer 1

So, here's the simplest solution I can find on my own!

@PeterN's answer is correct but could be hard to wrap a beginner's head around!

Here's my final code.

const express = require("express");
const fs = require("fs").promises; // !IMPORTANT Get the promises version of the File System Module
const path = require("path");
const os = require("os");

// Initialize Express
const app = express();

// PORT on which the app process should be started
const PORT = process.env.PORT || 5100;

// Setting Up the path to Projects folder dynamically
// !Currently only works(tested) on the Darwin(MacOS) systems PS. I don't own a Windows
// TODO: Test on Windows and Linux
const homedir = os.homedir();
const dir = `${homedir}/Projects/`;

// Home Route
app.get("/", async (req, res) => {
    let folders = [];

    // TODO: Proceed only if the path is correct and is a directory
    try {
        let files = await fs.readdir(dir);

        for (let i = 0; i < files.length; i++) {
            let file = await fs.lstat(path.join(dir, files[i]));

            if (file.isDirectory()) {
                folders.push(files[i]);
            }
        }
    } catch (error) {
        console.error(error);
    }

    res.send(folders);
});

// Start the express server
app.listen(PORT, err => {
    if (err) throw err;
    console.log(`Project Lister Running On PORT: ${PORT}`);
});

Take note, on the second line where I am importing the 'fs' modules, I now import it differently or rather say a different version!

I now import it as:

const fs = require("fs").promises;

The '.promises' added at the last imports the functions, methods of that module in their Promise based implementation. Thought you must note that it is stable only in version 11.x and up of NodeJs as of right now. I am using >12.x.

Now the rest of the process is rather straight forward assuming you are familiar with Async/Await and Promises. And if you're not I would highly suggest getting into it as it can save your day as it did with me.

Here's a great tutorial regarding it: Async/Await and Promise in JS

Ps. Use the for loop instead of 'array.forEach(e => //Do Something);' approach as it will again introduce the same problem as faced earlier because it is also callback-based!

Hope I helped you. Thanks!

Answer 2

The issue here is that fs.lstat is asynchronous.

If you use the sync version fs.lstatSync, then you can call res.send after the forEach loop.

app.get("/", (req, res) => {
  // TODO: Proceed only if the path is correct and it is a directory
  fs.readdir(dir, (err, files) => {
      console.log("READING:");

      if (err) throw err;

      files.forEach(file => {
          const add = folder => folders.push(folder);
          try {
            const stats = fs.lstatSync(path.join(dir, file))

            if (stats.isDirectory()) {
              add(file);
          }
          } catch (err) {
            throw err
          }
      });

      res.send(folders.length.toString());
  })
})

Or for a non-blocking way you could use Promise.all:

app.get("/", (req, res) => {
  // TODO: Proceed only if the path is correct and it is a directory
  fs.readdir(dir, (err, files) => {
    console.log("READING:");

    if (err) throw err;

    const promises = files.map(file => {
      return new Promise((resolve, reject) => {
        fs.lstat(path.join(dir, file), (err, stats) => {
          if (err) {
            reject(err);
          }
          if (stats.isDirectory()) {
            add(file);
            resolve();
          }
          console.log("INSIDE: " + folders);
        });
      });
    });

    Promise.all(promises, () => {
      res.send(folders.length.toString());
    });
  });
});