how to handle reading a directory tree recursively using promises

Question

I am writing attempting to write a function that accomplishes the same as the following written using a callback pattern with a promise pattern:

function readdirRecursive(path,handler,callback)  {
  var errs = [],
      tree = {};
  fs.readdir(path,function(err,dir)  {
    if(err)return callback(err);
    var pending = dir.length;
    if(!pending)return callback(null,tree);
    dir.forEach(function(file)  {
      var newPath = Path.join(path,file);
      fs.stat(newPath,function(err,stats)  {
        if(stats.isDirectory())  {
          readdirRecursive(newPath,handler,function(err,subtree)  {
            tree[file] = subtree
            handler(tree,newPath,file,"directory",function(err)  {
              if(err)errs.push(err);
              if(!--pending)return callback(errs.length>0?errs:null,tree);
            });
          });
        } else  {
          tree[file] = null; 
          handler(tree,newPath,file,"file",function(err)  {
            if(err)errs.push(err);
            if(!--pending)return callback(errs.length>0?errs:null,tree);
          });
        }
      });
    });
  });
};

this is my current attempt:

function readdirRecursive(path)  {
  var tree = {};
  return Q.Promise(function(resolve,reject,notify)  {
    return readdir(path)
    .then(function(dir)  {
      var futures = [];
      var pending = dir.length;
      if(!pending)return resolve(tree);
      dir.forEach(function(file)  {

        var deferred = Q.defer();
        var subPath = Path.join(path,file);
        futures.push(stat(subPath)
        .then(function(stats)  {
          if(stats.isDirectory())  {
            tree[file] = tree;
            var sub = readdirRecursive(subPath)
            sub
            .then(function(subtree)  {
              notify({
                path:subPath,
                name:file,
                type:"directory",
                done:deferred,
                pending:pending
              });
              //return subtree;
            },reject,notify);
          } else  {
            tree[file] = null;
            notify({
              tree:tree,
              path:subPath,
              name:file,
              type:"file",
              done:deferred,
              pending:pending
            });
            //return null;
          }
          //console.log("tree",tree);
          deferred.promise()
          .then(function()  {
            console.log("pending promise");
            if(!--pending)resolve(tree);
          }
          ,function(err)  {
            reject();
          });
        }));
      });
      return Q.all(futures)
      .then(function(futures)  {
        console.log("hi",futures);
      });
    });
  });
};

This code will iterate over the entire tree, but it will not return a tree, and the the notification action occurs, but the deferred promise does not ever resolve.

When the deferred promise is initiated before the notify event, nothing happens at all.

I know that I could solve this problem by handing a done function to the progress event instead of attempting to give a promise of some sort, but I want to make as full of use of promises here as possible, for example, this code does exactly what I want it to do:

function readdirRecursive(path)  {
  var tree = {};
  return Q.Promise(function(resolve,reject,notify)  {
    return readdir(path)
    .then(function(dir)  {
      var futures = [];
      var pending = dir.length;
      if(!pending)return resolve(tree);
      dir.forEach(function(file)  {

        var deferred = Q.defer();
        var subPath = Path.join(path,file);
        console.log("file",file);
        /*deferred.promise()
        .then(function()  {
          console.log("pending promise");
          if(!--pending)resolve(tree);
        }
        ,function(err)  {
          reject();
        });*/
        futures.push(stat(subPath)
        .then(function(stats)  {
          if(stats.isDirectory())  {
            var sub = readdirRecursive(subPath)
            sub
            .then(function(subtree)  {
              tree[file] = subtree
              notify({
                path:subPath,
                name:file,
                type:"directory",
                done:function(err)  {
                  console.log("pending promise");
                  if(err)return reject(err);
                  if(!--pending)resolve(tree);
                },
                pending:pending
              });
              //return subtree;
            },reject,notify);
          } else  {
            tree[file] = null;
            notify({
              tree:tree,
              path:subPath,
              name:file,
              type:"file",
              done:function(err)  {
                console.log("pending promise");
                if(err)return reject();
                if(!--pending)resolve(tree);
              },
              pending:pending
            });
            //return null;
          }
          //console.log("tree",tree);
        }));
      });
      return Q.all(futures)
      .then(function(futures)  {
        console.log("hi",futures);
      });
    });
  });
};

this is the code that will be executing these functions:

readdirRecursive("../").then(function(tree)  {
  console.log("TREE!!!",tree);
},function(err)  {
  console.log("ERROR",err);
},function(progress)  {
  console.log("PRGRESS WAS MADE",progress);
  progress.done();
});

Answer 1

My first thought was to simply to wrap your original function in a promise. This is normally the way I'd do this without re-engineering the underlying code:

function readdirRecursiveWithPromise (path, handler) {
    return new Promise((resolve, reject) => {
        readdirRecursive(path, handler, (err, tree) => {
            if (err) {
                reject(err);
            }
            else {
                resolve(tree);
            }
        });
    })
}

Unfortunately though when I tried to test this code I discovered a couple of underlying issues with your code.

Firstly, I have no idea what your 'handler' is supposed to do. You have not provided an explanation of this or described what it should do. It's kind of essential to the problem because it controls whether the ultimate callback is eventually called, so I can surmise that the 'handler' is in control of this operation and if your 'callback' is not being called it could be due to the logic in your 'handler'.

The next problem is that your 'pending' variable is set to the number of files and directories in total and yet it is only decremented for directories. So your 'pending' variable will never reach 0 and your conditional code that invokes the callback will never be called.

So I'm going to get rid of 'handler' and 'pending' and I'll show you how I'd rewrite this with promises from the ground up.

Here's the complete working code example: https://github.com/ashleydavis/read-directory-with-promises. Read on for an explanation.

Let's start with a promise-based version of readdir that is not recursive:

function readdir (path) { // Promise-based version of readdir.
    return new Promise((resolve, reject) => { // Wrap the underlying operation in a promise.
        fs.readdir(path, (err, files) => {
            if (err) {
                reject(err); // On error, reject the promise.
            }
            else {
                resolve(files); // On success, resolve the promise.
            }
        });    
    });
};

We also need a promise-based function that we can use to determine the type (file or directory) of a particular path:

function determineType (parentPath, childPath) { // Promise-based function to determine if the path is a file or directory.
    return new Promise((resolve, reject) => {
        fs.stat(path.join(parentPath, childPath), (err, stats) => {
            if (err) {
                reject(err);
            }
            else {
                resolve({ 
                    path: childPath,
                    type: stats.isDirectory() ? 'directory' : 'file' // Check if it's a directory or a file.
                }); 
            }
        });
    });
};

Now we can expand on determineType and create a function takes an array of paths and determines the type of each. This uses Promise.all to execute multiple async operations in parallel:

function determineTypes (parentPath, paths) { // Async function to determine if child paths are directories or files.

    return Promise.all(
            paths.map(
                childPath => determineType(parentPath, childPath) // Is the path a directory or a file?
            )
        );
};

Now we can build our promise-based recrusive version of readdir:

function readdirTree (rootPath) { // Read an entire directory tree, the promise-based recursive version.
    return readdir(rootPath) // Initial non-recursive directory read.
        .then(childPaths => determineTypes(rootPath, childPaths)) // Figure out the type of child paths.
        .then(children => {
            return Promise.all(children // Use Promise.all to figure out all sub-trees in a parallel.
                .filter(child => child.type === 'directory') // Filter so we only directories are remaining.
                .map(child => {
                    return readdirTree(path.join(rootPath, child.path)) // It's a directory, recurse to the next level down.
                        .then(subTree => {
                            return {
                                path: child.path,
                                subTree: subTree,
                            };
                        });
                })
            );
        })
        .then(children => {
            const tree = {}; // Reorganise the list of directories into a tree.
            children.forEach(directory => {
                tree[directory.path] = directory.subTree;
            });
            return tree;
        });
};

Here's an example of use:

readdirTree("c:\\some-directory")
    .then(tree => {
        console.log("tree:");
        console.log(tree);
    })
    .catch(err => {
        console.error("error:");
        console.error(err);
    });

I've got a complete working example for you in my Github: https://github.com/ashleydavis/read-directory-with-promises

Hope it helps you move forward.