How to list all subdirectories and files using nodeJS?

Question

Using node ssh2-sftp-client, I'd like to recursively list all directories (and in a second time, files) of a SSH-remote directory root.

I'm using nodejs

node v12.15.0

I've found a usefulness same but non answered StackOverflow question.

Documentation "basic usage" (see ssh2-sftp-client) gives a standard use of sftp connection and list of directory '/tmp' :

const Client = require('ssh2-sftp-client');
const sshConfig = {
    host: process.env.REMOTE_SSH_HOST,
    port: 22,
    username: process.env.REMOTE_SSH_USERNAME,
    password: process.env.REMOTE_SSH_PASSWORD,
    readyTimeout: 99999,
};
let sftp = new Client();
sftp.connect(sshConfig).then(() => {
    return sftp.list('/tmp');
}).then(data => {
    console.log(data, 'the data info');
}).catch(err => {
    console.log(err, 'catch error');
});

I've tried to add a function recursively reading every sub-directories, but I got an error on forEach sentence saying :

sftp.list(...).forEach is not a function

although sftp.list() returns an array.
I know there is something with sftp.list method that return a Promise and then an array, but cannot figure out what and where I should modify.

Here is my code :
sshList.js

const Client = require('ssh2-sftp-client');
const sshConfig = {
    host: process.env.REMOTE_SSH_HOST,
    port: 22,
    username: process.env.REMOTE_SSH_USERNAME,
    password: process.env.REMOTE_SSH_PASSWORD,
    readyTimeout: 99999,
};
let sftp = new Client();

// Function to read sub-directories
async function SubRead(sub, parentDirectory) {
    if (sub['type'] === 'd') {
        await Read(parentDirectory + '/' + sub['name']);
    }
}
async function Read(directory) {
    console.log('Read(' + directory + ')');
    const result = sftp.list(directory);
    result.forEach( x => SubRead(x, directory) );
}
async function main(directory) {
    try {
        console.log('Connecting...');
        await sftp.connect(sshConfig).then(async () => {
            console.log('Connected');
            await Read(directory);
        });
    } catch (e) {
        console.error(e.message);
    } finally {
        console.log('Closing session...');
        await sftp.end();
        console.log('Session closed.');
    }
}
console.log('Application started');
main('/home/user/path').then(r => {
    console.log('Application ended.');
});

Launch with

node sshList.js

---

Adding sample structure

Let say remote directory "/home/user/path" to read has this tree structure :

.
├── Level1_A
│   ├── Level2_AA
│   │   ├── Level3_AAA
│   │   ├── toto
│   │   ├── toto.txt
│   │   ├── txttoto
│   │   └── txt.toto
│   ├── Level2_AB
│   └── Level2_AC
├── Level1_B
│   ├── Level2_BA
│   ├── Level2_BB
│   └── Level2_BC
└── Level1_C
    ├── Level2_CA
    └── Level2_CB

Running node command (see upper) gives an incomplete result :

Application started
Connecting...
Connected
Reading(/home/iliad/alain)
Reading(/home/iliad/alain/Level1_B)
Reading(/home/iliad/alain/Level1_A)
Reading(/home/iliad/alain/Level1_C)
Closing session...
Session closed.
Application ended.

No way to find "Level 2" directories !
I was expecting (do not consider "# Missing" text) :

Application started
Connecting...
Connected
Reading(/home/iliad/alain)
Reading(/home/iliad/alain/Level1_B)
Reading(/home/iliad/alain/Level1_B/Level2_BA) # Missing
Reading(/home/iliad/alain/Level1_B/Level2_BB) # Missing
Reading(/home/iliad/alain/Level1_B/Level2_BC) # Missing
Reading(/home/iliad/alain/Level1_A)
Reading(/home/iliad/alain/Level1_A/Level2_AA) # Missing
Reading(/home/iliad/alain/Level1_A/Level2_AA/Level4_AAA) # Missing
Reading(/home/iliad/alain/Level1_A/Level2_AB) # Missing
Reading(/home/iliad/alain/Level1_A/Level2_AC) # Missing
Reading(/home/iliad/alain/Level1_C)
Reading(/home/iliad/alain/Level1_C/Level2_CA)
Reading(/home/iliad/alain/Level1_C/Level2_CB)
Closing session...
Session closed.
Application ended.

What am I missing ?
Any help will be welcome !

Answer 1

Iterative method to make an array for infinite layers of folders (folders inside folders inside folders, etc):

`use strict`
require('child_process').execSync('cls',{stdio:'inherit'})//console.clear(), but it always works!
console.log('Executed At: ',Date(),'\n')

const fs=require('fs')
,ListSubFolders=Path=>{
    let ToTest=fs.readdirSync(Path).filter(Item=>fs.lstatSync(Item).isDirectory())
    const SubFolders=[]
    while(ToTest.length!=0){
        const Folder=ToTest.at(-1);SubFolders.push(ToTest.pop())//DFS
        /*
            Depth First Search (DFS) is the default method because it uses .at(-1) & .pop() which are faster and more efficient because they check and remove items from the end and don't need to measure the length of an array and reindex each item when one is removed, unlike Breadth First Search (BFS) which uses [0] & .shift().
            Should you still wish to switch between the methods, I made it easy for you: just uncomment the line below and comment the line above
        */
        // const Folder=ToTest[0];SubFolders.push(ToTest.shift())//BFS

        let Contents=fs.readdirSync(Folder).filter(Item=>fs.lstatSync(`${Folder}/${Item}`).isDirectory())
        Contents=Contents.map(SubFolder=>SubFolder=`${Folder}/${SubFolder}`)
        ToTest=ToTest.concat(Contents).flat()
    }
    console.log(SubFolders.length+' folders across '+SubFolders.map(Path=>Path=Path.split('/').length).reduce((a,b)=>Math.max(a,b))+" layers inside the '"+process.cwd()+"' folder:\n",SubFolders)
    return SubFolders
}
ListSubFolders(process.cwd())

Use process.cwd() for Path if you want to check the current folder where the terminal is open.

Difference in order of searching and listing - BFS VS DFS:

Note: this code is so good it even lists hidden folders! (and can be used for that purpose...) If it lists a folder you can't see, now you know why 😉

This is my original code, I worked on this for hours, so I hope you appreciate it. Tried different approaches, this is my favourite. Find it here on Github: https://github.com/EhudKirsh/NodeJS-list-all-sub-directories-folders.git

Answer 2

sftp.list returns a promise so you need to await it before using it. Something like this should work

const Client = require('ssh2-sftp-client');
const sshConfig = {
    host: process.env.REMOTE_SSH_HOST,
    port: 22,
    username: process.env.REMOTE_SSH_USERNAME,
    password: process.env.REMOTE_SSH_PASSWORD,
    readyTimeout: 99999,
};
let sftp = new Client();

async function Read(directory) {
    console.log('Read(' + directory + ')');
    const result = await sftp.list(directory);
    for(const sub of result) {
      if (sub['type'] === 'd') {
          await Read(directory + '/ ' + sub['name']);
      }
    }
}

async function main(directory) {
    try {
        console.log('Connecting...');
        await sftp.connect(sshConfig).then(() => {
            console.log('Connected');
            await Read(directory);
        });
    } catch (e) {
        console.error(e.message);
    } finally {
        console.log('Closing session...');
        await sftp.end();
        console.log('Session closed.');
    }
}
console.log('Application started');
main('/home/user/path').then(r => {
    console.log('Application ended.');
});

Answer 3

sftp.list(directory).forEach(

have to be either :

 sftp.list(directory).then((myList)=> {myList.foreach()})//with your code
                     .catch((err)=> console.log(err))//never forget to catch

or declare you function async and

try{
const myList = await sftp.list(directory);
}catch(err){console.log(err)}//NEVER forget to catch