How to access 'response' parameter of app.get in callback functions

Question

I want to pass a list of files (obj) that I'm getting through a google drive API to EJS files.

i.e. I want to write

app.get('/',function(req,res){
  res.render('index',obj);
}

The problem is that I'm getting the js object through a few call back functions. This function is called

fs.readFile('client_secret.json',processClientSecrets );

which in turn calls,

function processClientSecrets(err,content) {
if (err) {
  console.log('Error loading client secret file: ' + err);
  return;
}else{
  authorize(JSON.parse(content),findFiles);
 }
}

which calls these two,

function authorise(credentials,callback) {
var clientSecret = credentials.installed.client_secret;
  var clientId = credentials.installed.client_id;
  var redirectUrl = credentials.installed.redirect_uris[0];
  var auth = new googleAuth();
  var oauth2Client = new auth.OAuth2(clientId, clientSecret, redirectUrl);

  // Check if we have previously stored a token.
  fs.readFile(TOKEN_PATH, function(err, token) {
    if (err) {
      getNewToken(oauth2Client, callback);
    } else {
      oauth2Client.credentials = JSON.parse(token);
      callback(oauth2Client);
    }
  });
}

[EDIT]

function findFiles(auth){
var obj ={};
var key = 'files';
obj[key]=[];
var drive = google.drive('v3');
drive.files.list({
                auth: auth,
                folderId: '****************',
                q: "mimeType contains 'application/pdf' and trashed = false"
                },
  function(err,response){
    var f = response.files;
    if (f.length == 0) {
    console.log('No files found.');
    }else {
        var i;
        for (i = 0; i < f.length; i++) {
        var file = f[i];
        //console.log('%s (%s)', file.name, file.id);
        obj[key].push(file.name + ' ' + file.id);
        }
        console.log(obj);
        return obj;
  }
                  });

}

This looks like a very basic question, however im not able to solve it as node.js is asynchronous in nature and all my attempts to return obj have resulted in rendering of obj before retrieving it.

Answer 1

Welcome to callback hell. :-) The old "Node" way would be to do nested callbacks, which gets very ugly very quickly.

The modern approach is to use promises, which makes it easier to compose multiple async operations together. Make your own async functions return promises, and for Node API functions (or add-on libs that don't provide promises yet), use wrappers to make them promise-enabled (manually, or by using something like promisify).

With promise-based functions, for instance, your call would look like this:

app.get('/',function(req,res){
    readFilePromise('client_secret.json')
        .then(content => JSON.parse(content))
        .then(authorise)
        .then(findFiles)
        .then(files => {
            res.render('index', files);
        })
        .catch(err => {
            // Render error here
        });
});

or since neither JSON.parse nor findFiles is asynchronous:

app.get('/',function(req,res){
    readFilePromise('client_secret.json')
        .then(content => authorise(JSON.parse(content)))
        .then(auth => {
            res.render('index', findFiles(auth));
        })
        .catch(err => {
            // Render error here
        });
});

It's fine to use non-async functions with then provided the function expects one parameter and returns the processed result, so the first version was fine too, though there is a bit of overhead involved.

In both cases, readFilePromise is a promisified version of readFile, and authorize looks something like this:

function authorise(credentials) {
    var clientSecret = credentials.installed.client_secret;
    var clientId = credentials.installed.client_id;
    var redirectUrl = credentials.installed.redirect_uris[0];
    var auth = new googleAuth();
    var oauth2Client = new auth.OAuth2(clientId, clientSecret, redirectUrl);

    // Check if we have previously stored a token.
    return readFilePromise(TOKEN_PATH)
        .then(token => {
            oauth2Client.credentials = JSON.parse(token);
            return oauth2Client;
        });
}

(Also note — subjectivity warning! — that because we don't end up with hellish deeply-nested callback structures, we can use a reasonable indentation width rather than the two spaces so many Node programmers felt the need to adopt.)

Moving forward further, if you're using Node V8.x+, you could use async/await syntax to consume those promises:

app.get('/', async function(req, res){
    try {
        const credentials = JSON.parse(await readFilePromise('client_secret.json'));
        const auth = await authorize(credentials);
        const files = findFiles(auth);
        res.render('index', files);
    } catch (e) {
        // Render error here
    }
});

Note the async before function and the await any time we're calling a function that returns a promise. An async function returns a promise under the covers, and await consumes promises under the covers. The code looks synchronous, but isn't. Every await is effectively a call to then registering a callback for when the promise completes. Similarly, the try/catch is effectively a call to the catch method on the promise chain.

We could condense that if we wanted:

app.get('/', async function(req, res){
    try {
        res.render('index', findFiles(await authorize(JSON.parse(await readFilePromise('client_secret.json'))));
    } catch (e) {
        // Render error here
    }
});

...but readability/debuggability suffer. :-)

Important note: When passing an async function into something (like app.get) that isn't expecting the function to return a promise, you must wrap it in a try/catch as above and handle any error, because if the calling code isn't expecting a promise, it won't handle promise rejections, and you need to do that; unhandled rejections are a bad thing (and in future versions of Node will cause your process to terminate).

If what you're passing an async function into does expect a function returning a process, best to leave the try/catch` off and allow errors to propagate.

---

You asked for help with findFiles. I recommend learning promisify or something similar. The right way (to my mind) to solve this is to give yourself a promisified version of drive.files.list, since drive.files.list uses Node-style callbacks instead.

But without promisifying it, we can do this:

function findFiles(auth) {
    var drive = google.drive('v3');
    return new Promise(function(resolve, reject) {
        drive.files.list({
            auth: auth,
            folderId: '****************',
            q: "mimeType contains 'application/pdf' and trashed = false"
        },
        function(err, response) {
            if (err) {
                reject(err);
                return;
            }
            var f = response.files;
            if (f.length == 0) {
                console.log('No files found.');
            }
            else {
                var key = 'files'; // Why this indirection??
                resolve({[key]: f.map(file => file.name + ' ' + file.id)});
                // Without the indirection it would be:
                // resolve({files: f.map(file => file.name + ' ' + file.id)});
            }
        });
    });
}

If we had a promisified version, and we did away with the key indirection which seems unnecessary, it would be simpler:

function findFiles(auth) {
    return drivePromisified.files.list({
        auth: auth,
        folderId: '****************',
        q: "mimeType contains 'application/pdf' and trashed = false"
    }).then(files => ({files: files.map(file => file.name + ' ' + file.id)}));
}

Or as an async function using await:

async function findFiles(auth) {
    const files = await drivePromisified.files.list({
        auth: auth,
        folderId: '****************',
        q: "mimeType contains 'application/pdf' and trashed = false"
    });
    return {files: files.map(file => file.name + ' ' + file.id)};
}