Is it possible to write asynchronous Node.js code "cleaner"?

Question

While coding in Node.js, I encountered many situations when it is so hard to implement some elaborated logic mixed with database queries (I/O).

Consider an example written in python. We need to iterate over an array of values, for each value we query the database, then, based on the results, we need to compute the average.

def foo:
  a = [1, 2, 3, 4, 5]
  result = 0
  for i in a:
    record = find_from_db(i) # I/O operation
    if not record:
      raise Error('No record exist for %d' % i)
    result += record.value

  return result / len(a)

The same task in Node.js

function foo(callback) {
  var a = [1, 2, 3, 4, 5];
  var result = 0;
  var itemProcessed = 0;
  var error;
  function final() {
    if (itemProcessed == a.length) {
      if (error) {
        callback(error);
      } else {
        callback(null, result / a.length);
      }
    }
  }
  a.forEach(function(i) {
    // I/O operation
    findFromDb(function(err, record) {
      itemProcessed++;
      if (err) {
        error = err;
      } else if (!record) {
        error = 'No record exist for ' + i;
      } else {
        result += record.value;
      }
      final();
    });
  });
}

You can see that such code much harder to write/read, and it is more prone to errors. My questions:

1. Is there a way to make above Node.js code cleaner?
2. Imagine more sophisticated logic. For example, when we obtained a record from the db, we might need do another db query based on some conditions. In Node.js that becomes a nightmare. What are common patterns for dealing with such tasks?
3. Based on your experience, does the performance gain deserves the productivity loss when you code with Node.js?
4. Is there other asynchronous I/O framework/language that is easier to work with?

Answer 1

"If you try to code bussiness db login using pure node.js, you go straight to callback hell"

I've recently created a simple abstraction named WaitFor to call async functions in sync mode (based on Fibers): https://github.com/luciotato/waitfor

check the database example:

Database example (pseudocode)

pure node.js (mild callback hell):

var db = require("some-db-abstraction");

function handleWithdrawal(req,res){  
    try {
        var amount=req.param("amount");
        db.select("* from sessions where session_id=?",req.param("session_id"),function(err,sessiondata) {
            if (err) throw err;
            db.select("* from accounts where user_id=?",sessiondata.user_ID),function(err,accountdata) {
                if (err) throw err;
                    if (accountdata.balance < amount) throw new Error('insufficient funds');
                    db.execute("withdrawal(?,?),accountdata.ID,req.param("amount"), function(err,data) {
                        if (err) throw err;
                        res.write("withdrawal OK, amount: "+ req.param("amount"));
                        db.select("balance from accounts where account_id=?", accountdata.ID,function(err,balance) {
                            if (err) throw err;
                            res.end("your current balance is "  + balance.amount);
                        });
                    });
                });
            });
        }
        catch(err) {
            res.end("Withdrawal error: "  + err.message);
    }  

Note: The above code, although it looks like it will catch the exceptions, it will not. Catching exceptions with callback hell adds a lot of pain, and i'm not sure if you will have the 'res' parameter to respond to the user. If somebody like to fix this example... be my guest.

using wait.for:

var db = require("some-db-abstraction"), wait=require('wait.for');

function handleWithdrawal(req,res){  
    try {
        var amount=req.param("amount");
        sessiondata = wait.forMethod(db,"select","* from session where session_id=?",req.param("session_id"));
        accountdata= wait.forMethod(db,"select","* from accounts where user_id=?",sessiondata.user_ID);
        if (accountdata.balance < amount) throw new Error('insufficient funds');
        wait.forMethod(db,"execute","withdrawal(?,?)",accountdata.ID,req.param("amount"));
        res.write("withdrawal OK, amount: "+ req.param("amount"));
        balance=wait.forMethod(db,"select","balance from accounts where account_id=?", accountdata.ID);
        res.end("your current balance is "  + balance.amount);
        }
    catch(err) {
        res.end("Withdrawal error: "  + err.message);
}  

Note: Exceptions will be catched as expected. db methods (db.select, db.execute) will be called with this=db

Your Code

In order to use wait.for, you'll have to STANDARDIZE YOUR CALLBACKS to function(err,data)

If you STANDARDIZE YOUR CALLBACKS, your code might look like:

var wait = require('wait.for');

//run in a Fiber
function process() {
  var a = [1, 2, 3, 4, 5];
  var result = 0;
  a.forEach(function(i) {
    // I/O operation
    var record = wait.for(findFromDb,i); //call & wait for async function findFromDb(i,callback)
    if (!record) throw new Error('No record exist for ' + i);
    result += record.value;
  });

  return result/a.length;
}

function inAFiber(){
   console.log('result is: ',process());
}

// run the loop in a Fiber (keep node spinning)
wait.launchFiber(inAFiber);

see? closer to python and no callback hell

Answer 2

The more I work with async the more I love it and I like node more. Let me give you a simple example of what I have for a server initialization.

async.parallel ({
    "job1": loadFromCollection1,
    "job2": loadFromCollection2,
},
function (initError, results) {
    if (initError) {
        console.log ("[INIT] Server initialization error occurred: " + JSON.stringify(initError, null, 3));
        return callback (initError);
    }
    // Do more stuff with the results
});

In fact, this very same approach can be followed and one can pass different arguments to the different functions that correspond to the various jobs; see for example Passing arguments to async.parallel in node.js.

To be perfectly honest with you, I prefer the node-way which is also non-blocking. I think node forces someone to have a better design and sometimes you spend time creating more definitions and grouping functions and objects in arrays so that you can write better code. The reason I think is that in the end you want to exploit some variant of async and mix and merge stuff accordingly. In my opinion, spending some extra time and thinking about the code a bit more is well worth it when you also take into account that node is asynchronous.

Other than that, I think it is a habit. The more one writes code for node, the more one improves and writes better asynchronous code. What is good on node is that it really forces someone to write more robust code since one starts respecting all the error codes from all the functions much more. For example, how often do people check, say if malloc or new have succeeded and one does not have an error handler for a NULL pointer after the command has been issued? Writing asynchronous code though forces one to respect the events and the error codes that the events have. I guess one obvious reason is that one respects the code that one writes and in the end we have to write code that returns errors so that caller knows what happened.

I really think that you need to give it more time and start working with async more. That's all.

Answer 3

To answer your questions:

1. There are libraries such as async which provide a variety of solutions for common scenarios when working with asynchronous tasks. For "callback hell" concerns, there are many ways to avoid that as well, including (but not limited to) naming your functions and pulling them out, modularizing your code, and using promises.

2. More or less what you currently have is a fairly common pattern: having counter and function index variables with an array of functions to call. Again, async can help here because it reduces this kind of boilerplate that you will probably find yourself repeating often. async currently doesn't have methods that really allow for skipping individual tasks, but you could easily do this yourself if you are writing the boilerplate (just increment the function index variable by 2 for example).

3. From my own experience, if you properly design your javascript code with asynchronous in mind and use a lot of tools like async, you will find it easier to develop with node. Writing for asynchronous vs synchronous in node is typically always going to be more complicated (although less so with generators, fibers, etc. as compared to callbacks/promises).

4. I personally think that deciding on a language based upon that single aspect is not worthwhile. You have to consider much much more than just the design of the language, for example the size of the community, availability of third party libraries, performance, technical support options, ease of code debugging, etc.

Answer 4

Just write your code more compactly:

// parallel version
function foo (cb) {
  var items = [ 1, 2, 3, 4, 5 ];
  var pending = items.length;
  var result = 0;

  items.forEach(function (item) {
    findFromDb(item, function (err, record) {
      if (err) return cb(err);
      if (!record) return cb(new Error('No record for: ' + item))
      result += record.value / items.length;
      if (-- pending === 0) cb(null, result);
    });
  });
}

That clocks in at 13 source lines of code compared to the 9 sloc for python that you posted. However, unlike the python that you posted, this code runs all the jobs in parallel.

To do the same thing in series, a trick I usually do is a next() function defined inline that invokes itself and pops a job off of an array:

// sequential version
function foo (cb) {
  var items = [ 1, 2, 3, 4, 5 ];
  var len = items.length;
  var result = 0;

  (function next () {
    if (items.length === 0) return cb(null, result);
    var item = items.shift();

    findFromDb(item, function (err, record) {
      if (err) return cb(err);
      if (!record) return cb(new Error('No record for: ' + item))
      result += record.value / len;
      next();
    });
  })();
}

This time, 15 lines. The nice thing is that you can easily control whether the actions should happen in parallel or sequentially or somewhere in between. That is not so easy in a language like python where everything is synchronous and you've got to do lots of work-arounds like threads or evented libraries to get things back up to asynchronous. Try implementing a parallel version of what you have in python! It would most certainly be longer than the node version.

As for the promise/async route: it's not actually all that hard or bad to use ordinary functions for these relatively simple kinds of tasks. In the future (or in node 0.11+ with --harmony) you can use generators and a library like co, but that feature isn't widely deployed yet.

Answer 5

Everyone here seems to be suggesting async, which is a great library. But to give another suggestion, you should take a look at Promises , which is a new built-in being introduced to the language (and currently has several very good polyfills). It allows you to write asynchronous code in a way that looks much more structured. For example, take a look at this code:

var items = [ 1, 2, 3, 4 ];
var processItem = function(item, callback) {
    // do something async ...
};
var values = [ ];
items.forEach(function(item) {
    processItem(item, function(err, value) {
        if (err) {
            // something went wrong
        }
        values.push(value);
        // all of the items have been processed, move on
        if (values.length === items.length) {
            doSomethingWithValues(values, function(err) {
                if (err) {
                    // something went wrong
                }
                // and we're done
            });
        }
    });
});
function doSomethingWithValues(values, callback) {
    // do something async ...
}

Using promises, it would be written something like this:

var items = [ 1, 2, 3, 4 ];
var processItem = function(item) {
    return new Promise(function(resolve, reject) {
        // do something async ...
    });
};
var doSomethingWithValues = function(values) {
    return new Promise(function(resolve, reject) {
        // do something async ...
    });
};
// promise.all returns a new promise that will resolve when all of the promises passed to it have resolved
Promise.all(items.map(processItem))
    .then(doSomethingWithValues)
    .then(function() {
        // and we're done
    })
    .catch(function(err) {
        // something went wrong
    });

The second version is much cleaner and simpler, and that barely even scratches the surface of promises real power. And, like I said, Promises are in es6 as a new language built-in, so (eventually) you won't even need to load in a library, it will just be available.

Answer 6

1. don't use anonymous (un-named) functions they make the code ugly and they make debugging much harder, so always name your functions and define them outside the function scope not inline.
2. that is a real issue with Node.js (it is called callback hell or pyramid of doom ,..) you can solve this issue by using promises or using async.js which have so many functions for handling different situations (waterfall, parallel, series, auto, ...)
3. well the performance gain is absolutely a good thing and it is not that much loss (when you start to master it) and also the Node.js community is great.
4. Check async.js, q.