nodejs parallel callback design pattern

I'm trying to find a good pattern to execute a bunch of parallel tasks.

Let me define some task to exemplify. Tasks a, b, c, d, e, f, g execute as a(function(er, ra){//task a returned, ra is result}), so do b to g

There are also some tasks that should be execute after some task is done, let's call them ab, bc, abc, bd, bcd, af, fg, means when a and b has returned ab(ra, rb) should be executed at once, and when b and c returned, bc(rb, rc) should be executed at once, and if a, b, c all returned, abc(ra, rb, rc) should be executed.

For the simplest case, if there is only a and b, I can do something like this:

(function(cb){
    var count = 2, _ra, _rb;
    function update(){if(--count == 0) cb(null, _ra, _rb)}
    a(function(er, ra){_ra = ra; update()});
    b(function(er, ra){_rb = rb; update()});
})(function(er, ra, rb){
    ab(ra, rb);
});

As you can see, a and b execute in parallel, and when both are done, ab(ra, rb) execute.

But how can I do more things for a lot of parallel tasks?

Upvotes: 5

Answers (6)

Khanh Nguyen

Reputation: 11132

nimble is another good choice.

Upvotes: 0

Dave Dopson

Reputation: 42716

You should check out Step ( https://github.com/creationix/step ). It's just over a hundred lines of code, so you can read the whole thing if needed.

My preferred pattern looks something like this:


function doABunchOfCrazyAsyncStuff() {
  Step (
    function stepA() {
      a(arg1, arg2, arg3, this); // this is the callback, defined by Step
    }
    ,function stepB(err, data) {
      if(err) throw err; // causes error to percolate to the next step, all the way to the end.  same as calling "this(err, null); return;"
      b(data, arg2, arg3, this);
    }
    ,function stepC(err, data) {
      if(err) throw err;
      c(data, arg2, arg3, this);
    }
    ,function stepDEF(err, data) {
      if(err) throw err;
      d(data, this.parallel());
      e(data, this.parallel());
      f(data, this.parallel());
    }
    ,function stepGGG(err, dataD, dataE, dataF) {
      if(err) throw err;
      var combined = magick(dataD, dataE, dataF);
      var group = this.group();  // group() is how you get Step to merge multiple results into an array
      _.map(combined, function (element) {
        g(element, group()); 
      });
    }
    ,function stepPostprocess(err, results) {
      if(err) throw err;
      var processed = _.map(results, magick);
      return processed; // return is a convenient alternative to calling "this(null, result)"
    }
    ,cb // finally, the callback gets (err, result) from the previous function, and we are done
  );
}

Notes

My example also uses the underscore library, "the tie to match JQuery's tux": http://documentcloud.github.com/underscore/
Naming each step function stepXXXXX is a good habit so that the stack traces are clear and readable.
Step lets you make powerful and elegant combinations of serial and parallel execution. These patterns are straightforward and comprehensible. If you need anything more complex like "when 3 of 5 of these methods completes, go to the next step", seriously RETHINK your design. do you really need such a complex pattern? (maybe you are waiting for a quorum set). Such a complex pattern deserves a function of it's own.

Upvotes: 8

Raynos

Reputation: 169471

What you actually want is a deferred pattern though like futures.

function defer(f) {
    // create a promise.
    var promise = Futures.promise();
    f(function(err, data) {
        if (err) {
            // break it
            promise.smash(err);
        } else {
            // fulfill it
            promise.fulfill(data);
        }
    });
    return promise;
}
var da = defer(a), db = defer(b), dc = defer(c), dd = defer(d), de = defer(e), df = defer(f), dg = defer(g);

// when a and b are fulfilled then call ab
// ab takes one parameter [ra, rb]
Futures.join(da, db).when(ab);
Futures.join(db, dc).when(bc);
// abc takes one parameter [ra, rb, rc]
Futures.join(da, db, dc).when(abc);
Futures.join(db, dd).when(bd);
Futures.join(db, dc, dd).when(bcd);
Futures.join(da, df).when(af);
// where's e ?
Futures.join(df,dg).when(fg);
Futures.join(da,db,dc,dd,de,df,dg).fail(function() {
    console.log(":(");
});

Upvotes: 14