node.js fall through data cache pattern

Question

I am looking for a clean way to structure my node.js code for the following situation. I thought of using EventEmitters to create a "workflow" type of thing. Also I thought of using one of the async libraries out there, that has been less thought out though.

Problem: Looking for a piece of data

1. check cache, if found return
2. check db, if found return (store in cache)
3. get live data and return, (store in db, cache)

I mocked something up quickly using event emitters below.

var util = require("util");
var events = require('events');

var CheckForData = function() {
    events.EventEmitter.call(this);

    this.checkForData = function(key) {
        this.emit("checkForDataRequest", key);
    }

    var _checkCache = function(key) {
        if (key === 'cache') {
            this.emit("found", {data:'cached data'});
        }
        else {
            console.log("not found in cache "+key);
            this.emit("checkDatastore", key);
        }
    }

    var _chechDatastore = function(key) {
        if (key === 'db') {
            this.emit("found", {data:'db data'});
            this.emit("storeCache", key, {data:'db data'});
        }
        else {
            console.log("not found in db "+key);
            this.emit("getData", key);
        }
    }

    var _getData = function(key) {
        if (key === 'live') {
            this.emit("found", {data:'live data'});
            this.emit("storeData", key, {data:'live data'});
        }
        else {
            console.log("not found in live "+key);
            this.emit("notFound", key);
        }
    }

    var _storeData = function(key, data) {
        this.emit("storeDb", key, data);
        this.emit("storeCache", key, data);
    }

    var _storeDb = function(key, data) {
        console.log("storing data in db. for "+key);
        console.log(data);
    }

    var _storeCache = function(key, data) {
        console.log("storing data in cache. for "+key);
        console.log(data);
    }

    var _found = function(data) {
        return data;
    }

    var _notFound = function(key) {
        return key;
    }

    this.on("checkForDataRequest", _checkCache);
    this.on("checkDatastore", _chechDatastore);
    this.on("getData", _getData);
    this.on("found", _found);
    this.on("notFound", _notFound);
    this.on("storeData", _storeData);
    this.on("storeDb", _storeDb);
    this.on("storeCache", _storeCache);
};

util.inherits(CheckForData, events.EventEmitter);
module.exports = new CheckForData();

To test it...

var checkForData = require('./check-for-data');

checkForData.on("found", function(data) {
    console.log("Found data ");
    console.log(data);
});

checkForData.on("notFound", function(key) {
    console.log("NOOO Found data for " + key);
});

console.log("-------");
checkForData.checkForData('cache');
console.log("-------");
checkForData.checkForData('db');
console.log("-------");
checkForData.checkForData('live');
console.log("-------");
checkForData.checkForData('other');
console.log("-------");

---

Then async.js, i made a quick checkSeries which is basically the async.detectSeries but instead of returning the item in the collection return the result. See below...

var async = require('async');

function check_cache(key) {
    console.log('checking cache');
    return null;
}

function check_datastore(key) {
    console.log('checking datastore');
    return null;//{data: "db data"};
}

function check_api(options) {
    console.log('calling api');
    return {data: "live data"};
}

function exists(item, callback) {
    callback(item());
}

async.checkSeries([check_cache, check_datastore, check_api], exists, function(result) {
    // result now equals the first function that return data
    console.log(result);
});

Any suggestions, hints, tips, ...? Is there a pattern or library that i am missing? Do you think it be possible/easier to do in Step, flow, ...? Memoize?

Answer 1

That seems like a lot of work for the caller and a lot of extra code that doesn't seem to be adding much value. I have something that looks like this in my model.

Foo.get = function (id, cb) {

  var self = this;
  // check the cache first
  cache.get('foo:' + id, function (err, cacheFoo) {

    // if found, just return it
    if (!err && cacheFoo) {
      return cb(null, cacheFoo);
    }

    // otherwise get from db
    self.findOne({id: id}, function (err, dbFoo) {

      if (err || !dbFoo) {
        return cb(new Error('Foo not found', 404));
        // you could do get live call here
      }

      // and store in cache
      cache.store('foo:' + id, dbFoo);
      return cb(null, dbFoo);
    });
  });
};

Callers can then always just call Foo.get(id, callback) and they don't have to care how it is actually retrieved. If it gets more complicated, you could use an async library (such as the aptly named async) to make the code more readable, but this should still be completely hidden from the caller.