CODEWITHSUNDEEP
javascriptnode.jsasync.js
xtx
xtx

Reputation: 4456

How to organize a loop of async.waterfall calls

How to organize a loop of async.waterfall calls so that every iteration starts only after the previous one is fully finished.

So far I have the following code:

var hasMoreData = true;
async.whilst(function () {
  // * has more data?
  return hasMoreData;
},
function processData(callback1) {
  async.waterfall([
    function readDataFromSource(callback2) {
      // * read data
      readData(function readFinished(data, hasMore) {
        // * got data 
        hasMoreData = hasMore;
        callback2(null, data);
      });
    }, 
    function writeDataToDest(data, callback2) {
      // * write data
      writeData(data, function writeFinished() {
        callback2();
      });
    },
  ], function (err) {
    callback1(err);
  });
},
function finished(err) {

});

It works like this:

I understand why it works in this order, and this may be totally correct in some cases. But in my specific case I need to start a new iteration only after the previous one is finished:

PS: I can't read all the data into array and then process the array with async.each, for example. Neither I can predict how much data the external source has.

Upvotes: 1

Views: 476

Answers (1)

T.J. Crowder
T.J. Crowder

Reputation: 1074335

I don't use async, but just looking at the code and description and API docs, it seems like this does what you want, not using waterfall at all:

var hasMoreData = true;
async.whilst(function () {
  // * has more data?
  return hasMoreData;
},
function processData(callback1) {
  // * read data
  // blocking i/o operation
  readData(function readFinished(data, hasMore) {
    // * got data
    hasMoreData = hasMore;
    // * write data
    // blocking i/o operation
    writeData(data, function writeFinished() {
      callback1();
    });
  });
},
function finished(err) {

});

Live Example (using shims for readData and writeData):

// Shims
var datacount = 0;
function readData(callback) {
  // since you said "blocking", busy-wait for a quarter second
  var done = Date.now() + 250;
  while (done > Date.now()) {
  }
  // but calls like this are almost always async, so we'll complete async
  ++datacount;
  setTimeout(function() {
    callback(datacount, datacount < 3);
  }, 0);
}
function writeData(data, callback) {
  // since you said "blocking", busy-wait for a quarter second
  var done = Date.now() + 250;
  while (done > Date.now()) {
  }
  // but calls like this are almost always async, so we'll complete async
  setTimeout(function() {
    callback();
  }, 0);
}

// The code
var hasMoreData = true;
async.whilst(function() {
  // * has more data?
  snippet.log("has more data?");
  return hasMoreData;
},
function processData(callback1) {
  // * read data
  snippet.log("read data");
  // blocking i/o operation
  readData(function readFinished(data, hasMore) {
    // * got data
    snippet.log("got data: " + data + " (more? " + hasMore + ")");
    hasMoreData = hasMore;
    // * write data
    snippet.log("write data");
    // blocking i/o operation
    writeData(data, function writeFinished() {
      callback1();
    });
  });
},
function finished(err) {
  snippet.log("finished");
});
<!-- Script provides the `snippet` object, see http://meta.stackexchange.com/a/242144/134069 -->
<script src="//tjcrowder.github.io/simple-snippets-console/snippet.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/async/1.5.0/async.min.js"></script>

Upvotes: 1

Related Questions