CODEWITHSUNDEEP
javascriptnode.jsasynchronous
sn4ke
sn4ke

Reputation: 619

Using node.js async and require to fetch multiple urls

Good day -- I've conducted many queries on this subject but can't find a good example that fits my situation.

Goal: To call multiple urls using request and async (these urls are different, some are xml and some are json), with the ability to scale the amount of links, with the end result of passing the data to the view for parsing.

Code:

var express = require('express');
var router = express.Router();
var request = require('request');
var parseString = require('xml2js').parseString;
var async = require('async');

var bis;

var url_bis = {
  url: 'https://www.bis.org/list/cbspeeches/index.rss',
  headers: {
    'User-Agent': 'request'
  }
};

function callback_bis(error, response, body) {
  if (!error && response.statusCode == 200) {
    bis = "";
    parseString(body, function (err, result) {
        bis = result;
        bis = bis["rdf:RDF"]["item"];
    });
  }
}

var doj;

var url_doj = {
  url: 'http://www.justice.gov/feeds/opa/justice-news.xml',
  headers: {
    'User-Agent': 'request'
  }
};

function callback_doj (error, response, body) {
  if (!error && response.statusCode == 200) {
    doj = "";
    parseString(body, function (err, result) {
        _doj = result;
        doj = _doj.rss.channel[0].item;

    });
  }
}   

function feed_doj() {
    request(url_doj, callback_doj);
}
function feed_bis() {
    request(url_bis, callback_bis);
}

router.get('/', function(req, res, next) {

async.parallel({
    bis: feed_bis,
    doj: feed_doj

}, function(err, results) {
    console.log(results)
    // how to get the bis/doj response here in results?
});

});

module.exports = router;

Upvotes: 1

Views: 4185

Answers (3)

David Knipe
David Knipe

Reputation: 3454

async needs to know when you've finished each task. To achieve this, the tasks are called with arguments which are callback functions. Your task functions, feed_doj and feed_bis, should call the callback functions. This is also the standard way of getting the output from the tasks - don't use global functions, it's widely considered bad form. Do this instead:

function feed_bis(callback) {
  function callback_bis(error, response, body) {
    if (!error && response.statusCode == 200) {
        parseString(body, function (err, result) {
        var bis = result["rdf:RDF"]["item"];
        callback(null, bis);
      });
    }
  }
  request(url_bis, callback_bis);
}

// And same again for doj

Once the callback functions have been called for both bis and doj, the main callback will be called with arguments (err, result); err will be null because you haven't done anything to handle error cases, and result will be an object with properties named bis and doj.

Upvotes: 1

edsadr
edsadr

Reputation: 1085

You can accomplish this using the parallel method included in the async module,you just push all request in an array and then implement this: https://github.com/caolan/async/blob/master/README.md#parallel it is designed to execute a final callback when all processes end.

Upvotes: 0

greim
greim

Reputation: 9437

This is super-easy with promises and the Promise.all() function:

var request = require('request-promise');

var calls = [
  request({ url: 'http://...', headers: { ... } }),
  request({ url: 'http://...', headers: { ... } }),
  // ...as many more as you want...
];

Promise.all(calls).then(function(results) {
  // do something with results[0]
  // do something with results[1]
});

The Promise API is native in newer versions of JS, and can be easily polyfilled in older JS engines, so it reduces the need for helper libs like async.

This example runs all the requests in parallel. Promise.all() waits for them all to finish, then gives you an array of results matching the original requests. Note also use of the request-promise module, which just piggybacks on top of the request module, but returns a promise instead of taking a callback.

Upvotes: 2

Related Questions