CODEWITHSUNDEEP
node.jsrequest
Illusionist
Illusionist

Reputation: 5499

Async map with headers for getting multiple url's parallel

I am learning async map . I want to download a bunch of URLs but i want to send a header along with the get request.

If i didn't have a header, I could have just done

var request = require('request');
var async = require('async');

var urls = ['http://myurl1.com', 'http://myurl2.com', 'http://myurl3.com'];

async.map(urls, request, function(err, results) {
        if (err) throw(err);          // handle error 
        console.log(results.length);  // == urls.length
});

Now I do need to send a header {"x-url-key":"myurlkey"} along with every get request. How do I modify the code above to do so?

Upvotes: 0

Views: 165

Answers (1)

Terry Lennox
Terry Lennox

Reputation: 30705

That should be straightforward enough to do, we can create a wrapper function requestWithHeader to pass to async.map, this will specify whichever headers (or other options) you wish.

I'm also specifying json: true here, you may not want to do that in your actual code.

In this example I'm using https://httpbin.org/get as the url, this will send back all the request parameters which is useful for test purposes as we can see which headers we populated.

var request = require('request');
var async = require('async');

var urls = ["https://httpbin.org/get?foo=bar", "https://httpbin.org/get?foo=baz"];

function requestWithHeader(uri, callback) {
    request(uri, { headers: {"x-url-key":"myurlkey"}, json:true }, callback)
}

async.map(urls, requestWithHeader, function(err, results) {
    if (err) throw(err);          // handle error 
    console.log("Results:", results.map(result => result.body));
});

To wait for async.map to finish you can create an asynchronous function to call it, e.g.

async function testMapWithPromise() {
    try { 
        let results = await async.map(urls, requestWithHeader);
        console.log("testMapWithPromise: Results:", results.map(result => result.body));
        // Do whatever with results here...
    } catch (error) {
        console.error("testMapWithPromise: An error occurred:", error);
    }
}

testMapWithPromise();

Upvotes: 1

Related Questions