CODEWITHSUNDEEP
javascriptnode.jsexpressrequest
John Verrone
John Verrone

Reputation: 307

Loop through multiple urls with Request (Node.js)

I currently have a app.get() method that uses request and cheerio to pull html from iTunes reviews.

// get all reviews
app.get('/api/reviews', function(req, res) {
  var options = {
    url: 'https://itunes.apple.com/WebObjects/MZStore.woa/wa/customerReviews',
    qs: {
     'displayable-kind': '11',
     'id': 'XXXXXXXXX',
     'page': i,
     'sort': '4'
    },
    headers: {
      'User-Agent': 'iTunes/10.3.1 (Macintosh; Intel Mac OS X 10.6.8) AppleWebKit/533.21.1',
      'X-Apple-Store-Front': '143441-1,12',
      'X-Apple-Tz': '-18000',
      'Accept-Language': 'en-us, en;q=0.50',
    }
  } 

  request(options, function(error, response, html) {
    // load html, parse, do work, etc.
    // res.send(results);
  }     
};

I want to iterate through this url call with the 'page' query being a variable like below. I am getting a "Error: Can't set headers after they are sent." error when trying it this way. How do you go about changing the headers and doing another url call?

app.get('/api/reviews', function(req, res) {
  for(var i = 0; i < 4; i++) {
    var options = {
      url: 'https://itunes.apple.com/WebObjects/MZStore.woa/wa/customerReviews',
      qs: {
       'displayable-kind': '11',
       'id': 'XXXXXXXXX',
       'page': '1',
       'sort': '4'
      },
      headers: {
        'User-Agent': 'iTunes/10.3.1 (Macintosh; Intel Mac OS X 10.6.8) AppleWebKit/533.21.1',
        'X-Apple-Store-Front': '143441-1,12',
        'X-Apple-Tz': '-18000',
        'Accept-Language': 'en-us, en;q=0.50',
      }
    } 

    request(options, function(error, response, html) {
      // load html, parse, do work, etc.
    }
  }     
};

Upvotes: 0

Views: 2114

Answers (1)

user142162
user142162

Reputation:

The issue you are having is that you are trying to send the result page in chunks, which does not work. You can solve this by collecting all of the request results then send whatever data you need to back to the client.

Here is an example of doing this using async:

var async = require('async');

app.get('/api/reviews', function(req, res) {
  async.times(4, function(i, cb) {
      var options = {
        url: 'https://itunes.apple.com/WebObjects/MZStore.woa/wa/customerReviews',
        qs: {
         'displayable-kind': '11',
         'id': 'XXXXXXXXX',
         'page': i,
         'sort': '4'
        },
        headers: {
          'User-Agent': 'iTunes/10.3.1 (Macintosh; Intel Mac OS X 10.6.8) AppleWebKit/533.21.1',
          'X-Apple-Store-Front': '143441-1,12',
          'X-Apple-Tz': '-18000',
          'Accept-Language': 'en-us, en;q=0.50',
        }
      } 
      request(options, function(error, response, html) {
        // load html, parse, do work, etc.
        // var result = ...;
        cb(null, result);
      });
  }, function(err, results) {
    // completed processing of all items
    // array of `request` results are stored in results
    // send data to client here
  });
});

Upvotes: 1

Related Questions