MattDionis
MattDionis

Reputation: 3616

Send synchronous requests in Node based on latest response

I'm using the async module on my Node server in order to loop through objects in my mongodb database, fire off requests to the Instagram API based on data within each object, and increment minTimestamp on each iteration until endTimestamp is reached.

The code below works great except for one big issue. If I increment minTimestamp by a hard-coded value (minTimestamp += 1000) everything runs great. However, when I change that one line of code to grab the latest created_time from the most recent response (minTimestamp = images[0].created_time) my loop runs once for each 'event' then stops. I get the correct incremented minTimestamp logged to my console, but the value never seems to make it to the next loop.

// modules =================================================
var express        = require('express.io');
var app            = express();
var port           = process.env.PORT || 6060;
var io             = require('socket.io').listen(app.listen(port));
var request        = require('request');
var Instagram      = require('instagram-node-lib');
var mongoose       = require('mongoose');
var async          = require('async');
var bodyParser     = require('body-parser');
var methodOverride = require('method-override');
var db             = require('./config/db');
var Event          = require('./app/models/event');

// configuration ===========================================
mongoose.connect(db.url); // connect to our mongoDB database

// get all data/stuff of the body (POST) parameters
app.use(bodyParser.json()); // parse application/json 
app.use(bodyParser.json({ type: 'application/vnd.api+json' })); //  parse        application/vnd.api+json as json
app.use(bodyParser.urlencoded({ extended: true })); // parse application/x-www-form- urlencoded

app.use(methodOverride('X-HTTP-Method-Override')); // override with the X-HTTP-Method- Override  header in the request. simulate DELETE/PUT
app.use(express.static(__dirname + '/public')); // set the static files location  /public/img  will be /img for users

var baseUrl = 'https://api.instagram.com/v1/media/search?lat=';
var clientId = CLIENT-ID;

Event.find({}, function(err, events) {

  async.eachSeries(events, function(event, seriesCallback) {

  var name = event.event;
  var latitude = event.latitude;
  var longitude = event.longitude;
  var distance = event.radius;
  var minTimestamp = Math.floor(new Date(event.start).getTime()/1000);
  var endTimestamp = Math.floor(new Date(event.end).getTime()/1000);

  async.whilst(
    function () { return minTimestamp < Math.floor(Date.now() / 1000) && minTimestamp <  endTimestamp; },
      function(requestFinishedCallback) {
        console.log('sending request to Instagram for ' + name + ' with min_timestamp: ' + minTimestamp);
        request(baseUrl + latitude + '&lng=' + longitude + '&distance=' + distance + '&min_timestamp=' + minTimestamp + '&client_id=' + clientId,
          function (error, response, body) {
            if (error) { 
              console.log('error');
              return;
            }

            //JSON object with all the info about the image
            var imageJson = JSON.parse(body);
            var images = imageJson.data;
            var numImages = images.length;
            if (numImages > 0) {
              console.log(numImages + ' images returned with starting time ' + images[(numImages - 1)].created_time + ' and ending time ' + images[0].created_time);
            }

            async.eachSeries(images, function(image, imageFinishedCallback) {

              //Save the new object to DB
              Event.findOneAndUpdate( { $and: [{latitude: latitude}, {radius: distance}] }, { $push: {'photos':
                { img: image.images.standard_resolution.url,
                  link: image.link,
                  username: image.user.username,
                  profile: image.user.profile_picture,
                  text: image.caption ? image.caption.text : '',
                  longitude: image.location.longitude,
                  latitude: image.location.latitude
                }}},
                { safe: true, upsert: false },
                function(err, model) {
                  console.log(err);  
                }
              );
              imageFinishedCallback();
            }, function(err){
                 // if any of the image processing produced an error, err would equal that error
                 if( err ) {
                   // One of the iterations produced an error.
                   // All processing will now stop.
                   console.log('Images failed to process');
                 } else {
                   console.log('Images processed');
                 }
               });
              // this works
              minTimestamp += 1000;
              // this does not
              // minTimestamp = images[0].created_time;
              if (numImages > 0) {
                console.log(numImages + 'images have been processed successfully and min_timestamp has been incremented to: ' + minTimestamp);
              }
            requestFinishedCallback();
          }
        );
         }, function(err){
               // if any of the image processing produced an error, err would equal that error
               if( err ) {
                 // One of the iterations produced an error.
                 // All processing will now stop.
                 console.log('Event failed to process');
               } else {
                 console.log(name + ' has been fully processed successfully with final min_timestamp of: ' + minTimestamp);
               }
          }
);
        seriesCallback();
         }, function(err){
                // if any of the image processing produced an error, err would equal that error
                if( err ) {
                  // One of the iterations produced an error.
                  // All processing will now stop.
                  console.log('Something failed to process');
                } else {
                  console.log('All events have been processed successfully');
                }
            }
);
});

// routes ==================================================
require('./app/routes')(app); // configure our routes

// start app ===============================================
console.log('Magic happens on port ' + port);           // shoutout to  the user
exports = module.exports = app;

Upvotes: 1

Views: 335

Answers (1)

TheIronDeveloper
TheIronDeveloper

Reputation: 3624

If you have a git repo I can look at, I can debug this much better, but... that said, I see two glaring issues:

  1. Although you are doing a async.eachSeries, you aren't waiting to finish the findOneAndUpdate call.

Your example:

Event.findOneAndUpdate( {}, {},
    function(err, model) {
        console.log(err);  
    }
);
imageFinishedCallback();

Should be turned into this:

Event.findOneAndUpdate( {}, {},
    function(err, model) {
        console.log(err);  
        imageFinishedCallback();
    }
);
  1. Similar to the first problem, but with the async.whilst callback. You are calling async.eachSeries, but are then immediately moving to the next loop.

Your code:

function (error, response, body) {
    // ...
    async.eachSeries(images, function(image, imageFinishedCallback) {/* ... */},
    function(err){
        // ...
    });

    // this works
    minTimestamp += 1000;
    // this does not
    // minTimestamp = images[0].created_time;
    if (numImages > 0) {
        console.log(numImages + 'images have been processed successfully and min_timestamp has been incremented to: ' + minTimestamp);
    }
    requestFinishedCallback();
}

Should be changed to:

function (error, response, body) {
    // ...
    async.eachSeries(images, function(image, imageFinishedCallback) {/* ... */},
    function(err){
        // ...
        console.log(numImages + 'images have been processed successfully and min_timestamp has been incremented to: ' + minTimestamp);
        minTimestamp = images[0].created_time;
        requestFinishedCallback();
    });
}

If you have a github repo to post up I can debug further, but... it looks like the issue is coming from not waiting for the async functions to finish.

Upvotes: 1

Related Questions