CODEWITHSUNDEEP
node.jsmongodbmongoose
ChickenWing24
ChickenWing24

Reputation: 205

How to handle callbacks in a for loop(Node.JS)

I am trying to write a code with NodeJS where I grab data from an external API and then populate them in MongoDB using Mongoose. In between that, I'll check to see if that particular already exists in Mongo or not. Below is my code.

router.route('/report') // the REST api address
  .post(function(req, res) // calling a POST 
  {
    console.log('calling report API');
    var object = "report/" + reportID; // related to the API
    var parameters = '&limit=100' // related to the API
    var url = link + object + apiKey + parameters; // related to the API

    var data = "";
    https.get(url, function callback(response)
    {
      response.setEncoding("utf8");
      response.on("data", function(chunk)
      {
        data += chunk.toString() + ""; 
      });

      response.on("end", function()
      {
        var jsonData = JSON.parse(data);
        var array = jsonData['results']; // data is return in array of objects. accessing only a particular array
        var length = array.length;
        console.log(length);

        for (var i = 0; i < length; i++) 
        {
          var report = new Report(array.pop()); // Report is the schema model defined. 
          console.log('^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^');
          console.log(i);
          console.log('*****************************');
          console.log(report);
          console.log('*****************************');
          // console.log(report['id']);

          /*report.save(function(err)
          {
            if(err)
              res.send(err);
          });*/

          Report.find({id:report['id']}).count(function(err, count) // checks if the id of that specific data already exists in Mongo
          {
            console.log(count);
            console.log('*****************************');
            if (count == 0) // if the count = 0, meaning not exist, then only save
            {
              report.save(function(err)
              {
                console.log('saved');
                if(err)
                  res.send(err);
              });
            }
          });
        };
        res.json({
                    message: 'Grabbed Report'
                  }); 
      });
      response.on("error", console.error);
    });
  })

My problem is that since NodeJS callbacks are parallel, it is not getting called sequentially. My end result would be something like this :

  1. Calling report API
  2. console.log(length) = 100
  3. ^^^^^^^^^^^^^^^^^^^^^^^^
  4. console.log(i) = starts with 0
  5. *******************************
  6. console.log(report) = the data which will be stored inside Mongo
  7. *******************************
  8. number 3 - 7 repeats 100 times as the length is equals to 100
  9. console.log(count) = either 0 or 1
  10. number 9 repeats 100 times
  11. console.log('saved')
  12. number 11 repeats 100 times
  13. Lastly, only the last out of 100 data is stored into Mongo

What I need is some sort of technique or method to handle these callbacks which are executing one after the other and not sequentially following the loop. I am pretty sure this is the problem as my other REST APIs are all working.

I have looked into async methods, promises, recursive functions and a couple others non which I could really understand how to solve this problem. I really hope someone can shed some light into this matter.

Feel free also to correct me if I did any mistakes in the way I'm asking the question. This is my first question posted in StackOverflow.

Upvotes: 0

Views: 899

Answers (2)

laggingreflex
laggingreflex

Reputation: 34627

This problem is termed as the "callback hell". There's lots of other approaches like using Promise and Async libraries you'll find.

I'm more excited about the native async ES7 will bring, which you can actually start using today with transpiler library Babel.

But by far the simplest approach I've found is the following: You take out the long callback functions and define them outside. 

router.route('/report') // the REST api address
    .post(calling_a_POST)

function calling_a_POST(req, res) {
    ...
    var data = "";
    https.get(url, function callback(response) {
        ...
        response.on("end", response_on_end_callback); // --> take out
        response.on("error", console.error);
    });
}

function response_on_end_callback() {                 // <-- define here
    ...
    for (var i = 0; i < length; i++) {
        var report = new Report(array.pop());
        ...
        Report.find({ id: report['id'] })
              .count(Report_find_count_callback);     // --> take out
    };
    res.json({
        message: 'Grabbed Report'
    });
}

function Report_find_count_callback(err, count) {     // <-- define here
    ...
    if (count == 0) {
        report.save(function(err) {                   // !! report is undefined here
            console.log('saved');
            if (err)
                res.send(err);                        // !! res is undefined here
        });
    }
}

A caveat is that you won't be able to access all the variables inside what used to be the callback, because you've taken them out of the scope.

This could be solved with a "dependency injection" wrapper of sorts to pass the required variables.

router.route('/report') // the REST api address
    .post(calling_a_POST)

function calling_a_POST(req, res) {
    ...
    var data = "";
    https.get(url, function callback(response) {
        ...
        response.on("end", function(err, data){       // take these arguments
            response_on_end(err, data, res);          // plus the needed variables
        });
        response.on("error", console.error);
    });
}

function response_on_end(err, data, res) {  // and pass them to function defined outside
    ...
    for (var i = 0; i < length; i++) {
        var report = new Report(array.pop());
        ...
        Report.find({ id: report['id'] })
            .count(function(err, count){
                Report_find_count(err, count, report, res);  // same here
            });
    };
    res.json({                                        // res is now available
        message: 'Grabbed Report'
    });
}

function Report_find_count(err, count, report, res) {        // same here
    ...
    if (count == 0) {
        report.save(function(err) {                   // report is now available
            console.log('saved');
            if (err)
                res.send(err);                        // res is now available
        });
    }
}

When I execute the response_on_end function, I am getting the undefined:1 unexpected token u error. I am pretty much sure it has something to do with this line: var jsonData = JSON.parse(data) My response_on_end is as below: var jsonData = JSON.parse(data); // problem here

I realize I made an error here:

function calling_a_POST(req, res) {
    ...
    var data = "";
    https.get(url, function callback(response) {
        ...
        //sponse.on("end", function(err, data){
        response.on("end", function(err){ // data shouldn't be here
            response_on_end(err, data, res);
        });
        response.on("error", console.error);
    });
}

Another problem I could forsee, which actually may not arise here but still would be better to talk about anyways. The data variable, since it's a string which is a primitive type unlike an object, it is "passed by value". More info

It's better to wrap the variable in an object and pass the object, because objects in javascript are always "passed by reference".

function calling_a_POST(req, res) {
    ...
    // var data = ""; // 
    var data_wrapper = {};
    data_wrapper.data = {};                                // wrap it in an object
    https.get(url, function callback(response) {
        ...
        response.on("data", function(chunk){
            data_wrapper.data += chunk.toString() + "";   // use the dot notation to reference
        });
        response.on("end", function(err){ 
            response_on_end(err, data_wrapper, res);      // and pass that object
        });
        response.on("error", console.error);
    });
}

function response_on_end_callback(err, data_wrapper, res) {
    var data = data_wrapper.data;                         // later redefine the variable
    ...
    for (var i = 0; i < length; i++) {
        var report = new Report(array.pop());
        ...

Upvotes: 5

Edgar
Edgar

Reputation: 1313

You can use async library for controlling your execution flows. And there are also iterators for working with arrays.

Upvotes: 0

Related Questions