issue using async module in Node.js

Question

Having an issue, appreciate any help.

I'm trying to gather all my async functions together. Tried async.parallel, async.each, gather-gm. Nothing makes the final callback work. Here's the updated code(but yet not working properly):

var calls = [];

async.each(parser.allHrefs,function (href,callback) {

    getHtml(href,function(err, add){
        console.log("Passing data: " + href);

        if(err){
            return callback(err);
        };

        if(add){
            calls.push(href);
        };

        return callback();
    });
}, function (err) {
    if(err){
        console.log('something went wrong');
    }else{
        console.log('finished');
    };
}); 

And the first function:

function getHtml(link, callback) {    
  httpreq.get(link, function(err, res) {

    if(err) {            
      return callback(err);      
    }

    if(res.statusCode >= 300) {      
      return callback(null, false);       
    } else {
      //cut parsing code here...
      return callback(null, true);          
    }
  });
}

p.s.:I've updated the code couple times. In this example i've tried to use async.parallel. And the thing is when i even get no errors, i still dont get to the "getLocations" function. Code looks much more better than first versions, but still refuses to work correctly.

Answer 1

Instead of deleting the parser.allHrefs while you go through it. You should refactor de getHtml just to know if you can add the link or not. Like this:

function getHtml (link, callback) {    
    httpreq.get(link, function(err, res) {
        //the request has already finished executing here
        if(err) {      
            return callback(err);      
        }

        if(res.statusCode >= 300) {
            return callback(null,false);
        } else {
            //cut parsing code here...
            return callback(null,true);
        }
    });
};

This way when you call the gatherSecondLevelData function you check if you add the link or not like this:

function gatherSecondLevelData (err) {
    var calls = [];

    async.each(parser.allHrefs,function (href,callback) {

        getHtml(href,function(err, add){

            if(err){
                return callback(err);
            };

            if(add){
                calls.push(href);
            };

            return callback();
        });
    }, function (err) {
        if(err){
            console.log('something went wrong');
        }else{
            console.log('finished');
        };
    });
};

Answer 2

There are several issues in your gethtml function you need to fix.

Make sure that everywhere you have now return, you call the callback, e.g.

return callback(err)  

when you want to communicate an error or

return callback(null,  result) 

when you want to communicate success and return a result. Never return without calling the callback.