Nodejs - Step multiple callbacks

Question

I'm using Step to control parallel executions. What I'm trying to do is:

1. Read a json file using http (the json file contains an array of events)
2. Loop the array of events and get the event details using http
3. Store the event details in a local variable "events"

What I'm doing is someting like this:

var events = []; // this is the local variable to hold event details
Step(
	function getEvents(){
		
		var options = {
			host: '95.85.14.142',
			port: 80,
			path: '/data/IT/config.json',
			method: 'GET'
		};
		
		http.request(options,this).end();
	
	},
	function getEventsResponse(res){
		
		if (res.statusCode==200){
				
			res.setEncoding('utf8');
			res.on('data',this);
			
		}
	},
	function getEventDetails(data){
		
		var events = JSON.parse(data).events;
		
		var group = this.group();
		for(e in events){
			var event = events[e];	
			var options = {
				host: '95.85.14.142',
				port: 80,
				path: '/data/IT/events/'+event.code+'.json',
				method: 'GET'
			};
			
			http.request(options,group()).end();
		
		}	
	
	},
	function getEventDetailsResponse(err,responses){
		

		// responses = [undefined,undefined,..] ???

		
	},
	...
);

In getEventDetailsResponse I'm expecting an array of objects but I get an array of undefined variables:

[undefined,undefined,..]

What am I doing wrong?

Answer 1

Here's a working version of your code that ends in a printout of the events variable. I'll go into details below.

var http = require('http');
var Step = require('step');

var events = []; // this is the local variable to hold event details
Step(
    function getEvents(){
        var options = {
            host: '95.85.14.142',
            port: 80,
            path: '/data/IT/config.json',
            method: 'GET'
        };
        http.request(options,this).end();
    },
    function getEventsResponse(res){
        if (res.statusCode==200){
            res.setEncoding('utf8');
            res.on('data',this);
        }
    },
    function getEventDetails(data){
        var events = JSON.parse(data).events;
        var group = this.group();
        for(e in events){
            var event = events[e];
            var options = {
                host: '95.85.14.142',
                port: 80,
                path: '/data/IT/events/'+event.code+'.json',
                method: 'GET'
            };

            (function(){
                var generated_callback = group();
                http.request(options,function(res){generated_callback(null,res);}).end();
            })();
        }
    },
    function getEventDetailsResponse(err,responses){
        var group = this.group();
        for(i in responses){
            var r = responses[i];
            if (r.statusCode==200){
                r.setEncoding('utf8');
                (function(){
                    var generated_callback = group();
                    r.on('data',function(data){
                        var parsed_data = JSON.parse(data);
                        events.push(parsed_data);
                        generated_callback(null,parsed_data);
                    });
                })();
            }
        }
    },
    function displayEvents(err,passed_events){
        console.log("Events Global Variable");
        console.log(events);
        console.log("Passed Events");
        console.log(passed_events);
    }
);

This returns:

Events Global Variable
[ { title: 'TITLE 1' },
  { title: 'TITLE 2' },
  { title: 'TITLE 3' } ]
Passed Events
[ { title: 'TITLE 1' },
  { title: 'TITLE 2' },
  { title: 'TITLE 3' } ]

The issue you were running into, which I had to dig into the Step code to find out, is that the group() method generates a callback which expects the first argument to be a(possibly null) error object and the second to be the actual response/data. http.request only passes the response object back and Step was interpreting that as an error.

To work around that, you can manually call the group function to generate your callback then manually call the callback with a null first argument. Since this is all async though, you'll need to wrap that up in a self calling closure to isolate and persist the generated callback, otherwise it will get overwritten each time and when the first request returns it will try to call the callback that was generated for the third - or whatever - request instead of its own. You can see that wrapped around the http.request in my version of the getEventDetails function as well as a slightly more complex example in the getEventDetailsResponse function.

As a side note, the events variable declared at the top of the script is a global variable, not local. The one declared in getEventDetails is a local variable. If you meant to set the data onto the global variable with var events = JSON.parse(data).events; then you can do that by removing the 'var' from that line.