Two HTTP Requests issue in nodejs

Question

I have created a function on Amazon Lambda. My scenario is that i have to send two HTTP requests simultaneously on two different URL's. So i have write two HTTP requests in my code with different URL's. The problem is that most of the time the first HTTP request is not called and 2nd HTTP request almost runs all the time. So please help me to figure out the problem. Here is my code

console.log('Loading function');

var aws = require('aws-sdk');
var http = require('http');
var s3 = new aws.S3({ apiVersion: '2006-03-01' });

exports.handler = function(event, context) {
    console.log('Received event:', JSON.stringify(event, null, 2));

    // Get the object from the event and show its content type
    var bucket = event.Records[0].s3.bucket.name;
    var key = event.Records[0].s3.object.key;
    var params = {
        Bucket: bucket,
        Key: decodeURIComponent(key)
    };
   s3.getObject(params, function(err, data) {
        if (err) {
            console.log("Error getting object " + key + " from bucket " + bucket +
                ". Make sure they exist and your bucket is in the same region as this function.");
            context.fail("Error getting file: " + err);
        } else {
            var userString = JSON.stringify(params);
            console.log(userString);

            var options = {
              hostname: 'example1.com',
              path: '/api/test1.cfm',
              method: 'POST',
              Port:'80',
              headers: {
                    'Content-Type': 'application/x-www-form-urlencoded',
                    'Content-Length': userString.length
              }
            };
            console.log("Start");
            var x = http.request(options,function(res){
                console.log("Connected");
                console.log('CONTENT TYPE:', data.ContentType);
                console.log("Start");
                console.log("x");
                context.succeed(data.ContentType);

            });

            x.write(userString);
            x.end();



            var optionsdev = {
              hostname: 'example2.com',
              path: '/api/test2.cfm',
              method: 'POST',
              headers: {
                    'Content-Type': 'application/x-www-form-urlencoded',
                    'Content-Length': userString.length
              }
            };
            console.log("Start");
            var y = http.request(optionsdev,function(res){
                console.log("Connected");
                console.log('CONTENT TYPE:', data.ContentType);
                console.log("Start");
                console.log("y");
                context.succeed(data.ContentType);

            });

            y.write(userString);
            y.end();



        }
    });
};

Answer 1

First off, if you look at the sample code in the node.js doc for http.request(), you get a callback, but in that callback, you are passed the response object and it's a stream that you have to set event handlers on.

Here's the basic structure from the documentation:

var req = http.request(options, function(res) {
  console.log('STATUS: ' + res.statusCode);
  console.log('HEADERS: ' + JSON.stringify(res.headers));
  res.setEncoding('utf8');
  res.on('data', function (chunk) {
    console.log('BODY: ' + chunk);
  });
  res.on('end', function() {
    console.log('No more data in response.')
  })
});

req.on('error', function(e) {
  console.log('problem with request: ' + e.message);
});

// write data to request body
req.write(postData);
req.end();

So, you have to process the response when you receive the 'data' and 'end' events on the response object.

For most requests, this is kind of inconvenient as you have to accumulate the response because the data event can be called more than one. What I would suggest is using the request module as it does more of the work for you.

Then, secondly if you want your operations to be processed serially one after the other, you need to not start the second operation until the first one has completed. There are many different ways to structure that, but the one most like the structure you already have is to just start the second request inside the completion of the first request.

Here's a way to do this using the request() module:

var aws = require('aws-sdk');
var http = require('http');
var s3 = new aws.S3({ apiVersion: '2006-03-01' });
var request = require('request');

exports.handler = function(event, context) {
    console.log('Received event:', JSON.stringify(event, null, 2));

    // Get the object from the event and show its content type
    var bucket = event.Records[0].s3.bucket.name;
    var key = event.Records[0].s3.object.key;
    var params = {
        Bucket: bucket,
        Key: decodeURIComponent(key)
    };
   s3.getObject(params, function(err, data) {
        if (err) {
            console.log("Error getting object " + key + " from bucket " + bucket +
                ". Make sure they exist and your bucket is in the same region as this function.");
            context.fail("Error getting file: " + err);
        } else {
            var sendData = JSON.stringify(params);
            request.post('http://example1.com/api/test1.cfm', {form: sendData}, function(err, httpResponse, body) {
                if (err) {
                    console.log("Error on first post");
                } else {
                    // do something with the response here
                    console.log("first request = ", body);
                    // context.succeed(...);

                    // now launch the second request
                    request.post('http://example2.com/api/test2.cfm', {form: sendData}, function(err, httpResponse, body) {
                        if (err) {
                            console.log("Error on second post");
                        } else {
                            // do something with the response here
                            console.log("second request = ", body)
                            // context.succeed(...);

                            // done with both requests here

                        }
                    });
                }
            });
        }
    });
};