AWS Lambda Get Image and Upload to S3

Question

I am working in a AWS Lambda function. I am successfully making an API call to the NASA APOD and getting back the values. I want to take the url for the image and download that image and then upload into S3. I am getting an error when I try to access the "test.jpg" image, "Error: EACCES: permission denied, open 'test.jpg'". If I move the S3bucket.putObject outside the http.request, I get data is equal to null. I know I am missing something simple. Thought?

function GetAPOD(intent, session, callback) {
    var nasa_api_key = 'demo-key'
    ,   nasa_api_path = '/planetary/apod?api_key=' + nasa_api_key;

    var options = {
        host: 'api.nasa.gov',
        port: 443,
        path: nasa_api_path,
        method: 'GET'
    };

    var req = https.request(options, function (res) {
        res.setEncoding('utf-8');

        var responseString = '';
        res.on('data', function (data) {
            responseString += data;
        });

        res.on('end', function () {
            console.log('API Response: ' + responseString);

            var responseObject = JSON.parse(responseString)
            ,   image_date = responseObject['date']
            ,   image_title = responseObject['title']
            ,   image_url = responseObject['url']
            ,   image_hdurl = responseObject['hdurl']
            ,   image_desc = responseObject['explanation'];

            var s3Bucket = new AWS.S3( { params: {Bucket: 'nasa-apod'} } );

            var fs = require('fs');

            var file = fs.createWriteStream("test.jpg");
            var request = http.get(image_url, function(response) {
                response.pipe(file);

                var data = {Key: "test.jpg", Body: file};
                s3Bucket.putObject(data, function(err, data) {
                    if (err) {
                        console.log('Error uploading data: ', data); 
                    }
                    else {
                        console.log('succesfully uploaded the image!');
                    }
                });
            });

        });
    });

    req.on('error', function (e) {
        console.error('HTTP error: ' + e.message);
    });

    //req.write();
    req.end();
}

Answer 1

I got it!! Thank you Mark B for the help. I was able to get the data from the stream without saving it locally and then writing to the bucket. I did have to change my IAM role to allow the putObject for S3.

function GetAPOD(intent, session, callback) {
    var nasa_api_key = 'demo-key'
    ,   nasa_api_path = '/planetary/apod?api_key=' + nasa_api_key;

    var options = {
        host: 'api.nasa.gov',
        port: 443,
        path: nasa_api_path,
        method: 'GET'
    };

    var req = https.request(options, function (res) {
        res.setEncoding('utf-8');

        var responseString = '';
        res.on('data', function (data) {
            responseString += data;
        });

        res.on('end', function () {
            // console.log('API Response: ' + responseString);

            var responseObject = JSON.parse(responseString)
            ,   image_date = responseObject['date']
            ,   image_title = responseObject['title']
            ,   image_url = responseObject['url']
            ,   image_hdurl = responseObject['hdurl']
            ,   image_desc = responseObject['explanation'];

            var image_name = image_date + '.jpg';

            var s3 = new AWS.S3();
            var s3Bucket = new AWS.S3( { params: {Bucket: 'nasa-apod'} } );

            var request = http.get(image_url, function(response) {
                var image_stream = null;
                response.on('data', function (data) {
                    image_stream = data;
                });

                response.on('end', function () {
                    var param_data = {Key: image_name, Body: image_stream, ContentType: "image/jpeg", ContentLength: response.headers['content-length']};
                    s3Bucket.putObject(param_data, function(err, output_data) {
                        if (err) {
                            console.log('Error uploading data to S3: ' + err); 
                        }
                    });
                });
            });
            request.end();

        });
    });

    req.on('error', function (e) {
        console.error('HTTP error: ' + e.message);
    });

    req.end();
}

Answer 2

You need to be writing the file to /tmp. That's the only directory in the Lambda environment that you will have write access to.