Append string to a text file Nodejs in AWS Lambda

Question

The scenario: A text file snapshot-ids.txt is located in a S3 bucket. I'm trying to create a Lambda function that run daily (Cron) that would use AWS CLI to take snapshot of a volume, then save that snapshotId to a text file in S3. On the next time another snapshot is created, the new snapshotId will be saved to the same text file on S3. The text file is a place holder for snapshotIds and when it reaches a threshold, it will delete the top snapshotIds and add the new one at the end (FIFO pipe).

For people who don't use AWS lambda, my question is what's the quickest way to append text to a variable and return the new variable with new lines in it.

For people who know Lambda, this is the basic code from AWS Lambda I have, I use fs.appendFile, but how do I use the file I got from s3.getObject() and eventually pass it to s3.putObject()?

EDIT: this is my progress:

console.log('Loading function');

var aws = require('aws-sdk');
var s3 = new aws.S3({ apiVersion: '2006-03-01' });
var fs = require('fs');

exports.handler = function(event, context) {
    //console.log('Received event:', JSON.stringify(event, null, 2));

    // Get the object from the event and show its content type
    var bucket = event.Records[0].s3.bucket.name;
    var key = decodeURIComponent(event.Records[0].s3.object.key.replace(/\+/g, ' '));
    var params = {
        Bucket: bucket,
        Key: key
    };
    s3.getObject(params, function(err, data) {
        if (err) {
            console.log(err);
            var message = "Error getting object " + key + " from bucket " + bucket +
                ". Make sure they exist and your bucket is in the same region as this function.";
            console.log(message);
            context.fail(message);
        } else {
            // fs.appendFile('snapshot-ids.txt', 'snap-001', function (err) {
            //     if (err) throw err;
            //     console.log('The "data to append" was appended to file!');
            // });
            console.log(params_new);
            console.log('CONTENT TYPE getObject:', data.ContentType);
            // context.succeed(data.Body.toString('ascii'));
        }
    });
    var params_new = {
        Bucket: bucket,
        Key: key,
        Body: 'snap-002'
    };
    s3.putObject(params_new, function(err, data) {
                console.log('put here');
                if (err) {
                    console.log(err);
                    var message = "Error getting object " + key + " from bucket " + bucket +
                        ". Make sure they exist and your bucket is in the same region as this function.";
                    console.log(message);
                    context.fail(message);
                } else {
                    console.log('CONTENT TYPE putObject:', data.ContentType);
                    context.succeed(data.ContentType);
                }
    });
};

Answer 1

A couple of things I noticed with your code so far...

1. You can't call s3.putObject until s3.getObject is finished and you have the file from s3.

2. You aren't dealing with the file system since you get the data from s3.getObject.

With those things in mind I modified your code(I haven't tried this but it should get you going in the right direction):

console.log('Loading function');

var aws = require('aws-sdk');
var s3 = new aws.S3({ apiVersion: '2006-03-01' });

exports.handler = function(event, context) {
    //console.log('Received event:', JSON.stringify(event, null, 2));

    // Get the object from the event and show its content type
    var bucket = event.Records[0].s3.bucket.name;
    var key = decodeURIComponent(event.Records[0].s3.object.key.replace(/\+/g, ' '));
    var params = {
        Bucket: bucket,
        Key: key
    };
    s3.getObject(params, function(err, data) {
        if (err) {
            console.log(err);
            var message = "Error getting object " + key + " from bucket " + bucket +
                ". Make sure they exist and your bucket is in the same region as this function.";
            console.log(message);
            context.fail(message);
        } else {
            console.log(params_new);
            console.log('CONTENT TYPE getObject:', data.ContentType);

            // convert body(file contents) to a string so we can append
            var body = data.Body.toString('utf-8');
            // append data
            body += 'snap-001\n';

            var params_new = {
                Bucket: bucket,
                Key: key,
                Body: body
            };
            //NOTE this call is now nested in the s3.getObject call so it doesn't happen until the response comes back
            s3.putObject(params_new, function(err, data) {
                        console.log('put here');
                        if (err) {
                            console.log(err);
                            var message = "Error getting object " + key + " from bucket " + bucket +
                                ". Make sure they exist and your bucket is in the same region as this function.";
                            console.log(message);
                            context.fail(message);
                        } else {
                            console.log('CONTENT TYPE putObject:', data.ContentType);
                            context.succeed(data.ContentType);
                        }
            });

        }
    });

};

Something else to keep in mind is if you have more than 1 of this Lambda running at the same time it is likely they will stomp on each others changes. Sounds like you will just be scheduling it once a day so it shouldn't be a big deal but its worth noting.