CODEWITHSUNDEEP
node.jsamazon-s3metadataaws-lambda
ian
ian

Reputation: 1002

Getting s3 object metadata then creating stream

I'm downloading an object from s3 and creating a read stream object from it to process a video:

s3.getObject(params).createReadStream()

However, I need to get the metadata from it which is possible when i just get the object by accessing its 'metadata' property:

s3.getObject()

How would I either:

  1. Get the object via s3.getObject(), grab the metadata from its metadata property, and then turn it into a read stream?

    var stream = fs.createReadStream(response); isn't working - input must be a string

-- OR --

  1. Get the stream via s3.getObject().createReadStream(), and extract the metadata from the stream?

    To my knowledge metadata isn't passed within streams.

Tell me if my assumptions are wrong, but I am currently stuck with these two needs:

Upvotes: 5

Views: 2602

Answers (2)

cementblocks
cementblocks

Reputation: 4596

You can get the metadata via the request's httpHeaders event.

let fs = require('fs')
let aws = require('aws-sdk')
let s3 = new aws.S3()

let request = s3.getObject({
    Bucket: 'my-bucket',
    Key: 'my-key'
})

let stream

request.on('httpHeaders', (statusCode, httpHeaders) => {
    // object metadata is represented by any header in httpHeaders starting with 'x-amz-meta-'
    // you can use the stream object that this point
    stream.pipe(fs.createWriteStream('./somepath'))
    stream.on('end', () => { 
        console.log('were done')
    })
})

stream = request.createReadStream()

Alternatively you can also call s3.headObject to get the metadata without downloading the object and then download the object using s3.getObject

Upvotes: 9

ian
ian

Reputation: 1002

So I kind of found a solution. This works for most files under 10 MB. If they are larger than that the buffer stream ends before the file is done being written. I've tried putting the bufferStream.end inside of the on finish function but then my call back doesnt go through....

function download(s3Event, srcKey, cb){

  console.log('Starting download');

  s3.getObject({
      Bucket: s3Event.bucket.name,
      Key: srcKey
  }, cb);
}



function writeToFile(data, cb){

  var dlFile = path.join(tempDir, 'download');

  console.log('data = ', data);
  console.log('data.Body = ', data.Body);
  var stream = bufferStream.pipe(fs.createWriteStream(dlFile)).on('finish', function () {
     console.log('finished writing stream');
     cb(null, data);
  });

  bufferStream.end(data.Body);
}


exports.handler = function(event, context) {

// Read options from the event.

console.log("Reading options from event:\n", util.inspect(event, {depth: 5}));

var s3Event = event.Records[0].s3;
var srcKey = decodeURIComponent(s3Event.object.key);
var keyPrefix = srcKey.replace(/\.[^/.]+$/, '');         
var dstBucket = "jump-lambda";


async.waterfall([

    function (cb){
        download(s3Event, srcKey, cb);
    },

    function (data, cb){
        writeToFile(data, cb);
    },

    function (data, cb){
        fluentffmpegProcess(data, cb);
    },

    function (data, cb){
        transform(data, cb);
    },

    function (data, buffer, cb){
        thumbnailUpload(data, buffer, dstBucket, keyPrefix, cb);
    },

    function (data, cb){
        updateParse(data, srcKey, keyPrefix, cb);
    },

],

    function (err) {
        if (err) {
            console.error(
                'Unable to convert video to scene object, with error code: ' + err.description
            );
        } else {
            console.log(
                'Successfully created scene object, updated venue, and created thumbnail'
            );
        }
    }
);
};

Upvotes: 0

Related Questions