CODEWITHSUNDEEP
node.jsamazon-web-services
Nathan
Nathan

Reputation: 7709

Trying to download a file from aws s3 into nodejs writeStream

I am trying to download a file from s3 and directly put into into a file on the filesystem using a writeStream in nodejs. This is my code:

downloadFile = function(bucketName, fileName, localFileName) {
    //Donwload the file
    var bucket = new AWS.S3({
        params: { Bucket: bucketName },
        signatureVersion: 'v4'
    });
    var file = require('fs').createWriteStream(localFileName);
    var request = bucket.getObject({ Key: fileName });
    request.createReadStream().pipe(file);
    request.send();
    return request.promise();
}

Running this function I get this error:

Uncaught Error: write after end

What is happening? Is the file closed before the write is finished? Why?

Upvotes: 3

Views: 16025

Answers (3)

cameck
cameck

Reputation: 2098

Also AWS notes an example of using promises like this:

const s3 = new aws.S3({apiVersion: '2006-03-01'});
const params = { Bucket: 'yourBucket', Key: 'yourKey' };
const file = require('fs').createWriteStream('./local_file_path');

const s3Promise = s3.getObject(params).promise();

s3Promise.then((data) => {
  file.write(data.Body, () => {
    file.end();
    fooCallbackFunction();
  });
}).catch((err) => {
  console.log(err);
});

This works perfect for me.
https://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/using-promises.html

EDIT: (15 Feb 2018) Updated the code, as you have to end the write stream (file.end()).

Upvotes: 4

loretoparisi
loretoparisi

Reputation: 16281

I have combined the above response with a typical gunzip operation in pipe:

            var s3DestFile = "./archive.gz";
            var s3UnzippedFile    = './deflated.csv';

            var gunzip = zlib.createGunzip();
            var file = fs.createWriteStream( s3DestFile );
            s3.getObject({ Bucket: Bucket, Key: Key })
            .on('error', function (err) {
                console.log(err);
            })
            .on('httpData', function (chunk) {
                file.write(chunk);
            })
            .on('httpDone', function () {
                file.end();

                console.log("downloaded file to" + s3DestFile);
                fs.createReadStream( s3DestFile )
                .on('error', console.error)
                .on('end', () => {
                    console.log("deflated to "+s3UnzippedFile)
                })
                .pipe(gunzip)
                .pipe(fs.createWriteStream( s3UnzippedFile ))
            })
            .send();

Upvotes: 0

KibGzr
KibGzr

Reputation: 2093

var s3 = new AWS.S3({
    accessKeyId: accessKeyId,
    secretAccessKey: secretAccessKey
}),
file = fs.createWriteStream(localFileName);
s3
.getObject({
    Bucket: bucketName,
    Key: fileName
})
.on('error', function (err) {
    console.log(err);
})
.on('httpData', function (chunk) {
    file.write(chunk);
})
.on('httpDone', function () {
    file.end();
})
.send();

Upvotes: 10

Related Questions