CODEWITHSUNDEEP
javascriptnode.jsamazon-s3
Augustin Riedinger
Augustin Riedinger

Reputation: 22270

Can't download files from S3 using nodeJS: few files don't finish

This is driving me nuts.

I need my server to download 100 relatively small files files from S3 (max 2 Mb).

It always work for ~95% of the files, but it blocks for the last 6-8 files. The resolve nor the reject callback gets never called...

I tried to download files in parallel ...

Here's the code, which technically works in 95% of the cases :

  let singleGetFromS3 = (bucket, fileName) => {
    return new Promise((resolve, reject) => {
      let extension = getFileNameExtension(fileName);
      fs.stat(`./${fileName}`, (err, stat) => {
        if (err === null) {
          console.log(`${fileName} exists locally`);
          resolve(fileName);
        } else if(err.code === 'ENOENT') {
          let params = {Bucket: bucket, Key: fileName};
          let file = require('fs').createWriteStream(`./tmp-${fileName}`);
          s3.getObject(params).createReadStream()
            .on('error', (error) => { return reject(error); })
            .on('end', () => {
              fs.rename(`./tmp-${fileName}`, `./${fileName}`, reject);
              return resolve(fileName);
            })
            .pipe(file);
        } else {
          reject(err);
        }
      });
    });
  };

Using:

"aws-sdk": "^2.23.0",
node --version
v4.5.0

Upvotes: 1

Views: 612

Answers (1)

peteb
peteb

Reputation: 19428

Looks like you are listening for the wrong event for when to resolve. You don't want to resolve when all bytes have been read from S3's Readable, you want to resolve when file is done writing.

let singleGetFromS3 = (bucket, fileName) => {
    return new Promise((resolve, reject) => {
      let extension = getFileNameExtension(fileName);
      fs.stat(`./${fileName}`, (err, stat) => {
        if (err === null) {
          console.log(`${fileName} exists locally`);
          resolve(fileName);
        } else if(err.code === 'ENOENT') {
          let params = {Bucket: bucket, Key: fileName};
          let file = require('fs').createWriteStream(`./tmp-${fileName}`);    

          // Listen for the file to be done writing, then resolve
          file.on('finish', () => {
              fs.rename(`./tmp-${fileName}`, `./${fileName}`, reject);
              return resolve(fileName);
            })

          s3.getObject(params).createReadStream()
            .on('error', (error) => { return reject(error); })
            .pipe(file);
        } else {
          reject(err);
        }
      });
    });
  };

Upvotes: 3

Related Questions