CODEWITHSUNDEEP
javascriptnode.jsamazon-web-servicesexpressamazon-s3
Charlie Fish
Charlie Fish

Reputation: 20566

Node.js AWS S3 Upload Never Calls End Function

I'm using this AWS S3 NPM Package to handle S3 uploads from my Express server. I'm having an issue where the uploader never calls the end function.

var key = utils.createID(Date.now());
var s3 = require('s3');
var client = s3.createClient({
  maxAsyncS3: 20,     // this is the default 
  s3RetryCount: 3,    // this is the default 
  s3RetryDelay: 1000, // this is the default 
  multipartUploadThreshold: 20971520, // this is the default (20 MB) 
  multipartUploadSize: 15728640, // this is the default (15 MB) 
  s3Options: {
    accessKeyId: "ACTUALKEYHERE",
    secretAccessKey: "ACTUALSECRETHERE",
    // any other options are passed to new AWS.S3() 
    // See: http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/Config.html#constructor-property 
  },
});
var params = {
  localFile: req.file.path,

  s3Params: {
    Bucket: "ACTUALBUCKETHERE",
    Key: key,
    // other options supported by putObject, except Body and ContentLength. 
    // See: http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#putObject-property 
  },
};
var uploader = client.uploadFile(params);
console.log(key);

uploader.on('error', function(err) {
  console.error("unable to sync:", err.stack);
});
uploader.on('progress', function() {
  console.log("progress", uploader.progressAmount, uploader.progressTotal);
});
uploader.on('end', function() {
  // **NEVER GETS CALLED**
  console.log("done uploading");
});

I have replaced the bucket name and AWS keys with other strings but other then that everything is the same. The console prints the following code.

205d3e63ed92f6edbae59465b4769e5feb2560a7
progress 16384 93619
progress 32768 93619
progress 49152 93619
progress 65536 93619
progress 81920 93619
progress 93619 93619
progress 93619 93619
progress 16384 93619
progress 32768 93619
progress 49152 93619
progress 65536 93619
progress 81920 93619
progress 93619 93619
progress 93619 93619
progress 16384 93619
progress 32768 93619
progress 49152 93619
progress 65536 93619
progress 81920 93619
progress 93619 93619
progress 93619 93619
unable to sync: RequestTimeout: Your socket connection to the server was not read from or written to within the timeout period. Idle connections will be closed.
    at Request.extractError (/Code/node_modules/s3/node_modules/aws-sdk/lib/services/s3.js:343:35)
    at Request.callListeners (/Code/node_modules/s3/node_modules/aws-sdk/lib/sequential_executor.js:100:18)
    at Request.emit (/Code/node_modules/s3/node_modules/aws-sdk/lib/sequential_executor.js:77:10)
    at Request.emit (/Code/node_modules/s3/node_modules/aws-sdk/lib/request.js:604:14)
    at Request.transition (/Code/node_modules/s3/node_modules/aws-sdk/lib/request.js:21:12)
    at AcceptorStateMachine.runTo (/Code/node_modules/s3/node_modules/aws-sdk/lib/state_machine.js:14:12)
    at /Code/node_modules/s3/node_modules/aws-sdk/lib/state_machine.js:26:10
    at Request.<anonymous> (/Code/node_modules/s3/node_modules/aws-sdk/lib/request.js:22:9)
    at Request.<anonymous> (/Code/node_modules/s3/node_modules/aws-sdk/lib/request.js:606:12)
    at Request.callListeners (/Code/node_modules/s3/node_modules/aws-sdk/lib/sequential_executor.js:104:18)
    at Request.emit (/Code/node_modules/s3/node_modules/aws-sdk/lib/sequential_executor.js:77:10)
    at Request.emit (/Code/node_modules/s3/node_modules/aws-sdk/lib/request.js:604:14)
    at Request.transition (/Code/node_modules/s3/node_modules/aws-sdk/lib/request.js:21:12)
    at AcceptorStateMachine.runTo (/Code/node_modules/s3/node_modules/aws-sdk/lib/state_machine.js:14:12)
    at /Code/node_modules/s3/node_modules/aws-sdk/lib/state_machine.js:26:10
    at Request.<anonymous> (/Code/node_modules/s3/node_modules/aws-sdk/lib/request.js:22:9)
    at Request.<anonymous> (/Code/node_modules/s3/node_modules/aws-sdk/lib/request.js:606:12)
    at Request.callListeners (/Code/node_modules/s3/node_modules/aws-sdk/lib/sequential_executor.js:104:18)
    at callNextListener (/Code/node_modules/s3/node_modules/aws-sdk/lib/sequential_executor.js:90:14)
    at IncomingMessage.onEnd (/Code/node_modules/s3/node_modules/aws-sdk/lib/event_listeners.js:183:11)
    at emitNone (events.js:91:20)
    at IncomingMessage.emit (events.js:185:7)

So it the progress indicates it has completed but the end function never gets called. Any ideas?

Upvotes: 0

Views: 839

Answers (1)

George Whitaker
George Whitaker

Reputation: 1688

Looking at your log messages it almost appears to be retrying... then timing out. Which makes me think it failing MD5 validation.

If you look at https://www.npmjs.com/package/s3#clientuploadfileparams you'll see it retries on MD5 issues. If you look at the totals reported in your logging you'll also see that it hits the total at least 3 times which matches the retry count you have set in your client params.

That Node module supports logging another value uploader.progressMd5Amount, try adding that to your logger. I wonder if those amounts will report differently. And try checking the contents of req.file.path.

Upvotes: 1

Related Questions