How do you stream an upload to S3 via node.js and show progress to user?

Question

The goal is to:

1. Be able to upload a file from browser
2. Be able to stream the upload directly to S3
3. Relay the upload progress to browser

It took me a while to figure this out (and didn't quite find an answer to do all 3 on StackOverflow yet) so posting my answer in case anyone else has the same problem.

I'm open to suggestions to better solutions (the speeds on mine are pretty horrible).

Answer 1

On Frontend HTML

<input type="file" class="file" name="files[]">

On Frontend JS

require('blueimp-file-upload');
var socket = require('socket.io-client');

var fileSize;
var socketId;

// Socket will be used to transmit the percent uploaded from server
socket.connect()
  // when the socket gives us a unique ID we will save it so that we
  // can identify the client
  .on('id', function (data) {
    socketId = data;
  })

  .on('uploadProgress', function (data){
    // you can show a progress bar with this percentage 
    var pct = data.loaded / fileSize;
  });

$('.file')
    .change(function (){
        // save the file size so that we can calculate perfectage
        fileSize = this.files[0].size;  
    })
    .fileupload({
        dataType: 'json',
    })
    .fileupload('option', {
        url: '/upload?socketId=' + socketId
    });

On Server

var Busboy = require('busboy');
var AWS = require('aws-sdk');
var socket = require('socket.io');
var express = require('express');
var http = require('http');

// Set up Express
var app = express();
var server = http
    .Server(app)
    .listen(80);

var io = socket(server);

// keeps track of all the open sockets
var sockets = {};

io.on('connection', function (socket) {
  var uniqueId = _.random(0, 100000000);
  app.socket[uniqueId] = socket;
  socket.emit('id', uniqueId);
});

// Set up upload route
app.post('/upload', function (req, res) {
        AWS.config.update({
            appKey: '',
            jsKey: '',
        });
        var s3 = new AWS.S3();

        var busboy = new Busboy({
            headers: req.headers 
        });

    busboy.on('file', function(fieldname, file, filename) {
        s3.upload({
            Bucket: 'bucketname',
            Key: new Date().getTime() + filename,
            Body: file //stream
        }, function(err, file){
            res.json({
                success: true
            });
        }).on('httpUploadProgress', function(evt) { 
            //emit progress
            sockets[req.query.socketId].emit('uploadProgress', evt);
        });
    });

    req.pipe(busboy);
});