expressjs stress testing

Question

I am having quite a difficult time trying to identify a performance bottleneck between my webserver (Cherokee) and expressjs websites.

Due to a growing amount of POST traffic, we are starting to see performance issues from time -to-time. So I designed a test to see what the breaking point of 1, 2, 3 and 4 instances of the website would be and was shocked by poor performance (working well at 5 per second and very poorly at 10 per second).

According to my performance logging a full request start to end takes begins at between 1000-4000 ms, and after a short time starts increase with a linear trend so that after about 5 minutes the requests are taking 40,000-60,0000 ms. Performance logging in the expressjs website records that start to end a request takes between 100-400 ms (most of the time 90-390 ms of this time is spent waiting for a Java API to respond).

I have investigated connection pooling/max sockets, however do not understand how to configure this using express, and with ever growing traffic am starting to feel the heat from every reducing performance of the wesite.

Right now I have two theories: 1. There is something really wrong with the stress testing code; 2. There is a small connection pool either when sending requests from the stress testing app or within expressjs.

If either of the above are true I have no idea on how to resolve these issues.

-- stress testing app --

/*
 * POST stress testing
 */
var rest            = require( "restler");
var u               = require( "../underscore")._;
var uuid            = require( "node-guid" );
var fs              = require( "fs" );
var quoteContent    = fs.readFileSync( __dirname + "/data/quoteList.json" );
var quotes          = JSON.parse( quoteContent ).quotes;
var feedListContent = fs.readFileSync( __dirname + "/data/feedList.json" );
var feeds           = JSON.parse( feedListContent ).feeds;
var feedLength      = feeds.length;

var experiment      = { "users": 2, "frequency": 1000 };

// Clear memory out  bit
quoteContent        = null;
feedListContent     = null;

console.log("Concurrent users:", experiment.users);
console.log("Every          :", (experiment.frequency / 1000) + " seconds" );

setInterval(
    function()
        {
            var rndQuote;

            for(i=0;i<experiment.users;i++)
                {
                    delayPost( (experiment.frequency) / (experiment.users + 1) );
                }
        }
, experiment.frequency);

function delayPost( delay )
    {
        setTimeout(
            function()
                {
                    rndQuote = getRandomQuote();
                    postToRandomFeed( rndQuote );
                }
        , delay );
    }

function getRandomQuote()
    {
        var n = Math.floor( Math.random() * feedLength );
        return quotes[n];
    }

function postToRandomFeed( theQuote )
    {
        var n           = Math.floor( Math.random() * feedLength );
        var feed        = feeds[n];

        var     kloudId = feed.kloudId
            ,   feedId  = feed._id
            ,   uid     = feed.owner["$oid"]
            ,   url     = "quote://" + uuid.new() // Generate random URL
            ,   quote   = theQuote.quote
            ,   author  = theQuote.author;

        var     data    = { "url" : url, "title": quote, "text": quote, "tags": [author], "feeds": [ feedId ] }
            ,   postTo  = [ "http://127.0.0.1:4100/api/content/post/" + uid + "/wintermead" /*, "http://127.0.0.1:4101/api/content/post/" + uid + "/wintermead"*/ ]
            ,   ports = [ 4100 ];

        var st = gt();

        var postUrl = postTo[ Math.floor(Math.random()*postTo.length) ];

        console.log( postUrl );

        rest.post( postUrl,
            {
                  "data": data
                , "headers":
                    { "kloudid": kloudId, 'Connection':'close', 'Connection-Timeout': 10 }
            }
        ).on('complete',
            function( data, response )
                {
                    gt( "end: ",st );
                    if (response.statusCode == 202 ||response.statusCode == 201 || response.statusCode == 200)
                        {
                            try {
                                data =JSON.parse( data );
                            } catch(e) {}

                            if( data && data.error )
                                console.log( "1", data.message );
                            else
                                console.log( "2", data._id );
                            // you can get at the raw response like this...
                        }
                    else
                        {
                            console.log( "! Error ", response.statusCode, data );
                        }
                }
        );
    }

function gt( name, lastTime )
    {
        if( !name || name=="start" )
            {
                var d = new Date().getTime();
                var uu = uuid.new();
                return (d + "|" + uu);
            }
        else if( lastTime )
            {
                var props   = lastTime.split("|");
                var time    = parseInt(props[0]);
                var uu      = props[1];
                console.log( uu, name, ( new Date().getTime() - time ));
            }
        else
            {
                console.log( "logging: invalid start time  " + name);
            }
    }

Answer 1

Have you looked at using existing stress testing tools like ab (apache bench) or siege?

I know it's tempting to write your own, but existing ones are solid and you can focus on fixing your site instead of your load-testing tool.