NGINX 502 under high load with NodeJS

Question

Here is my nginx conf file for my site:

server {
    listen 80;
    listen [::]:80;

    server_name domain.com;

    proxy_http_version 1.1;
    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
    proxy_set_header Upgrade $http_upgrade;
    proxy_set_header Connection 'upgrade';
    proxy_set_header Host $host;
    proxy_cache_bypass $http_upgrade;
    proxy_intercept_errors on;

    location / {
        proxy_pass http://127.0.0.1:10000;
    }
}

While using nginx under high-load performance testing (1200 RPS) I get a 50% error rate and it comes back as a 502. I have no error logs in the application in nodejs as it's rejected straight away by nginx.

If I modify the iptables to redirect port 80 to 10000 (application port) I get zero errors; and much better performance.

I would like to avoid this as in future I would use let's encrypt and nginx to serve https; but I need to maintain the raw performance.

I've tried searching for answers and seen a few for PHP but nothing for node.

Has anyone come across this or maybe has an idea how to fine-tune nginx? The nginx.conf is right now set to it's default install.

nginx.conf

user www-data;
worker_processes auto;
pid /run/nginx.pid;

events {
    worker_connections 768;
    # multi_accept on;
}

http {

    ##
    # Basic Settings
    ##

    sendfile on;
    tcp_nopush on;
    tcp_nodelay on;
    keepalive_timeout 65;
    types_hash_max_size 2048;
    # server_tokens off;

    # server_names_hash_bucket_size 64;
    # server_name_in_redirect off;

    include /etc/nginx/mime.types;
    default_type application/octet-stream;

    ##
    # SSL Settings
    ##

    ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # Dropping SSLv3, ref: POODLE
    ssl_prefer_server_ciphers on;

    ##
    # Logging Settings
    ##

    access_log /var/log/nginx/access.log;
    error_log /var/log/nginx/error.log;

    ##
    # Gzip Settings
    ##

    gzip on;
    gzip_disable "msie6";

    # gzip_vary on;
    # gzip_proxied any;
    # gzip_comp_level 6;
    # gzip_buffers 16 8k;
    # gzip_http_version 1.1;
    # gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript;

    ##
    # Virtual Host Configs
    ##

    include /etc/nginx/conf.d/*.conf;
    include /etc/nginx/sites-enabled/*;
}


#mail {
#   # See sample authentication script at:
#   # http://wiki.nginx.org/ImapAuthenticateWithApachePhpScript
# 
#   # auth_http localhost/auth.php;
#   # pop3_capabilities "TOP" "USER";
#   # imap_capabilities "IMAP4rev1" "UIDPLUS";
# 
#   server {
#       listen     localhost:110;
#       protocol   pop3;
#       proxy      on;
#   }
# 
#   server {
#       listen     localhost:143;
#       protocol   imap;
#       proxy      on;
#   }
#}

Update:

In addition to following this guide (https://www.linode.com/docs/websites/nginx/configure-nginx-for-optimized-performance) I had to:

edit sysctl.conf and add the following:

sysctl net.core.somaxconn=65536
sysctl net.ipv4.tcp_max_tw_buckets=1440000
sysctl net.ipv4.ip_local_port_range=1024 65000
sysctl net.ipv4.tcp_fin_timeout=15
sysctl net.ipv4.tcp_window_scaling=1    
sysctl net.ipv4.tcp_max_syn_backlog=3240000

Edit the /etc/security/limits.conf and add:

www-data    hard    nofile    65536
www-data    soft    nofile    65536

Edit the lib/systemd/system/nginx.service config and add the following under KillMode

LimitNOFILE=65536

For performance I replaced /etc/nginx/proxy_params with:

proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;

proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection 'upgrade';
proxy_cache_bypass $http_upgrade;
proxy_intercept_errors on;

proxy_redirect off;
proxy_connect_timeout 90;
proxy_send_timeout 90;
proxy_read_timeout 90;
proxy_buffer_size 4k;
proxy_buffers 4 32k;
proxy_busy_buffers_size 64k;
proxy_temp_file_write_size 64k;
proxy_temp_path /etc/nginx/proxy_temp;

Answer 1

For the future references. Turn off access logs by access_log off for production.

This will hardly improve performance.