nodejs to redis ECONNRESET error

Question

For our Node.js to Redis connections, we are using the Redis npm module.

Once in a while, we are getting the following error,

message: read ECONNRESET, stack: Error: read ECONNRESET
at exports._errnoException (util.js:1020:11)
at TCP.onread (net.js:568:26)

The Redis setup is, One redis server in intranet and 2 Node js servers each with 8 PM2 instances running in DMZ. There is a Firewall between the node servers and the Redis server.

NODE version - 6.11.2 REDIS version - 3.2.9 PM2 version - 2.4.6

We did the TCP dump as well. The TCP dump shows some RST/ACK packets. TCP Dump

In the nodeJS, we are creating a single redis connection and are trying to use the same redis connection for all requests.

    const Redis = require('redis');
    const Config = require('../../config');
    const Logger = require('../helpers/logger');

    const redisClient = (function() {

       // Start with a fake client so that we have a client that works
       // even when Redis server is down
      let client = {
        get: function(key, callback) {
          callback(null, null);
        },
        setex: function(key, time, value) {
          Logger.info('Value:',value);
          // Do nothing in particular
        }
      };
       // Attempt to create a new instance of an actual redis client
      const redisConfig = Config.get('/redis');
      const tempClient = Redis.createClient(redisConfig.port,redisConfig.host, {
        //eslint-disable-next-line
        // console.log('[redis]','Creating the retry strategy');
        retry_strategy: function(options) { //eslint-disable-line
          //eslint-disable-next-line
          console.log('[redis]','Creating the retry strategy');
          if (options.error && options.error.code === 'ECONNREFUSED') {
              // End reconnecting on a specific error and flush all commands with
              // a individual error
            //eslint-disable-next-line
            console.log('[redis,error]','Connection refused error');
            return new Error('The server refused the connection');
          }
          if (options.error && options.error.code === 'NR_CLOSED') {
            //eslint-disable-next-line
            console.log('[redis,error]','Connection closed error');
            return new Error('The server closed the connection');
          }
          if (options.attempt > 5) {
              // End reconnecting with built in error
            //eslint-disable-next-line
            console.log('Exceeded attempts');
            return undefined;
          }
          if (options.total_retry_time > 1000 * 60 * 60) {
            // End reconnecting after a specific timeout and flush all commands
            // with a individual error
            //eslint-disable-next-line
            console.log('Retrial time:' + options.total_retry_time);
            return 1000;
          }
          // reconnect after
          return Math.min(options.attempt * 100, 3000);
        }
      });

       // Set the "client" variable to the actual redis client instance
       // once a connection is established with the Redis server
      tempClient.on('ready', () => {
        client = tempClient;
      });
      tempClient.on('error', (error) => {
        Logger.info(['redis','error'],'Redis client error:', error);
        if (error.code === 'NR_CLOSED') {
          tempClient.end();
          client = Redis.createClient(redisConfig.port,redisConfig.host, {
            retry_strategy: function(options) { //eslint-disable-line
              if (options.error && options.error.code === 'NR_CLOSED') {
                Logger.info(['redis','error'],'Connection closed error');
                return new Error('The server refused the connection');
              }
            }
          });
        }
      });

       /**
        * Get a redis client
        * @return {Object} client - eventually a proper redis client object
        * (if redis is up) or a fake client object (if redis is down)
        */
      const getClient = function() {
        Logger.info('Getting the client ' + client);
        return client;
      };

      return {
        getClient: getClient
      };

    }());

    module.exports = redisClient;

We would like to know on what exactly is causing the connection issues and why and what is the resolution.

Answer 1

in the /etc/redis/redis.conf file ,

Replace bind 127.0.0.1 with bind 0.0.0.0 Replace protected-mode yes with protected-mode no

and then Allow port 6379 using ufw allow 6379 and ufw allow 6379/tcp

iptables -A INPUT -p tcp --dport 6379 -j ACCEPT iptables -A INPUT -p udp --dport 6379 -j ACCEPT

finally

sudo systemctl stop redis sudo systemctl start redis

Answer 2

I had the same problem with Google Firebase Cloud Functions. The problem was that latest version of Redis keep the connections alive forever. If the process that calls Redis ends shortly after it calls Redis as in our case with the Cloud Functions, you have to set the timeout setting to something different than the default 0.

You can do this by changing the timeout value in the Redis configuration file called redis.conf

The documentation says you can also set the timeout setting without restarting the instance which is not possible if your Redis instance is protected and thus cannot run something like CONFIG SET timeout 15 through the redis-cli as you would get the following error:

(error) ERR unknown command CONFIG