long loop in node.js leak memory

Question

please help me, I want to make my script to access to multiple URL in the same time to get some resource. This script is running normally,but the problem is that it leak memory and server will down. Pleas anyone help to give me better solution. Thank you! Please help to check source code below:

var fs = require('fs');
var request = require('request');
var dir = './';
var dateTime = require('node-datetime');

const express = require('express');
const app = express();

var urls = [
  'http://www.google.com',
  'https://www.bing.com',
  'http://www.yahoo.com',
  "http://stackoverflow.com",
  "http://github.com",
  "http://www.yahoo.co.jp"
];

var urlLength=urls.length;
function remoteControllerPress(){
    var dt = dateTime.create();
    var newCurrentTimeDir = dt.format('Y-m-d-H-M-S').toString();
    var dirName = dir+newCurrentTimeDir;
    if (!fs.existsSync(dirName)){
        fs.mkdirSync(dirName);
    }
    console.log("==============calling at "+newCurrentTimeDir + '\n');

    for(var i = 0; i<urlLength; i++) {
     (function() {
      var j = i;
      var url = urls[j];
      process.nextTick(function() {


        request(url, function(error, response, body) {
          console.log("---------URL: "+url);
          if (!error && response.statusCode == 200) {
            console.log("URL: " + url + " statusCode: " + response.statusCode);

            fs.writeFile(dirName+"/file_"+j, body.toString(), function(err) {
                if(err) {
                    return console.log(err);
                }
                console.log(dirName+"/file_"+ j +" was saved!\n\n");
            }); 


          } else if (error) {
            console.log("error.code:", error.code);
          }
        });

      });
     })();
    }

    setTimeout(function(){
        remoteControllerPress();
    },6000);
}
console.log("==============running...\n ");
remoteControllerPress();

app.post('/', function (req, res) {
  res.send('hello world');
});

app.listen(
    3000, 
    () => console.log('Server start listening on port 3000...')
);

Answer 1

Use async operator to do async loop https://caolan.github.io/async/docs.html

Use setInterval with setImmediate instead of using setTimeout in order to clear context at each call

var async = require("async");

function remoteControllerPress(){

    var dt = dateTime.create();
    var newCurrentTimeDir = dt.format('Y-m-d-H-M-S').toString();
    var dirName = dir+newCurrentTimeDir;
    if (!fs.existsSync(dirName)){
        fs.mkdirSync(dirName);
    }
    console.log("==============calling at "+newCurrentTimeDir + '\n');

    // TO do it 5 by 5
    async.eachOfLimit(urls, 5, function(url, i, cb){

       request(url, function(error, response, body) {

          console.log("---------URL: "+url);


          if (!error && response.statusCode == 200) {

            console.log("URL: " + url + " statusCode: " + response.statusCode);

            fs.writeFile(dirName+"/file_"+j, body.toString(), function(err) {
                if(err)
                {
                   console.log(err);
                }
                console.log(dirName+"/file_"+ j +" was saved!\n\n");

                return cb();
            }); 


          }
          else if (error)
          {
            console.log("error.code:", error.code);
            return cb();
          }
          else
          {
            return cb();
          }
        });
    }, function(err){

        return;

    });


}
console.log("==============running...\n ");

setInterval(function(){

  setImmediate(remoteControllerPress);

},6000);

Answer 2

Your remoteControllerPress function seems to have a reference to itself since you call it recursively with the timeout. Thus the garbage collector have trouble to clean memory.

A simple (ugly ?) workaround should be to use setInterval

     }

    /*setTimeout(function(){
        remoteControllerPress();
    },6000); <-- remove that */ 
}
console.log("==============running...\n ");
setInterval(remoteControllerPress, 6000); // <- change this