How to add json data to csv file one by one?

Question

I need to start adding json data to csv file from beginning until the end. But each row will be added every minute. In my situation it adds all of them in the first minute. This is my code.

var unirest = require('unirest');
var fs = require('fs');
var outFilename = 'mert_akel_minutereadings.csv';

var interval = setInterval(function () { post(); }, 60000); //Write to file every minute

function post(){
    unirest.post('https://power.ivyiot.com/Thingworx/Things/GamaNetworkServices/Services/GetNetworkData')
    .headers({'Accept': 'application/json', 'appKey': 'XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX'})
    .end(function (response) {
      writeToCsv(response.body);
    });
}


function writeToCsv(inJSON){    

        var outCSV = inJSON.rows;
        
    //  console.log(outCSV)
        
        var csv = [];

        for(var k in outCSV) {
            var items = [[outCSV[k].PowerPlant , outCSV[k].gen1min , outCSV[k].gen1minDateTime, timeConverter(outCSV[k].gen1minDateTime)]];
             for (index = 0; index < items.length; ++index) {
                csv.push(items[index].join(', ') + '\n');
             }
             
             fs.writeFile(outFilename, csv, function (err) {
                if (err) {
                    return console.log(err);
                }
                
                console.log("Added one row!")
                
             });
        }
        
    }

function timeConverter(UNIX_timestamp){
             var a = new Date(UNIX_timestamp*1000);
             var hour = a.getUTCHours();
             var min = a.getUTCMinutes();
             var sec = a.getUTCSeconds();
             var time = hour+':'+min+':'+sec ;
             return time;
    }

Example output:

PowerPlant0006_GamaEnerji, 0.04, 1513493037055, 12:30:55
,PowerPlant0005_GamaEnerji, 0.52, 1513498329715, 18:41:55
,PowerPlant0004_GamaEnerji, 0.165, 1513498330065, 18:47:45
,PowerPlant0007_GamaEnerji, 0.72, 1513498329103, 18:31:43
,PowerPlant0002_GamaEnerji, 0.01, 1513498323020, 16:50:20
,PowerPlant0003_GamaEnerji, 0.019, 1513498322780, 16:46:20
,PowerPlant0001_GamaEnerji, 0.368, 1513498339155, 21:19:15

If anyone can help I would be appreciated.

Answer 1

The setInterval on the post function isn't required, this is just going to keep retrieving the data. On writing the rows every minute you're basically looking at adding a delay in an iterative fashion. You need to use setTimeout for this but rework the logic behind the loop so they don't trip over each other as setTimeout will return immediately.

function writeToCsv(rows) {
  var lineNum = 0;
  addCSVLine(); //Run the function below;
  function addCSVline() {
    var line = rows[lineNum]; //Current line
    /*
       Your code here to create the data and add this row to the file
    */
    lineNum++ //Increment our line variable
    if (lineNum < rows.length) setTimeout(addCSVline, 60000); //recursion, if we haven't reached the last row, run again;
  }
}

Here is an example of the delay, i've added a smaller delay for obvious reasons. - https://jsfiddle.net/ssc8xhjq/

Answer 2

First remove setInterval from post method.(you don't want to get all rows every minute) intead call in once. Then set interval, for every minute, to consume a single row. the method will consume a row and will increment a k offset of the last consumed row for next iteration.

Note: the code is not tested but should do the work.

hope it help a bit

function post(){
    unirest.post('https://power.ivyiot.com/Thingworx/Things/GamaNetworkServices/Services/GetNetworkData')
    .headers({'Accept': 'application/json', 'appKey': 'XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX'})
    .end(function (response) {
      setInterval(writeToCsv(response.body.rows),60000);
    });
}

function writeToCsv(outCSV){
    var k = 0;
  return function(){
    if(k > outCsv.length){
    // TODO: stop interval, since no more rows to consume
    }
    var items = [[outCSV[k].PowerPlant , outCSV[k].gen1min , outCSV[k].gen1minDateTime, timeConverter(outCSV[k].gen1minDateTime)]];
             for (index = 0; index < items.length; ++index) {
                csv.push(items[index].join(', ') + '\n');
             }

              fs.writeFile(outFilename, csv, function (err) {
                if (err) {
                    return console.log(err);
                }

                console.log("Added one row!")

             });
             k++;
  }
}

function timeConverter(UNIX_timestamp){
             var a = new Date(UNIX_timestamp*1000);
             var hour = a.getUTCHours();
             var min = a.getUTCMinutes();
             var sec = a.getUTCSeconds();
             var time = hour+':'+min+':'+sec ;
             return time;
}

post();

`