CODEWITHSUNDEEP
javascriptnode.js
Connor Leech
Connor Leech

Reputation: 18873

how to read a file, store data and then write it

I have a text file with a ton of values that I want to convert to meaningful JSON using node.js fs module.

I want to store the first value of every line in an array unless the value is already present.

7000111,-1.31349,36.699959,1004,
7000111,-1.311739,36.698589,1005,
8002311,-1.262245,36.765884,2020,
8002311,-1.261135,36.767544,2021,

So for this case, I'd like to write to a file:

[7000111, 8002311]

Here's what I have so far. It writes [] to the file.

var fs = require('fs');
var through = require('through');
var split = require('split');
var shape_ids = [];

var source = fs.createReadStream('data/shapes.txt');
var target = fs.createWriteStream('./output3.txt');

var tr = through(write, end);

source
    .pipe(split())
    .pipe(tr)

// Function definitions

function write(line){
    var line = line.toString();
    var splitted = line.split(',');

    // if it's not in array
    if (shape_ids.indexOf(splitted[0]) > -1){
        shape_ids.push(splitted[0]);
    }
}   

function end(){
    shape_ids = JSON.stringify(shape_ids);
    target.write(shape_ids);
    console.log('data written');
}

The code is using the split and through modules

How do I store values in the array and write the populated array to the file?

== === ====== =================

Update: This is what I want to do, but it's in Ruby:

shape_ids = []

File.open("data/shapes.txt").readlines.each do |line|
   data = line.split(',')
   shape_id = data.first

   if !shape_ids.include? shape_id
       shape_ids.push(shape_id)
   end
end

puts shape_ids  # array of unique shape_ids

Can I do this in javascript?

Upvotes: 0

Views: 125

Answers (1)

srquinn
srquinn

Reputation: 10501

Unless you are super comfortable with the new Stream API in node, use the event-stream module to accomplish this:

var fs = require('fs');
var es = require('event-stream');

function getIds(src, target, callback) {
  var uniqueIDs = [];
  es.pipeline(
    fs.createReadStream(src),
    es.split(),
    es.map(function (line, done) {
      var id = line.split(',').shift();
      if (uniqueIDs.indexOf(id) > -1) return done();
      uniqueIDs.push(id);
      done(null);
    }),
    es.wait(function (err, text) {
      // Here we create our JSON — keep in mind that valid JSON starts
      // as an object, not an array
      var data = JSON.stringify({ ids: uniqueIDs});
      fs.writeFile(target, data, function (err) {
        if ('function' == typeof callback) callback(err);
      });
    })
  );
}

getIds('./values.txt', './output.json');

Unfortunately there is no "easy" way to keep this as a pure stream flow so you have to "wait" until the data is done filtering before turning into a JSON string. Hope that helps!

Upvotes: 1

Related Questions