CODEWITHSUNDEEP
javascriptjsonnode.jscsvnpm
javascript novice
javascript novice

Reputation: 777

csvtojson conversion using Nodejs gives gibberish data

I am trying to convert a csv file to json using csvtojson converter in nodejs.My code is as shown below. I get an output of data as shown below. I am not sure why this is happening and how to prevent it.

var Converter  = require("csvtojson").Converter;
var fileStream = fs.createReadStream("input.csv");

var converter = new Converter({constructResult:false});

converter.on("end_parsed", function (jsonObj) {
  var jsonfile = require('jsonfile');
  var file     = 'output.json';
  jsonfile.writeFile(file, jsonObj, function (err) { console.error(err); });});


fileStream.pipe(converter);

{"��P\u0000a\u0000c\u0000k\u0000a\u0000g\u0000e\u0000 \u0000N\u0000a\u0000m\u0000e\u0000":"\u0000c\u0000o\u0000m\u0000.\u0000t\u0000r\u0000i\u0000n\u0000e\u0000t\u0000.\u0000h\u0000r\u0000p\u0000m\u0000o\u0000b\u0000i\u0000l\u0000e\u0000","\u0000A\u0000p\u0000p\u0000 \u0000V\u0000e\u0000r\u0000s\u0000i\u0000o\u0000n\u0000 \u0000C\u0000o\u0000d\u0000e\u0000":"\u00006\u00002\u0000","\u0000R\u0000e\u0000v\u0000i\u0000e\u0000w\u0000e\u0000r\u0000 \u0000L\u0000a\u0000n\u0000g\u0000u\u0000a\u0000g\u0000e\u0000":"\u0000e\u0000n\u0000","\u0000D\u0000e\u0000v\u0000i\u0000c\u0000e\u0000":"\u0000","\u0000R\u0000e\u0000v\u0000i\u0000e\u0000w\u0000 \u0000S\u0000u\u0000b\u0000m\u0000i\u0000t\u0000 \u0000D\u0000a\u0000t\u0000e\u0000 \u0000a\u0000n\u0000d\u0000 \u0000T\u0000i\u0000m\u0000e\u0000":"\u00002\u00000\u00001\u00005\u0000-\u00001\u00002\u0000-\u00002\u00002\u0000T\u00000\u00003\u0000:\u00003\u00002\u0000:\u00003\u00008\u0000Z\u0000","\u0000R\u0000e\u0000v\u0000i\u0000e\u0000w\u0000 \u0000S\u0000u\u0000b\u0000m\u0000i\u0000t\u0000 \u0000M\u0000i\u0000l\u0000l\u0000i\u0000s\u0000 \u0000S\u0000i\u0000n\u0000c\u0000e\u0000 \u0000E\u0000p\u0000o\u0000c\u0000h\u0000":"\u00001\u00004\u00005\u00000\u00007\u00005\u00005\u00001\u00005\u00008\u00002\u00006\u00002\u0000","\u0000R\u0000e\u0000v\u0000i\u0000e\u0000w\u0000 \u0000L\u0000a\u0000s\u0000t\u0000 \u0000U\u0000p\u0000d\u0000a\u0000t\u0000e\u0000 \u0000D\u0000a\u0000t\u0000e\u0000 \u0000a\u0000n\u0000d\u0000 \u0000T\u0000i\u0000m\u0000e\u0000":"\u00002\u00000\u00001\u00005\u0000-\u00001\u00002\u0000-\u00002\u00002\u0000T\u00000\u00003\u0000:\u00003\u00002\u0000:\u00003\u00008\u0000Z\u0000","\u0000R\u0000e\u0000v\u0000i\u0000e\u0000w\u0000 \u0000L\u0000a\u0000s\u0000t\u0000 \u0000U\u0000p\u0000d\u0000a\u0000t\u0000e\u0000 \u0000M\u0000i\u0000l\u0000l\u0000i\u0000s\u0000 \u0000S\u0000i\u0000n\u0000c\u0000e\u0000 \u0000E\u0000p\u0000o\u0000c\u0000h\u0000":"\u00001\u00004\u00005\u00000\u00007\u00005\u00005\u00001\u00005\u00008\u00002\u00006\u00002\u0000","\u0000S\u0000t\u0000a\u0000r\u0000 \u0000R\u0000a\u0000t\u0000i\u0000n\u0000g\u0000":"\u00005\u0000","\u0000R\u0000e\u0000v\u0000i\u0000e\u0000w\u0000 \u0000T\u0000i\u0000t\u0000l\u0000e\u0000":"\u0000","\u0000R\u0000e\u0000v\u0000i\u0000e\u0000w\u0000 \u0000T\u0000e\u0000x\u0000t\u0000":"\u0000","\u0000D\u0000e\u0000v\u0000e\u0000l\u0000o\u0000p\u0000e\u0000r\u0000 \u0000R\u0000e\u0000p\u0000l\u0000y\u0000 \u0000D\u0000a\u0000t\u0000e\u0000 \u0000a\u0000n\u0000d\u0000 \u0000T\u0000i\u0000m\u0000e\u0000":"\u0000","\u0000D\u0000e\u0000v\u0000e\u0000l\u0000o\u0000p\u0000e\u0000r\u0000 \u0000R\u0000e\u0000p\u0000l\u0000y\u0000 \u0000M\u0000i\u0000l\u0000l\u0000i\u0000s\u0000 \u0000S\u0000i\u0000n\u0000c\u0000e\u0000 \u0000E\u0000p\u0000o\u0000c\u0000h\u0000":"\u0000","\u0000D\u0000e\u0000v\u0000e\u0000l\u0000o\u0000p\u0000e\u0000r\u0000 \u0000R\u0000e\u0000p\u0000l\u0000y\u0000 \u0000T\u0000e\u0000x\u0000t\u0000":"\u0000","\u0000R\u0000e\u0000v\u0000i\u0000e\u0000w\u0000 \u0000L\u0000i\u0000n\u0000k\u0000":"\u0000"}, {"��P\u0000a\u0000c\u0000k\u0000a\u0000g\u0000e\u0000 \u0000N\u0000a\u0000m\u0000e\u0000":"\u0000"}

Upvotes: 2

Views: 3471

Answers (3)

javascript novice
javascript novice

Reputation: 777

I resolved this issue: It was an encoding issue: correct code was "utf16 " 

   var csvEncoding = { encoding: 'utf16le' }; 
    var csvString = fs.readFileSync(csvfile, csvEncoding).toString(); 
    converter.fromString(csvString, function(err,result){ 
    //your code here console.log(err); 
    console.log(result); 

});

Upvotes: 7

Amaranadh Meda
Amaranadh Meda

Reputation: 724

var Converter  = require("csvtojson").Converter;

var fs  = require('fs');
var fileStream = fs.createReadStream("input.csv");

var converter = new Converter({constructResult:true});
converter.on("end_parsed", function (jsonObj) {
  var jsonfile = require('jsonfile');
  var file     = 'output.json';
  console.log(jsonObj);
  jsonfile.writeFile(file, jsonObj, function (err,result) {
    console.error(err);
    console.log(result) ;
  });
});

fileStream.pipe(converter);

Upvotes: 0

joel.software
joel.software

Reputation: 1515

I had a very similar issue to OP but was using csv-parse with a file coming out of S3.

Thanks to OP I got on the right path with an encoding issue, I was able to resolve my issue by using utf16le in my stream coupled with iconv-lite like so:

s3
  .getObject(getObjectParams)
  .createReadStream()
  .on('end', () => cb(null))
  .pipe(iconv.decodeStream('utf16le'))
  .pipe(parse({ delimiter: '\t', columns: true }))
  .pipe(transformer);

Hopefully this helps others in the same boat!

Upvotes: 1

Related Questions