CODEWITHSUNDEEP
javascriptnode.jsfast-csv
Zag Gol
Zag Gol

Reputation: 1076

nodejs - streaming csv to string variable

I have a code that accepts a list of nested objects, each of which should be converted to a log row.

The code goes through a loop on each object, then an inner loop on each property, and extracts its property (there are hundreds of properties), then puts all the information of a row - as a map of the object's name and its value, into a variable called returnVar. We use the 'fast-csv' library, with WriteStream that is named csvStream. also with a fs.createWriteStream pipe.

Finally, we loop over each object and write it with csvStream.write(), that will insert the properties name in the first line of the file, and the logs (in the same order) in the other lines.

I need to change the code so that instead of doing pipe to file stream, it will print to a string type variable.

This is the code: 

let Promise = require('bluebird');
let csv = require('fast-csv');
let fs = Promise.promisifyAll(require('fs'));

...

return new Promise(function (resolve, reject) {
    var csvStream = csv.createWriteStream({ headers: propNames })
        .transform(function (item) { // every item is a nested object that contains data for a log line
            var returnVar = {}; // every returnVar will represents a map of property and value, that will be transform to a log line
            for (var prop in item) { 
                if (item.hasOwnProperty(prop)) {
                    if (propNames.indexOf(prop) >= 0) {
                        if (typeof item[prop] === 'object') {
                            returnVar[prop] = JSON.stringify(item[prop]);
                        }
                        else {
                            returnVar[prop] = item[prop];
                        }
                    }
                    //the object might be a complex item that contains some properties that we want to export...
                    else if (typeof item[prop] === 'object') {
                        var nestedItem = item[prop];
                        for (var nestedProp in nestedItem) {
                            if (propNames.indexOf(prop + "_" + nestedProp) >= 0) {
                                returnVar[prop + "_" + nestedProp] = nestedItem[nestedProp];
                            }
                        }
                    }
                }
            }

            return returnVar; // return log line
        });

    // create file path
    var fileId = "Report_" + cryptoService.generateRandomPassword(16) + ".csv";
    var filePath = tempPath + fileId;

    getOrCreateTempDirectory().then(function () {
        var writableStream = fs.createWriteStream(filePath);

        writableStream.on("finish", function () {
            resolve({
                fileId: fileId
            });
        });

        csvStream.pipe(writableStream);

        _.each(results.records, function (result) {
            // write line to file
            csvStream.write(result._source);
        });

        csvStream.end();
    });
});

Upvotes: 0

Views: 1980

Answers (1)

user120242
user120242

Reputation: 15268

https://c2fo.io/fast-csv/docs/formatting/methods#writetobuffer
https://c2fo.io/fast-csv/docs/formatting/methods#writetostring

Change csvStream.write(result._source); to
csvStream.writeToString(result._source).then(data => console.log(data)); 

Promise.all(_.map(results.records, result => csvStream.writeToString(result._source)))
  .then(rows=>console.log(rows))
// rows should be an array of strings representing all the results

You can also use async/await

Upvotes: 1

Related Questions