How to merge two csv files rows in node js

Question

I have 2 csv files which have different different data but having a same header eg. FILE 1 data is

"CODE","NAME","SUB_USER","SCORE"
"01","TEST","1","5"
"01","TEST","2","6"

other file FILE2 have data like this

"CODE","NAME","SUB_USER","SCORE"
    "02","TEST2","3","5"
    "02","TEST2","4","6"

so i want to merge both file create FILE3 output like this

"CODE","NAME","SUB_USER","SCORE"
"01","TEST","1","5"
"01","TEST","2","6"
"02","TEST2","3","5"
"02","TEST2","4","6"

I have tried below code

var express = require('express');
var router = express.Router();
var fs = require('fs');
var parse = require('csv-parse');
var async = require('async');
var csv = require("fast-csv");

var file1 = appRoot + '\\csvFiles\\details1.csv';
var file2 = appRoot + '\\csvFiles\\details2.csv';
var stream = fs.createReadStream(file1);
var stream2 = fs.createReadStream(file2);
var fileData1 = [],
    fileData2 = [];
csv
    .fromStream(stream)
    .on("data", function(data) {
        fileData1.push(data);
    })
    .on("end", function() {
        console.log("done");
    });

csv
    .fromStream(stream2)
    .on("data", function(data) {
        fileData2.push(data);
    })
    .on("end", function() {
        console.log("done");
    });
var fileData3 = fileData1.concat(fileData2);

csv.writeToPath("outputfile.csv", fileData3).on("finish", function() {
    console.log("END");
});

But not working don't know why?? Please help me ///**********************************************************************// Thax for help but i got new problem here

After some changes above code start working

var file1 = appRoot + '\\csvFiles\\details1.csv';
var file2 = appRoot + '\\csvFiles\\idetails2.csv';
var stream = fs.createReadStream(file1);
var stream2 = fs.createReadStream(file2);
var fileData1 = [],
    fileData2 = [],
    i = 0;

csv.fromStream(stream).on("data", function(data) {
    fileData1.push(data);
}).on("end", function() {
    csv.fromStream(stream2).on("data", function(data) {
        if (i != 0) {
            fileData2.push(data);
        }
        i++;
    }).on("end", function() {
        console.log("done");
        var fileData3 = fileData1.concat(fileData2);
        csv.writeToPath("outputfile.csv", fileData3).on("finish", function() {
            res.send('Done merge');
        });
    });
});

But problem is that what if my number of file increase then how i will handle that thing

Answer 1

The biggest problem here is a quite common one. You do async tasks but you don't wait for them to finish before you are using their result.

You concat the file data before the "end" callback for each tasks was called. The solution is to wait for every callback to be called and THEN working with the data.

I created a small example using Promises

const file1 = 'one.csv';
const file2 = 'two.csv';
const stream = fs.createReadStream(file1);
const stream2 = fs.createReadStream(file2);
const fileData1 = [];
const fileData2 = [];

const file1Promise = new Promise((resolve) => {
  csv
      .parseFile(file1, {headers: true})
      .on('data', function(data) {
        fileData1.push(data);
      })
      .on('end', function() {
        console.log('done');
        resolve();
      });
});

const file2Promise = new Promise((resolve) => {
  csv
      .parseFile(file2, {headers: true})
      .on('data', function(data) {
        fileData2.push(data);
      })
      .on('end', function() {
        console.log('done');
        resolve();
      });
});

Promise.all([
  file1Promise,
  file2Promise,
])
    .then(() => {
      const fileData3 = fileData1.concat(fileData2);
      console.log(fileData3);

      const csvStream = csv.format({headers: true});
      const writableStream = fs.createWriteStream('outputfile.csv');

      writableStream.on('finish', function() {
        console.log('DONE!');
      });

      csvStream.pipe(writableStream);
      fileData3.forEach((data) => {
        csvStream.write(data);
      });
      csvStream.end();
    });

---

I created a function with which you can easily merge multiple files:

function concatCSVAndOutput(csvFilePaths, outputFilePath) {
  const promises = csvFilePaths.map((path) => {
    return new Promise((resolve) => {
      const dataArray = [];
      return csv
          .parseFile(path, {headers: true})
          .on('data', function(data) {
            dataArray.push(data);
          })
          .on('end', function() {
            resolve(dataArray);
          });
    });
  });

  return Promise.all(promises)
      .then((results) => {

        const csvStream = csv.format({headers: true});
        const writableStream = fs.createWriteStream(outputFilePath);

        writableStream.on('finish', function() {
          console.log('DONE!');
        });

        csvStream.pipe(writableStream);
        results.forEach((result) => {
          result.forEach((data) => {
            csvStream.write(data);
          });
        });
        csvStream.end();

      });
}

example usage

concatCSVAndOutput(['one.csv', 'two.csv'], 'outputfile.csv')
    .then(() => ...doStuff);