CODEWITHSUNDEEP
javascriptnode.jsfilestreamnode-csv-parse
llandino
llandino

Reputation: 75

Read csv file with nodejs with a max number rows

I'm trying to read a CSV file with node.js using the csv-parser library.

Since it's a big file, I need to check the header and the first 100 rows and the stop the method and return true if everything is ok or false if the data doesn't respect the condition.

How can I achieve this?

This is what I have so far:

const csv = require('csv-parser');
const fs = require('fs');    
exports.checkFileFormat = (file) => {
  let stream = fs.createReadStream(file.tempFilePath)
    .pipe(csv())
    .on('headers', (headers) => {
      /*...some logic...*/
    })
    .on('data', (row) => {
      if (!typeof (row["USAGE"]) == 'number'
          || !moment(row["START_DATE"], 'YYYYMMDD', true).isValid()
          || !moment(row["END_DATE"], 'YYYYMMDD', true).isValid()) {
        stream.unpipe(csv());
        return false;
      }       
    })
    .on('end', () => {
       console.log('CSV file successfully processed');
    });
    return true;
}

In a previous version I had also declared: var num = 100 and tested it inside .on('data', (row) => {...} but it didn't work.

Upvotes: 1

Views: 4385

Answers (2)

TKoL
TKoL

Reputation: 13892

Following up from my comment

make the function checkFileFormat return a promise. Inside the promise, resolve(false) instead of return false and resolve(true) in the '.on('end') callback. I'm not completely sure this will work, but that's how I would approach it

const csv = require('csv-parser');
const fs = require('fs');

exports.checkFileFormat = (file) => {
    return new Promise((resolve, reject) => {
        let stream = fs.createReadStream(file.tempFilePath)
            .pipe(csv())
            .on('headers', (headers) => {
                /*...some logic...*/
            })
            .on('data', (row) => {
                if (!typeof (row["USAGE"]) == 'number'
                    || !moment(row["START_DATE"], 'YYYYMMDD', true).isValid()
                    || !moment(row["END_DATE"], 'YYYYMMDD', true).isValid()) {
                    stream.end(); // stream.unpipe(csv());
                    resolve(false);
                }
            })
            .on('end', () => {
                console.log('CSV file successfully processed');
                resolve(true);
            });
    });    
}

Upvotes: 2

eol
eol

Reputation: 24555

If you want to read a certain amount of lines and then break, you can try the following:

const csv = require('csv-parser');
const fs = require('fs');
let count = 0;
let maxLines = 3;
let fsStream = fs.createReadStream('./data.csv');
let csvStream = csv();

fsStream.pipe(csvStream)
    .on('headers', (headers) => {
        console.log(headers)
    })
    .on('data', (data) => {
        if (count >= maxLines) {
            fsStream.unpipe(csvStream);
            csvStream.end();
            fsStream.destroy();             
        } else {
            console.log(data);
            count++;
        }
    });

Basically you just count each read line and when the max is reached, you unpipe the csv-stream from the fs-stream, then end the csv-stream and finally destroy the fs-stream.

Upvotes: 1

Related Questions