Download huge amount of data from mongodb using node js

Question

I want to download huge data from mongodb database using nodejs. But it is failing, how to fix it? Please help.

Node service:

function getDwl (req, res) {
  var queryObject = url.parse(req.url, true).query
  db.collection('history').
    find({ 'timestamp': { '$gte': queryObject.fromdate, '$lt': queryObject.todate } }, { 'type': 1, 'value': 1 }).
    toArray(function (err, result) {
      var index = 0, resultset = []
      var count = result.length
      if (count == 0) {
        res.writeHead(200, {
          'Content-Type': 'application/json',
        })
        res.write('')
        res.end()
      } else {
        result.forEach(function (doc) {
          if (doc != null) {
            var valdata = doc.alarms
            var fields = []
            var queryString = 'SELECT field1 FROM details c inner join ldetails l on c.loc_id=l.loc_id where no=\'' + doc.no + '\';'
            var dtfield1 = null
            connection.query(queryString, function (err, result) {
              index++
              console.log('result query')
              if (err) {
                console.log('err', err)
              } else {
                if (result.length > 0) {
                  dtfield1 = result[ 0 ].datafield1

                  if (dtfield1 != null) {
                    for (var x in valdata) {
                      resultset.push({ 'Name': dtfield1 })
                    }
                  }
                }

                if (index == count) {
                  console.log('result data')

                  res.writeHead(200, {
                    'Content-Type': 'application/json',
                  })
                  var resultData = resultset
                  res.write(JSON.stringify(resultData))
                  res.end()
                }
              }
            })
          }
        })
      }
    })
}

How to get huge data download in nodejs service call from http request from MONGODB database.

Answer 1

You could use paging and download data in batches. This will reduce amount of memory and CPU you need to perform this operation and distribute it over time. You could send those data in batches using node.js stream Like this:

const batchSize = 10000;
db.collection("history").find(query).count()
    .then(count => {
        const operations = [];
        for (let i = 0; i < count; i += batchSize) {
            operations.push(
                db.collection("history")
                    .find(query)
                    .skip(i)
                    .limit(i + batchSize)
                    .then(batchHandler) // batchHandler will contain your stream logic
            )
        }
        return Promise.all(operations);
    })
    .then(() => console.log("Everything is handled"));

Or, if you have a control over client application, you could simply paginate over the collection using client-side request params. For example:

// http://example.com/get-huge-collection?skip=0&limit=10000

const hugeCollectionController = (req, res) => {
    const { skip = 0, limit = 10000 } = req.query;
    return db.getCollection("history")
        .find(query)
        .skip(+skip)
        .limit(+limit)
        .then(collection => {
            res.write(JSON.stringify(collection));
            res.end();
        })
}