CODEWITHSUNDEEP
javascriptnode.jsmongodb
santosh
santosh

Reputation: 190

insert large data in 2 collections at a time in mongodb

I am trying to insert large data around 20,000 in both the collection. I have a list of the path in an array which gives me an XML data where I am filtering it and pushing it to the database. "results" is the array of path.

Here is the code

const https = require('https');
var xpath = require('xpath')
var dom = require('xmldom').DOMParser

    results.forEach(ress => {
        const options = {
          hostname: 'abcd.com',
          port: 443,
          path: '/local/content' + ress,
          method: 'GET',
          rejectUnauthorized: false,
          requestCert: true,
          agent: false
        };
        https.request(options, (res) => {
          res.setEncoding('utf-8');
          res.on('data', (d) => {  //here i am getting XML data
            var doc = new dom().parseFromString(d);
            var nodes = xpath.select("//content", doc);
              nodes.forEach((n, i) => {
                pathvalue.push({
                   ........
                })
                projectdep.push({
                 .........
                })
              });
              dbo.collection("comp").insertMany(pathvalue, (err, post) => {
              });
              dbo.collection("pro").insertMany(projectdep, (err, post) => {
              });
          });
        }).on('error', (e) => {
          console.error(e);
        }).end();
      });

please help me with insert large data which is loop inside a loop.

Upvotes: 1

Views: 93

Answers (1)

chridam
chridam

Reputation: 103365

You may want to promisify the request method as

const https = require('https');

function httpRequest(params, postData) {
    return new Promise(function(resolve, reject) {
        var req = https.request(params, function(res) {
            // reject on bad status
            if (res.statusCode < 200 || res.statusCode >= 300) {
                return reject(new Error('statusCode=' + res.statusCode));
            }
            // cumulate data
            var body = [];
            res.on('data', function(chunk) {
                body.push(chunk);
            });
            // resolve on end
            res.on('end', function() {
                try {
                    body = JSON.parse(Buffer.concat(body).toString());
                } catch(e) {
                    reject(e);
                }
                resolve(body);
            });
        });
        // reject on request error
        req.on('error', function(err) {
            reject(err);
        });

        if (postData) {
            req.write(postData);
        }
        // IMPORTANT
        req.end();
    });
}

Then create a list of promises that you can resolve in an async/wait as

const xpath = require('xpath')
const DOMParser = require('xmldom').DOMParser;

const docsPromises = results.map(res => {
    const options = {
        hostname: 'abcd.com',
        port: 443,
        path: '/local/content' + res,
        method: 'GET',
        rejectUnauthorized: false,
        requestCert: true,
        agent: false
    };

    return httpRequest(options, null);
});

(async () => {
    try {
        const docs = await Promise.all(docsPromises);
        const pathvalue = []
        const projectdep = []

        docs.forEach(d => {
            const doc = new DOMParser().parseFromString(d);
            const nodes = xpath.select("//content", doc);
            nodes.forEach((node, i) => {
                pathvalue.push({ ...node })
                projectdep.push({ ...node })
            });
        })

        await dbo.collection("comp").insertMany(pathvalue);
        await dbo.collection("pro").insertMany(projectdep);
    } catch (err) {
        console.error(err)
    }
})();

Upvotes: 1

Related Questions