Bulk insert in MongoDB with mongoose for multiple collections

Question

I have 2 collections(data, metaData)

data schema is

{
_id: ......,
name: ......, //not unique
mobile: ......, // unique or null
email: ......, // unique or null
uniqueId: ......, // unique or null
}

at least one of unique data is required for insert

metaData schema is

{
_id: ......,
dataId: ......,//refrence from _id of data collection
key: ......,
value: ......
}

JSON array is getting from client

[{
  name: "abc",
  mobile: 9999999999,
  mData: {
    c1: 123,
    c2: "xyz"
  }
},
{
  name: "qwerty",
  email: 'qwerty@mail.com',
  mData: {
    c1: 123,
    c2: "zxc"
  }
}
......
]

I am iterating through the array and inserting each of them in both collections into MongoDB.

let Bulk = Data.collection.initializeUnorderedBulkOp();
dataArr.forEach(function(item) {
  let data = service.generateData(item);
  // data.query: {mobile: ..., email: ..., uniqueId: ...}
  // if value exists then keys is also exists for mobile, email, uniqueId in query
  Bulk.find(data.query).upsert().updateOne(data.doc);
});
Bulk.execute((e, d) => {
  let metaBulk = MetaData.collection.initializeOrderedBulkOp();
  let length = dataArr.length;
  dataArr.forEach(function(data) {
    Data.findOne(data.query).exec(function(err, data) {
      length--;      
      for(let key in data["mData"]) {
        let value = data["mData"][key] || "";
        let mData = service.generateMdata(key, value, data._id);
        metaBulk.find(mData.query).upsert().updateOne(mData.doc);
      }
      if(length == 0) {
        metaBulk.execute();
      }
    });
  });
});

my solution is working fine right now but it's taking so much time to iterating data collection for finding ids for metaData collection.

I need a way of inserting the data in bulk into MongoDB without find query for data id. Is there any option to perform bulk upserts with mongoose for multiple collections in a single query.

Answer 1

No multiple collection update in a single command for your scenario. In your case if you can include metadata array inside parent collection it can insert data with single command with updateMany(). MongoDB also supports bulk insert through the db.collection.insertMany().

db.data.insertMany( [{ name: "abc",mobile: 9999999999, mData: { c1: 123, c2: "xyz"} },
                                            {name: "qwerty",email: 'qwerty@mail.com',mData: { c1: 123, c2: "zxc" }}]);

Also you can use db.collection.bulkWrite() as well.

Answer 2

I think what you can do is:

async.each(jsonArray, function(jsonData,callback){
  //first insert data in data schema
  var data = new data(jsonData);
  data.save(function(err){
    if err throw err;
    //then you save the data in metaData collection
    async.each(jsonData.mData, function(metadata, callback2){
      var metaDataObj = new metaData(metadata);
      metaDataObj.dataId = data._id;
      metaDataObj.save(function(err){
       callback2();
      });
    }, function(err, results1){
      callback();
    });
  });
}, function(err, results){
   console.log('Data is saved');
});