CODEWITHSUNDEEP
node.jsmongodb
BugHunterUK
BugHunterUK

Reputation: 8968

MongoDB Bulk Insert Ignore Duplicate

I've Googled around and can't find any solid information on how to ignore duplicate errors when using bulk insert.

Here's the code I'm currently using:

MongoClient.connect(mongoURL, function(err, db) {
      if(err) console.err(err)
      let col = db.collection('user_ids')
      let batch = col.initializeUnorderedBulkOp()

      ids.forEach(function(id) {
        batch.insert({ userid: id, used: false, group: argv.groupID })
      })

      batch.execute(function(err, result) {
        if(err) {
          console.error(new Error(err))
          db.close()
        }

        // Do some work

        db.close()
      })
    })

Is it possible? I've tried adding {continueOnError: true, safe: true} to bulk.insert(...) but that didn't work.

Any ideas?

Upvotes: 5

Views: 3385

Answers (1)

chridam
chridam

Reputation: 103445

An alternative is to use bulk.find().upsert().replaceOne() instead:

MongoClient.connect(mongoURL, function(err, db) {
    if(err) console.err(err)
    let col = db.collection('user_ids')
    let batch = col.initializeUnorderedBulkOp()

    ids.forEach(function(id) {        
        batch.find({ userid: id }).upsert().replaceOne({ 
            userid: id, 
            used: false,  
            group: argv.groupID 
        });
    });

    batch.execute(function(err, result) {
        if(err) {
            console.error(new Error(err))
            db.close()
        }

        // Do some work

        db.close()
    });
});

With the above, if a document matches the query { userid: id } it will be replaced with the new document, otherwise it will be created hence there are No duplicate key errors thrown.

For MongoDB server versions 3.2+, use bulkWrite as:

MongoClient.connect(mongoURL, function(err, db) {

    if(err) console.err(err)

    let col = db.collection('user_ids')
    let ops = []
    let counter = 0

    ids.forEach(function(id) {
        ops.push({
            "replaceOne": {
                "filter": { "userid": id },
                "replacement": { 
                    userid: id, 
                    used: false,  
                    group: argv.groupID 
                },
                "upsert": true
            }
        })

        counter++

        if (counter % 500 === 0) {
            col.bulkWrite(ops, function(err, r) {
                // do something with result
                db.close()
            })
            ops = []
        }
    })

    if (counter % 500 !== 0) {
        col.bulkWrite(ops, function(err, r) {
            // do something with result
            db.close()
        }
    } 
})

Upvotes: 5

Related Questions