Why is MongoDB ignoring some of my updates?

Question

I've been building an application in Node.JS using the native MongoDB driver - it includes contacts, and when a user accepts a contact, it should remove from "pending" and "sent" contacts, then add to "contacts".

Example code and documents:

    /*
=============================
User "john"
=============================
{
  username: "john",
  contacts: ["jim"],
  pending_contacts: ["bob"]
}

=============================
User "bob"
=============================
{
  username: "bob",
  contacts: ["dave"],
  sent_contacts: ["john"]
}

=============================
What SHOULD happen
=============================
{
  username: "bob",
  contacts: ["dave", "john"],
  sent_contacts: []
},
{
  username: "john",
  contacts: ["jim", "bob"],
  pending_contacts: []
}

=============================
What ACTUALLY happens
=============================
{
  username: "john",
  contacts: ["jim", "bob"],
  pending_contacts: ["bob"]
},
{
  username: "bob",
  contacts: ["dave", "john"],
  sent_contacts: ["john"]
}

*/

  var col = this.db.collection('users');
  var contact = "bob", username = "john";
  var who = [contact, username];
  var finishCount = 0;
  // finish will run 3 times before callback
  function finish(name) {
    console.log(name, ' has finished');
    finishCount++;
    if(finishCount<3) return;
    callback(false, null);
  }
  // run if there's an error
  function failed(err) {
    callback(err, null)
  }
  console.log('removing %s and %s from pending and sent', username, contact)
  col.update(
    {username: { $in: who }},
    {
      $pullAll: {
        sent_contacts: who,
        pending_contacts: who
      }
    }, {multi: 1},
    function(err,data) {
      if(err) return failed(err);
      finish('REMOVE_CONTACTS');
    }
  );
  col.update(
    {username: username}, {$addToSet: {contacts: contact}},
    function(err,res) {
      if(err) return failed(err);
      console.log('added 1');
      finish('ADD_TO_USER');
    }
  );
  col.update(
    {username: contact}, {$addToSet: {contacts: username}},
    function(err,res) {
      if(err) return failed(err);
      console.log('added 2');
      finish('ADD_TO_CONTACT');
    }
  );

The first update removes the contact and the owner from each-others pending/sent list, the second and third update add the owner to the contact's contact list and vice versa.

The issue is, the final result appears to be as if the removal never happened, though the removal query works perfectly fine by itself. I don't know if this is a problem with MongoDB itself (or if it's intended), or if it's an issue with the driver, so I hope someone can at least clarify this for me.

NOTE: Yes I know they run asynchronously. Running them one after the other by putting each update in the previous callback does NOT make a difference. Before anyone complains about how awful this code looks, I previously had it set up within Async.JS but I removed it from this code sample to ensure that Asyn.cJS was not responsible for the issues.

Answer 1

Using the node native driver this works for me every time:

var mongodb = require('mongodb'),
    async = require('async'),
    MongoClient = mongodb.MongoClient;

var user = "john",
    contact  = "bob";

var contactsList = [
  {
    "username": "john",
    "contacts": [
      "jim"
    ],
    "pending_contacts": [
      "bob"
    ]
  },
  {
    "username": "bob",
    "contacts": [
      "dave"
    ],
    "sent_contacts": [
      "john"
    ]
  }
];

MongoClient.connect('mongodb://localhost/test',function(err,db) {

  var coll = db.collection("contacts");

  async.series(
    [
      // Wipe clean
      function(callback) {
        coll.remove({},callback)
      },

      // Init collection
      function(callback) {
        async.each(contactsList,function(contact,callback) {
          coll.insert(contact,callback);
        },callback);
      },

      // Do updates
      function(callback) {
        // Init batch
        var bulk = coll.initializeOrderedBulkOp();

        // Add to user and pull from pending
        bulk.find({
          "username": user,
          "contacts": { "$ne": contact },
        }).updateOne({
          "$push": { "contacts": contact },
          "$pull": { "pending_contacts": contact }
        });

        // Add to contact and pull from sent
        bulk.find({
          "username": contact,
          "contacts": { "$ne": user },
          "sent_contacts": user
        }).updateOne({
          "$push": { "contacts": user },
          "$pull": { "sent_contacts": user }
        });

        // Execute
        bulk.execute(function(err,response) {
          console.log( response.toJSON() );
          callback(err);
        });

      },

      // List collection
      function(callback) {
        coll.find({}).toArray(function(err,results) {
          console.log(results);
          callback(err);
        });
      }

    ],
    function(err) {
      if (err) throw err;
      db.close();
    }

  );

});

And the output:

{ ok: 1,
  writeErrors: [],
  writeConcernErrors: [],
  insertedIds: [],
  nInserted: 0,
  nUpserted: 0,
  nMatched: 2,
  nModified: 2,
  nRemoved: 0,
  upserted: [] }
[ { _id: 55b0c16934fadce812cdcf9d,
    username: 'john',
    contacts: [ 'jim', 'bob' ],
    pending_contacts: [] },
  { _id: 55b0c16934fadce812cdcf9e,
    username: 'bob',
    contacts: [ 'dave', 'john' ],
    sent_contacts: [] } ]

Improvements here are basically to use the Bulk Operations API and send all updates at once to the server and get a single response. Also note the use of operators in the updates and the query selection as well.

Simply put, you already know the "user" as well as the "contact" they are accepting. The contact to be accepted is "pending" and the contact themselves have the user in "sent".

These are really just simple $push and $pull operations on either array as is appropriate. Rather than using $addToSet here, the query conditions make sure that the expected values are present when performing the update. This also preserves "order" which $addToSet can basically not guarantee, because it's a "set", which is un-ordered.

One send to the server and one callback response, leaving both users updated correctly. Makes more sense then sending multiple updates and waiting for the callback response from each.

Anyhow, this is a complete self contained listing with only the two named dependencies, so you can easily run it yourself and confirm the results.

---

When I say "Complete and self contained" it means start a new project and simply run the code. Here's the complete instruction:

mkdir sample
cd sample
npm init
npm install mongodb --save
npm install async --save

Then create a file with the code listing in that folder, say test.js and then run:

node test.js