CODEWITHSUNDEEP
node.jsmongodb
VarsMolta
VarsMolta

Reputation: 452

Insertion order of array elements in MongoDB

I am having trouble preserving insertion order of a bulk insert into mongodb.

My applications requires posting data continuously (via HTTP POST @ once a second) to a server. On the server side, the HTTP POST is handled and this data is stored in a capped collection in mongodb v2.4. The size of this capped collection is large (50MB). The format of this data is JSON and it has arrays in it like this:

{"Data":[{"Timestamp":"2014-08-02 13:38:18:852","Rabbit":[{"a":-0.0010663296561688185,"c":-0.0021321268286556005,"b":-0.0010663296561688185}],"Monkec":[{"a":17.511783599853516,"c":-0.42092469334602356,"b":-0.42092469334602356}]},{"Timestamp":"2014-08-02 13:38:18:858","Rabbit":[{"a":-0.0021321268286556005,"c":-0.0010663296561688185,"b":0.0010657970560714602}],"Monkec":[{"a":9.892329216003418,"c":-0.2339634746313095,"b":-0.2342628538608551}]},{"Timestamp":"2014-08-02 13:38:18:863","Rabbit":[{"a":-0.0010663296561688185,"c":-0.0021321268286556005,"b":0.0021315941121429205}],"Monkec":[{"a":9.702523231506348,"c":-0.24264541268348694,"b":-0.2148033082485199}]},{"Timestamp":"2014-08-02 13:38:18:866","Rabbit":[{"a":-0.0010663296561688185,"c":-0.0010663296561688185,"b":0.0010657970560714602}],"Monkec":[{"a":9.665101051330566,"c":-0.23366409540176392,"b":-0.2197430431842804}]},{"Timestamp":"2014-08-02 13:38:18:868","Rabbit":[{"a":-0.0021321268286556005,"c":-0.0010663296561688185,"b":0.0010657970560714602}],"Monkec":[{"a":9.693991661071777,"c":-0.2936892807483673,"b":-0.22857467830181122}]},{"Timestamp":"2014-08-02 13:38:18:872","Rabbit":[{"a":-0.0010663296561688185,"c":-0.0010663296561688185,"b":0.0010657970560714602}],"Monkec":[{"a":9.684710502624512,"c":-0.2296224981546402,"b":-0.13786330819129944}]},{"Timestamp":"2014-08-02 13:38:18:873","Rabbit":[{"a":-0.0021321268286556005,"c":-0.0010663296561688185,"b":0.0010657970560714602}],"Monkec":[{"a":9.67707633972168,"c":-0.31255003809928894,"b":-0.1902543604373932}]},{"Timestamp":"2014-08-02 13:38:18:875","Rabbit":[{"a":-0.0021321268286556005,"c":-0.0010663296561688185,"b":0}],"Monkec":[{"a":9.739496231079102,"c":-0.1899549812078476,"b":-0.18845809996128082}]},{"Timestamp":"2014-08-02 13:38:18:878","Rabbit":[{"a":-0.003197923768311739,"c":-0.0010663296561688185,"b":0.0010657970560714602}],"Monkec":[{"a":9.721234321594238,"c":-0.19205063581466675,"b":-0.17318984866142273}]},{"Timestamp":"2014-08-02 13:38:18:881","Rabbit":[{"a":-0.003197923768311739,"c":-0.003197923768311739,"b":0.0010657970560714602}],"Monkec":[{"a":9.78545093536377,"c":-0.2501298487186432,"b":-0.1953437775373459}]},{"Timestamp":"2014-08-02 13:38:18:882","Rabbit":[{"a":0,"c":-0.0010663296561688185,"b":0.0021315941121429205}],"Monkec":[{"a":9.686058044433594,"c":-0.21630020439624786,"b":-0.18247054517269135}]},{"Timestamp":"2014-08-02 13:38:18:884","Rabbit":[{"a":-0.0010663296561688185,"c":0,"b":0.0010657970560714602}],"Monkec":[{"a":9.67198657989502,"c":-0.18546432256698608,"b":-0.23156845569610596}]},{"Timestamp":"2014-08-02 13:38:18:887","Rabbit":[{"a":-0.0010663296561688185,"c":-0.0010663296561688185,"b":0.0010657970560714602}],"Monkec":[{"a":9.640103340148926,"c":-0.23276595771312714,"b":-0.25686585903167725}]},{"Timestamp":"2014-08-02 13:38:18:889","Rabbit":[{"a":-0.0010663296561688185,"c":0,"b":0}],"Monkec":[{"a":9.739346504211426,"c":-0.19130218029022217,"b":-0.22602996230125427}]},{"Timestamp":"2014-08-02 13:38:18:891","Rabbit":[{"a":-0.0021321268286556005,"c":-0.0010663296561688185,"b":0}],"Monkec":[{"a":9.716594696044922,"c":-0.22543121874332428,"b":-0.19728973507881165}]},{"Timestamp":"2014-08-02 13:38:18:898","Rabbit":[{"a":-0.0010663296561688185,"c":-0.0010663296561688185,"b":0.0010657970560714602}],"Monkec":[{"a":9.682914733886719,"c":-0.28680360317230225,"b":-0.1740879863500595}]},{"Timestamp":"2014-08-02 13:38:18:904","Rabbit":[{"a":-0.0010663296561688185,"c":0,"b":0.0021315941121429205}],"Monkec":[{"a":9.693093299865723,"c":-0.20866607129573822,"b":-0.2586621046066284}]},{"Timestamp":"2014-08-02 13:38:18:907","Rabbit":[{"a":-0.0021321268286556005,"c":-0.0010663296561688185,"b":0}],"Monkec":[{"a":9.690997123718262,"c":-0.18681152164936066,"b":-0.23216719925403595}]},{"Timestamp":"2014-08-02 13:38:18:910","Rabbit":[{"a":-0.003197923768311739,"c":-0.0010663296561688185,"b":0.0010657970560714602}],"Monkec":[{"a":9.671688079833984,"c":-0.15388000011444092,"b":-0.2588118016719818}]},{"Timestamp":"2014-08-02 13:38:19:055","Rabbit":[{"a":-0.0010663296561688185,"c":-0.0010663296561688185,"b":0}],"Monkec":[{"a":9.689650535583496,"c":-0.23605911433696747,"b":-0.1989363133907318}]}],"Serial":"35689"}

I am inserting this in mongodb (using NodeJs MongoClient driver) using a bulk insert command:

    var length = 20; // only doing 20 inserts for testing purposes
    for (var i = 0; i <length;i++) {        
    var bulk = col.initializeUnorderedBulkOp();
    bulk.insert(data["Data"][i]); // data is my JSON data of interest

    bulk.execute(function(err) {
    if (err) {
      return cb(err);
    }

    if (++inserted == length) {
       cb(); // callback (not seen in this code snippet)
    }
  }); // end function
  } // end of for loop

However, when I examine the entries in the database, they are not inserted in the order in which the data resides in the originating JSON array. My source data is in ascending Timestamp order, but a few entries in the mongodb capped collection are out of order. For instance, I see this:

{ "Timestamp" : "2014-08-02 13:38:18:910", "Rabbit" : [     {   "a" : -0.003197923768311739,    "c" : -0.0010663296561688185,   "b" : 0.0010657970560714602 } ], "Monkec" : [   {   "a" : 9.671688079833984,    "c" : -0.15388000011444092,     "b" : -0.2588118016719818 } ], "_id" : ObjectId("548e67a683946a5d25bc6d1a") }

{ "Timestamp" : "2014-08-02 13:38:18:884", "Rabbit" : [  {  "a" : -0.0010663296561688185,  "c" : 0,  "b" : 0.0010657970560714602 } ], "Monkec" : [  {   "a" : 9.67198657989502,     "c" : -0.18546432256698608,     "b" : -0.23156845569610596 } ], "_id" : ObjectId("548e67a683946a5d25bc6d13") }

{ "Timestamp" : "2014-08-02 13:38:18:904", "Rabbit" : [  {  "a" : -0.0010663296561688185,  "c" : 0,  "b" : 0.0021315941121429205 } ], "Monkec" : [  {   "a" : 9.693093299865723,    "c" : -0.20866607129573822,     "b" : -0.2586621046066284 } ], "_id" : ObjectId("548e67a683946a5d25bc6d18") }

so Timestamp" : "2014-08-02 13:38:18:910 is stored before "Timestamp" : "2014-08-02 13:38:18:884" even though it is the other way around in the source JSON.

How to ensure mongodb inserts data in the correct order? I also tried non bulk inserts (db.col.insert or db.col.insertOne) but still get this inconsistency. Thank You

Upvotes: 1

Views: 174

Answers (1)

Max Noel
Max Noel

Reputation: 8910

If your queries aren't asking for any specific sorting/ordering, MongoDB makes no guarantees as to in which order they'll be returned.

How you insert your data is irrelevant. What you need to do is write your find query like this:

// Sort by ascending timestamp
db.my_collection.find({ ... }).sort({"TimeStamp": 1})

See http://docs.mongodb.org/manual/reference/method/cursor.sort/#cursor.sort for more information on how sorting works.

Of course, if you want to do that, you'll greatly benefit from adding an index on Timestamp to your collection (see http://docs.mongodb.org/manual/core/indexes/).

Upvotes: 1

Related Questions