CODEWITHSUNDEEP
mongodbexpressmongoosemongodb-query
Mohammad Zuha Khalid
Mohammad Zuha Khalid

Reputation: 87

Is there any efficient way to paginate given mongodb data in latest first order and avoid sort exceed memory limit for huge data?

I have given sample data collection as below. I am trying to get result in Latest first order so for that I have used pagination and sorted it respectively. But when I am trying to sort huge data say for API hit limit 100 . Then it is throwing me error as

"message": "Executor error during find command :: caused by :: Sort exceeded memory limit of 33554432 bytes, but did not opt in to external sorting. Aborting operation. Pass allowDiskUse:true to opt in."

{
  "success": true,
  "message": "",
  "status": {},
  "data": {
    "docs": [
 {
        "_id": "5d1dbe9cad2de907c4a6c888",
        "requestedBy": {
          "adminId": "1",
          "adminName": "JJ"
        },
        "reason": "Testing",
        "payload": [
          {
            "recipient": {
              "to": "[email protected]",
              "from": "[email protected]",
              "bcc": "[email protected]"
            },
            "data": {
              "b_name": "Cloud",
              "amount": 100,
              "l_id": 10,
              "date": "10",
              "month": "Jul",
              "year": "2019"
            },
            "template": {
              "name": "Reminder",
              "type": "EMAIL"
            }
          }
],
        "createdAt": "2019-07-04T08:53:48.129Z",
        "updatedAt": "2019-07-04T08:53:48.129Z",
        "__v": 0,
        "id": "5d1dbe9cad2de907c4a6c888"
      }
    ]
}

}

const mongoose = require("mongoose");
const mongoosePaginate = require("mongoose-paginate-v2");

mongoosePaginate.paginate.options = {
  lean: true,
  limit: 200,
  sort:{
    createdAt: -1
  }
};
const MODEL_NAME = "data";

const dSchema = mongoose.Schema(
  {
    requestedBy: {
      adminId: String,
      adminName: String
    },
    reason: String,
    payload: mongoose.Mixed
  },
  { timestamps: true }
);

// Plugins
dSchema.plugin(mongoosePaginate);

// Methods to interact with models
dSchema.statics.getTasks = (query, options) => {
  return mongoose.model(MODEL_NAME).paginate(query, options);
};

Upvotes: 1

Views: 482

Answers (1)

Tushar Gupta - curioustushar
Tushar Gupta - curioustushar

Reputation: 57105

https://mongoosejs.com/docs/guide.html#indexes

https://mongoosejs.com/docs/guide.html#timestamps

https://docs.mongodb.com/manual/tutorial/optimize-query-performance-with-indexes-and-projections/

Indexes also improve efficiency on queries that routinely sort on a given field.

https://docs.mongodb.com/manual/indexes/#create-an-index

Add an index to createdAt to improve the query performance

dSchema.index({ "createdAt" : -1 });

https://www.mongodb.com/blog/post/performance-best-practices-indexing

Upvotes: 2

Related Questions