Reputation: 11
I have built a web scraper that gets multiple URLs at once collects data and pushes it to a MongoDB. The total number of urls is 400k plus and it collects data of 100 urls at once and then uploads it to MongoDB atlas in a single collection, but after exactly 7399 documents it throws a maximum call stack size exceeded. And after I restart the loop from 7399 it then stops at 58998 exactly.
The Code
successCount += success.length;
failedCount += failed.length;
// FeedDataToMedData(success, failed, "drugs");
console.log(success.length, failed.length);
if (success.length != 0) {
// await (await DB())
// .collection("drugs-success")
// .insertMany(success, { ordered: false });
const toInsert = success.map((item) => ({
insertOne: { ...item },
}));
await (await DB()).collection("drugs-success").bulkWrite(toInsert, {
ordered: false,
});
}
if (failed.length != 0) {
// await (await DB())
// .collection("drugs-failed")
// .insertMany(failed, { ordered: false });
const toInsert = failed.map((item) => ({
insertOne: { ...item },
}));
await (await DB()).collection("drugs-failed").bulkWrite(toInsert, {
ordered: false,
});
}
}
The error
RangeError: Maximum call stack size exceeded
at calculateElement (/Users/eshansingh/dev/farmako/1mg-scraper/node_modules/bson/lib/parser/calculate_size.js:29:26)
at calculateObjectSize (/Users/eshansingh/dev/farmako/1mg-scraper/node_modules/bson/lib/parser/calculate_size.js:22:28)
at calculateElement (/Users/eshansingh/dev/farmako/1mg-scraper/node_modules/bson/lib/parser/calculate_size.js:158:21)
at calculateObjectSize (/Users/eshansingh/dev/farmako/1mg-scraper/node_modules/bson/lib/parser/calculate_size.js:22:28)
at calculateElement (/Users/eshansingh/dev/farmako/1mg-scraper/node_modules/bson/lib/parser/calculate_size.js:158:21)
at calculateObjectSize (/Users/eshansingh/dev/farmako/1mg-scraper/node_modules/bson/lib/parser/calculate_size.js:22:28)
at calculateElement (/Users/eshansingh/dev/farmako/1mg-scraper/node_modules/bson/lib/parser/calculate_size.js:158:21)
at calculateObjectSize (/Users/eshansingh/dev/farmako/1mg-scraper/node_modules/bson/lib/parser/calculate_size.js:22:28)
at calculateElement (/Users/eshansingh/dev/farmako/1mg-scraper/node_modules/bson/lib/parser/calculate_size.js:158:21)
at calculateObjectSize (/Users/eshansingh/dev/farmako/1mg-scraper/node_modules/bson/lib/parser/calculate_size.js:22:28)
RangeError: Maximum call stack size exceeded
at calculateElement (/Users/eshansingh/dev/farmako/1mg-scraper/node_modules/bson/lib/parser/calculate_size.js:29:26)
at calculateObjectSize (/Users/eshansingh/dev/farmako/1mg-scraper/node_modules/bson/lib/parser/calculate_size.js:22:28)
at calculateElement (/Users/eshansingh/dev/farmako/1mg-scraper/node_modules/bson/lib/parser/calculate_size.js:158:21)
at calculateObjectSize (/Users/eshansingh/dev/farmako/1mg-scraper/node_modules/bson/lib/parser/calculate_size.js:22:28)
at calculateElement (/Users/eshansingh/dev/farmako/1mg-scraper/node_modules/bson/lib/parser/calculate_size.js:158:21)
at calculateObjectSize (/Users/eshansingh/dev/farmako/1mg-scraper/node_modules/bson/lib/parser/calculate_size.js:22:28)
at calculateElement (/Users/eshansingh/dev/farmako/1mg-scraper/node_modules/bson/lib/parser/calculate_size.js:158:21)
at calculateObjectSize (/Users/eshansingh/dev/farmako/1mg-scraper/node_modules/bson/lib/parser/calculate_size.js:22:28)
at calculateElement (/Users/eshansingh/dev/farmako/1mg-scraper/node_modules/bson/lib/parser/calculate_size.js:158:21)
at calculateObjectSize (/Users/eshansingh/dev/farmako/1mg-scraper/node_modules/bson/lib/parser/calculate_size.js:22:28)
Upvotes: 1
Views: 367