Reputation: 616
I am implementing a nodejs app where I am using worker threads for parallel data processing which requires mongo connection. I am using mongoose here for mongo operation but if I create connection in each worker its working but if don't create the query is getting timed out. So, how to fix this, as below is sample code but my code spawns 20 workers and like these around 50 pods might get created which can create more mongo connections.
db.js:
const mongoose = require("mongoose");
const mongoURI =
"mongodb+srv://xxxxxxxx.mongodb.net/opensearch_migration"; // Replace with your MongoDB connection string
mongoose.connect(mongoURI, { useNewUrlParser: true, useUnifiedTopology: true });
console.log("Connected to MongoDB");
const db = mongoose.connection;
db.on("error", console.error.bind(console, "MongoDB connection error:"));
module.exports = db;
worker.js:
const { parentPort } = require("worker_threads");
const { Migration } = require("./openSearchMigrationModel"); // Import the model
const { v4: uuidv4 } = require("uuid"); // For generating random thread IDs
const threadId = uuidv4(); // Generate a unique thread ID
async function queryDatabase() {
try {
while (true) {
const result = await Migration.find({ status: "pending" });
parentPort.postMessage(
`Thread ID: ${threadId}, Time: ${new Date().toISOString()}, Query Result: ${JSON.stringify(
result
)}`
);
await new Promise((resolve) => setTimeout(resolve, 5000)); // Wait for 5 seconds
}
} catch (error) {
parentPort.postMessage(`Thread ID: ${threadId}, Error: ${error.message}`);
}
}
queryDatabase();
app.js
const { Worker } = require("worker_threads");
const db = require("./db"); // Import the database connection
function spawnWorkers(numberOfWorkers) {
for (let i = 0; i < numberOfWorkers; i++) {
const worker = new Worker("./worker.js");
worker.on("message", (message) =>
console.log(`Worker Message: ${message}`)
);
}
}
spawnWorkers(4);
Upvotes: 1
Views: 287