Reputation: 15
Here's my code that I'm trying to refactor and improve performance. I've tried tips from Fastest Way of Inserting in Entity Framework with 1000 commit count and recreate context. I have 50 collection to export from MongoDB to Sql Server. Some collection have more that milion documents up to 30mil which takes around 3 hours to move/save to sql. Any idea on how to improve performance here?
int chunkSize = 1_000;
var client = new MongoClient("mongo");
IMongoDatabase mongoDatabase = client.GetDatabase("mongoConnectionString");
await Export(mongoDatabase, chunkSize);
static async Task Export(IMongoDatabase mongoDatabase, int chunkSize)
{
IMongoCollection<ProfileDto> profileCollection = mongoDatabase.GetCollection<ProfileDto>("Profile");
var documentCount = await profileCollection.CountDocumentsAsync(FilterDefinition<ProfileDto>.Empty);
Logger.Info($"Profile collection count: {documentCount}");
var apacProfiles = profileCollection.Find(new BsonDocument()).ToList();
foreach (var profiles in apacProfiles.Batch(chunkSize))
{
var sqlDbContext = new SqlDbContext();
sqlDbContext.Configuration.AutoDetectChangesEnabled = false;
sqlDbContext.Configuration.AutoDetectChangesEnabled = true;
foreach (var profile in profiles)
{
ProfileTable profileTable = new ProfileTable();
profileTable.Name = profile.Name;
sqlDbContext.Profiles.Add(profile);
}
await sqlDbContext.SaveChangesAsync();
sqlDbContext.Dispose();
}
var sqlDbContext = new SqlDbContext();
var sqlCount = sqlDbContext.Profiles.Count();
Logger.Info($"Profiles count: {sqlCount}");
if (documentCount != sqlCount)
Logger.Info($"Document count - Profiles count: {documentCount - sqlCount}");
sqlDbContext.Dispose();
}
Upvotes: -2
Views: 88