Aragorn
Aragorn

Reputation: 15

C# Performance with Entity Framework

Here's my code that I'm trying to refactor and improve performance. I've tried tips from Fastest Way of Inserting in Entity Framework with 1000 commit count and recreate context. I have 50 collection to export from MongoDB to Sql Server. Some collection have more that milion documents up to 30mil which takes around 3 hours to move/save to sql. Any idea on how to improve performance here?


int chunkSize = 1_000;

var client = new MongoClient("mongo");
IMongoDatabase mongoDatabase = client.GetDatabase("mongoConnectionString");

await Export(mongoDatabase, chunkSize);

static async Task Export(IMongoDatabase mongoDatabase, int chunkSize)
{
    IMongoCollection<ProfileDto> profileCollection = mongoDatabase.GetCollection<ProfileDto>("Profile");

    var documentCount = await profileCollection.CountDocumentsAsync(FilterDefinition<ProfileDto>.Empty);
    Logger.Info($"Profile collection count: {documentCount}");

    var apacProfiles = profileCollection.Find(new BsonDocument()).ToList();

    foreach (var profiles in apacProfiles.Batch(chunkSize))
    {
        var sqlDbContext = new SqlDbContext();
        sqlDbContext.Configuration.AutoDetectChangesEnabled = false;
        sqlDbContext.Configuration.AutoDetectChangesEnabled = true;

        foreach (var profile in profiles)
        {
            ProfileTable profileTable = new ProfileTable();
            profileTable.Name = profile.Name;

            sqlDbContext.Profiles.Add(profile);
        }
        await sqlDbContext.SaveChangesAsync();
        sqlDbContext.Dispose();
    }

    var sqlDbContext = new SqlDbContext();
    var sqlCount = sqlDbContext.Profiles.Count();
    Logger.Info($"Profiles count: {sqlCount}");

    if (documentCount != sqlCount)
        Logger.Info($"Document count - Profiles count: {documentCount - sqlCount}");
    sqlDbContext.Dispose();
}

Upvotes: -2

Views: 88

Answers (0)

Related Questions