High memory consumption of .Net 6 app on Linux

Question

I encountered a very serious problem when switching from Windows to Linux (Ubuntu). The problem is very high memory consumption. I first noticed an increase in memory consumption on Windows when moving from .Net Framework to .Net Core 5. But then this was not much of a problem and was acceptable since the increase was not large. The .Net Framework application consumed approximately 150-200 megabytes and the .Net Core 5 application consumed approximately 250-350 megabytes. But when I switched to Linux, the consumption increased many times over. To make the transition, I switched to .Net 6. After running the application on Linux, I noticed that memory grows over time and does not decrease; with long-term operation of the application, memory consumption can reach 6-8 gigabytes. After doing a memory dump, I noticed that almost 90% was taken up by unmanagement memory. There are about 50 instances of this application running on my server, and they quickly consume all the memory, which leads to application crashes. I have no idea why this is happening, it seems that Garbage collection is not freeing the reserved memory. For information, on Windows under IIS the application worked in 32-bit on Linux in 64-bit.

I tried changing Garbage collection settings such as (DOTNET_GCConserveMemory and DOTNET_gcServer), but it didn't help.

My application is essentially a CMS hosted using nginx. Basically this is a site using liquid (https://mtirion.medium.com/using-liquid-for-text-base-templates-with-net-80ae503fa635) to parse content. It also runs several hosted services that periodically run tasks such as retrieving data from the database to send email if any, checking for changes in data in a file. Even when no requests are sent to my site (for example at night), memory usage does not decrease. I have done a few things like getting memory dump, dotnet-gcdump, dottrace and dotnetdump but I am not very knowledgeable about it and can provide as per need. https://i.sstatic.net/edFAS.png

Here is the code from one of my hosted services:

public override Task Execute()
{
    if (_isProcessingEmailSender)
        return Task.CompletedTask;
    _isProcessingEmailSender = true;
    return Task.Run(() =>
    {                             
        try
        {
            using (var scope = ServiceActivator.GetScope())
            {
                var dataContext = scope?.ServiceProvider?.GetRequiredService<IDataContext>();
                if (dataContext == null) return;
                var readyToSent = GetReadyToSend(dataContext);
                if (readyToSent.Any())
                {
                    var emailService = scope?.ServiceProvider?.GetRequiredService<IEmailService>();
                    if (emailService == null)
                        return;
                    var messagesToSend = new Queue<IEmailQueue>(readyToSent);
                    SendMessages(messagesToSend, dataContext, emailService);
                }
            }
            _isProcessingEmailSender = false;
        }
        catch (Exception ex)
        {
            _isProcessingEmailSender = false;
            Log.Error(ex, $"Error while sending emails. Error: {ex.Message}");
        }
        finally
        {
            _isProcessingEmailSender = false;
        }
    });
}       

private void SendMessages(Queue<IEmailQueue> messages, IDataContext dataContext, IEmailService emailService)
{
    while (messages.Any())
    {
        var email = messages.Dequeue();
        Send(email, dataContext, emailService);
    }
}
private IList<IEmailQueue> GetReadyToSend(IDataContext dataContext)
{
    var maxAttempts = _treeplSettings.EmailSender.MaxAttemptsCount;
    var updatedDateTime = DateTime.UtcNow.AddSeconds(-15);
    var countLimit = 15;
    var virtualPoints = dataContext.DatabaseProvider.GetPreparedEmail(maxAttempts, updatedDateTime, countLimit);
    if (!virtualPoints.Any())
        return new List<IEmailQueue>();
    return dataContext.Query<IEmailQueue>()
         .Where(e => virtualPoints.Any(vp => e.VirtualPointer.InstanceId == vp)).ToList();           
}
private void Send(IEmailQueue message, IDataContext dataContext, IEmailService emailService)
{
    if (message == null)
    {
        return;
    }
    message.SendAttempts++;
    try
    {
        Log.Information($"Sending email. Email Id {message?.VirtualPointer.Pointer.ToString()}, message to: {message?.To}, message body: {message?.Body}");
        if (TrySend(message.ToMailMessage(), emailService, out string errorMessage, out string sentServer))
        {
            message.State = EmailMessageState.Sent;
            message.SentServer = sentServer;
        }
        else
        {
            message.State = EmailMessageState.Failed;
            message.ErrorMessage = errorMessage;
        }
    }
    catch (Exception ex)
    {
        message.State = EmailMessageState.Failed;
        message.ErrorMessage = ex.Message;
        Log.Error($"Error while sending email. Email Id {message?.VirtualPointer.Pointer.ToString()}, message to: {message?.To}, message body: {message?.Body}");
        throw;
    }
    finally
    {
        dataContext.SerializeAll();
    }
}
public bool TrySend(MailMessage mail, IEmailService emailService, out string errorMessage, out string sentServer)
{
    SmtpSettings settings;
    var isSent = true;
    errorMessage = null;
    sentServer = null;

    if (CheckMailDomainIsVerified(mail.From, emailService))
    {
        settings = _treeplSettings.EmailSender.AwsSmtpSettings;
        sentServer = "AWS";
    }
    else
    {
        settings = _treeplSettings.EmailSender.CustomSmtpSettings;
        sentServer = "CUSTOM";
    }
    using (var client = GetClient(settings))
    {
        try
        {
            client.Send(mail);
        }
        catch (Exception ex)
        {
            errorMessage = ex.Message;
            isSent = false;
            throw new Exception(
                $"Error while sent email. Subject: {mail.Subject}, From: {mail.From}, To: {string.Join(",", mail.To.Select(x => x.Address))}", ex);
        }
    }
    return isSent;
}
private SmtpClient GetClient(SmtpSettings settings)
{
    var client = new SmtpClient(settings.Host, settings.Port)
    {
        Credentials = new NetworkCredential(settings.Username, settings.Password),
        EnableSsl = settings.EnableSsl
    };

    return client;
}
private bool CheckMailDomainIsVerified(MailAddress from, IEmailService emailService)
{
    try
    {
        var domainKey = $"{from.Host}-MailDomain";
        if (_emailDomainCache.TryGetValue(domainKey, out IMailDomain val))
        {
            return val != null && val.TxtStatus.Equals(SUCCESS);
        }
        else
        {
            var domain = emailService.GetIdentityVerificationAttribute(from.Host);
            _emailDomainCache.AddOrUpdate(domainKey, domain, (k, v) => domain);
            return domain != null && domain.TxtStatus.Equals(SUCCESS);
        }
    }
    catch
    {
        return false;
    }
}