How can I change the log format of HTTP logging middleware in .NET 8

Question

I have a requirement for creating Request/Response Logging. I am using default Logger implementation of Microsoft.

//Add Http Logging
builder.Services.AddHttpLogging(options =>
{
    options.CombineLogs = true;

    options.LoggingFields = HttpLoggingFields.RequestQuery | HttpLoggingFields.RequestPath |
        HttpLoggingFields.RequestBody | HttpLoggingFields.ResponseStatusCode | HttpLoggingFields.Duration;
});


//Using Middleware
app.UseHttpLogging();

In my appSettings.json file:

"Logging": {
    "LogLevel": {
      "Default": "Information",
      "Microsoft.AspNetCore": "Warning",
      "Microsoft.AspNetCore.HttpLogging.HttpLoggingMiddleware": "Information"
    }
  }

My issue is the default Log format of HTTP logging Middleware. This is what printed on my console:

info: Microsoft.AspNetCore.HttpLogging.HttpLoggingMiddleware[9]
      Request and Response:
      PathBase:
      Path: /api/film/rating
      QueryString:
      StatusCode: 200
      RequestBody: {
  "title": "G K Mon",
  "rating": "G"
}
      RequestBodyStatus: [Completed]
      Duration: 1595.698

Can I customize this default log format of HTTP Logging Middleware? Additional question is that how should I store logs in file without using Third party library.

https://stackoverflow.com/a/68363461/22971430

Answer 1

I had same requirement where log events from HttpLoggingMiddleware needs to be in JSON format so I went with following.

• Created custom formatter inspired from Serilog.Formatting.Compact to handle Microsoft.AspNetCore.HttpLogging.HttpLoggingMiddleware events separatly.
• With Serilog Expressions, I have written conditional sinks to route log events to different formatter sink.
• In addition, I had created custom interceptor to not to enable HttpLoggingMiddleware for all non-json requests.

I have written full blog post on handling my requirement at https://ranbook.cloud/posts/serilog-http-payload-logging/ with sample GitHub demo app to showcase the results.

Hopefully, this helps you with your requirement.

appsettings.json serilog configuration.

{
    "Serilog": {
        "MinimumLevel": "Information",
        "WriteTo": [
            {
                "Name": "Conditional",
                "Args": {
                    "expression": "SourceContext <> 'Microsoft.AspNetCore.HttpLogging.HttpLoggingMiddleware'",
                    "configureSink": {
                        "Console": {
                            "Name": "Console",
                            "Args": {
                                "formatter": "Serilog.Formatting.Compact.CompactJsonFormatter, Serilog.Formatting.Compact"
                            }
                        }
                    }
                }
            },
            {
                "Name": "Conditional",
                "Args": {
                    "expression": "SourceContext = 'Microsoft.AspNetCore.HttpLogging.HttpLoggingMiddleware'",
                    "configureSink": {
                        "Console": {
                            "Name": "Console",
                            "Args": {
                                "formatter": {
                                    "type": "serilog_payload_demo.Configuration.PayloadLogFormatter, serilog-payload-demo"
                                }
                            }
                        }
                    }
                }
            }
        ]
    },
    "Logging": {
        "LogLevel": {
            "Default": "Information",
            "Microsoft.AspNetCore": "Warning"
        }
    }
}

Below is the formatter code.

using System.Globalization;
using Newtonsoft.Json;
using Serilog.Events;
using Serilog.Formatting;
using Serilog.Formatting.Json;
using Serilog.Parsing;

namespace serilog_payload_demo.Configuration;

public class PayloadLogFormatter : ITextFormatter
{
    readonly JsonValueFormatter _valueFormatter;

    public PayloadLogFormatter(JsonValueFormatter? valueFormatter = null)
    {
        _valueFormatter = valueFormatter ?? new JsonValueFormatter(typeTagName: "$type");
    }

    public void Format(LogEvent logEvent, TextWriter output)
    {
        FormatEvent(logEvent, output, _valueFormatter);
        output.WriteLine();
    }

    public static void FormatEvent(LogEvent logEvent, TextWriter output, JsonValueFormatter valueFormatter)
    {
        if (logEvent == null) throw new ArgumentNullException(nameof(logEvent));
        if (output == null) throw new ArgumentNullException(nameof(output));
        if (valueFormatter == null) throw new ArgumentNullException(nameof(valueFormatter));

        output.Write("{\"@t\":\"");
        output.Write(logEvent.Timestamp.UtcDateTime.ToString("O"));
        output.Write("\",\"@m\":\"PayloadEvent\"");

        output.Write(",\"@l\":\"");
        output.Write(logEvent.Level);
        output.Write('\"');

        if (logEvent.TraceId != null)
        {
            output.Write(",\"@tr\":\"");
            output.Write(logEvent.TraceId.Value.ToHexString());
            output.Write('\"');
        }

        if (logEvent.SpanId != null)
        {
            output.Write(",\"@sp\":\"");
            output.Write(logEvent.SpanId.Value.ToHexString());
            output.Write('\"');
        }


        foreach (var property in logEvent.Properties)
        {
            var name = property.Key;
            if (name.Length > 0 && name[0] == '@')
            {
                // Escape first '@' by doubling
                name = '@' + name;
            }
            //TO-DO: We can write the code in programmatic way to allow which properties to be logged part of this event.
            switch (name)
            {
                case "HttpLog":
                    break;
                case "SourceContext":
                    output.Write(',');
                    JsonValueFormatter.WriteQuotedJsonString("@sc", output);
                    output.Write(':');
                    valueFormatter.Format(property.Value, output);
                    break;
                case "Duration":
                    output.Write(',');
                    JsonValueFormatter.WriteQuotedJsonString("duration", output);
                    output.Write(':');
                    valueFormatter.Format(property.Value, output);
                    break;
                case "StatusCode":
                    output.Write(',');
                    JsonValueFormatter.WriteQuotedJsonString("statusCode", output);
                    output.Write(':');
                    valueFormatter.Format(property.Value, output);
                    break;
                case "ResponseBody":
                    output.Write(',');
                    JsonValueFormatter.WriteQuotedJsonString("responseBody", output);
                    output.Write(':');
                    output.Write(JsonConvert.DeserializeObject(property.Value.ToString()).ToString().Replace("\n", ""));
                    break;
                case "RequestBody":
                    output.Write(',');
                    JsonValueFormatter.WriteQuotedJsonString("requestBody", output);
                    output.Write(':');
                    output.Write(JsonConvert.DeserializeObject(property.Value.ToString()).ToString().Replace("\n", ""));
                    break;
                case "RequestPath":
                    output.Write(',');
                    JsonValueFormatter.WriteQuotedJsonString("requestPath", output);
                    output.Write(':');
                    valueFormatter.Format(property.Value, output);
                    break;
                case "Method":
                    output.Write(',');
                    JsonValueFormatter.WriteQuotedJsonString("requestMethod", output);
                    output.Write(':');
                    valueFormatter.Format(property.Value, output);
                    break;
                default:
                    output.Write(',');
                    JsonValueFormatter.WriteQuotedJsonString(name, output);
                    output.Write(':');
                    valueFormatter.Format(property.Value, output);
                    break;
            }
        }

        output.Write('}');
    }
}

Answer 2

HttpLogging has pre-defined format/order and doesn't have a built-in way to change. You could try implement a custom loggerProvider to modify the logmessage before log and then write to a file.

Reformat the logMessage before output.

    public class CustomHttpLogger : ILogger
    {
        private readonly string _filePath;

        public CustomHttpLogger(string filePath)
        {
            _filePath = filePath;
        }
        public IDisposable BeginScope<TState>(TState state) => null;

        public bool IsEnabled(LogLevel logLevel) => true;

        public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception exception, Func<TState, Exception, string> formatter)
        {
            var logMessage = formatter(state, exception);
            if (eventId.Name == "RequestResponseLog")
            {
                // Example: Reorder fields manually (string manipulation, regex, etc.)
                string reorderedLogMessage = ReorderLogFields(logMessage);

                // Output the reordered log message ,  if you use console without clearproviders you will find the default provider also logging.
                //Console.WriteLine(reorderedLogMessage);
                File.AppendAllText(_filePath, reorderedLogMessage);
            }
            //else
            //{
            //    Console.WriteLine(logMessage);
            //}
        }

        private string ReorderLogFields(string logMessage)
        {
            var messages=logMessage.Split("\r\n");
            string newMessage = "";

            newMessage += messages.FirstOrDefault(x => x.StartsWith("QueryString")) + "\r\n";
            newMessage += messages.FirstOrDefault(x => x.StartsWith("PathBase"))+ "\r\n";
            newMessage += messages.FirstOrDefault(x => x.StartsWith("RequestBody")) + "\r\n";
            newMessage += messages.FirstOrDefault(x => x.StartsWith("StatusCode")) + "\r\n";
            newMessage += messages.FirstOrDefault(x => x.StartsWith("Duration")) + "\r\n";

            return newMessage; 
        }
    }

Warp this logger to a logger provider

    public class CustomHttpLoggerProvider : ILoggerProvider
    {
        private readonly string _filePath;

        public CustomHttpLoggerProvider(string filePath)
        {
            _filePath = filePath;
        }
        public ILogger CreateLogger(string categoryName)
        {
            return new CustomHttpLogger(_filePath);
        }

        public void Dispose() { }
    }

Then added to logging providers

//This will clear the default providers, then only your custom provider will work.
//builder.Logging.ClearProviders();
builder.Logging.AddProvider(new CustomHttpLoggerProvider("E:\\log1.txt"));
builder.Services.AddHttpLogging(options =>
{
    
    options.CombineLogs = true;

    options.LoggingFields = HttpLoggingFields.RequestQuery | HttpLoggingFields.RequestPath |
        HttpLoggingFields.RequestBody | HttpLoggingFields.ResponseStatusCode | HttpLoggingFields.Duration;
});

Test result