Serilog HTTP Sink custom formatting for Logstash

Question

I am using Serilog HTTP sink for logging to Logstash in my .Net Core Project. In startup.cs I have following code to enable serilog.

 Log.Logger = new LoggerConfiguration()
        .Enrich.FromLogContext()
        .WriteTo.Http("http://mylogstashhost.com:5000").Enrich.WithProperty("user", "xxx").Enrich.WithProperty("serviceName", "yyy")
        .MinimumLevel.Warning()
        .CreateLogger();

And this code sends logs to the given http address. I can see on fiddler that following json is being posted to the logstash and logstash returns "ok" message.

{"events":[{"Timestamp":"2018-10-19T18:16:27.6561159+01:00","Level":"Warning","MessageTemplate":"abc","RenderedMessage":"abc","user":"xxx","serviceName":"yyy","Properties":{"ActionId":"b313b8ed-0baf-4d75-a6e2-f0dbcb941f67","ActionName":"MyProject.Controllers.HomeController.Index","RequestId":"0HLHLQMV1EBCJ:00000003","RequestPath":"/"}}]}

But when I checked on Kibana, I can not see this log. I tried to figure out what causes it and i realized that if I send the json as following format I can see the Log.

{"Timestamp":"2018-10-19T18:16:27.6561159+01:00","Level":"Warning","MessageTemplate":"abc","RenderedMessage":"abc","user":"xxx","serviceName":"yyy","Properties":{"ActionId":"b313b8ed-0baf-4d75-a6e2-f0dbcb941f67","ActionName":"MyProject.Controllers.HomeController.Index" ,"RequestId":"0HLHLQMV1EBCJ:00000003","RequestPath":"/"}}

So Logstash doesnt like the event to be in Events{} and also it wants "user" and "ServiceName" tags out of "Properties". Is there a way to format my Json like this?

Answer 1

You can simply process the batch file in logstash changing the pipeline as recommended by https://blog.romanpavlov.me/logging-serilog-elk/

# Http input listening port 8080
input {
    http {  
        #default host 0.0.0.0:8080
        codec => json
    }
}

# Separate the logs
filter {
    split {
        field => "events"
        target => "e"
        remove_field => "events"
    }
}

# Send the logs to Elasticsearch
output {
    elasticsearch {
        hosts => "elasticsearch:9200"
        index=>"customer-%{+xxxx.ww}"
    }
}

Answer 2

I would like to extend @nooaa answer with this variation. Instead of manipulating the string to add new objects, I would suggest using Newtonsoft.Json.Linq. This way you can append, add or remove existing properties of the object itself.

Also, instead of doing output.write after each event, you can combine all the output from the events and do output.write once at the end (a bit of performance)

public override void Format(IEnumerable<string> logEvents, TextWriter output)
{
    if (logEvents == null) throw new ArgumentNullException(nameof(logEvents));
    if (output == null) throw new ArgumentNullException(nameof(output));

    // Abort if sequence of log events is empty
    if (!logEvents.Any())
    {
        return;
    }
    
    List<object> updatedEvents = new List<object>();
    foreach (string logEvent in logEvents)
    {
        if (string.IsNullOrWhiteSpace(logEvent))
        {
            continue;
        }

        // Parse the log event
        var obj = JObject.Parse(logEvent);

        // Add New entries 
        obj["@source_host"] = obj["fields"]["MachineName"].Value<string>().ToLower();

        // Remove any entries you are not interested in
        ((JObject)obj["fields"]).Remove("MachineName");
                
        // Default tags for any log that goes out of your app.
        obj["@tags"] = new JArray() { "appName", "api" };

        // Additional tags from end points (custom based on routes)
        if (obj["fields"]["tags"] != null) 
        {
            ((JArray)obj["@tags"]).Merge((JArray)obj["fields"]["tags"]);
            ((JObject)obj["fields"]).Remove("tags");
        }
       
        updatedEvents.Add(obj);
    }

    output.Write(JsonConvert.SerializeObject(updatedEvents));
}

Update Release Notes v8.0.0

1. With latest release, you dont override the method anymore.

namespace Serilog.Sinks.Http.BatchFormatters {
  public class MyCustomFormatter: IBatchFormatter {
    public void Format(IEnumerable<string> logEvents, TextWriter output) {
      ...
    }
  }
}

2. you don't provide any Contructors for it either.
3. Add queueLimitBytes along with batchFormatter and textFormatter

WriteTo.Http(new Uri(), 
  batchFormatter: new MyCustomFormatter(),
  queueLimitBytes: 50 * ByteSize.MB,
  textFormatter: new ElasticsearchJsonFormatter());

Answer 3

Ok after some research and help, basically to achieve custom formats, one should implement interfaces like ITextFormatter, BatchFormatter etc.

I could achieve the format i need, by modifying ArrayBatchFormatter a little:

public class MyFormat : BatchFormatter
{
    /// <summary>
    /// Initializes a new instance of the <see cref="ArrayBatchFormatter"/> class.
    /// </summary>
    /// <param name="eventBodyLimitBytes">
    /// The maximum size, in bytes, that the JSON representation of an event may take before it
    /// is dropped rather than being sent to the server. Specify null for no limit. Default
    /// value is 256 KB.
    /// </param>
    public MyFormat(long? eventBodyLimitBytes = 256 * 1024): base(eventBodyLimitBytes)
    {

    }

    /// <summary>
    /// Format the log events into a payload.
    /// </summary>
    /// <param name="logEvents">
    /// The events to format.
    /// </param>
    /// <param name="output">
    /// The payload to send over the network.
    /// </param>
    public override void Format(IEnumerable<string> logEvents, TextWriter output)
    {
        if (logEvents == null) throw new ArgumentNullException(nameof(logEvents));
        if (output == null) throw new ArgumentNullException(nameof(output));

        // Abort if sequence of log events is empty
        if (!logEvents.Any())
        {
            return;
        }

        output.Write("[");

        var delimStart = string.Empty;

        foreach (var logEvent in logEvents)
        {
            if (string.IsNullOrWhiteSpace(logEvent))
            {
                continue;
            }
            int index = logEvent.IndexOf("{");

            string adjustedString = "{\"user\":\"xxx\",\"serviceName\" : \"yyy\"," + logEvent.Substring(1);
            if (CheckEventBodySize(adjustedString))
            {
                output.Write(delimStart);
                output.Write(adjustedString);
                delimStart = ",";
            }
        }

        output.Write("]");
    }
}