Best way to write custom json messages using log4j2

Question

I have been using log4j for different kind of projects and have some experience with log4j2. All implementations used the default appender and layout. Currently i need to write a application which writes in json format. So i tried the log4j2 JSONLayout layout by setting up a very simple log4j2 logger.

public class JSONLogger {

    private static final Logger LOGGER = LogManager.getLogger();

    public static void main(String[] args) {
        JSONLogger jsonlogger = new JSONLogger() ;
    }

    public JSONLogger() {
        LOGGER.log(Level.FATAL, "hi mum!") ;

         int val1 = 10, val2 = 11, val3 = 12;

         LOGGER.log(Level.FATAL,"val1={}, val2={}, val3={}", val1, val2, val3);
    }

}

jsonLoggerProperties.xml

<?xml version="1.0" encoding="UTF-8"?>
<Configuration status="DEBUG">
    <Properties>
        <Property name="log-path">/Users/petervannes/NetBeansProjects/JSONLogger/logfiles</Property>
    </Properties>

    <Appenders>
        <RollingFile name="json_file_appender" fileName="${log-path}/jsonlogger.json"
                     filePattern="${log-path}/%d{yyyyMMdd}_jsonlogger-%i.json" >
            <JSONLayout complete="true" compact="false"/>
            <Policies>
                <SizeBasedTriggeringPolicy size="1 KB" />
            </Policies>
            <DefaultRolloverStrategy max="4"/>
        </RollingFile>

    </Appenders>


    <Loggers>
        <root level="debug" additivity="false">
            <AppenderRef ref="json_file_appender"/>
        </root>
    </Loggers>
</Configuration>

Resulting in an log entry similar to;

, {
  "timeMillis" : 1474573600359,
  "thread" : "main",
  "level" : "FATAL",
  "loggerName" : "JSONLogger",
  "message" : "val1=10, val2=11, val3=12",
  "contextStack" : [ "fieldName2" ],
  "endOfBatch" : false,
  "loggerFqcn" : "org.apache.logging.log4j.spi.AbstractLogger",
  "threadId" : 1,
  "threadPriority" : 5
}

What i need is to log to a JSON format like this;

, {
  "DateTime" : "08/01/2016 21:33:22.334",
  "level" : "FATAL",
  "Summary" : "Something has gone wrong",
  "ChainManager" : "Manager A",
  "Optionals" : { "Key_1": "Value1",
                  "Key_2": "Value2" }
}

Is this possibile with the log4j2 JSONLayout or is there any other layout i can use to get this format ?

Answer 1

If you want to log a custom object as plain JSON, without any Log4j event fields:

• create Java class that extends from org.apache.logging.log4j.message.MapMessage . The class should also properly construct an instance of the MapMessage. See constructor implementation for details:

    public class MyClass extends MapMessage<MyClass, MyClass> {
    
      UUID id;
      String type;

      public MyClass(UUID id, String type) {
        super();
        this.id = id;
        this.type = type;
        this.put("id", id.toString());
        this.put("type", type);
      }

    }

• when logging, create an instance of the class and use it in the log statement, without any text message:

    org.apache.logging.log4j.Logger LOGGER = LogManager.getLogger("MY-JSON-LOGGER");

    var id = UUID.randomUUID();
    var type = "myType";

    var logMessageObject = new MyClass(id, type);

    LOGGER.info(logMessageObject);

• The important part in the Log4J 2 configuration file is eventTemplate that uses a map resolver - this is the part that will render MapMessage object as JSON:

    <?xml version="1.0" encoding="UTF-8"?>
    <Configuration status="INFO">
    
      <Appenders>
        <Console name="ConsoleTest" target="SYSTEM_OUT" follow="true">
            <JsonTemplateLayout eventTemplate='{"$resolver": "map", "stringified": true}'/>
        </Console>
        
    </Appenders>
    <Loggers>
        <Root level="info">
            <AppenderRef ref="Console" />
        </Root>
        <Logger name="MY-JSON-LOGGER" level="INFO" additivity="false">
            <AppenderRef ref="ConsoleTest"/>
        </Logger>
    </Loggers>
</Configuration>

• log output sample:

    {"id":"11111111-1111-1111-1111-111111111111","type":"myType"}

[UPDATE] looks like this works only for flat JSON models, that have only one level keys. Perhaps this is due to the way Log4j's MapMessage is implemented, because basically its API only accepts String values.

Answer 2

It is possible using MapMessage, FormattedMessage (or ParameterizedMessage if you only use {} placeholders and want a better performance) and Map Resolver Template.

1. Define a helper method to build log message:

public static MapMessage buildMessage(
        final String message,
        final Object messageArgument,
        final String supplementalDataKey,
        final String supplementalDataValue) {
    return new MapMessage<>(Map.of(
        "message", new FormattedMessage(message, messageArgument),
        supplementalDataKey, supplementalDataValue));
}

2. Use it with logger:

LOGGER.log(
    Level.FATAL, 
    buildMessage(
        "Message: {}", "argumentValue",
        "SupplementalDataKey", "SupplementalDataValue");

3. Define template:

{
  ...
  "message": {
    "$resolver": "map",
    "key": "message",
    "stringified": true
  },
  "SupplementalData": {
    "SupplementalDataKey": {
      "$resolver": "map",
      "key": "SupplementalDataKey"
    }
  },
  ...
}

As you can see, the trick here is to redefine message in the template as part of the MessageMap, and then include supplemental data field.

Answer 3

I know this is an old question but I think there is a better way to do it.

You should use the JSON Template Layout.

And then you will be able to configure your JsonLayout with a template like this one:

{
  "mdc": {
    "$resolver": "mdc"
  },
  "exception": {
    "exception_class": {
      "$resolver": "exception",
      "field": "className"
    },
    "exception_message": {
      "$resolver": "exception",
      "field": "message"
    },
    "stacktrace": {
      "$resolver": "exception",
      "field": "stackTrace",
      "stackTrace": {
        "stringified": true
      }
    }
  },
  "line_number": {
    "$resolver": "source",
    "field": "lineNumber"
  },
  "class": {
    "$resolver": "source",
    "field": "className"
  },
  "@version": 1,
  "source_host": "${hostName}",
  "message": {
    "$resolver": "message",
    "stringified": true
  },
  "thread_name": {
    "$resolver": "thread",
    "field": "name"
  },
  "@timestamp": {
    "$resolver": "timestamp"
  },
  "level": {
    "$resolver": "level",
    "field": "name"
  },
  "file": {
    "$resolver": "source",
    "field": "fileName"
  },
  "method": {
    "$resolver": "source",
    "field": "methodName"
  },
  "logger_name": {
    "$resolver": "logger",
    "field": "name"
  }
}

There are many configurations available using templates.

See more about it here:

https://logging.apache.org/log4j/2.x/manual/json-template-layout.html

Answer 4

If you're looking for a way to generate customized JSON log messages without any of the "noise" added by log4j2 you could create an implementation of the Message interface and use a different layout - for example PatternLayout. Below is an example of this approach:

First a class to implement Message inferface

import java.util.HashMap;
import java.util.Map;

import org.apache.logging.log4j.message.Message;

import com.google.gson.Gson;
import com.google.gson.GsonBuilder;

public class JSONMessage implements Message {

    private static final long serialVersionUID = 538439258494853324L;
    private String messageString;
    private static final Gson GSON = new GsonBuilder()
            .setPrettyPrinting()
            .create();

    public JSONMessage(){
        this(null);
    }

    public JSONMessage(Object msgObj){
        parseMessageAsJson(msgObj);
    }

    public JSONMessage(String msgStr){
        Map<String,String> msgObj = new HashMap<>();
        msgObj.put("message", msgStr);
        parseMessageAsJson(msgObj);
    }

    private void parseMessageAsJson(Object msgObj){
        messageString = GSON.toJson(msgObj);
    }

    @Override
    public String getFormattedMessage() {
        return messageString;
    }

    @Override
    public String getFormat() {
        return messageString;
    }

    @Override
    public Object[] getParameters() {
        return null;
    }

    @Override
    public Throwable getThrowable() {
        return null;
    }

}

Next the log4j2.xml configuration:

<?xml version="1.0" encoding="UTF-8"?>
<Configuration status="WARN" name="App">
    <Appenders>
        <Console name="Console" target="SYSTEM_OUT">
                <PatternLayout pattern="%m%n" />
        </Console>
    </Appenders>

    <Loggers>
        <Root level="trace">
            <AppenderRef ref="Console" />
        </Root>
    </Loggers>
</Configuration>

Now a simple application class to generate a log event

import java.util.HashMap;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;


public class App {

    private static final Logger logger = LogManager.getLogger();

    public static void main( String[] args )
    {
        HashMap<String,Object> msgMap = new HashMap<>();
        msgMap.put("someInt", 123);
        msgMap.put("note", "Maybe you put a message here");

        HashMap<String,Object> anotherMap = new HashMap<>();
        anotherMap.put("key1", "value1");
        anotherMap.put("key2", "value2");
        msgMap.put("map", anotherMap);
        logger.info(new JSONMessage(msgMap));
    }
}

Here is some sample output:

{
  "note": "Maybe you put a message here",
  "map": {
    "key1": "value1",
    "key2": "value2"
  },
  "someInt": 123
}

Note that I'm using Gson to create the JSON output, but you could use any library you want. Also note that this code does not generate "complete" JSON in that it does not add the square brackets at the start and end of the log or the comma between message objects.

Answer 5

The question is about writing a custom json messages using log4j2.

This is possible since version 2.11 of log4j2 version:

https://issues.apache.org/jira/browse/LOG4J2-2190

The new parameter of for JSONLayout is called

objectMessageAsJsonObject

. Sample project files;

log4j2.properties

status = error


appender.ana_whitespace.type = RollingFile
appender.ana_whitespace.name = ana_whitespace
appender.ana_whitespace.fileName = ${sys:es.logs.base_path:-target}${sys:file.separator}ana_whitespace.log
appender.ana_whitespace.layout.type = JsonLayout
appender.ana_whitespace.layout.propertiesAsList = false
appender.ana_whitespace.layout.compact = false
appender.ana_whitespace.layout.eventEol = true
appender.ana_whitespace.layout.objectMessageAsJsonObject = true
appender.ana_whitespace.layout.complete= true
appender.ana_whitespace.layout.properties= true
appender.ana_whitespace.filePattern = ${sys:es.logs.base_path:-target}${sys:file.separator}ana_whitespace-%d{yyyy-MM-dd}.log
appender.ana_whitespace.filter.1.type = MarkerFilter
appender.ana_whitespace.filter.1.onMismatch=DENY
appender.ana_whitespace.filter.1.onMatch=ACCEPT
appender.ana_whitespace.filter.1.marker=ANA_WHITESPACE
appender.ana_whitespace.policies.type = Policies
appender.ana_whitespace.policies.time.type = TimeBasedTriggeringPolicy
appender.ana_whitespace.policies.time.interval = 1
appender.ana_whitespace.policies.time.modulate = true
appender.ana_whitespace.policies.size.type = SizeBasedTriggeringPolicy
appender.ana_whitespace.policies.size.size = 10 MB

rootLogger.level = info
rootLogger.appenderRef.ana_whitespace.ref = ana_whitespace

Example Java code

package de.es.stemmer;

import java.io.IOException;
import java.util.Map;
import java.util.TreeMap;

import org.apache.http.client.ClientProtocolException;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.Marker;
import org.apache.logging.log4j.MarkerManager;
import org.apache.logging.log4j.ThreadContext;
import org.apache.logging.log4j.core.LoggerContext;
import org.apache.logging.log4j.message.ObjectMessage;

public class JsonLoggerTest {
    final static Logger log = LogManager.getLogger(JsonLoggerTest.class);
    final static Marker MARKER_WHITESPACE = MarkerManager.getMarker("ANA_WHITESPACE");

    public static void main(String[] args) throws ClientProtocolException, IOException {
        System.setProperty("es.logs.base_path", "target");
        System.setProperty("es.logs.cluster_name", "_cluster");
        LoggerContext.getContext().reconfigure();
        ThreadContext.put("orig", "MDC_origValue");
        ThreadContext.put("source", "MDC_sourceSnippet");
        Map<String, String> map = new TreeMap<>();
        map.put("orig", "msg_origValue");
        map.put("source", "msg_sourceSnippet");
        ObjectMessage msg = new ObjectMessage(map);
        log.info(MARKER_WHITESPACE, msg);
        ThreadContext.remove("orig");
        ThreadContext.remove("source");
    }

}

JSON Log Entry

[
{
  "thread" : "main",
  "level" : "INFO",
  "loggerName" : "de.es.stemmer.JsonLoggerTest",
  "marker" : {
    "name" : "ANA_WHITESPACE"
  },
  "message" : {
    "orig" : "msg_origValue",
    "source" : "msg_sourceSnippet"
  },
  "endOfBatch" : false,
  "loggerFqcn" : "org.apache.logging.log4j.spi.AbstractLogger",
  "instant" : {
    "epochSecond" : 1526576578,
    "nanoOfSecond" : 184000000
  },
  "contextMap" : {
    "orig" : "MDC_origValue",
    "source" : "MDC_sourceSnippet"
  },
  "threadId" : 1,
  "threadPriority" : 5
}

]

Answer 6

I found a solution which works for me; slf4j-json-logger. It is a slf4j framework, so should be included in the pom.xml. Sample project files;

pom.xml

  <?xml version="1.0" encoding="UTF-8"?>
  <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <modelVersion>4.0.0</modelVersion>
     <groupId>com.reddipped</groupId>
     <artifactId>JSONLogger_2</artifactId>
     <version>1.0-SNAPSHOT</version>
     <packaging>jar</packaging>
     <properties>

        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
        <maven.compiler.source>1.6</maven.compiler.source>
        <maven.compiler.target>1.6</maven.compiler.target>

        <mainClass>com.reddipped.jsonlogger_2.Test</mainClass>

        <slf4j.version>1.7.21</slf4j.version>
        <!-- current log4j 2 release -->
        <log4j.version>2.6.2</log4j.version> 

     </properties>

     <dependencies>
        <dependency>
           <groupId>org.slf4j</groupId>
           <artifactId>slf4j-api</artifactId>
           <version>${slf4j.version}</version>
        </dependency>
        <!-- Binding for Log4J -->
        <dependency>
           <groupId>org.apache.logging.log4j</groupId>
           <artifactId>log4j-slf4j-impl</artifactId>
           <version>${log4j.version}</version>
        </dependency>
        <!-- Log4j API and Core implementation required for binding -->
        <dependency>
           <groupId>org.apache.logging.log4j</groupId>
           <artifactId>log4j-api</artifactId>
           <version>${log4j.version}</version>
        </dependency>
        <dependency>
           <groupId>org.apache.logging.log4j</groupId>
           <artifactId>log4j-core</artifactId>
           <version>${log4j.version}</version>
        </dependency>
        <!-- Logger slf4j-json-logger -->
        <dependency>
           <groupId>com.savoirtech.logging</groupId>
           <artifactId>slf4j-json-logger</artifactId>
           <version>2.0.2</version>
        </dependency>
     </dependencies> 
  </project>

log4j2.xml

  <?xml version="1.0" encoding="UTF-8"?>

  <!--

  Use java property log4j.configurationFile to specify log4j2.xml location
  if not available in classpath

  -    Dlog4j.configurationFile="/Users/petervannes/NetBeansProjects/JSONLogger_2/src/mann/java/resources/log4j2.xml"

  -->
  <configuration status="trace">
     <Properties>
        <Property name="log-path">/Users/petervannes/NetBeansProjects/JSONLogger_2/logfiles</Property>
     </Properties>
     <appenders>
        <RollingFile name="RollingFile" fileName="${log-path}/jsonlogger.json"
                  filePattern="${log-path}/%d{yyyyMMdd}_jsonlogger-%i.json" >
           <PatternLayout>
              <pattern>%m%n</pattern>
           </PatternLayout> 
           <Policies>
              <TimeBasedTriggeringPolicy />
              <SizeBasedTriggeringPolicy size="1 KB" />
           </Policies>
           <DefaultRolloverStrategy max="4"/>
        </RollingFile>
     </appenders>
     <Loggers>
        <Logger name="JSONLogger" level="debug" additivity="false">
           <AppenderRef ref="RollingFile" />
        </Logger>
        <Root level="debug">
           <AppenderRef ref="RollingFile" />
        </Root>
     </Loggers>
  </configuration>

Example Java code

  package com.reddipped.jsonlogger_2;

  import com.savoirtech.logging.slf4j.json.LoggerFactory;
  import java.util.HashMap;
  import java.util.Map;

  /**
   *
   * @author petervannes
   */
  public class Test {
      public static void main(String[] args) {   
        Test t = new Test() ;
     }


     public Test() {

        LoggerFactory.setIncludeLoggerName(false);
        LoggerFactory.setDateFormatString("yyyy-MM-dd HH:mm:ss.SSS");

         com.savoirtech.logging.slf4j.json.logger.Logger LOGGER =  LoggerFactory.getLogger("JSONLogger");

     Map<String, String> optionalFields = new HashMap();
     optionalFields.put("CaseNumber", "C12.12343");
     optionalFields.put("Step","Assignment") ;
     optionalFields.put("Department","BPM") ;

     String LOB = "Business Administration" ;
     String Service = "DocumentService" ;
     String Process = "AddAttachements" ;
     String Reason = "Technical" ; 

     LOGGER.error().message("Conversion error 'incompatible PDF document'")
           .field("LOB",LOB)
           .field("Service", Service)
           .field("Process",Process)
           .field("Reason", Reason)
           .map("OptionalFields", optionalFields).log() ;
     }

  }

JSON Log Entry

  {
    "message": "Conversion error  'incompatible PDF document'",
    "LOB": "Business Administration",
    "Service": "DocumentService",
    "Process": "AddAttachements",
    "Reason": "Technical",
    "OptionalFields": {
     "Step": "Assignment",
     "Department": "BPM",
     "CaseNumber": "C12.12343"
    },
    "level": "ERROR",
    "thread_name": "main",
    "class": "com.reddipped.jsonlogger_2.Test",
    "@timestamp": "2016-09-23 10:18:06.623"
  }