Caio Boratto
Reputation: 46
Apache NiFi redirect all logs to stdout
I have been trying to redirect all Apache NiFi logs to stdout configuring a new STDOUT appender into its ./conf/logback.xml.
That looks like this:
<configuration scan="true" scanPeriod="30 seconds">
<contextListener class="ch.qos.logback.classic.jul.LevelChangePropagator">
<resetJUL>true</resetJUL>
</contextListener>
<appender name="STDOUT" class="ch.qos.logback.core.rolling.RollingFileAppender">
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%date %level [%thread] %logger{40} %msg%n</pattern>
</encoder>
</appender>
<appender name="APP_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${org.apache.nifi.bootstrap.config.log.dir}/nifi-app.log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
<fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-app_%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern>
<maxFileSize>100MB</maxFileSize>
<!-- keep 30 log files worth of history -->
<maxHistory>30</maxHistory>
</rollingPolicy>
<immediateFlush>true</immediateFlush>
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%date %level [%thread] %logger{40} %msg%n</pattern>
</encoder>
</appender>
<appender name="USER_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${org.apache.nifi.bootstrap.config.log.dir}/nifi-user.log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-user_%d.log</fileNamePattern>
<!-- keep 30 log files worth of history -->
<maxHistory>30</maxHistory>
</rollingPolicy>
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%date %level [%thread] %logger{40} %msg%n</pattern>
</encoder>
</appender>
<appender name="BOOTSTRAP_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${org.apache.nifi.bootstrap.config.log.dir}/nifi-bootstrap.log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<!--
For daily rollover, use 'user_%d.log'.
For hourly rollover, use 'user_%d{yyyy-MM-dd_HH}.log'.
To GZIP rolled files, replace '.log' with '.log.gz'.
To ZIP rolled files, replace '.log' with '.log.zip'.
-->
<fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-bootstrap_%d.log</fileNamePattern>
<!-- keep 5 log files worth of history -->
<maxHistory>5</maxHistory>
</rollingPolicy>
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%date %level [%thread] %logger{40} %msg%n</pattern>
</encoder>
</appender>
<appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%date %level [%thread] %logger{40} %msg%n</pattern>
</encoder>
</appender>
<!-- valid logging levels: TRACE, DEBUG, INFO, WARN, ERROR -->
<logger name="org.apache.nifi" level="INFO"/>
<logger name="org.apache.nifi.processors" level="WARN"/>
<logger name="org.apache.nifi.processors.standard.LogAttribute" level="INFO"/>
<logger name="org.apache.nifi.processors.standard.LogMessage" level="INFO"/>
<logger name="org.apache.nifi.controller.repository.StandardProcessSession" level="WARN" />
<logger name="org.apache.zookeeper.ClientCnxn" level="ERROR" />
<logger name="org.apache.zookeeper.server.NIOServerCnxn" level="ERROR" />
<logger name="org.apache.zookeeper.server.NIOServerCnxnFactory" level="ERROR" />
<logger name="org.apache.zookeeper.server.quorum" level="ERROR" />
<logger name="org.apache.zookeeper.ZooKeeper" level="ERROR" />
<logger name="org.apache.zookeeper.server.PrepRequestProcessor" level="ERROR" />
<logger name="org.apache.calcite.runtime.CalciteException" level="OFF" />
<logger name="org.apache.curator.framework.recipes.leader.LeaderSelector" level="OFF" />
<logger name="org.apache.curator.ConnectionState" level="OFF" />
<!-- Logger for managing logging statements for nifi clusters. -->
<logger name="org.apache.nifi.cluster" level="INFO"/>
<!-- Logger for logging HTTP requests received by the web server. -->
<logger name="org.apache.nifi.server.JettyServer" level="INFO"/>
<!-- Logger for managing logging statements for jetty -->
<logger name="org.eclipse.jetty" level="INFO"/>
<!-- Suppress non-error messages due to excessive logging by class or library -->
<logger name="com.sun.jersey.spi.container.servlet.WebComponent" level="ERROR"/>
<logger name="com.sun.jersey.spi.spring" level="ERROR"/>
<logger name="org.springframework" level="ERROR"/>
<!-- Suppress non-error messages due to known warning about redundant path annotation (NIFI-574) -->
<logger name="com.sun.jersey.spi.inject.Errors" level="ERROR"/>
<!--
Logger for capturing user events. We do not want to propagate these
log events to the root logger. These messages are only sent to the
user-log appender.
-->
<logger name="org.apache.nifi.web.security" level="INFO" additivity="false">
<appender-ref ref="USER_FILE"/>
</logger>
<logger name="org.apache.nifi.web.api.config" level="INFO" additivity="false">
<appender-ref ref="USER_FILE"/>
</logger>
<logger name="org.apache.nifi.authorization" level="INFO" additivity="false">
<appender-ref ref="USER_FILE"/>
</logger>
<logger name="org.apache.nifi.cluster.authorization" level="INFO" additivity="false">
<appender-ref ref="USER_FILE"/>
</logger>
<logger name="org.apache.nifi.web.filter.RequestLogger" level="INFO" additivity="false">
<appender-ref ref="USER_FILE"/>
</logger>
<!--
Logger for capturing Bootstrap logs and NiFi's standard error and standard out.
-->
<logger name="org.apache.nifi.bootstrap" level="INFO" additivity="false">
<appender-ref ref="BOOTSTRAP_FILE" />
</logger>
<logger name="org.apache.nifi.bootstrap.Command" level="INFO" additivity="false">
<appender-ref ref="CONSOLE" />
<appender-ref ref="BOOTSTRAP_FILE" />
</logger>
<!-- Everything written to NiFi's Standard Out will be logged with the logger org.apache.nifi.StdOut at INFO level -->
<logger name="org.apache.nifi.StdOut" level="INFO" additivity="false">
<appender-ref ref="BOOTSTRAP_FILE" />
</logger>
<!-- Everything written to NiFi's Standard Error will be logged with the logger org.apache.nifi.StdErr at ERROR level -->
<logger name="org.apache.nifi.StdErr" level="ERROR" additivity="false">
<appender-ref ref="BOOTSTRAP_FILE" />
</logger>
<root level="INFO">
<appender-ref ref="APP_FILE"/>
<appender-ref ref="STDOUT"/>
</root>
</configuration>
The result so far is not what I expected. It keeps printing just its bootstrap and some other logging stuff:
14:27:51,395 |-INFO in ch.qos.logback.classic.LoggerContext[default] - Could NOT find resource [logback-test.xml]
14:27:51,395 |-INFO in ch.qos.logback.classic.LoggerContext[default] - Could NOT find resource [logback.groovy]
14:27:51,396 |-INFO in ch.qos.logback.classic.LoggerContext[default] - Found resource [logback.xml] at [file:/C:/nifi-1.4.0/conf/logback.xml]
14:27:51,465 |-INFO in ch.qos.logback.classic.joran.action.ConfigurationAction - debug attribute not set
14:27:51,472 |-INFO in ch.qos.logback.classic.joran.action.ConfigurationAction - Will scan for changes in [file:/C:/nifi-1.4.0/conf/logback.xml]
14:27:51,472 |-INFO in ch.qos.logback.classic.joran.action.ConfigurationAction - Setting ReconfigureOnChangeTask scanning period to 30 seconds
14:27:51,474 |-INFO in ch.qos.logback.classic.joran.action.LoggerContextListenerAction - Adding LoggerContextListener of type [ch.qos.logback.classic.jul.LevelChangePropagator] to the object stack
14:27:51,485 |-INFO in ch.qos.logback.classic.jul.LevelChangePropagator@58ceff1 - Propagating DEBUG level on Logger[ROOT] onto the JUL framework
14:27:51,485 |-INFO in ch.qos.logback.classic.joran.action.LoggerContextListenerAction - Starting LoggerContextListener
14:27:51,485 |-INFO in ch.qos.logback.core.joran.action.AppenderAction - About to instantiate appender of type [ch.qos.logback.core.rolling.RollingFileAppender]
14:27:51,489 |-INFO in ch.qos.logback.core.joran.action.AppenderAction - Naming appender as [STDOUT]
14:27:51,545 |-WARN in ch.qos.logback.core.rolling.RollingFileAppender[STDOUT] - No TriggeringPolicy was set for the RollingFileAppender named STDOUT
14:27:51,545 |-WARN in ch.qos.logback.core.rolling.RollingFileAppender[STDOUT] - For more information, please visit http://logback.qos.ch/codes.html#rfa_no_tp
14:27:51,545 |-INFO in ch.qos.logback.core.joran.action.AppenderAction - About to instantiate appender of type [ch.qos.logback.core.rolling.RollingFileAppender]
14:27:51,545 |-INFO in ch.qos.logback.core.joran.action.AppenderAction - Naming appender as [APP_FILE]
14:27:51,552 |-INFO in c.q.l.core.rolling.SizeAndTimeBasedRollingPolicy@2083562754 - Archive files will be limited to [100 MB] each.
14:27:51,554 |-INFO in c.q.l.core.rolling.SizeAndTimeBasedRollingPolicy@2083562754 - No compression will be used
14:27:51,555 |-INFO in c.q.l.core.rolling.SizeAndTimeBasedRollingPolicy@2083562754 - Will use the pattern C:/NIFI-1~1.0/bin/..//logs/nifi-app_%d{yyyy-MM-dd_HH}.%i.log for the active file
14:27:51,557 |-INFO in ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP@49e4cb85 - The date pattern is 'yyyy-MM-dd_HH' from file name pattern 'C:/NIFI-1~1.0/bin/..//logs/nifi-app_%d{yyyy-MM-dd_HH}.%i.log'.
14:27:51,557 |-INFO in ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP@49e4cb85 - Roll-over at the top of every hour.
14:27:51,559 |-INFO in ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP@49e4cb85 - Setting initial period to Tue Feb 20 12:14:09 BRT 2018
14:27:51,563 |-INFO in ch.qos.logback.core.rolling.RollingFileAppender[APP_FILE] - Active log file name: C:\NIFI-1~1.0\bin\..\\logs/nifi-app.log
14:27:51,563 |-INFO in ch.qos.logback.core.rolling.RollingFileAppender[APP_FILE] - File property is set to [C:\NIFI-1~1.0\bin\..\\logs/nifi-app.log]
14:27:51,564 |-INFO in ch.qos.logback.core.joran.action.AppenderAction - About to instantiate appender of type [ch.qos.logback.core.rolling.RollingFileAppender]
14:27:51,564 |-INFO in ch.qos.logback.core.joran.action.AppenderAction - Naming appender as [USER_FILE]
14:27:51,565 |-INFO in c.q.l.core.rolling.TimeBasedRollingPolicy@557041912 - No compression will be used
14:27:51,565 |-INFO in c.q.l.core.rolling.TimeBasedRollingPolicy@557041912 - Will use the pattern C:/NIFI-1~1.0/bin/..//logs/nifi-user_%d.log for the active file
14:27:51,566 |-INFO in c.q.l.core.rolling.DefaultTimeBasedFileNamingAndTriggeringPolicy - The date pattern is 'yyyy-MM-dd' from file name pattern 'C:/NIFI-1~1.0/bin/..//logs/nifi-user_%d.log'.
14:27:51,566 |-INFO in c.q.l.core.rolling.DefaultTimeBasedFileNamingAndTriggeringPolicy - Roll-over at midnight.
14:27:51,566 |-INFO in c.q.l.core.rolling.DefaultTimeBasedFileNamingAndTriggeringPolicy - Setting initial period to Tue Feb 20 11:38:18 BRT 2018
14:27:51,567 |-INFO in ch.qos.logback.core.rolling.RollingFileAppender[USER_FILE] - Active log file name: C:\NIFI-1~1.0\bin\..\\logs/nifi-user.log
14:27:51,567 |-INFO in ch.qos.logback.core.rolling.RollingFileAppender[USER_FILE] - File property is set to [C:\NIFI-1~1.0\bin\..\\logs/nifi-user.log]
14:27:51,568 |-INFO in ch.qos.logback.core.joran.action.AppenderAction - About to instantiate appender of type [ch.qos.logback.core.rolling.RollingFileAppender]
14:27:51,568 |-INFO in ch.qos.logback.core.joran.action.AppenderAction - Naming appender as [BOOTSTRAP_FILE]
14:27:51,568 |-INFO in c.q.l.core.rolling.TimeBasedRollingPolicy@1134712904
14:27:51,577 |-INFO in ch.qos.logback.classic.jul.LevelChangePropagator@58ceff1 - Propagating INFO level on Logger[org.apache.nifi.server.JettyServer] onto the JUL framework
14:27:51,577 |-INFO in ch.qos.logback.classic.joran.action.LoggerAction - Setting level of logger [org.eclipse.jetty] to INFO
14:27:51,577 |-INFO in ch.qos.logback.classic.jul.LevelChangePropagator@58ceff1 - Propagating INFO level on Logger[org.eclipse.jetty] onto the JUL framework
14:27:51,577 |-INFO in ch.qos.logback.classic.joran.action.LoggerAction - Setting level of logger [com.sun.jersey.spi.container.servlet.WebComponent] to ERROR
14:27:51,577 |-INFO in ch.qos.logback.classic.jul.LevelChangePropagator@58ceff1 - Propagating ERROR level on Logger[com.sun.jersey.spi.container.servlet.WebComponent] onto the JUL framework
14:27:51,577 |-INFO in ch.qos.logback.classic.joran.action.LoggerAction - Setting level of logger [com.sun.jersey.spi.spring] to ERROR
14:27:51,577 |-INFO in ch.qos.logback.classic.jul.LevelChangePropagator@58ceff1 - Propagating ERROR level on Logger[com.sun.jersey.spi.spring] onto the JUL framework
14:27:51,577 |-INFO in ch.qos.logback.classic.joran.action.LoggerAction - Setting level of logger [org.springframework] to ERROR
14:27:51,577 |-INFO in ch.qos.logback.classic.jul.LevelChangePropagator@58ceff1 - Propagating ERROR level on Logger[org.springframework] onto the JUL framework
14:27:51,577 |-INFO in ch.qos.logback.classic.joran.action.LoggerAction - Setting level of logger [com.sun.jersey.spi.inject.Errors] to ERROR
14:27:51,577 |-INFO in ch.qos.logback.classic.jul.LevelChangePropagator@58ceff1 - Propagating ERROR level on Logger[com.sun.jersey.spi.inject.Errors] onto the JUL framework
14:27:51,577 |-INFO in ch.qos.logback.classic.joran.action.LoggerAction - Setting level of logger [org.apache.nifi.web.security] to INFO
14:27:51,577 |-INFO in ch.qos.logback.classic.jul.LevelChangePropagator@58ceff1 - Propagating INFO level on Logger[org.apache.nifi.web.security] onto the JUL framework
14:27:51,577 |-INFO in ch.qos.logback.classic.joran.action.LoggerAction - Setting additivity of logger [org.apache.nifi.web.security] to false
14:27:51,577 |-INFO in ch.qos.logback.core.joran.action.AppenderRefAction - Attaching appender named [USER_FILE] to Logger[org.apache.nifi.web.security]
14:27:51,578 |-INFO in ch.qos.logback.classic.joran.action.LoggerAction - Setting level of logger [org.apache.nifi.web.api.config] to INFO
14:27:51,578 |-INFO in ch.qos.logback.classic.jul.LevelChangePropagator@58ceff1 - Propagating INFO level on Logger[org.apache.nifi.web.api.config] onto the JUL framework
14:27:51,578 |-INFO in ch.qos.logback.classic.joran.action.LoggerAction - Setting additivity of logger [org.apache.nifi.web.api.config] to false
14:27:51,578 |-INFO in ch.qos.logback.core.joran.action.AppenderRefAction - Attaching appender named [USER_FILE] to Logger[org.apache.nifi.web.api.config]
14:27:51,578 |-INFO in ch.qos.logback.classic.joran.action.LoggerAction - Setting level of logger [org.apache.nifi.authorization] to INFO
14:27:51,578 |-INFO in ch.qos.logback.classic.jul.LevelChangePropagator@58ceff1 - Propagating INFO level on Logger[org.apache.nifi.authorization] onto the JUL framework
14:27:51,578 |-INFO in ch.qos.logback.classic.joran.action.LoggerAction - Setting additivity of logger [org.apache.nifi.authorization] to false
14:27:51,578 |-INFO in ch.qos.logback.core.joran.action.AppenderRefAction - Attaching appender named [USER_FILE] to Logger[org.apache.nifi.authorization]
14:27:51,578 |-INFO in ch.qos.logback.classic.joran.action.LoggerAction - Setting level of logger [org.apache.nifi.cluster.authorization] to INFO
14:27:51,578 |-INFO in ch.qos.logback.classic.jul.LevelChangePropagator@58ceff1 - Propagating INFO level on Logger[org.apache.nifi.cluster.authorization] onto the JUL framework
14:27:51,578 |-INFO in ch.qos.logback.classic.joran.action.LoggerAction - Setting additivity of logger [org.apache.nifi.cluster.authorization] to false
14:27:51,578 |-INFO in ch.qos.logback.core.joran.action.AppenderRefAction - Attaching appender named [USER_FILE] to Logger[org.apache.nifi.cluster.authorization]
14:27:51,578 |-INFO in ch.qos.logback.classic.joran.action.LoggerAction - Setting level of logger [org.apache.nifi.web.filter.RequestLogger] to INFO
14:27:51,578 |-INFO in ch.qos.logback.classic.jul.LevelChangePropagator@58ceff1 - Propagating INFO level on Logger[org.apache.nifi.web.filter.RequestLogger] onto the JUL framework
14:27:51,578 |-INFO in ch.qos.logback.classic.joran.action.LoggerAction - Setting additivity of logger [org.apache.nifi.web.filter.RequestLogger] to false
14:27:51,578 |-INFO in ch.qos.logback.core.joran.action.AppenderRefAction - Attaching appender named [USER_FILE] to Logger[org.apache.nifi.web.filter.RequestLogger]
14:27:51,578 |-INFO in ch.qos.logback.classic.joran.action.LoggerAction - Setting level of logger [org.apache.nifi.bootstrap] to INFO
14:27:51,578 |-INFO in ch.qos.logback.classic.jul.LevelChangePropagator@58ceff1 - Propagating INFO level on Logger[org.apache.nifi.bootstrap] onto the JUL framework
14:27:51,578 |-INFO in ch.qos.logback.classic.joran.action.LoggerAction - Setting additivity of logger [org.apache.nifi.bootstrap] to false
14:27:51,578 |-INFO in ch.qos.logback.core.joran.action.AppenderRefAction - Attaching appender named [BOOTSTRAP_FILE] to Logger[org.apache.nifi.bootstrap]
14:27:51,578 |-INFO in ch.qos.logback.classic.joran.action.LoggerAction - Setting level of logger [org.apache.nifi.bootstrap.Command] to INFO
14:27:51,578 |-INFO in ch.qos.logback.classic.jul.LevelChangePropagator@58ceff1 - Propagating INFO level on Logger[org.apache.nifi.bootstrap.Command] onto the JUL framework
14:27:51,578 |-INFO in ch.qos.logback.classic.joran.action.LoggerAction - Setting additivity of logger [org.apache.nifi.bootstrap.Command] to false
14:27:51,578 |-INFO in ch.qos.logback.core.joran.action.AppenderRefAction - Attaching appender named [CONSOLE] to Logger[org.apache.nifi.bootstrap.Command]
14:27:51,578 |-INFO in ch.qos.logback.core.joran.action.AppenderRefAction - Attaching appender named [BOOTSTRAP_FILE] to Logger[org.apache.nifi.bootstrap.Command]
14:27:51,578 |-INFO in ch.qos.logback.classic.joran.action.LoggerAction - Setting level of logger [org.apache.nifi.StdOut] to INFO
14:27:51,578 |-INFO in ch.qos.logback.classic.jul.LevelChangePropagator@58ceff1 - Propagating INFO level on Logger[org.apache.nifi.StdOut] onto the JUL framework
14:27:51,579 |-INFO in ch.qos.logback.classic.joran.action.LoggerAction - Setting additivity of logger [org.apache.nifi.StdOut] to false
14:27:51,579 |-INFO in ch.qos.logback.core.joran.action.AppenderRefAction - Attaching appender named [BOOTSTRAP_FILE] to Logger[org.apache.nifi.StdOut]
14:27:51,579 |-INFO in ch.qos.logback.classic.joran.action.LoggerAction - Setting level of logger [org.apache.nifi.StdErr] to ERROR
14:27:51,579 |-INFO in ch.qos.logback.classic.jul.LevelChangePropagator@58ceff1 - Propagating ERROR level on Logger[org.apache.nifi.StdErr] onto the JUL framework
14:27:51,579 |-INFO in ch.qos.logback.classic.joran.action.LoggerAction - Setting additivity of logger [org.apache.nifi.StdErr] to false
14:27:51,579 |-INFO in ch.qos.logback.core.joran.action.AppenderRefAction - Attaching appender named [BOOTSTRAP_FILE] to Logger[org.apache.nifi.StdErr]
14:27:51,579 |-INFO in ch.qos.logback.classic.joran.action.RootLoggerAction - Setting level of ROOT logger to INFO
14:27:51,579 |-INFO in ch.qos.logback.classic.jul.LevelChangePropagator@58ceff1 - Propagating INFO level on Logger[ROOT] onto the JUL framework
14:27:51,579 |-INFO in ch.qos.logback.core.joran.action.AppenderRefAction - Attaching appender named [APP_FILE] to Logger[ROOT]
14:27:51,579 |-INFO in ch.qos.logback.core.joran.action.AppenderRefAction - Attaching appender named [STDOUT] to Logger[ROOT]
14:27:51,579 |-INFO in ch.qos.logback.classic.joran.action.ConfigurationAction - End of configuration.
14:27:51,579 |-INFO in ch.qos.logback.classic.joran.JoranConfigurator@3ac3fd8b - Registering current configuration as safe fallback point
2018-02-20 14:27:52,303 INFO [main] org.apache.nifi.bootstrap.Command Starting Apache NiFi...
2018-02-20 14:27:52,306 INFO [main] org.apache.nifi.bootstrap.Command Working Directory: C:\NIFI-1~1.0
2018-02-20 14:27:52,308 INFO [main] org.apache.nifi.bootstrap.Command Command: C:\java\jdk1.8.0_131\bin\java.exe -classpath C:\NIFI-1~1.0\.\conf;C:\NIFI-1~1.0\.\lib\javax.servlet-api-3.1.0.jar;C:\NIFI-1~1.0\.\lib\jcl-over-slf4j-1.7.25.jar;C:\NIFI-1~1.0\.\lib\jetty-schemas-3.1.jar;C:\NIFI-1~1.0\.\lib\jul-to-slf4j-1.7.25.jar;C:\NIFI-1~1.0\.\lib\log4j-over-slf4j-1.7.25.jar;C:\NIFI-1~1.0\.\lib\logback-classic-1.2.3.jar;C:\NIFI-1~1.0\.\lib\logback-core-1.2.3.jar;C:\NIFI-1~1.0\.\lib\nifi-api-1.4.0.jar;C:\NIFI-1~1.0\.\lib\nifi-framework-api-1.4.0.jar;C:\NIFI-1~1.0\.\lib\nifi-nar-utils-1.4.0.jar;C:\NIFI-1~1.0\.\lib\nifi-properties-1.4.0.jar;C:\NIFI-1~1.0\.\lib\nifi-runtime-1.4.0.jar;C:\NIFI-1~1.0\.\lib\ojdbc7.jar;C:\NIFI-1~1.0\.\lib\slf4j-api-1.7.25.jar -Dorg.apache.jasper.compiler.disablejsr199=true -Xmx512m -Xms512m -Djava.security.egd=file:/dev/urandom -Dsun.net.http.allowRestrictedHeaders=true -Djava.net.preferIPv4Stack=true -Djava.awt.headless=true -XX:+UseG1GC -Djava.protocol.handler.pkgs=sun.net.www.protocol -Dnifi.properties.file.path=C:\NIFI-1~1.0\.\conf\nifi.properties -Dnifi.bootstrap.listen.port=53032 -Dapp=NiFi -Dorg.apache.nifi.bootstrap.config.log.dir=C:\NIFI-1~1.0\bin\..\\logs org.apache.nifi.NiFi
2018-02-20 14:27:52,732 WARN [main] org.apache.nifi.bootstrap.Command Failed to set permissions so that only the owner can read pid file C:\NIFI-1~1.0\bin\..\run\nifi.pid; this may allows others to have access to the key needed to communicate with NiFi. Permissions should be changed so that only the owner can read this file
2018-02-20 14:27:52,748 WARN [main] org.apache.nifi.bootstrap.Command Failed to set permissions so that only the owner can read status file C:\NIFI-1~1.0\bin\..\run\nifi.status; this may allows others to have access to the key needed to communicate with NiFi. Permissions should be changed so that only the owner can read this file
2018-02-20 14:27:52,759 INFO [main] org.apache.nifi.bootstrap.Command Launched Apache NiFi with Process ID 5924
And when I go to the ./logs/nifi-app.log, what I really wanted to see in my stdout is:
2018-02-20 14:28:23,830 INFO [StandardProcessScheduler Thread-1] o.a.n.c.s.TimerDrivenSchedulingAgent Scheduled PutS3Object[id=c6b4a824-015f-1000-18a6-cc48e1317763] to run with 1 threads
2018-02-20 14:28:23,869 INFO [main] o.e.jetty.server.handler.ContextHandler Started o.e.j.w.WebAppContext@41a4135e{/,file:///C:/nifi-1.4.0/work/jetty/nifi-web-error-1.4.0.war/webapp/,AVAILABLE}{.\work\nar\framework\nifi-framework-nar-1.4.0.nar-unpacked\META-INF\bundled-dependencies\nifi-web-error-1.4.0.war}
2018-02-20 14:28:23,905 INFO [main] o.eclipse.jetty.server.AbstractConnector Started ServerConnector@6abd88fe{HTTP/1.1,[http/1.1]}{0.0.0.0:8080}
2018-02-20 14:28:23,906 INFO [main] org.eclipse.jetty.server.Server Started @31510ms
2018-02-20 14:28:24,827 INFO [main] org.apache.nifi.web.server.JettyServer NiFi has started. The UI is available at the following URLs:
2018-02-20 14:28:24,828 INFO [main] org.apache.nifi.web.server.JettyServer http://172.22.5.70:8080/nifi
2018-02-20 14:28:24,828 INFO [main] org.apache.nifi.web.server.JettyServer http://192.168.56.1:8080/nifi
2018-02-20 14:28:24,828 INFO [main] org.apache.nifi.web.server.JettyServer http://127.0.0.1:8080/nifi
2018-02-20 14:28:24,828 INFO [main] org.apache.nifi.web.server.JettyServer http://192.168.99.1:8080/nifi
2018-02-20 14:28:24,830 INFO [main] org.apache.nifi.BootstrapListener Successfully initiated communication with Bootstrap
2018-02-20 14:28:24,830 INFO [main] org.apache.nifi.NiFi Controller initialization took 26283906190 nanoseconds (26 seconds).
2018-02-20 14:28:27,081 ERROR [Timer-Driven Process Thread-9] o.a.nifi.processors.aws.s3.PutS3Object PutS3Object[id=c6b4a824-015f-1000-18a6-cc48e1317763] Failed to put StandardFlowFileRecord[uuid=107879bd-8675-471b-ba13-471c6d4b8df8,claim=StandardContentClaim [resourceClaim=StandardResourceClaim[id=1519147703257-1, container=default, section=1], offset=0, length=153971],offset=0,name=274138647934808,size=153811] to Amazon S3 due to com.amazonaws.services.s3.model.AmazonS3Exception: Invalid id (Service: Amazon S3; Status Code: 400; Error Code: InvalidArgument; Request ID: E93DF68048C87611), S3 Extended Request ID: 5MBwIoUlrjjsuH7VRa/uKzLm3kQe0UFYH+ZRZSyTRHEc3FBauH+s0ZOiCzc/4VBHoZy92yeQ0Ts=: {}
com.amazonaws.services.s3.model.AmazonS3Exception: Invalid id (Service: Amazon S3; Status Code: 400; Error Code: InvalidArgument; Request ID: E93DF68048C87611)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1545)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1183)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:964)
at com.amazonaws.java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
All this because NiFi is running into Mesos and by default, all our containers logs are captured in stdout.
Question: Does anyone know how to print all Apache NiFi logs in STDOUT?
Upvotes: 1
Views: 4184
Answers (3)
S'chn T'gai Spock
Reputation: 1366
Adding the CONSOLE appender to the 'org.apache.nifi.StdOut' and 'org.apache.nifi.StdErr' loggers would do the trick.
Below is my complete logback.xml
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<configuration scan="true" scanPeriod="30 seconds">
<contextListener class="ch.qos.logback.classic.jul.LevelChangePropagator">
<resetJUL>true</resetJUL>
</contextListener>
<appender name="APP_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${org.apache.nifi.bootstrap.config.log.dir}/nifi-app.log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
<!--
For daily rollover, use 'app_%d.log'.
For hourly rollover, use 'app_%d{yyyy-MM-dd_HH}.log'.
To GZIP rolled files, replace '.log' with '.log.gz'.
To ZIP rolled files, replace '.log' with '.log.zip'.
-->
<fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-app_%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern>
<maxFileSize>100MB</maxFileSize>
<!-- keep 30 log files worth of history -->
<maxHistory>30</maxHistory>
</rollingPolicy>
<immediateFlush>true</immediateFlush>
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%date %level [%thread] %logger{40} %msg%n</pattern>
</encoder>
</appender>
<appender name="USER_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${org.apache.nifi.bootstrap.config.log.dir}/nifi-user.log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<!--
For daily rollover, use 'user_%d.log'.
For hourly rollover, use 'user_%d{yyyy-MM-dd_HH}.log'.
To GZIP rolled files, replace '.log' with '.log.gz'.
To ZIP rolled files, replace '.log' with '.log.zip'.
-->
<fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-user_%d.log</fileNamePattern>
<!-- keep 30 log files worth of history -->
<maxHistory>30</maxHistory>
</rollingPolicy>
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%date %level [%thread] %logger{40} %msg%n</pattern>
</encoder>
</appender>
<appender name="BOOTSTRAP_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${org.apache.nifi.bootstrap.config.log.dir}/nifi-bootstrap.log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<!--
For daily rollover, use 'user_%d.log'.
For hourly rollover, use 'user_%d{yyyy-MM-dd_HH}.log'.
To GZIP rolled files, replace '.log' with '.log.gz'.
To ZIP rolled files, replace '.log' with '.log.zip'.
-->
<fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-bootstrap_%d.log</fileNamePattern>
<!-- keep 5 log files worth of history -->
<maxHistory>5</maxHistory>
</rollingPolicy>
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%date %level [%thread] %logger{40} %msg%n</pattern>
</encoder>
</appender>
<appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%date %level [%thread] %logger{40} %msg%n</pattern>
</encoder>
</appender>
<!-- valid logging levels: TRACE, DEBUG, INFO, WARN, ERROR -->
<logger name="org.apache.nifi" level="INFO">
</logger>
<logger name="org.apache.nifi.processors" level="WARN"/>
<logger name="org.apache.nifi.processors.standard.LogAttribute" level="INFO"/>
<logger name="org.apache.nifi.processors.standard.LogMessage" level="INFO"/>
<logger name="org.apache.nifi.controller.repository.StandardProcessSession" level="WARN" />
<logger name="org.apache.zookeeper.ClientCnxn" level="ERROR" />
<logger name="org.apache.zookeeper.server.NIOServerCnxn" level="ERROR" />
<logger name="org.apache.zookeeper.server.NIOServerCnxnFactory" level="ERROR" />
<logger name="org.apache.zookeeper.server.quorum" level="ERROR" />
<logger name="org.apache.zookeeper.ZooKeeper" level="ERROR" />
<logger name="org.apache.zookeeper.server.PrepRequestProcessor" level="ERROR" />
<logger name="org.apache.calcite.runtime.CalciteException" level="OFF" />
<logger name="org.apache.curator.framework.recipes.leader.LeaderSelector" level="OFF" />
<logger name="org.apache.curator.ConnectionState" level="OFF" />
<!-- Logger for managing logging statements for nifi clusters. -->
<logger name="org.apache.nifi.cluster" level="INFO"/>
<!-- Logger for logging HTTP requests received by the web server. -->
<logger name="org.apache.nifi.server.JettyServer" level="INFO"/>
<!-- Logger for managing logging statements for jetty -->
<logger name="org.eclipse.jetty" level="INFO"/>
<!-- Suppress non-error messages due to excessive logging by class or library -->
<logger name="org.springframework" level="ERROR"/>
<!-- Suppress non-error messages due to known warning about redundant path annotation (NIFI-574) -->
<logger name="org.glassfish.jersey.internal.Errors" level="ERROR"/>
<!-- Suppress non-error messages due to Jetty AnnotationParser emitting a large amount of WARNS. Issue described in NIFI-5479. -->
<logger name="org.eclipse.jetty.annotations.AnnotationParser" level="ERROR"/>
<!-- Suppress non-error messages from SSHJ which was emitting large amounts of INFO logs by default -->
<logger name="net.schmizz.sshj" level="WARN" />
<logger name="com.hierynomus.sshj" level="WARN" />
<!--
Logger for capturing user events. We do not want to propagate these
log events to the root logger. These messages are only sent to the
user-log appender.
-->
<logger name="org.apache.nifi.web.security" level="INFO" additivity="false">
<appender-ref ref="USER_FILE"/>
</logger>
<logger name="org.apache.nifi.web.api.config" level="INFO" additivity="false">
<appender-ref ref="USER_FILE"/>
</logger>
<logger name="org.apache.nifi.authorization" level="INFO" additivity="false">
<appender-ref ref="USER_FILE"/>
</logger>
<logger name="org.apache.nifi.cluster.authorization" level="INFO" additivity="false">
<appender-ref ref="USER_FILE"/>
</logger>
<logger name="org.apache.nifi.web.filter.RequestLogger" level="INFO" additivity="false">
<appender-ref ref="USER_FILE"/>
</logger>
<logger name="org.apache.nifi.web.api.AccessResource" level="INFO" additivity="false">
<appender-ref ref="USER_FILE"/>
</logger>
<!--
Logger for capturing Bootstrap logs and NiFi's standard error and standard out.
-->
<logger name="org.apache.nifi.bootstrap" level="INFO" additivity="false">
<appender-ref ref="CONSOLE" />
<appender-ref ref="BOOTSTRAP_FILE" />
</logger>
<logger name="org.apache.nifi.bootstrap.Command" level="INFO" additivity="false">
<appender-ref ref="CONSOLE" />
<appender-ref ref="BOOTSTRAP_FILE" />
</logger>
<!-- Everything written to NiFi's Standard Out will be logged with the logger org.apache.nifi.StdOut at INFO level -->
<logger name="org.apache.nifi.StdOut" level="INFO" additivity="false">
<appender-ref ref="CONSOLE" />
<appender-ref ref="BOOTSTRAP_FILE" />
</logger>
<!-- Everything written to NiFi's Standard Error will be logged with the logger org.apache.nifi.StdErr at ERROR level -->
<logger name="org.apache.nifi.StdErr" level="ERROR" additivity="false">
<appender-ref ref="CONSOLE" />
<appender-ref ref="BOOTSTRAP_FILE" />
</logger>
<root level="INFO">
<appender-ref ref="CONSOLE" />
</root>
</configuration>
Upvotes: 0
Caio Boratto
Reputation: 46
I tried to use the solution mentioned by @Jacob Hughes changing to CONSOLE appender and adding their root levels, but couldn't make it work.
So I found out a workaround: As my main objective is just to send the output to stdout I'm using tail -f ./logs/nifi-app.log. This resolved my issue.
Upvotes: 0
Jacob Hughes
Reputation: 70
I think you may have the wrong class on STDOUT appender.
I have just been using the default CONSOLE appender and I believe that logs everything to the console.
<appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%date %level [%thread] %logger{40} %msg%n</pattern>
</encoder>
</appender>
Then towards the end of your file you need to update the log levels to use this appender:
<root level="INFO">
<appender-ref ref="APP_FILE"/>
<appender-ref ref="CONSOLE"/>
</root>
<root level="ERRROR">
<appender-ref ref="CONSOLE"/>
</root>
<root level="DEBUG">
<appender-ref ref="CONSOLE"/>
</root>
This other question may help you file and stdout appenders in logback.xml
Alternatively, you can edit these lines:
<!-- Everything written to NiFi's Standard Out will be logged with the logger org.apache.nifi.StdOut at INFO level -->
<logger name="org.apache.nifi.StdOut" level="INFO" additivity="false">
<appender-ref ref="BOOTSTRAP_FILE" />
</logger>
<!-- Everything written to NiFi's Standard Error will be logged with the logger org.apache.nifi.StdErr at ERROR level -->
<logger name="org.apache.nifi.StdErr" level="ERROR" additivity="false">
<appender-ref ref="BOOTSTRAP_FILE" />
</logger>
to use CONSOLE instead of BOOTSTRAP_FILE.
Upvotes: 3
Related Questions
- Logback messages getting funneled to a single logger
- Logging to seperate file in Apache NiFi Executescript processor (python)
- How to get logs using rest api in apache nifi
- How do I separate out the logs for different dataflows in Apache NiFi
- Making log files in NiFi
- logback.xml:How to make 'debug' logs in nifi
- NIFI:how to get custom log files by tailFail?
- Logging bad response in nifi
- Separate log file for custom processor
- How to get the error messages wrote into the logs using the NIFI processors