Include additional JMX metrics in Micrometer / Prometheus info exposed by a Spring Boot Camel application

Question

I've configured a Spring Boot 2 application with Camel 3.9 to expose metrics, using Micrometer, through the actuator/prometheus endpoint, returning some Camel metrics correctly:

# HELP CamelExchangesFailed_total  
# TYPE CamelExchangesFailed_total counter
CamelExchangesFailed_total{camelContext="camel-1",routeId="route3",serviceName="MicrometerRoutePolicyService",} 0.0
CamelExchangesFailed_total{camelContext="camel-1",routeId="route2",serviceName="MicrometerRoutePolicyService",} 0.0
CamelExchangesFailed_total{camelContext="camel-1",routeId="route1",serviceName="MicrometerRoutePolicyService",} 0.0
# HELP CamelExchangesTotal_total  
# TYPE CamelExchangesTotal_total counter
CamelExchangesTotal_total{camelContext="camel-1",routeId="route3",serviceName="MicrometerRoutePolicyService",} 0.0
CamelExchangesTotal_total{camelContext="camel-1",routeId="route2",serviceName="MicrometerRoutePolicyService",} 0.0
CamelExchangesTotal_total{camelContext="camel-1",routeId="route1",serviceName="MicrometerRoutePolicyService",} 0.0
# HELP CamelExchangesExternalRedeliveries_total  
# TYPE CamelExchangesExternalRedeliveries_total counter
CamelExchangesExternalRedeliveries_total{camelContext="camel-1",routeId="route3",serviceName="MicrometerRoutePolicyService",} 0.0
CamelExchangesExternalRedeliveries_total{camelContext="camel-1",routeId="route2",serviceName="MicrometerRoutePolicyService",} 0.0
CamelExchangesExternalRedeliveries_total{camelContext="camel-1",routeId="route1",serviceName="MicrometerRoutePolicyService",} 0.0
# HELP CamelExchangesSucceeded_total  
# TYPE CamelExchangesSucceeded_total counter
CamelExchangesSucceeded_total{camelContext="camel-1",routeId="route3",serviceName="MicrometerRoutePolicyService",} 0.0
CamelExchangesSucceeded_total{camelContext="camel-1",routeId="route2",serviceName="MicrometerRoutePolicyService",} 0.0
CamelExchangesSucceeded_total{camelContext="camel-1",routeId="route1",serviceName="MicrometerRoutePolicyService",} 0.0
# HELP CamelExchangesFailuresHandled_total  
# TYPE CamelExchangesFailuresHandled_total counter
CamelExchangesFailuresHandled_total{camelContext="camel-1",routeId="route3",serviceName="MicrometerRoutePolicyService",} 0.0
CamelExchangesFailuresHandled_total{camelContext="camel-1",routeId="route2",serviceName="MicrometerRoutePolicyService",} 0.0
CamelExchangesFailuresHandled_total{camelContext="camel-1",routeId="route1",serviceName="MicrometerRoutePolicyService",} 0.0

Exploring the mbeans, I can see there are some other metrics I am also interested (e.g. route mean processing time)

The question is, how can I make Spring Boot Actuator + Micrometer + Prometheus include those additional metrics?

The following article describes a way to use a JMX agent to publish on Prometheus using a config.yml file:

src/main/resources/config.yml

rules:
  - pattern: 'fis.metrics<name=os.(.*)><>(.+):'
    name: os_$1
    help: some help

  - pattern: 'org.apache.camel<context=camel, type=routes, name=\"(.*)\"><>LastProcessingTime'
    name: camel_last_processing_time
    help: Last Processing Time [milliseconds]
    type: GAUGE
    labels:
      route: $1

However I can't find a way to do it with the current infrastructure of Spring Boot Actuator + Micrometer/Prometheus.

This is my config:

Dependencies:

plugins {
    id "org.springframework.boot" version "2.4.4"
    id "com.github.lkishalmi.gatling" version "3.3.4"
}

apply plugin: 'eclipse'
apply plugin: 'com.github.lkishalmi.gatling'

description = """sle-sync"""

ext {
    springCloudVersion = '2020.0.2'
    orikaVersion = '1.5.2'
    junitVersion = '5.2.0'
    junitPlatformVersion = '1.2.0'
    camelVersion = '3.9.0'
}

repositories {
    mavenLocal()
}

dependencyManagement {
    imports {
        mavenBom "org.apache.camel.springboot:camel-spring-boot-bom:${camelVersion}"
        mavenBom "org.springframework.cloud:spring-cloud-dependencies:${springCloudVersion}"
    }
}

dependencies {
    implementation 'org.apache.camel.springboot:camel-spring-boot-starter',
            'org.apache.camel.springboot:camel-servlet-starter',
            'org.apache.camel.springboot:camel-http-starter',
            'org.apache.camel.springboot:camel-metrics-starter',
            "org.apache.camel.springboot:camel-micrometer-starter",
            'com.playtika.sleuth:sleuth-camel-core:2.1.0',
            "org.springframework.boot:spring-boot-starter-web",
            "org.springframework.boot:spring-boot-starter-actuator",
            "org.springframework.cloud:spring-cloud-starter-config",
            "org.springframework.cloud:spring-cloud-starter-kubernetes-fabric8-config",
            "org.springframework.cloud:spring-cloud-starter-sleuth",
            "io.micrometer:micrometer-registry-prometheus",
            "ma.glasnost.orika:orika-core:${orikaVersion}",
            'org.projectlombok:lombok',
            "jakarta.xml.bind:jakarta.xml.bind-api:2.3.2",
            "org.glassfish.jaxb:jaxb-runtime:2.3.2",
            'org.apache.camel:camel-management'
    testCompile 'org.apache.camel:camel-test-spring',
            "org.springframework.boot:spring-boot-starter-test",
            'com.github.sbrannen:spring-test-junit5:1.0.2',
            "org.junit.jupiter:junit-jupiter-api:${junitVersion}"
    testRuntimeOnly "org.junit.platform:junit-platform-launcher:${junitPlatformVersion}",
            "org.junit.jupiter:junit-jupiter-engine:${junitVersion}"
    annotationProcessor "org.projectlombok:lombok:1.18.10"
    testAnnotationProcessor "org.projectlombok:lombok:1.18.10"
}

CamelContextConfiguration:

    @Bean
    public CamelContextConfiguration camelContextConfiguration() {

        return new CamelContextConfiguration() {
            @Override
            public void beforeApplicationStart(CamelContext camelContext) {
                camelContext.addRoutePolicyFactory(new MicrometerRoutePolicyFactory());
                camelContext.setMessageHistoryFactory(new MicrometerMessageHistoryFactory());
            }

            @Override
            public void afterApplicationStart(CamelContext camelContext) {

            }
        };
    }

application.yml

camel:
  component:
    servlet:
      mapping:
        context-path: /api/*
    metrics:

      metric-registry: prometheusMeterRegistry


management:
  endpoints:
    web:
      exposure:
        include: info, health, prometheus

Answer 1

Please check the camel metrics config first if there are things you can enable. If Camel creates distributions or timers, you can enable reporting percentiles on micrometer side. With Spring Boot, check the management.metrics property, e.g.:

management.metrics.distribution.percentiles.all=0.95, 0.98, 0.99, 0.999, 0.9999, 0.99999
management.metrics.distribution.percentiles-histogram.all=true

The Prometheus JMX Exporter can be a quick and dirty solution for this problem but I'm not sure I would recommend it wholeheartedly:

• The JVM agent complicates things since you need to add it to your deployment solution
• JMX is hierarchical, Prometheus is dimensional so you need to write and maintain the regular expressions to do the conversion (been there, it's not a very pleasant exercise)
• If you want to use another backend, Prometheus JMX Exporter will not work with them

As an alternative, you can write your own MeterBinder. It's a component that can register Meters to you registry (exactly what you need). Micrometer has a few of them that are registering Meters by using data from JMX, e.g.: KafkaConsumerMetrics, CommonsObjectPool2Metrics, you can take a look.

Also, since the Camel instrumentation is provided by Camel, you can open an issue for camel-metrics or camel-micrometer to add the missing Meters. Please feel free to cc me if you create an issue (https://github.com/jonatan-ivanov) for them.