From 1c0be58e252e588b5624969742d4ef6cda9735f2 Mon Sep 17 00:00:00 2001 From: jomin mathew <> Date: Wed, 29 Apr 2026 16:41:33 +0100 Subject: [PATCH 1/2] Add Grafana observability stack with Prometheus and Tempo to observability example --- observability/README.adoc | 144 ++- .../main/docker/docker-compose-grafana.yml | 81 ++ .../provisioning/dashboards/dashboards.yml | 30 + .../provisioning/datasources/datasources.yml | 48 + .../src/main/docker/prometheus/prometheus.yml | 35 + observability/src/main/docker/tempo/tempo.yml | 56 + .../src/main/resources/application.properties | 1 + .../camel-observability-dashboard.json | 1132 +++++++++++++++++ 8 files changed, 1519 insertions(+), 8 deletions(-) create mode 100644 observability/src/main/docker/docker-compose-grafana.yml create mode 100644 observability/src/main/docker/grafana/provisioning/dashboards/dashboards.yml create mode 100644 observability/src/main/docker/grafana/provisioning/datasources/datasources.yml create mode 100644 observability/src/main/docker/prometheus/prometheus.yml create mode 100644 observability/src/main/docker/tempo/tempo.yml create mode 100644 observability/src/main/resources/grafana/dashboards/camel-observability-dashboard.json diff --git a/observability/README.adoc b/observability/README.adoc index 96cd438c..4b855078 100644 --- a/observability/README.adoc +++ b/observability/README.adoc @@ -146,27 +146,155 @@ You can also directly leverage MicroProfile Health APIs to create checks. Class To be able to diagnose problems in Camel Quarkus applications, it's useful to instrument method calls, HTTP interactions etc with OpenTelemetry. -If you are running example in the Development mode, then Observability dev service is enabled (for more information, visit https://quarkus.io/guides/observability-devservices-lgtm[Observability Dev Services with Grafana OTel LGTM]). -Observability dev service will run OTel collector in the background and the `quarkus.otel.exporter.otlp.endpoint` property is automatically set to the OTel collector endpoint as seen from the outside of the Docker container. +This example is pre-configured to send traces to Tempo via OTLP on `http://localhost:4317`. The behavior differs based on how you run the application: -In production you would rather configure real OTel collector and you would configure the OpenTelemetry exporter in `application.properties` as follows: +==== Dev Mode (quarkus:dev) + +When running `mvn clean compile quarkus:dev`, the Quarkus Observability dev service is automatically enabled (see https://quarkus.io/guides/observability-devservices-lgtm[Observability Dev Services with Grafana OTel LGTM]). + +The dev service automatically: +- Starts a Grafana LGTM container with Tempo for tracing +- Overrides the `quarkus.otel.exporter.otlp.traces.endpoint` property to point to the dev service container +- Assigns a random port to Grafana (check logs for `grafana.endpoint=http://localhost:XXXXX`) + +To view traces in dev mode, find the Grafana endpoint in logs and browse to it. + +==== Docker Compose Mode (Recommended) + +When running the packaged application with Docker Compose (see Grafana Dashboards section above), traces are sent to the Tempo container at `localhost:4317`. + +The OpenTelemetry configuration in `application.properties` is already set: [source, text] ---- -# We are using a property placeholder to be able to use this example in convenient way in a cloud environment -quarkus.otel.exporter.otlp.traces.endpoint = http://${TELEMETRY_COLLECTOR_COLLECTOR_SERVICE_HOST:localhost}:4317 -# To enable tracing (it is disabled by default via camel-quarkus-observability-services) +quarkus.otel.exporter.otlp.traces.endpoint = http://localhost:4317 quarkus.otel.sdk.disabled=false ---- +Traces are automatically visible in Grafana at http://localhost:3000. + +==== Cloud/Kubernetes Deployment + +For cloud or Kubernetes deployments, override the OTLP endpoint using an environment variable: + +[source, shell] +---- +export QUARKUS_OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://tempo-service:4317 +---- + +Or use property placeholders in `application.properties`: + +[source, text] +---- +quarkus.otel.exporter.otlp.traces.endpoint = http://${TEMPO_HOST:localhost}:4317 +---- + NOTE: For information about other OpenTelemetry exporters, refer to the Camel Quarkus OpenTelemetry https://camel.apache.org/camel-quarkus/next/reference/extensions/opentelemetry.html#extensions-opentelemetry-usage-exporters[extension documentation]. -To view tracing events find `grafana.endpoint` in your log (it will use random port) browse to eg. http://localhost:32768. -Then follow instructions at https://quarkus.io/guides/observability-devservices-lgtm#explore. +==== Understanding Traces The `platform-http` consumer route introduces a random delay to simulate latency, hence the overall time of each trace should be different. When viewing a trace, you should see a hierarchy of 8 spans showing the progression of the message exchange through each endpoint. +=== Grafana Dashboards + +This example includes a pre-configured Grafana dashboard to visualize metrics, distributed traces, and JVM statistics. + +==== Quick Start with Docker Compose (Recommended) + +The easiest way to visualize the dashboard with working metrics is using Docker Compose with static ports: + +[source,shell] +---- +# Start the observability stack (Grafana, Prometheus, Tempo) +$ docker-compose -f src/main/docker/docker-compose-grafana.yml up -d + +# In another terminal, package and run the application +$ mvn clean package -DskipTests +$ java -jar target/quarkus-app/quarkus-run.jar +---- + +Access Grafana at http://localhost:3000 (credentials: `admin/admin`). The dashboard is automatically provisioned and ready to use! + +NOTE: To stop the observability stack: `docker-compose -f src/main/docker/docker-compose-grafana.yml down -v` + +==== Alternative: Dev Mode with Manual Dashboard Import + +When running in dev mode, Quarkus automatically provisions a Grafana instance with dynamic ports: + +[source,shell] +---- +$ mvn clean compile quarkus:dev +---- + +IMPORTANT: The LGTM dev services Grafana can display *distributed traces* but cannot display *metrics* from the dashboard panels. This is because the Prometheus instance inside the LGTM container cannot scrape metrics from `localhost:9876` on the host machine. For full dashboard functionality, use the Docker Compose setup above. + +To view traces in dev mode: + +1. Find the Grafana endpoint in logs: `grafana.endpoint=http://localhost:XXXXX` +2. Access Grafana (credentials: `admin/admin`) +3. Navigate to *Explore* → Select *Tempo* datasource +4. Query traces with: `{name="POST /greeting"}` + +==== Dashboard Contents + +The Camel Quarkus Observability Dashboard includes four main sections: + +*Custom Application Metrics* + +- *Greeting Counter*: Tracks calls to the `/greeting` endpoint (incremented via CDI MeterRegistry) +- *Greeting Provider Counter*: Tracks calls to `/greeting-provider` endpoint (incremented via Camel micrometer component) +- *Timer Counter*: Automatically increments every 10 seconds (incremented via @Counted annotation) +- *Metrics Comparison*: Shows correlation between timer, greeting, and provider calls + +*Camel Route Metrics* + +- *Camel Exchanges by Route*: Total number of messages processed per route +- *Camel Exchange Rate*: Messages per second across all routes + +*Distributed Tracing* + +- *Distributed Traces*: Information panel with link to Tempo Explore for viewing detailed traces + +*JVM & System Metrics* + +- *JVM Memory Usage*: Heap and non-heap memory consumption +- *CPU Usage*: Process and system CPU utilization +- *Garbage Collection Pause Time*: GC pause duration by type +- *Active HTTP Connections*: Current number of active requests + +==== What You'll See + +When the application is running, the dashboard will show: + +- *Timer Counter*: Increments automatically every 10 seconds (from the timer route) +- *Greeting Metrics*: Increase when you call the endpoint: ++ +[source,shell] +---- +$ curl localhost:8080/greeting +---- + +- *Distributed Traces*: Click "Open Tempo Explore" to view complete trace hierarchy with 8 spans showing message flow through Camel endpoints +- *JVM Metrics*: Real-time memory, CPU, and GC statistics + +TIP: The dashboard auto-refreshes every 5 seconds. Generate some traffic using `curl` in a loop to see the metrics change in real-time. + +==== Docker Compose Details + +The Docker Compose setup includes: + +- *Grafana* on http://localhost:3000 with the dashboard automatically provisioned +- *Prometheus* on http://localhost:9090 scraping metrics from the application every 5 seconds +- *Tempo* on http://localhost:3200 receiving distributed traces via OTLP + +The application exposes: +- Metrics at `http://localhost:9876/observe/metrics` (scraped by Prometheus) +- Traces sent to Tempo at `http://localhost:4317` (OTLP gRPC) +- Health checks at `http://localhost:9876/observe/health/live` and `/observe/health/ready` + +All services use **static ports** for easy access and documentation. + === Jolokia & Hawtio It can be useful to leverage Camel's JMX management features to manage and introspect the application. diff --git a/observability/src/main/docker/docker-compose-grafana.yml b/observability/src/main/docker/docker-compose-grafana.yml new file mode 100644 index 00000000..36b563ee --- /dev/null +++ b/observability/src/main/docker/docker-compose-grafana.yml @@ -0,0 +1,81 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: '3.8' + +services: + prometheus: + image: prom/prometheus:v2.50.1 + container_name: camel-quarkus-prometheus + command: + - '--config.file=/etc/prometheus/prometheus.yml' + - '--storage.tsdb.path=/prometheus' + - '--web.console.libraries=/usr/share/prometheus/console_libraries' + - '--web.console.templates=/usr/share/prometheus/consoles' + ports: + - "9090:9090" + volumes: + - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml + - prometheus-data:/prometheus + networks: + - observability + restart: unless-stopped + + tempo: + image: grafana/tempo:2.4.0 + container_name: camel-quarkus-tempo + command: [ "-config.file=/etc/tempo.yml" ] + ports: + - "3200:3200" # tempo + - "4317:4317" # otlp grpc + - "4318:4318" # otlp http + volumes: + - ./tempo/tempo.yml:/etc/tempo.yml + - tempo-data:/var/tempo + networks: + - observability + restart: unless-stopped + + grafana: + image: grafana/grafana:10.3.3 + container_name: camel-quarkus-grafana + ports: + - "3000:3000" + environment: + - GF_SECURITY_ADMIN_USER=admin + - GF_SECURITY_ADMIN_PASSWORD=admin + - GF_PATHS_PROVISIONING=/etc/grafana/provisioning + - GF_AUTH_ANONYMOUS_ENABLED=false + volumes: + - ./grafana/provisioning:/etc/grafana/provisioning + - ../resources/grafana/dashboards:/var/lib/grafana/dashboards + - grafana-data:/var/lib/grafana + networks: + - observability + depends_on: + - prometheus + - tempo + restart: unless-stopped + +networks: + observability: + driver: bridge + +volumes: + prometheus-data: + tempo-data: + grafana-data: diff --git a/observability/src/main/docker/grafana/provisioning/dashboards/dashboards.yml b/observability/src/main/docker/grafana/provisioning/dashboards/dashboards.yml new file mode 100644 index 00000000..3b0f6d1e --- /dev/null +++ b/observability/src/main/docker/grafana/provisioning/dashboards/dashboards.yml @@ -0,0 +1,30 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: 1 + +providers: + - name: 'Camel Quarkus Dashboards' + orgId: 1 + folder: 'Camel Quarkus' + type: file + disableDeletion: false + updateIntervalSeconds: 10 + allowUiUpdates: true + options: + path: /var/lib/grafana/dashboards + foldersFromFilesStructure: false diff --git a/observability/src/main/docker/grafana/provisioning/datasources/datasources.yml b/observability/src/main/docker/grafana/provisioning/datasources/datasources.yml new file mode 100644 index 00000000..a234086f --- /dev/null +++ b/observability/src/main/docker/grafana/provisioning/datasources/datasources.yml @@ -0,0 +1,48 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: true + jsonData: + httpMethod: POST + exemplarTraceIdDestinations: + - name: traceID + datasourceUid: tempo + + - name: Tempo + type: tempo + access: proxy + url: http://tempo:3200 + editable: true + uid: tempo + jsonData: + httpMethod: GET + serviceMap: + datasourceUid: 'prometheus' + nodeGraph: + enabled: true + search: + hide: false + tracesToMetrics: + datasourceUid: 'prometheus' diff --git a/observability/src/main/docker/prometheus/prometheus.yml b/observability/src/main/docker/prometheus/prometheus.yml new file mode 100644 index 00000000..fc3111ea --- /dev/null +++ b/observability/src/main/docker/prometheus/prometheus.yml @@ -0,0 +1,35 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +global: + scrape_interval: 15s + scrape_timeout: 10s + evaluation_interval: 15s + +scrape_configs: + - job_name: 'camel-quarkus-observability' + metrics_path: '/observe/metrics' + static_configs: + - targets: ['host.docker.internal:9876'] + labels: + application: 'camel-quarkus-observability' + environment: 'local' + scrape_interval: 5s + + - job_name: 'prometheus' + static_configs: + - targets: ['localhost:9090'] diff --git a/observability/src/main/docker/tempo/tempo.yml b/observability/src/main/docker/tempo/tempo.yml new file mode 100644 index 00000000..164d15e7 --- /dev/null +++ b/observability/src/main/docker/tempo/tempo.yml @@ -0,0 +1,56 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +server: + http_listen_port: 3200 + +distributor: + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + +ingester: + max_block_duration: 5m + +compactor: + compaction: + block_retention: 1h + +metrics_generator: + registry: + external_labels: + source: tempo + storage: + path: /var/tempo/generator/wal + remote_write: + - url: http://prometheus:9090/api/v1/write + send_exemplars: true + +storage: + trace: + backend: local + wal: + path: /var/tempo/wal + local: + path: /var/tempo/blocks + +overrides: + metrics_generator_processors: [service-graphs, span-metrics] diff --git a/observability/src/main/resources/application.properties b/observability/src/main/resources/application.properties index 036e4931..b7254004 100644 --- a/observability/src/main/resources/application.properties +++ b/observability/src/main/resources/application.properties @@ -29,6 +29,7 @@ quarkus.application.name = camel-quarkus-observability # quarkus.camel.opentelemetry2.trace-processors=true # For OTLP +quarkus.otel.exporter.otlp.traces.endpoint = http://localhost:4317 quarkus.otel.exporter.otlp.traces.timeout = 30s # To enable tracing (it is disabled by default via camel-quarkus-observability-services) quarkus.otel.sdk.disabled=false diff --git a/observability/src/main/resources/grafana/dashboards/camel-observability-dashboard.json b/observability/src/main/resources/grafana/dashboards/camel-observability-dashboard.json new file mode 100644 index 00000000..8f990a24 --- /dev/null +++ b/observability/src/main/resources/grafana/dashboards/camel-observability-dashboard.json @@ -0,0 +1,1132 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Comprehensive observability dashboard for Apache Camel Quarkus example demonstrating metrics, health checks, distributed tracing, and JVM monitoring", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [ + { + "asDropdown": false, + "icon": "external link", + "includeVars": false, + "keepTime": false, + "tags": [], + "targetBlank": true, + "title": "Camel Quarkus Documentation", + "tooltip": "Apache Camel Quarkus Observability Guide", + "type": "link", + "url": "https://camel.apache.org/camel-quarkus/latest/reference/extensions/observability-services.html" + } + ], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 100, + "panels": [], + "title": "Custom Application Metrics", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Counter metric incremented each time the /greeting endpoint is called. Uses CDI-injected MeterRegistry.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 1 + }, + "id": 1, + "options": { + "legend": { + "calcs": [ + "last", + "max" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "org_acme_observability_greeting_total{purpose=\"example\"}", + "legendFormat": "Greeting Counter", + "range": true, + "refId": "A" + } + ], + "title": "Greeting Counter", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Counter metric incremented when /greeting-provider endpoint is called. Uses Camel micrometer component. Includes random 1-5 second delay.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 1 + }, + "id": 2, + "options": { + "legend": { + "calcs": [ + "last", + "max" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "org_acme_observability_greeting_provider_total{purpose=\"example\"}", + "legendFormat": "Greeting Provider Counter", + "range": true, + "refId": "A" + } + ], + "title": "Greeting Provider Counter", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Counter metric incremented every 10 seconds by the timer route. Uses @Counted annotation via Micrometer.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 1 + }, + "id": 3, + "options": { + "legend": { + "calcs": [ + "last", + "max" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "org_acme_observability_timer_counter_total{purpose=\"example\"}", + "legendFormat": "Timer Counter", + "range": true, + "refId": "A" + } + ], + "title": "Timer Counter (Every 10s)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Comparison of all three custom metrics showing correlation. Timer triggers greeting, which calls greeting-provider.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 4, + "options": { + "legend": { + "calcs": [ + "last", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "org_acme_observability_timer_counter_total{purpose=\"example\"}", + "legendFormat": "Timer (every 10s)", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "org_acme_observability_greeting_total{purpose=\"example\"}", + "legendFormat": "Greeting Endpoint", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "org_acme_observability_greeting_provider_total{purpose=\"example\"}", + "legendFormat": "Greeting Provider", + "range": true, + "refId": "C" + } + ], + "title": "Metrics Comparison - Request Flow", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 101, + "panels": [], + "title": "Camel Route Metrics", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Total number of messages exchanged through each Camel route", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 100 + }, + { + "color": "red", + "value": 1000 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 18 + }, + "id": 5, + "options": { + "orientation": "auto", + "reduceOptions": { + "values": false, + "calcs": [ + "lastNotNull" + ], + "fields": "" + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "camel_exchanges_total", + "legendFormat": "{{routeId}}", + "range": true, + "refId": "A" + } + ], + "title": "Camel Exchanges by Route", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Exchange completion rate over time for all Camel routes", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 6, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(camel_exchanges_total[1m])", + "legendFormat": "{{routeId}}", + "range": true, + "refId": "A" + } + ], + "title": "Camel Exchange Rate", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 26 + }, + "id": 102, + "panels": [], + "title": "Distributed Tracing", + "type": "row" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "description": "View distributed traces showing the complete request flow from timer \u2192 /greeting \u2192 /greeting-provider with detailed span information including Camel routes, HTTP calls, and timing data.", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 27 + }, + "id": 7, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "## View Traces in Tempo\n\nDistributed traces are available in Grafana Explore with detailed span attributes:\n\n- **Camel routes**: timer, /greeting, /greeting-provider\n- **HTTP calls**: POST requests with status codes\n- **Span attributes**: camel.uri, component, exchangeId, http.route, etc.\n- **Timing**: Complete request flow with 1-5s random delay\n\n### How to View Traces\n\n1. Click **\"Open Tempo Explore\"** button below\n2. Or navigate to **Explore** \u2192 Select **Tempo** datasource\n3. Search with: `{name=\"POST /greeting\"}` or `{name=\"timer\"}`\n4. Click any trace to see the span hierarchy\n5. Expand spans to view detailed attributes\n\n### Example Trace Attributes\n- `http.request.method`: POST\n- `http.response.status_code`: 200 \n- `http.route`: /greeting\n- `camel.uri`: platform-http:///greeting\n- `component`: camel-platform-http\n- `exchangeId`: Camel exchange ID", + "mode": "markdown" + }, + "pluginVersion": "10.0.0", + "title": "Distributed Traces", + "type": "text", + "links": [ + { + "title": "Open Tempo Explore", + "url": "/explore?left=%7B%22datasource%22:%22tempo%22,%22queries%22:%5B%7B%22query%22:%22%7Bname%3D~%5C%22.%2A%5C%22%7D%22,%22queryType%22:%22traceql%22,%22limit%22:20%7D%5D,%22range%22:%7B%22from%22:%22now-15m%22,%22to%22:%22now%22%7D%7D", + "targetBlank": true + } + ] + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 47 + }, + "id": 103, + "panels": [], + "title": "JVM & System Metrics", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "JVM memory usage including heap and non-heap memory", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 48 + }, + "id": 9, + "options": { + "legend": { + "calcs": [ + "last", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "jvm_memory_used_bytes{area=\"heap\"}", + "legendFormat": "Heap Used", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "jvm_memory_max_bytes{area=\"heap\"}", + "legendFormat": "Heap Max", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "jvm_memory_used_bytes{area=\"nonheap\"}", + "legendFormat": "Non-Heap Used", + "range": true, + "refId": "C" + } + ], + "title": "JVM Memory Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Process CPU usage percentage", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.7 + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 48 + }, + "id": 10, + "options": { + "legend": { + "calcs": [ + "last", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "process_cpu_usage", + "legendFormat": "CPU Usage", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "system_cpu_usage", + "legendFormat": "System CPU", + "range": true, + "refId": "B" + } + ], + "title": "CPU Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Garbage collection pause time across all GC types", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 56 + }, + "id": 11, + "options": { + "legend": { + "calcs": [ + "last", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(jvm_gc_pause_seconds_sum[1m])", + "legendFormat": "{{action}} - {{cause}}", + "range": true, + "refId": "A" + } + ], + "title": "Garbage Collection Pause Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of currently active HTTP connections", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 50 + }, + { + "color": "red", + "value": 100 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 56 + }, + "id": 12, + "options": { + "orientation": "auto", + "reduceOptions": { + "values": false, + "calcs": [ + "lastNotNull" + ], + "fields": "" + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "http_server_active_requests", + "legendFormat": "Active Requests", + "range": true, + "refId": "A" + } + ], + "title": "Active HTTP Connections", + "type": "gauge" + } + ], + "refresh": "5s", + "schemaVersion": 38, + "style": "dark", + "tags": [ + "camel", + "quarkus", + "observability", + "example" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "includeAll": false, + "label": "Prometheus", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": false, + "text": "Tempo", + "value": "Tempo" + }, + "hide": 0, + "includeAll": false, + "label": "Tempo", + "multi": false, + "name": "DS_TEMPO", + "options": [], + "query": "tempo", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Camel Quarkus Observability Dashboard", + "uid": "camel-quarkus-observability", + "version": 1, + "weekStart": "" +} \ No newline at end of file From b75fa0dd9fba099930b089dad05e14be8771cbf3 Mon Sep 17 00:00:00 2001 From: jomin mathew <> Date: Thu, 30 Apr 2026 10:37:07 +0100 Subject: [PATCH 2/2] Replaced docker-compose with docker compose and retested functionality --- observability/README.adoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/observability/README.adoc b/observability/README.adoc index 4b855078..377de4c7 100644 --- a/observability/README.adoc +++ b/observability/README.adoc @@ -207,7 +207,7 @@ The easiest way to visualize the dashboard with working metrics is using Docker [source,shell] ---- # Start the observability stack (Grafana, Prometheus, Tempo) -$ docker-compose -f src/main/docker/docker-compose-grafana.yml up -d +$ docker compose -f src/main/docker/docker-compose-grafana.yml up -d # In another terminal, package and run the application $ mvn clean package -DskipTests @@ -216,7 +216,7 @@ $ java -jar target/quarkus-app/quarkus-run.jar Access Grafana at http://localhost:3000 (credentials: `admin/admin`). The dashboard is automatically provisioned and ready to use! -NOTE: To stop the observability stack: `docker-compose -f src/main/docker/docker-compose-grafana.yml down -v` +NOTE: To stop the observability stack: `docker compose -f src/main/docker/docker-compose-grafana.yml down -v` ==== Alternative: Dev Mode with Manual Dashboard Import