feat: introduce a few env vars to config otel collector (#864)

- Support `CLICKHOUSE_ENDPOINT` to switch aio clickhouse endpoint (Ref: HDX-1758)
- Support `HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE ` (Ref: HDX-1786)

Support command like
```
docker run -e CLICKHOUSE_ENDPOINT=<CH-CLOUD-ENDPOINT> -e CLICKHOUSE_USER=default -e CLICKHOUSE_PASSWORD='BLABLA' -e HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE =hyperdx -p 8080:8080 -p 4317:4317 -p 4318:4318 hyperdx/hyperdx-local:2-nightly
```
So users can export data to other services like clickhouse cloud

Author: wrn14897

.env

@@ -20,5 +20,8 @@ HYPERDX_APP_URL=http://localhost
 HYPERDX_LOG_LEVEL=debug
 HYPERDX_OPAMP_PORT=4320
 
+# Otel/Clickhouse config
+HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE=default
+
 # TEMPORARY: local development
 HYPERDX_API_KEY=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx

docker-compose.ci.yml

@@ -6,6 +6,7 @@ services:
       target: dev
     environment:
       CLICKHOUSE_ENDPOINT: 'tcp://ch-server:9000?dial_timeout=10s'
+      HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE: ${HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE}
       HYPERDX_API_KEY: ${HYPERDX_API_KEY}
       HYPERDX_LOG_LEVEL: ${HYPERDX_LOG_LEVEL}
     volumes:

docker-compose.dev.yml

@@ -11,7 +11,7 @@ services:
       service.name: 'hdx-oss-dev-db'
     image: mongo:5.0.14-focal
     volumes:
-      - .volumes/db:/data/db
+      - .volumes/db_dev:/data/db
     ports:
       - 27017:27017
     networks:
@@ -24,8 +24,9 @@ services:
       context: ./docker/otel-collector
       target: dev
     environment:
-      CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT: 'ch-server:9363'
       CLICKHOUSE_ENDPOINT: 'tcp://ch-server:9000?dial_timeout=10s'
+      CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT: 'ch-server:9363'
+      HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE: ${HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE}
       HYPERDX_API_KEY: ${HYPERDX_API_KEY}
       HYPERDX_LOG_LEVEL: ${HYPERDX_LOG_LEVEL}
       OPAMP_SERVER_URL: 'http://host.docker.internal:${HYPERDX_OPAMP_PORT}'
@@ -55,8 +56,8 @@ services:
     volumes:
       - ./docker/clickhouse/local/config.xml:/etc/clickhouse-server/config.xml
       - ./docker/clickhouse/local/users.xml:/etc/clickhouse-server/users.xml
-      - .volumes/ch_data:/var/lib/clickhouse
-      - .volumes/ch_logs:/var/log/clickhouse-server
+      - .volumes/ch_data_dev:/var/lib/clickhouse
+      - .volumes/ch_logs_dev:/var/log/clickhouse-server
     restart: on-failure
     networks:
       - internal

docker-compose.yml

@@ -29,6 +29,7 @@ services:
     image: ${HDX_IMAGE_REPO}/${OTEL_COLLECTOR_IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION}
     environment:
       CLICKHOUSE_ENDPOINT: 'tcp://ch-server:9000?dial_timeout=10s'
+      HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE: ${HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE}
       HYPERDX_LOG_LEVEL: ${HYPERDX_LOG_LEVEL}
       OPAMP_SERVER_URL: 'http://app:${HYPERDX_OPAMP_PORT}'
     ports:

docker/hyperdx/entry.local.base.sh

@@ -12,7 +12,8 @@ export FRONTEND_URL="${FRONTEND_URL:-${HYPERDX_APP_URL:-http://localhost}:${HYPE
 export OPAMP_PORT=${HYPERDX_OPAMP_PORT:-4320}
 
 # Internal Services
-export CLICKHOUSE_ENDPOINT="tcp://ch-server:9000?dial_timeout=10s"
+export HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE="${HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE:-default}"
+export CLICKHOUSE_ENDPOINT="${CLICKHOUSE_ENDPOINT:-tcp://ch-server:9000?dial_timeout=10s}"
 export MONGO_URI="mongodb://db:27017/hyperdx"
 export OPAMP_SERVER_URL="http://127.0.0.1:${OPAMP_PORT}"
 
@@ -32,12 +33,14 @@ echo "127.0.0.1      db" >> /etc/hosts
 
 echo "Visit the HyperDX UI at $FRONTEND_URL/search"
 echo ""
-echo "Local App Mode: $IS_LOCAL_APP_MODE"
-echo ""
 echo "Send OpenTelemetry data via"
 echo "http/protobuf: OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318"
 echo "gRPC: OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317"
 echo ""
+echo "Exporting data to ClickHouse:
+  Endpoint: $CLICKHOUSE_ENDPOINT
+  Database: $HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE
+"
 echo ""
 
 # Start Clickhouse Server

docker/otel-collector/config.yaml

@@ -112,10 +112,10 @@ exporters:
     sampling_initial: 5
     sampling_thereafter: 200
   clickhouse/rrweb:
+    database: ${env:HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE}
     endpoint: ${env:CLICKHOUSE_ENDPOINT}
-    database: default
-    username: ${env:CLICKHOUSE_USER}
     password: ${env:CLICKHOUSE_PASSWORD}
+    username: ${env:CLICKHOUSE_USER}
     ttl: 72h
     logs_table_name: hyperdx_sessions
     timeout: 5s
@@ -125,10 +125,10 @@ exporters:
       max_interval: 30s
       max_elapsed_time: 300s
   clickhouse:
+    database: ${env:HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE}
     endpoint: ${env:CLICKHOUSE_ENDPOINT}
-    database: default
-    username: ${env:CLICKHOUSE_USER}
     password: ${env:CLICKHOUSE_PASSWORD}
+    username: ${env:CLICKHOUSE_USER}
     ttl: 72h
     timeout: 5s
     retry_on_failure:

smoke-tests/otel-collector/docker-compose.yaml

@@ -31,6 +31,7 @@ services:
     environment:
       - CLICKHOUSE_ENDPOINT=tcp://ch-server:9000?dial_timeout=10s
       - CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT=ch-server:9363
+      - HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE=default
       - HYPERDX_LOG_LEVEL=info
     ports:
       - 4318:4318 # OTLP http receiver