From 53318bd749ee76b67bc123d8a119bbcb8c3b2f71 Mon Sep 17 00:00:00 2001 From: Vineeth Pothulapati Date: Wed, 6 Jul 2022 12:45:02 +0530 Subject: [PATCH 1/3] Add e2e docker-compose --- docker-compose/promscale-demo/dashboards.yaml | 23 +++ .../promscale-demo/datasources.yaml | 41 +++++ .../promscale-demo/docker-compose.yaml | 154 ++++++++++++++++++ docker-compose/promscale-demo/grafana.ini | 7 + .../promscale-demo/otel-collector-config.yaml | 26 +++ 5 files changed, 251 insertions(+) create mode 100644 docker-compose/promscale-demo/dashboards.yaml create mode 100644 docker-compose/promscale-demo/datasources.yaml create mode 100644 docker-compose/promscale-demo/docker-compose.yaml create mode 100644 docker-compose/promscale-demo/grafana.ini create mode 100644 docker-compose/promscale-demo/otel-collector-config.yaml diff --git a/docker-compose/promscale-demo/dashboards.yaml b/docker-compose/promscale-demo/dashboards.yaml new file mode 100644 index 0000000000..71195efee3 --- /dev/null +++ b/docker-compose/promscale-demo/dashboards.yaml @@ -0,0 +1,23 @@ +apiVersion: 1 + +providers: + # an unique provider name. Required + - name: 'demo' + # Org id. Default to 1 + orgId: 1 + # name of the dashboard folder. + folder: '' + # folder UID. will be automatically generated if not specified + folderUid: '' + # provider type. Default to 'file' + type: file + # disable dashboard deletion + disableDeletion: false + # how often Grafana will scan for changed dashboards + updateIntervalSeconds: 10 + # allow updating provisioned dashboards from the UI + allowUiUpdates: true + options: + # path to dashboard files on disk. Required when using the 'file' type + path: /etc/grafana/provisioning/dashboards + # use folder names from filesystem to create folders in Grafana diff --git a/docker-compose/promscale-demo/datasources.yaml b/docker-compose/promscale-demo/datasources.yaml new file mode 100644 index 0000000000..701b08927a --- /dev/null +++ b/docker-compose/promscale-demo/datasources.yaml @@ -0,0 +1,41 @@ +# config file version +apiVersion: 1 + +# list of datasources that should be deleted from the database +deleteDatasources: + - name: Graphite + orgId: 1 + +# list of datasources to insert/update depending +# what's available in the database +datasources: + - name: Promscale-metrics + type: prometheus + url: http://promscale:9201 + editable: true + isDefault: false + + - name: Promscale-Tracing + type: jaeger + uid: a01e209e82a346dfbecd10bcb6143ee4 + url: http://promscale:9201 + editable: true + isDefault: false + + - name: Promscale-SQL + type: postgres + uid: d7b6774ba1c1420aafd3b7ddfc9064c8 + url: timescaledb:5432 + database: postgres + user: postgres + editable: true + isDefault: true + secureJsonData: + password: password + jsonData: + sslmode: 'disable' + maxOpenConns: 10 + maxIdleConns: 2 + connMaxLifetime: 14400 + postgresVersion: 1400 + timescaledb: true \ No newline at end of file diff --git a/docker-compose/promscale-demo/docker-compose.yaml b/docker-compose/promscale-demo/docker-compose.yaml new file mode 100644 index 0000000000..bd01d8ef89 --- /dev/null +++ b/docker-compose/promscale-demo/docker-compose.yaml @@ -0,0 +1,154 @@ +version: '3.0' + +services: + timescaledb: + image: timescale/timescaledb-ha:pg14-latest + ports: + - 5432:5432/tcp + volumes: + - timescaledb-data:/var/lib/postgresql/data + environment: + POSTGRES_PASSWORD: password + POSTGRES_USER: postgres + + prometheus: + image: prom/prometheus:latest + depends_on: + - promscale + ports: + - 9090:9090/tcp + volumes: + - ${PWD}/../prometheus.yml:/etc/prometheus/prometheus.yml + + alertmanager: + image: prom/alertmanager:latest + ports: + - 9093:9093/tcp + + promscale: + image: timescale/promscale:latest + ports: + - 9201:9201/tcp + - 9202:9202/tcp + restart: on-failure + depends_on: + - timescaledb + volumes: + - ${PWD}/../promscale_prometheus.yml:/prometheus.yml + - ${PWD}/../rules.yml:/rules.yml + - ${PWD}/../alerts.yml:/alerts.yml + environment: + PROMSCALE_DB_URI: postgres://postgres:password@timescaledb:5432/postgres?sslmode=allow + PROMSCALE_TELEMETRY_TRACE_JAEGER_ENDPOINT: "http://collector:14268/api/traces" + PROMSCALE_TELEMETRY_TRACE_SAMPLING_RATIO: "0.1" + PROMSCALE_METRICS_RULES_CONFIG_FILE: /prometheus.yml + + collector: + platform: linux/amd64 + image: "otel/opentelemetry-collector:0.43.0" + command: [ "--config=/etc/otel-collector-config.yaml" ] + volumes: + - ${PWD}/otel-collector-config.yaml:/etc/otel-collector-config.yaml + depends_on: + - promscale + ports: + - 14268:14268/tcp # jaeger http + - 4317:4317/tcp + - 4318:4318/tcp + + jaeger: + image: jaegertracing/jaeger-query:1.33.0 + environment: + SPAN_STORAGE_TYPE: grpc-plugin + command: [ + "--grpc-storage.server=promscale:9202", + ] + ports: + - "16686:16686" + + grafana: + image: grafana/grafana-oss:8.5.0 + volumes: + - grafana-data:/var/lib/grafana + ports: + - 3000:3000/tcp + restart: on-failure + volumes: + - ${PWD}/grafana.ini:/etc/grafana/grafana.ini + - ${PWD}/datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml + - ${PWD}/dashboards.yaml:/etc/grafana/provisioning/dashboards/dashboards.yaml + - ${PWD}/../../docs/mixin/dashboards:/etc/grafana/provisioning/dashboards/demo + depends_on: + - timescaledb + - jaeger + + node_exporter: + image: quay.io/prometheus/node-exporter + ports: + - "9100:9100" + + upper: + image: jamesgresql/opentelemetry-demo_upper:0.2 + restart: on-failure + depends_on: + - collector + ports: + - 5054:5000/tcp + environment: + - OTEL_EXPORTER_OTLP_ENDPOINT=collector:4317 + + lower: + image: jamesgresql/opentelemetry-demo_lower:0.2 + restart: on-failure + depends_on: + - collector + ports: + - 5053:5000/tcp + environment: + - OTEL_EXPORTER_OTLP_ENDPOINT=http://collector:4318 + + special: + image: jamesgresql/opentelemetry-demo_special:0.2 + restart: on-failure + depends_on: + - collector + ports: + - 5052:5000/tcp + environment: + - OTEL_EXPORTER_OTLP_ENDPOINT=collector:4317 + + digit: + image: jamesgresql/opentelemetry-demo_digit:0.2 + restart: on-failure + depends_on: + - collector + ports: + - 5051:5000/tcp + environment: + - OTEL_EXPORTER_OTLP_ENDPOINT=collector:4317 + + generator: + image: jamesgresql/opentelemetry-demo_generator:0.2 + restart: on-failure + depends_on: + - upper + - lower + - special + - digit + ports: + - 5050:5000/tcp + environment: + - OTEL_EXPORTER_OTLP_ENDPOINT=collector:4317 + + load: + image: jamesgresql/opentelemetry-demo_load:0.2 + restart: on-failure + depends_on: + - generator + deploy: + mode: replicated + replicas: 3 + +volumes: + timescaledb-data: + grafana-data: \ No newline at end of file diff --git a/docker-compose/promscale-demo/grafana.ini b/docker-compose/promscale-demo/grafana.ini new file mode 100644 index 0000000000..8dab2fbbc6 --- /dev/null +++ b/docker-compose/promscale-demo/grafana.ini @@ -0,0 +1,7 @@ +[security] +admin_user = admin +admin_password = admin +allow_embedding = true + +[plugins] +plugin_admin_enabled = true \ No newline at end of file diff --git a/docker-compose/promscale-demo/otel-collector-config.yaml b/docker-compose/promscale-demo/otel-collector-config.yaml new file mode 100644 index 0000000000..5ba1f2eac2 --- /dev/null +++ b/docker-compose/promscale-demo/otel-collector-config.yaml @@ -0,0 +1,26 @@ +receivers: + otlp: + protocols: + grpc: + http: + +processors: + batch: + +exporters: + logging: + otlp: + endpoint: "promscale:9202" + compression: none + tls: + insecure: true + +service: + telemetry: + logs: + level: "debug" + pipelines: + traces: + receivers: [otlp] + processors: [batch] + exporters: [logging, otlp] \ No newline at end of file From 220864025346dff6b2fcd8086173d8df068bd4f0 Mon Sep 17 00:00:00 2001 From: Vineeth Pothulapati Date: Fri, 8 Jul 2022 08:38:57 +0530 Subject: [PATCH 2/3] refactor --- docker-compose/promscale-demo/dashboards.yaml | 23 ----------- .../promscale-demo/datasources.yaml | 41 ------------------- .../promscale-demo/docker-compose.yaml | 25 +++++------ docker-compose/promscale-demo/grafana.ini | 7 ---- .../promscale-demo/otel-collector-config.yaml | 26 ------------ 5 files changed, 10 insertions(+), 112 deletions(-) delete mode 100644 docker-compose/promscale-demo/dashboards.yaml delete mode 100644 docker-compose/promscale-demo/datasources.yaml delete mode 100644 docker-compose/promscale-demo/grafana.ini delete mode 100644 docker-compose/promscale-demo/otel-collector-config.yaml diff --git a/docker-compose/promscale-demo/dashboards.yaml b/docker-compose/promscale-demo/dashboards.yaml deleted file mode 100644 index 71195efee3..0000000000 --- a/docker-compose/promscale-demo/dashboards.yaml +++ /dev/null @@ -1,23 +0,0 @@ -apiVersion: 1 - -providers: - # an unique provider name. Required - - name: 'demo' - # Org id. Default to 1 - orgId: 1 - # name of the dashboard folder. - folder: '' - # folder UID. will be automatically generated if not specified - folderUid: '' - # provider type. Default to 'file' - type: file - # disable dashboard deletion - disableDeletion: false - # how often Grafana will scan for changed dashboards - updateIntervalSeconds: 10 - # allow updating provisioned dashboards from the UI - allowUiUpdates: true - options: - # path to dashboard files on disk. Required when using the 'file' type - path: /etc/grafana/provisioning/dashboards - # use folder names from filesystem to create folders in Grafana diff --git a/docker-compose/promscale-demo/datasources.yaml b/docker-compose/promscale-demo/datasources.yaml deleted file mode 100644 index 701b08927a..0000000000 --- a/docker-compose/promscale-demo/datasources.yaml +++ /dev/null @@ -1,41 +0,0 @@ -# config file version -apiVersion: 1 - -# list of datasources that should be deleted from the database -deleteDatasources: - - name: Graphite - orgId: 1 - -# list of datasources to insert/update depending -# what's available in the database -datasources: - - name: Promscale-metrics - type: prometheus - url: http://promscale:9201 - editable: true - isDefault: false - - - name: Promscale-Tracing - type: jaeger - uid: a01e209e82a346dfbecd10bcb6143ee4 - url: http://promscale:9201 - editable: true - isDefault: false - - - name: Promscale-SQL - type: postgres - uid: d7b6774ba1c1420aafd3b7ddfc9064c8 - url: timescaledb:5432 - database: postgres - user: postgres - editable: true - isDefault: true - secureJsonData: - password: password - jsonData: - sslmode: 'disable' - maxOpenConns: 10 - maxIdleConns: 2 - connMaxLifetime: 14400 - postgresVersion: 1400 - timescaledb: true \ No newline at end of file diff --git a/docker-compose/promscale-demo/docker-compose.yaml b/docker-compose/promscale-demo/docker-compose.yaml index bd01d8ef89..2654e1f571 100644 --- a/docker-compose/promscale-demo/docker-compose.yaml +++ b/docker-compose/promscale-demo/docker-compose.yaml @@ -10,6 +10,8 @@ services: environment: POSTGRES_PASSWORD: password POSTGRES_USER: postgres + POSTGRES_DB: otel_demo + POSTGRES_HOST_AUTH_METHOD: trust prometheus: image: prom/prometheus:latest @@ -38,17 +40,15 @@ services: - ${PWD}/../rules.yml:/rules.yml - ${PWD}/../alerts.yml:/alerts.yml environment: - PROMSCALE_DB_URI: postgres://postgres:password@timescaledb:5432/postgres?sslmode=allow + PROMSCALE_DB_URI: postgres://postgres:password@timescaledb:5432/otel_demo?sslmode=allow PROMSCALE_TELEMETRY_TRACE_JAEGER_ENDPOINT: "http://collector:14268/api/traces" PROMSCALE_TELEMETRY_TRACE_SAMPLING_RATIO: "0.1" PROMSCALE_METRICS_RULES_CONFIG_FILE: /prometheus.yml collector: platform: linux/amd64 - image: "otel/opentelemetry-collector:0.43.0" + image: "timescale/promscale-demo-collector" command: [ "--config=/etc/otel-collector-config.yaml" ] - volumes: - - ${PWD}/otel-collector-config.yaml:/etc/otel-collector-config.yaml depends_on: - promscale ports: @@ -67,17 +67,12 @@ services: - "16686:16686" grafana: - image: grafana/grafana-oss:8.5.0 + image: timescale/promscale-demo-grafana volumes: - grafana-data:/var/lib/grafana ports: - 3000:3000/tcp restart: on-failure - volumes: - - ${PWD}/grafana.ini:/etc/grafana/grafana.ini - - ${PWD}/datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml - - ${PWD}/dashboards.yaml:/etc/grafana/provisioning/dashboards/dashboards.yaml - - ${PWD}/../../docs/mixin/dashboards:/etc/grafana/provisioning/dashboards/demo depends_on: - timescaledb - jaeger @@ -88,7 +83,7 @@ services: - "9100:9100" upper: - image: jamesgresql/opentelemetry-demo_upper:0.2 + image: timescale/promscale-demo-upper restart: on-failure depends_on: - collector @@ -108,7 +103,7 @@ services: - OTEL_EXPORTER_OTLP_ENDPOINT=http://collector:4318 special: - image: jamesgresql/opentelemetry-demo_special:0.2 + image: timescale/promscale-demo-special restart: on-failure depends_on: - collector @@ -118,7 +113,7 @@ services: - OTEL_EXPORTER_OTLP_ENDPOINT=collector:4317 digit: - image: jamesgresql/opentelemetry-demo_digit:0.2 + image: timescale/promscale-demo-digit restart: on-failure depends_on: - collector @@ -128,7 +123,7 @@ services: - OTEL_EXPORTER_OTLP_ENDPOINT=collector:4317 generator: - image: jamesgresql/opentelemetry-demo_generator:0.2 + image: timescale/promscale-demo-generator restart: on-failure depends_on: - upper @@ -141,7 +136,7 @@ services: - OTEL_EXPORTER_OTLP_ENDPOINT=collector:4317 load: - image: jamesgresql/opentelemetry-demo_load:0.2 + image: timescale/promscale-demo-load restart: on-failure depends_on: - generator diff --git a/docker-compose/promscale-demo/grafana.ini b/docker-compose/promscale-demo/grafana.ini deleted file mode 100644 index 8dab2fbbc6..0000000000 --- a/docker-compose/promscale-demo/grafana.ini +++ /dev/null @@ -1,7 +0,0 @@ -[security] -admin_user = admin -admin_password = admin -allow_embedding = true - -[plugins] -plugin_admin_enabled = true \ No newline at end of file diff --git a/docker-compose/promscale-demo/otel-collector-config.yaml b/docker-compose/promscale-demo/otel-collector-config.yaml deleted file mode 100644 index 5ba1f2eac2..0000000000 --- a/docker-compose/promscale-demo/otel-collector-config.yaml +++ /dev/null @@ -1,26 +0,0 @@ -receivers: - otlp: - protocols: - grpc: - http: - -processors: - batch: - -exporters: - logging: - otlp: - endpoint: "promscale:9202" - compression: none - tls: - insecure: true - -service: - telemetry: - logs: - level: "debug" - pipelines: - traces: - receivers: [otlp] - processors: [batch] - exporters: [logging, otlp] \ No newline at end of file From 564f3c8a1512f08b486ec99de20d4d44b2db699d Mon Sep 17 00:00:00 2001 From: Vineeth Pothulapati Date: Tue, 12 Jul 2022 14:14:57 +0530 Subject: [PATCH 3/3] update the images --- docker-compose/docker-compose.yaml | 2 +- docker-compose/otel-collector-config.yml | 8 ++++---- .../promscale-demo/docker-compose.yaml | 17 ++++++++++------- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/docker-compose/docker-compose.yaml b/docker-compose/docker-compose.yaml index 408a346cf5..beb5cdd008 100644 --- a/docker-compose/docker-compose.yaml +++ b/docker-compose/docker-compose.yaml @@ -39,7 +39,7 @@ services: PROMSCALE_WEB_TELEMETRY_PATH: /metrics-text PROMSCALE_DB_URI: postgres://postgres:password@db:5432/postgres?sslmode=allow PROMSCALE_TRACING_OTLP_SERVER_ADDRESS: ":9202" - PROMSCALE_TELEMETRY_TRACE_JAEGER_ENDPOINT: "http://otel-collector:14268/api/traces" + PROMSCALE_TELEMETRY_TRACE_OTEL_ENDPOINT: "otel-collector:4317" PROMSCALE_TELEMETRY_TRACE_SAMPLING_RATIO: "0.1" PROMSCALE_METRICS_RULES_CONFIG_FILE: /prometheus.yml diff --git a/docker-compose/otel-collector-config.yml b/docker-compose/otel-collector-config.yml index a06b9c856e..e47fe3631e 100644 --- a/docker-compose/otel-collector-config.yml +++ b/docker-compose/otel-collector-config.yml @@ -1,8 +1,8 @@ receivers: - jaeger: + otlp: protocols: - thrift_http: - endpoint: "0.0.0.0:14268" + grpc: + http: exporters: logging: @@ -21,7 +21,7 @@ service: pipelines: traces: - receivers: [jaeger] + receivers: [otlp] exporters: [otlp, logging] processors: [batch] diff --git a/docker-compose/promscale-demo/docker-compose.yaml b/docker-compose/promscale-demo/docker-compose.yaml index 2654e1f571..f7e30f1ced 100644 --- a/docker-compose/promscale-demo/docker-compose.yaml +++ b/docker-compose/promscale-demo/docker-compose.yaml @@ -40,24 +40,27 @@ services: - ${PWD}/../rules.yml:/rules.yml - ${PWD}/../alerts.yml:/alerts.yml environment: + PROMSCALE_WEB_TELEMETRY_PATH: /metrics-text PROMSCALE_DB_URI: postgres://postgres:password@timescaledb:5432/otel_demo?sslmode=allow - PROMSCALE_TELEMETRY_TRACE_JAEGER_ENDPOINT: "http://collector:14268/api/traces" + PROMSCALE_TELEMETRY_TRACE_OTEL_ENDPOINT: "collector:4317" PROMSCALE_TELEMETRY_TRACE_SAMPLING_RATIO: "0.1" PROMSCALE_METRICS_RULES_CONFIG_FILE: /prometheus.yml + TOBS_PROMSCALE_QUICK_START: "true" collector: - platform: linux/amd64 - image: "timescale/promscale-demo-collector" - command: [ "--config=/etc/otel-collector-config.yaml" ] + image: "otel/opentelemetry-collector:0.50.0" + command: [ "--config=/etc/otel-collector-config.yml" ] depends_on: - promscale ports: - 14268:14268/tcp # jaeger http - 4317:4317/tcp - 4318:4318/tcp + volumes: + - ${PWD}/../otel-collector-config.yml:/etc/otel-collector-config.yml jaeger: - image: jaegertracing/jaeger-query:1.33.0 + image: jaegertracing/jaeger-query:1.35.0 environment: SPAN_STORAGE_TYPE: grpc-plugin command: [ @@ -67,7 +70,7 @@ services: - "16686:16686" grafana: - image: timescale/promscale-demo-grafana + image: vineeth97/promscale-demo-grafana volumes: - grafana-data:/var/lib/grafana ports: @@ -93,7 +96,7 @@ services: - OTEL_EXPORTER_OTLP_ENDPOINT=collector:4317 lower: - image: jamesgresql/opentelemetry-demo_lower:0.2 + image: timescale/promscale-demo-lower restart: on-failure depends_on: - collector