From ffe647d00220ce945d6dd59e32fe37d0ae373c6d Mon Sep 17 00:00:00 2001 From: Emil Djupvik Date: Mon, 23 Mar 2026 20:01:45 +0100 Subject: [PATCH 01/13] add grafana --- containers/nginx/default.conf | 29 +++++++++++++++++++++++++++++ docker-compose.prod.yml | 29 +++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/containers/nginx/default.conf b/containers/nginx/default.conf index 38f386f41..7a9471683 100644 --- a/containers/nginx/default.conf +++ b/containers/nginx/default.conf @@ -3,6 +3,35 @@ upstream backend { server ${PROXY_PASS}; } +server { + listen 80; + server_name grafana.metrics.sctomega.com; + return 301 https://$host$request_uri; +} + +server { + listen 443 ssl; + server_name grafana.metrics.sctomega.com; + + ssl_certificate /etc/letsencrypt/live/grafana.metrics.sctomega.com/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/grafana.metrics.sctomega.com/privkey.pem; + + ssl_protocols TLSv1.1 TLSv1.2; + ssl_prefer_server_ciphers on; + ssl_ciphers ECDH+AESGCM:ECDH+AES256:ECDH+AES128:DHE+AES128:!ADH:!AECDH:!MD5; + + add_header Strict-Transport-Security "max-age=31536000" always; + + location / { + proxy_pass http://grafana:3000/; + proxy_http_version 1.1; + proxy_set_header Connection 'upgrade'; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Host $host; + proxy_cache_bypass $http_upgrade; + } +} + server { listen 80; server_name ${DOMAIN}; diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index b049f4671..e3c1d0611 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -46,6 +46,29 @@ services: entrypoint: "/bin/sh -c 'trap exit TERM; while :; do certbot renew; sleep 14d & wait $${!}; done;'" restart: always + grafana: + image: grafana/grafana:latest + volumes: + - grafana_data:/var/lib/grafana + - ./grafana/provisioning:/etc/grafana/provisioning:ro + restart: unless-stopped + + loki: + image: grafana/loki:latest + volumes: + - ./loki/config.yml:/etc/loki/config.yml:ro + - loki_data:/data/loki + command: -config.file=/etc/loki/config.yml + restart: unless-stopped + + mimir: + image: grafana/mimir:latest + volumes: + - ./mimir/config.yml:/etc/mimir/config.yml:ro + - mimir_data:/data/mimir + command: -config.file=/etc/mimir/config.yml + restart: unless-stopped + postfix: build: ${PROJECT_ROOT:-.}/containers/postfix/ ports: @@ -76,3 +99,9 @@ volumes: driver: local dobbelOmegaManifest: driver: local + grafana_data: + driver: local + loki_data: + driver: local + mimir_data: + driver: local From d55d723948ccdbaadd5f249d27ef6b066687753b Mon Sep 17 00:00:00 2001 From: Emil Djupvik Date: Mon, 23 Mar 2026 20:12:14 +0100 Subject: [PATCH 02/13] add .well-known --- containers/nginx/default.conf | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/containers/nginx/default.conf b/containers/nginx/default.conf index 7a9471683..0d0c1b75e 100644 --- a/containers/nginx/default.conf +++ b/containers/nginx/default.conf @@ -22,6 +22,10 @@ server { add_header Strict-Transport-Security "max-age=31536000" always; + location ^~ /.well-known/acme-challenge/ { + alias /var/www/certbot/.well-known/acme-challenge/; + } + location / { proxy_pass http://grafana:3000/; proxy_http_version 1.1; From e920e15c52c9c75d14ceab847e31149bbbdb5472 Mon Sep 17 00:00:00 2001 From: Emil Djupvik Date: Mon, 23 Mar 2026 20:19:36 +0100 Subject: [PATCH 03/13] move .well-known to http block --- containers/nginx/default.conf | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/containers/nginx/default.conf b/containers/nginx/default.conf index 0d0c1b75e..d3a81f6a0 100644 --- a/containers/nginx/default.conf +++ b/containers/nginx/default.conf @@ -6,7 +6,14 @@ upstream backend { server { listen 80; server_name grafana.metrics.sctomega.com; - return 301 https://$host$request_uri; + + location ^~ /.well-known/acme-challenge/ { + alias /var/www/certbot/.well-known/acme-challenge/; + } + + location / { + return 301 https://$host$request_uri; + } } server { @@ -22,10 +29,6 @@ server { add_header Strict-Transport-Security "max-age=31536000" always; - location ^~ /.well-known/acme-challenge/ { - alias /var/www/certbot/.well-known/acme-challenge/; - } - location / { proxy_pass http://grafana:3000/; proxy_http_version 1.1; From 4daad7332d60a20310707e28df6cff3b80b07545 Mon Sep 17 00:00:00 2001 From: Emil Djupvik Date: Mon, 23 Mar 2026 21:00:16 +0100 Subject: [PATCH 04/13] move well known to 443 block --- containers/nginx/default.conf | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/containers/nginx/default.conf b/containers/nginx/default.conf index d3a81f6a0..0d0c1b75e 100644 --- a/containers/nginx/default.conf +++ b/containers/nginx/default.conf @@ -6,14 +6,7 @@ upstream backend { server { listen 80; server_name grafana.metrics.sctomega.com; - - location ^~ /.well-known/acme-challenge/ { - alias /var/www/certbot/.well-known/acme-challenge/; - } - - location / { - return 301 https://$host$request_uri; - } + return 301 https://$host$request_uri; } server { @@ -29,6 +22,10 @@ server { add_header Strict-Transport-Security "max-age=31536000" always; + location ^~ /.well-known/acme-challenge/ { + alias /var/www/certbot/.well-known/acme-challenge/; + } + location / { proxy_pass http://grafana:3000/; proxy_http_version 1.1; From f3029c7a6afa16bfdedaf831afb97f7f7ca7ec4b Mon Sep 17 00:00:00 2001 From: Emil Djupvik Date: Mon, 23 Mar 2026 21:15:40 +0100 Subject: [PATCH 05/13] add config files for mimir and postfix --- containers/loki/config.yml | 28 ++++++++++++++++++++++++++++ containers/mimir/config.yml | 26 ++++++++++++++++++++++++++ docker-compose.prod.yml | 14 +++++++------- 3 files changed, 61 insertions(+), 7 deletions(-) create mode 100644 containers/loki/config.yml create mode 100644 containers/mimir/config.yml diff --git a/containers/loki/config.yml b/containers/loki/config.yml new file mode 100644 index 000000000..41da9ff03 --- /dev/null +++ b/containers/loki/config.yml @@ -0,0 +1,28 @@ +auth_enabled: false + +server: + http_listen_port: 3100 + grpc_listen_port: 9096 + +common: + instance_addr: 127.0.0.1 + path_prefix: /data/loki + storage: + filesystem: + chunks_directory: /data/loki/chunks + rules_directory: /data/loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + +schema_config: + configs: + - from: 2020-10-24 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + diff --git a/containers/mimir/config.yml b/containers/mimir/config.yml new file mode 100644 index 000000000..2aa80b1be --- /dev/null +++ b/containers/mimir/config.yml @@ -0,0 +1,26 @@ +target: all + +server: + http_listen_port: 9009 + grpc_listen_port: 9095 + +ingester: + ring: + instance_addr: 127.0.0.1 + kvstore: + store: inmemory + replication_factor: 1 + +blocks_storage: + backend: filesystem + filesystem: + dir: /data/mimir/blocks + tsdb: + dir: /data/mimir/tsdb + +compactor: + data_dir: /data/mimir/compactor + +store_gateway: + sharding_ring: + replication_factor: 1 diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index e3c1d0611..b31babae9 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -30,7 +30,7 @@ services: - 443:443 environment: DOMAIN: ${DOMAIN} - PROXY_PASS: 'projectnext:3000' + PROXY_PASS: "projectnext:3000" volumes: - ./containers/nginx/nginx.conf:/etc/nginx/nginx.conf:ro - ./containers/nginx/default.conf:/etc/nginx/templates/default.conf.template:ro @@ -50,13 +50,13 @@ services: image: grafana/grafana:latest volumes: - grafana_data:/var/lib/grafana - - ./grafana/provisioning:/etc/grafana/provisioning:ro + - ./containers/grafana/provisioning:/etc/grafana/provisioning:ro restart: unless-stopped loki: image: grafana/loki:latest volumes: - - ./loki/config.yml:/etc/loki/config.yml:ro + - ./containers/loki/config.yml:/etc/loki/config.yml:ro - loki_data:/data/loki command: -config.file=/etc/loki/config.yml restart: unless-stopped @@ -64,7 +64,7 @@ services: mimir: image: grafana/mimir:latest volumes: - - ./mimir/config.yml:/etc/mimir/config.yml:ro + - ./containers/mimir/config.yml:/etc/mimir/config.yml:ro - mimir_data:/data/mimir command: -config.file=/etc/mimir/config.yml restart: unless-stopped @@ -72,10 +72,10 @@ services: postfix: build: ${PROJECT_ROOT:-.}/containers/postfix/ ports: - - '587:587' - - '25:25' + - "587:587" + - "25:25" environment: - POSTGRES_HOST: 'db' + POSTGRES_HOST: "db" POSTGRES_DB: ${DB_NAME} POSTGRES_USER: ${DB_USERNAME} POSTGRES_PASSWORD: ${DB_PASSWORD} From 8095665ba54b86ffd5cdf05f36a9f06edb194d8a Mon Sep 17 00:00:00 2001 From: Emil Djupvik Date: Mon, 23 Mar 2026 21:17:38 +0100 Subject: [PATCH 06/13] remove rules directory --- containers/loki/config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/containers/loki/config.yml b/containers/loki/config.yml index 41da9ff03..8cae79bf5 100644 --- a/containers/loki/config.yml +++ b/containers/loki/config.yml @@ -10,7 +10,6 @@ common: storage: filesystem: chunks_directory: /data/loki/chunks - rules_directory: /data/loki/rules replication_factor: 1 ring: kvstore: From eb8af615989617e3acc67e0302f9a556ecb57747 Mon Sep 17 00:00:00 2001 From: Emil Djupvik Date: Mon, 23 Mar 2026 21:19:32 +0100 Subject: [PATCH 07/13] disable ruler --- containers/loki/config.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/containers/loki/config.yml b/containers/loki/config.yml index 8cae79bf5..43747656d 100644 --- a/containers/loki/config.yml +++ b/containers/loki/config.yml @@ -15,6 +15,9 @@ common: kvstore: store: inmemory +ruler: + enabled: false + schema_config: configs: - from: 2020-10-24 From b17522ed3284abecc3c2a2f085948d91e6e32d61 Mon Sep 17 00:00:00 2001 From: Emil Djupvik Date: Mon, 23 Mar 2026 21:21:58 +0100 Subject: [PATCH 08/13] fix loki --- containers/loki/config.yml | 3 --- docker-compose.prod.yml | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/containers/loki/config.yml b/containers/loki/config.yml index 43747656d..8cae79bf5 100644 --- a/containers/loki/config.yml +++ b/containers/loki/config.yml @@ -15,9 +15,6 @@ common: kvstore: store: inmemory -ruler: - enabled: false - schema_config: configs: - from: 2020-10-24 diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index b31babae9..865ec3c57 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -58,7 +58,7 @@ services: volumes: - ./containers/loki/config.yml:/etc/loki/config.yml:ro - loki_data:/data/loki - command: -config.file=/etc/loki/config.yml + command: -config.file=/etc/loki/config.yml -target=all,-ruler restart: unless-stopped mimir: From d9732f9b9ef9041ac1ebe8c9980358ff9abd0010 Mon Sep 17 00:00:00 2001 From: Emil Djupvik Date: Mon, 23 Mar 2026 21:23:37 +0100 Subject: [PATCH 09/13] fix loki --- containers/loki/config.yml | 6 ++++++ docker-compose.prod.yml | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/containers/loki/config.yml b/containers/loki/config.yml index 8cae79bf5..8dde74156 100644 --- a/containers/loki/config.yml +++ b/containers/loki/config.yml @@ -15,6 +15,12 @@ common: kvstore: store: inmemory +ruler: + storage: + type: local + local: + directory: /tmp/loki-rules + schema_config: configs: - from: 2020-10-24 diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 865ec3c57..b31babae9 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -58,7 +58,7 @@ services: volumes: - ./containers/loki/config.yml:/etc/loki/config.yml:ro - loki_data:/data/loki - command: -config.file=/etc/loki/config.yml -target=all,-ruler + command: -config.file=/etc/loki/config.yml restart: unless-stopped mimir: From 49f2cb2d6f87b21bdab325854437b9222f9c657f Mon Sep 17 00:00:00 2001 From: Emil Djupvik Date: Mon, 23 Mar 2026 21:24:45 +0100 Subject: [PATCH 10/13] fix loki by making him root --- docker-compose.prod.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index b31babae9..60903a43d 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -55,6 +55,7 @@ services: loki: image: grafana/loki:latest + user: root volumes: - ./containers/loki/config.yml:/etc/loki/config.yml:ro - loki_data:/data/loki From e5cce4c8183f6895c0f797e052c34f1b2313d3f9 Mon Sep 17 00:00:00 2001 From: Emil Djupvik Date: Mon, 13 Apr 2026 19:03:05 +0200 Subject: [PATCH 11/13] update nginx config --- containers/nginx/auth/metrics.htpasswd | 1 + containers/nginx/default.conf | 68 ++++++++++++++++++++++++++ 2 files changed, 69 insertions(+) create mode 100644 containers/nginx/auth/metrics.htpasswd diff --git a/containers/nginx/auth/metrics.htpasswd b/containers/nginx/auth/metrics.htpasswd new file mode 100644 index 000000000..7b25837a0 --- /dev/null +++ b/containers/nginx/auth/metrics.htpasswd @@ -0,0 +1 @@ +remote_writer:$apr1$baokJMMP$NH1P3Xbin2H9GqtJ8IZL2/ diff --git a/containers/nginx/default.conf b/containers/nginx/default.conf index 0d0c1b75e..c246a5299 100644 --- a/containers/nginx/default.conf +++ b/containers/nginx/default.conf @@ -36,6 +36,74 @@ server { } } +server { + listen 80; + server_name mimir.metrics.sctomega.com; + return 301 https://$host$request_uri; +} + +server { + listen 443 ssl; + server_name mimir.metrics.sctomega.com; + + # ssl_certificate /etc/letsencrypt/live/mimir.metrics.sctomega.com/fullchain.pem; + # ssl_certificate_key /etc/letsencrypt/live/mimir.metrics.sctomega.com/privkey.pem; + ssl_certificate /etc/letsencrypt/live/grafana.metrics.sctomega.com/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/grafana.metrics.sctomega.com/privkey.pem; + + ssl_protocols TLSv1.1 TLSv1.2; + ssl_prefer_server_ciphers on; + ssl_ciphers ECDH+AESGCM:ECDH+AES256:ECDH+AES128:DHE+AES128:!ADH:!AECDH:!MD5; + + add_header Strict-Transport-Security "max-age=31536000" always; + + location ^~ /.well-known/acme-challenge/ { + alias /var/www/certbot/.well-known/acme-challenge/; + } + + location / { + proxy_pass http://mimir:9009; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto https; + } +} + +server { + listen 80; + server_name loki.metrics.sctomega.com; + return 301 https://$host$request_uri; +} + +server { + listen 443 ssl; + server_name loki.metrics.sctomega.com; + + # ssl_certificate /etc/letsencrypt/live/loki.metrics.sctomega.com/fullchain.pem; + # ssl_certificate_key /etc/letsencrypt/live/loki.metrics.sctomega.com/privkey.pem; + ssl_certificate /etc/letsencrypt/live/grafana.metrics.sctomega.com/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/grafana.metrics.sctomega.com/privkey.pem; + + ssl_protocols TLSv1.1 TLSv1.2; + ssl_prefer_server_ciphers on; + ssl_ciphers ECDH+AESGCM:ECDH+AES256:ECDH+AES128:DHE+AES128:!ADH:!AECDH:!MD5; + + add_header Strict-Transport-Security "max-age=31536000" always; + + location ^~ /.well-known/acme-challenge/ { + alias /var/www/certbot/.well-known/acme-challenge/; + } + + location / { + proxy_pass http://loki:3100; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto https; + } +} + server { listen 80; server_name ${DOMAIN}; From dbe1ca1363988ec00485937e920e3bcfd55efabf Mon Sep 17 00:00:00 2001 From: Emil Djupvik Date: Mon, 13 Apr 2026 19:12:36 +0200 Subject: [PATCH 12/13] update default.conf --- containers/nginx/default.conf | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/containers/nginx/default.conf b/containers/nginx/default.conf index c246a5299..9d6e47ca4 100644 --- a/containers/nginx/default.conf +++ b/containers/nginx/default.conf @@ -46,10 +46,8 @@ server { listen 443 ssl; server_name mimir.metrics.sctomega.com; - # ssl_certificate /etc/letsencrypt/live/mimir.metrics.sctomega.com/fullchain.pem; - # ssl_certificate_key /etc/letsencrypt/live/mimir.metrics.sctomega.com/privkey.pem; - ssl_certificate /etc/letsencrypt/live/grafana.metrics.sctomega.com/fullchain.pem; - ssl_certificate_key /etc/letsencrypt/live/grafana.metrics.sctomega.com/privkey.pem; + ssl_certificate /etc/letsencrypt/live/mimir.metrics.sctomega.com/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/mimir.metrics.sctomega.com/privkey.pem; ssl_protocols TLSv1.1 TLSv1.2; ssl_prefer_server_ciphers on; @@ -80,10 +78,8 @@ server { listen 443 ssl; server_name loki.metrics.sctomega.com; - # ssl_certificate /etc/letsencrypt/live/loki.metrics.sctomega.com/fullchain.pem; - # ssl_certificate_key /etc/letsencrypt/live/loki.metrics.sctomega.com/privkey.pem; - ssl_certificate /etc/letsencrypt/live/grafana.metrics.sctomega.com/fullchain.pem; - ssl_certificate_key /etc/letsencrypt/live/grafana.metrics.sctomega.com/privkey.pem; + ssl_certificate /etc/letsencrypt/live/loki.metrics.sctomega.com/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/loki.metrics.sctomega.com/privkey.pem; ssl_protocols TLSv1.1 TLSv1.2; ssl_prefer_server_ciphers on; From d4b85141fff4d3bcde2d49d47bb5a6e8c1cfe47a Mon Sep 17 00:00:00 2001 From: Emil Djupvik Date: Mon, 13 Apr 2026 19:44:30 +0200 Subject: [PATCH 13/13] add basic auth --- containers/nginx/default.conf | 4 ++++ docker-compose.prod.yml | 1 + 2 files changed, 5 insertions(+) diff --git a/containers/nginx/default.conf b/containers/nginx/default.conf index 9d6e47ca4..71bbb4e80 100644 --- a/containers/nginx/default.conf +++ b/containers/nginx/default.conf @@ -54,6 +54,8 @@ server { ssl_ciphers ECDH+AESGCM:ECDH+AES256:ECDH+AES128:DHE+AES128:!ADH:!AECDH:!MD5; add_header Strict-Transport-Security "max-age=31536000" always; + auth_basic "Restricted"; + auth_basic_user_file /etc/nginx/auth/metrics.htpasswd; location ^~ /.well-known/acme-challenge/ { alias /var/www/certbot/.well-known/acme-challenge/; @@ -86,6 +88,8 @@ server { ssl_ciphers ECDH+AESGCM:ECDH+AES256:ECDH+AES128:DHE+AES128:!ADH:!AECDH:!MD5; add_header Strict-Transport-Security "max-age=31536000" always; + auth_basic "Restricted"; + auth_basic_user_file /etc/nginx/auth/metrics.htpasswd; location ^~ /.well-known/acme-challenge/ { alias /var/www/certbot/.well-known/acme-challenge/; diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 60903a43d..7cc4f5f9c 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -34,6 +34,7 @@ services: volumes: - ./containers/nginx/nginx.conf:/etc/nginx/nginx.conf:ro - ./containers/nginx/default.conf:/etc/nginx/templates/default.conf.template:ro + - ./containers/nginx/auth:/etc/nginx/auth:ro - store:/usr/store:ro - ./certs/conf:/etc/letsencrypt:ro - ./certs/www/:/var/www/certbot:ro