Provision Ansible Grafana with Grafana Cloud datasources, etc.

ansible/host_vars/healthchecks/docker_values.yml

@@ -1,6 +1,8 @@
 ---
 healthchecks:
   port: "8000"
+  project_id: "{{ opconnect_results['healthchecks'].project_id }}"
+  readonly_apikey: "{{ opconnect_results['healthchecks'].readonly_apikey }}"
   healthcheck: "{{ opconnect_results['healthchecks'].healthcheck }}"
   email:
     host: smtp.gmail.com

ansible/host_vars/monitoring/cloudflare.yml

@@ -1,23 +1,11 @@
 ---
 cloudflare_dns_records:
-  - zone: "{{ domain }}"
-    record: "{{ compose_project_name }}.{{ domain }}"
-    type: CNAME
-    value: tunnel-{{ ansible_host }}.{{ domain }}
-    proxied: true
-    state: present
   - zone: "{{ domain }}"
     record: grafana.{{ domain }}
     type: CNAME
     value: tunnel-{{ ansible_host }}.{{ domain }}
     proxied: true
     state: present
-  - zone: "{{ domain }}"
-    record: prometheus.{{ domain }}
-    type: CNAME
-    value: tunnel-{{ ansible_host }}.{{ domain }}
-    proxied: true
-    state: present
   - zone: "{{ domain }}"
     record: influxdb.{{ domain }}
     type: CNAME

ansible/host_vars/monitoring/cloudflared.yml

@@ -1,8 +1,6 @@
 ---
 additional_ingress:
   - hostname: grafana.{{ domain }}
-    service: http://localhost:{{ compose_project_ports.grafana[0] }}
-  - hostname: prometheus.{{ domain }}
-    service: http://localhost:{{ compose_project_ports.prometheus[0] }}
+    service: http://localhost:{{ grafana.port }}
   - hostname: influxdb.{{ domain }}
-    service: http://localhost:{{ compose_project_ports.influxdb[0] }}
+    service: http://localhost:{{ influxdb.port }}

ansible/host_vars/monitoring/docker-compose.yaml

@@ -1,32 +1,18 @@
 ---
 version: "3"
 services:
-  prometheus:
-    image: prom/prometheus:latest@sha256:1a3e9a878e50cd339ae7cf5718fda08381dda2d4ccd28e94bbaa3190d1a566c2
-    restart: always
-    volumes:
-      - "{{ docker_volume_dirs.prometheus[0] }}/prometheus.yml:/etc/prometheus/prometheus.yml"
-      - "{{ docker_volume_dirs.prometheus[1] }}:/prometheus" # todo: chown this after creation on host: chown 65534:65534
-    ports:
-      - "{{ compose_project_ports.prometheus[0] }}:{{ prometheus.internal_port }}"
-  loki:
-    image: grafana/loki:latest@sha256:262d91088fb8cc31d3a6a0591aea52f74fa448884c961e63abd35decd3570a88
-    restart: always
-    volumes:
-      - "{{ docker_volume_dirs.loki[0] }}/local-config.yaml:/etc/loki/local-config.yaml"
-    ports:
-      - "{{ compose_project_ports.loki[0] }}:3100"
   influxdb:
-    image: influxdb:2.6@sha256:d7c48214e57f77b8eba4e29d0c9b2a0c10393e7a9f9ee5d015da2d75b845faca
+    image: influxdb:1.8
     restart: always
     volumes:
       - "{{ docker_volume_dirs.influxdb[0] }}:/var/lib/influxdb"
     ports:
-      - "{{ compose_project_ports.influxdb[0] }}:{{ influxdb.internal_port }}"
+      - "{{ influxdb.port }}:{{ influxdb.internal_port }}"
   varken:
     image: boerderij/varken:latest@sha256:784aaf577c822a3e69c5b2792e0422896c8cdb0455e584cd5a4e46bca789848d
     restart: always
     depends_on:
+      # Currently varken requires influxdb to be on v1.8.x
       - influxdb
     volumes:
       - "{{ docker_volume_dirs.varken[0] }}:/config"
@@ -40,11 +26,10 @@ services:
       - "{{ docker_volume_dirs.grafana[1] }}:/var/lib/grafana"
       - "{{ docker_volume_dirs.grafana[2] }}:/var/lib/grafana/plugins"
       - "{{ docker_volume_dirs.grafana[3] }}:/etc/grafana/provisioning"
+      - "{{ docker_volume_dirs.grafana[4] }}:/var/lib/grafana/dashboards"
     ports:
-      - "{{ compose_project_ports.grafana[0] }}:3000"
+      - "{{ grafana.port }}:3000"
     depends_on:
-      - prometheus
-      - loki
       - influxdb
       - varken
     environment:

ansible/host_vars/monitoring/docker.yml

@@ -1,10 +1,6 @@
 ---
 compose_project_name: monitoring
 compose_project_ports:
-  prometheus:
-    - "{{ prometheus.port }}"
-  loki:
-    - "{{ loki.port }}"
   influxdb:
     - "{{ influxdb.port }}"
   grafana:
@@ -16,11 +12,6 @@ docker_group_id:
 docker_volume_dirs:
   monitoring:
     - /mnt/{{ compose_project_name }}
-  prometheus:
-    - /mnt/{{ compose_project_name }}/prometheus/config
-    - /mnt/{{ compose_project_name }}/prometheus/data
-  loki:
-    - /mnt/{{ compose_project_name }}/loki/config
   influxdb:
     - /mnt/{{ compose_project_name }}/influxdb/config
   varken:
@@ -30,7 +21,4 @@ docker_volume_dirs:
     - /mnt/{{ compose_project_name }}/grafana/data
     - /mnt/{{ compose_project_name }}/grafana/plugins
     - /mnt/{{ compose_project_name }}/grafana/provisioning
-    - /mnt/{{ compose_project_name }}/grafana/provisioning/datasources
-    - /mnt/{{ compose_project_name }}/grafana/provisioning/plugins
-    - /mnt/{{ compose_project_name }}/grafana/provisioning/notifiers
-    - /mnt/{{ compose_project_name }}/grafana/provisioning/dashboards
+    - /mnt/{{ compose_project_name }}/grafana/dashboards

ansible/host_vars/monitoring/docker_values.yml

@@ -1,27 +1,29 @@
 ---
-prometheus:
-  host: prometheus
-  port: "9000"
-  internal_port: "9090"
-  cloudflared:
-    - firefly.{{ domain }}
-    - firefly-importer.{{ domain }}
-    - healthchecks.{{ domain }}
-    - prometheus.{{ domain }}
-    - grafana.{{ domain }}
-    - drive.thepi.cloud
-    - pihole.{{ domain }}
-    - vault.{{ domain }}
-    - allaboutsecurity.xyz
-    - securemy.life
-  healthchecks:
-    project_id: "{{ opconnect_results['healthchecks'].project_id }}"
-    readonly_apikey: "{{ opconnect_results['healthchecks'].readonly_apikey }}"
-loki:
-  host: loki
-  port: "3100"
 grafana:
   port: "3000"
+  cloud:
+    enabled: false
+    org: "{{ opconnect_results['grafana'].cloud_tenant_name }}"
+    host: "{{ opconnect_results['grafana'].cloud_tenant_name }}.grafana.net"
+    services:
+      - prefix: am
+        name: "Alertmanager"
+      - prefix: hl
+        name: "Loki"
+      - prefix: hm
+        name: "Metrics"
+        subservices:
+          - prefix: Graphite
+            name: "Graphite"
+          - prefix: Prom
+            name: "Prometheus"
+      - prefix: ht
+        name: "Traces"
+    provisioning:
+      datasources:
+        api_path: "datasources"
+      dashboards:
+        api_path: "dashboards/home"
   plugins: grafana-piechart-panel,grafana-worldmap-panel,grafana-clock-panel
   admin:
     username: "{{ opconnect_results['grafana'].username }}"
@@ -33,17 +35,10 @@ grafana:
     password: "{{ opconnect_results['grafana'].smtp_password }}"
     from: grafana@{{ domain }}
   provisioning:
+    alerting:
     datasources:
       apiVersion: 1
       datasources:
-        - name: Loki
-          type: loki
-          access: proxy
-          url: http://{{ loki.host }}:{{ loki.port }}
-        - name: Prometheus
-          type: prometheus
-          access: proxy
-          url: http://{{ prometheus.host }}:{{ prometheus.internal_port }}
         - name: Influxdb
           type: influxdb
           access: proxy
@@ -58,7 +53,7 @@ grafana:
           database: varken
           username: "{{ influxdb.username }}"
           password: "{{ influxdb.password }}"
-    dashboardProviders:
+    dashboards:
       apiVersion: 1
       providers:
         - name: default
@@ -85,41 +80,43 @@ grafana:
           editable: true
           options:
             path: /var/lib/grafana/dashboards/media
+    notifiers:
+    plugins:
   dashboards:
     default:
       prometheus:
         gnetId: 2
         revision: 2
-        datasource: Prometheus
+        datasource: "grafanacloud-{{ opconnect_results['grafana'].cloud_tenant_name }}-prom"
       pihole:
         gnetId: 10176
         revision: 2
-        datasource: Prometheus
+        datasource: "grafanacloud-{{ opconnect_results['grafana'].cloud_tenant_name }}-prom"
     unifi:
       client-insights:
         gnetId: 11315
         revision: 8
-        datasource: Prometheus
+        datasource: "grafanacloud-{{ opconnect_results['grafana'].cloud_tenant_name }}-prom"
       uap-insights:
         gnetId: 11314
         revision: 9
-        datasource: Prometheus
+        datasource: "grafanacloud-{{ opconnect_results['grafana'].cloud_tenant_name }}-prom"
       network-sites:
         gnetId: 11311
         revision: 4
-        datasource: Prometheus
+        datasource: "grafanacloud-{{ opconnect_results['grafana'].cloud_tenant_name }}-prom"
       usw-insights:
         gnetId: 11312
         revision: 8
-        datasource: Prometheus
+        datasource: "grafanacloud-{{ opconnect_results['grafana'].cloud_tenant_name }}-prom"
       usg-insights:
         gnetId: 11313
         revision: 8
-        datasource: Prometheus
+        datasource: "grafanacloud-{{ opconnect_results['grafana'].cloud_tenant_name }}-prom"
       client-dpi:
         gnetId: 11310
         revision: 4
-        datasource: Prometheus
+        datasource: "grafanacloud-{{ opconnect_results['grafana'].cloud_tenant_name }}-prom"
     media:
       varken-official:
         gnetId: 9585

ansible/playbooks/codeserver.yml

@@ -23,4 +23,5 @@
     - dns
     - healthchecks
     - cron
+    - grafana_agent
     - codeserver

ansible/playbooks/gaming.yml

@@ -21,4 +21,5 @@
     - iptables
     - dns
     - healthchecks
+    - grafana_agent
     - docker

ansible/playbooks/healthchecks.yml

@@ -9,6 +9,8 @@
         - password
         - smtp_username
         - smtp_password
+        - project_id
+        - readonly_apikey
   roles:
     - opconnect
 
@@ -21,4 +23,5 @@
     - cloudflare
     - iptables
     - dns
+    - grafana_agent
     - docker

ansible/playbooks/monitoring.yml

@@ -8,6 +8,9 @@
         - password
         - smtp_username
         - smtp_password
+        - cloud_tenant_name
+        - cloud_admin_token
+        - cloud_service_account_token
       influxdb:
         - username
         - password
@@ -24,8 +27,6 @@
         - ombi_apikey
         - maxmind_license_key
       healthchecks:
-        - project_id
-        - readonly_apikey
         - apikey
   roles:
     - opconnect
@@ -41,8 +42,7 @@
     - dns
     - healthchecks
     - cron
-    - loki
-    - prometheus
+    - grafana_agent
     - grafana
     - varken
     - docker

ansible/playbooks/pihole.yml

@@ -21,5 +21,6 @@
     - iptables
     - dns
     - healthchecks
+    - grafana_agent
     - pihole
     - docker

ansible/playbooks/roles/grafana/tasks/dashboard.yml

@@ -0,0 +1,63 @@
+---
+# The grafana_dashboard.value might be a couple types of dashboards
+# Currently, only dashboards with the gnetId and revision values are supported
+# For dashboards with the gnetId and revision, we want to download the dashboard as JSON from https://grafana.com/api/dashboards/<gnetId>/revisions/<revision>/download
+- name: Download dashboard {{ grafana_dashboard.key }} from Grafana.com
+  ansible.builtin.uri:
+    url: https://grafana.com/api/dashboards/{{ grafana_dashboard.value.gnetId }}/revisions/{{ grafana_dashboard.value.revision }}/download
+    return_content: yes
+    status_code: 200
+  register: dashboard_json_result
+  no_log: true
+  when: grafana_dashboard.value.gnetId is defined and grafana_dashboard.value.revision is defined
+
+- name: Parse dashboard JSON
+  ansible.builtin.set_fact:
+    dashboard_json: "{{ dashboard_json_result.json }}"
+
+# For some dashboards, the datasource is configured in a datasource annotation in the annotations list
+# This list should be updated to match the configured datasource (if set)
+
+# First, get list of annotations
+- name: Get list of annotations
+  ansible.builtin.set_fact:
+    grafana_dashboard_annotations: "{{ dashboard_json.annotations.list | default([]) | list }}"
+
+# Then, find any annotations that have the 'datasource' key
+- name: Find annotations with datasource key
+  when: grafana_dashboard_annotations | length > 0
+  ansible.builtin.set_fact:
+    grafana_dashboard_annotations: "{{ grafana_dashboard_annotations | selectattr('datasource', 'defined') | list }}"
+
+- name: Set updated_grafana_dashboard_annotations to empty list
+  ansible.builtin.set_fact:
+    updated_grafana_dashboard_annotations: []
+
+- name: Create datasource fact
+  ansible.builtin.set_fact:
+    grafana_dashboard_datasource:
+      datasource: "{{ grafana_dashboard.value.datasource }}"
+
+# Then, update the datasource value to match the configured datasource
+# We want to override the datasource value in place and in order too
+- name: Update datasource value in annotations
+  when: grafana_dashboard_annotations | length > 0
+  ansible.builtin.set_fact:
+    updated_grafana_dashboard_annotations: "{{ updated_grafana_dashboard_annotations + [grafana_dashboard_annotation | combine(grafana_dashboard_datasource) ] }}"
+  loop: "{{ grafana_dashboard_annotations }}"
+  loop_control:
+    loop_var: grafana_dashboard_annotation
+
+# Finally, update the dashboard JSON with the updated annotations
+- name: Update dashboard JSON with updated annotations
+  ansible.builtin.set_fact:
+    dashboard_json: "{{ dashboard_json | combine({'annotations': dashboard_json.annotations | combine({'list': updated_grafana_dashboard_annotations})}, recursive=True) }}"
+  when: updated_grafana_dashboard_annotations is defined and updated_grafana_dashboard_annotations | length > 0
+
+# For dashboards with the gnetId and revision, we want to write the JSON to a file
+# The file will be written to the provider directory in /mnt/monitoring/grafana/dashboards/<provider>
+- name: Write dashboard {{ grafana_dashboard.key }} to file
+  ansible.builtin.copy:
+    dest: "{{ grafana_dashboard_provider_path }}/{{ grafana_dashboard.key }}.json"
+    content: "{{ dashboard_json | to_nice_json(indent=2) }}"
+  when: grafana_dashboard.value.gnetId is defined and grafana_dashboard.value.revision is defined