CCD-6510 Upgrade docker to version 9.1.2

compose/elasticsearch.yml

@@ -1,7 +1,6 @@
-version: '2.4'
 services:
   ccd-elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch:7.17.0
+    image: docker.elastic.co/elasticsearch/elasticsearch:9.1.2
     container_name: ccd-elasticsearch
     depends_on:
       - "ccd-data-store-api"

compose/logstash.yml

@@ -1,8 +1,7 @@
-version: '2.4'
 services:
 
   ccd-logstash:
-    image: "logstash:7.16.3"
+    image: "logstash:9.1.2"
     container_name: ccd-logstash
     environment:
       - XPACK_MONITORING_ENABLED=false

logstash/config/logstash.yml

@@ -1,5 +1,5 @@
 dead_letter_queue.enable: true
-http.host: 0.0.0.0
+pipeline.ecs_compatibility: disabled
 queue.type: persisted
-xpack.monitoring.elasticsearch.hosts: ["${ES_HOSTS}"]
-xpack.monitoring.enabled: false
+xpack.monitoring.elasticsearch.hosts: ${ES_HOSTS}
+xpack.monitoring.enabled: ${XPACK_MONITORING_ENABLED}

logstash/pipeline/dead_letter_indexing_pipeline.conf

@@ -6,23 +6,28 @@ input {
     }
 }
 filter {
-    # capture the entire event, and write it to a new field; we 'll call that field `failed_case`
-    ruby {
-        code => "event.set('failed_case', event.to_json())"
-    }
-    # prune every field off the event except for the one we 've just created. Note that this does not prune event metadata.
-    prune {
-        whitelist_names => ["^failed_case$"]
-    }
-    ruby {
-        code => "event.set('timestamp', event.get('[@metadata][dead_letter_queue][entry_time]'))"
-    }
-    # pull useful information out of the event metadata provided by the dead letter queue, and add it to the new event.
-    mutate {
-        add_field => {
-            "reason" => "%{[@metadata][dead_letter_queue][reason]}"
-        }
-    }
+  # Capture the failed event safely inside an allowed field
+  ruby {
+    code => "
+      # Serialize the entire failed event into the 'description' field
+      event.set('description', 'Failed event: ' + event.to_json)
+
+      # Add a few safe, permitted fields
+      event.set('last_modified', LogStash::Timestamp.now.time)
+      event.set('pipeline', event.get('[@metadata][dead_letter_queue][pipeline_id]') || 'main')
+      event.set('username', 'logstash')
+    "
+  }
+
+  # Remove everything else so only allowed fields remain
+  prune {
+    whitelist_names => [
+      '^description$',
+      '^last_modified$',
+      '^pipeline$',
+      '^username$'
+    ]
+  }
 }
 output {
     elasticsearch {