From d0a1ba6fea6a5376878a75dbb5abe9b1eb87e3cc Mon Sep 17 00:00:00 2001 From: markdathornehmcts Date: Thu, 17 Jul 2025 15:48:42 +0100 Subject: [PATCH 1/6] CCD-5849 Elasticsearch 9 upgraded --- compose/elasticsearch.yml | 3 +-- compose/logstash.yml | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/compose/elasticsearch.yml b/compose/elasticsearch.yml index 27c31939..4ca68446 100644 --- a/compose/elasticsearch.yml +++ b/compose/elasticsearch.yml @@ -1,7 +1,6 @@ -version: '2.4' services: ccd-elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:7.17.0 + image: docker.elastic.co/elasticsearch/elasticsearch:9.0.3 container_name: ccd-elasticsearch depends_on: - "ccd-data-store-api" diff --git a/compose/logstash.yml b/compose/logstash.yml index a77e0f39..318bd9c7 100644 --- a/compose/logstash.yml +++ b/compose/logstash.yml @@ -1,8 +1,7 @@ -version: '2.4' services: ccd-logstash: - image: "logstash:7.16.3" + image: "logstash:9.0.3" container_name: ccd-logstash environment: - XPACK_MONITORING_ENABLED=false From a6d804fda34b4ffe1f583c747e4ffcfa6529d20c Mon Sep 17 00:00:00 2001 From: markdathornehmcts Date: Mon, 28 Jul 2025 11:19:44 +0100 Subject: [PATCH 2/6] CCD-5849 Elasticsearch 9 upgrade minor version to 9.0.4 --- compose/elasticsearch.yml | 2 +- compose/logstash.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/compose/elasticsearch.yml b/compose/elasticsearch.yml index 4ca68446..20d7060b 100644 --- a/compose/elasticsearch.yml +++ b/compose/elasticsearch.yml @@ -1,6 +1,6 @@ services: ccd-elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:9.0.3 + image: docker.elastic.co/elasticsearch/elasticsearch:9.0.4 container_name: ccd-elasticsearch depends_on: - "ccd-data-store-api" diff --git a/compose/logstash.yml b/compose/logstash.yml index 318bd9c7..c441e1cc 100644 --- a/compose/logstash.yml +++ b/compose/logstash.yml @@ -1,7 +1,7 @@ services: ccd-logstash: - image: "logstash:9.0.3" + image: "logstash:9.0.4" container_name: ccd-logstash environment: - XPACK_MONITORING_ENABLED=false From ce0fc8dc89752a7758b2f2eeca07e807906eb70c Mon Sep 17 00:00:00 2001 From: markdathornehmcts Date: Mon, 28 Jul 2025 12:03:18 +0100 Subject: [PATCH 3/6] CCD-5849 http.host no longer required. Breaks if present. --- logstash/config/logstash.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/logstash/config/logstash.yml b/logstash/config/logstash.yml index 66cf3a87..5872cf1d 100644 --- a/logstash/config/logstash.yml +++ b/logstash/config/logstash.yml @@ -1,5 +1,4 @@ dead_letter_queue.enable: true -http.host: 0.0.0.0 queue.type: persisted -xpack.monitoring.elasticsearch.hosts: ["${ES_HOSTS}"] -xpack.monitoring.enabled: false +xpack.monitoring.elasticsearch.hosts: ${ES_HOSTS} +xpack.monitoring.enabled: ${XPACK_MONITORING_ENABLED} \ No newline at end of file From 8931fe02979d74251466ea948be8ce7fd23ee7a8 Mon Sep 17 00:00:00 2001 From: markdathornehmcts Date: Thu, 4 Sep 2025 10:40:02 +0100 Subject: [PATCH 4/6] CCD-6510 set ES version to 9.1.2 --- compose/elasticsearch.yml | 2 +- compose/logstash.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/compose/elasticsearch.yml b/compose/elasticsearch.yml index 20d7060b..a916b794 100644 --- a/compose/elasticsearch.yml +++ b/compose/elasticsearch.yml @@ -1,6 +1,6 @@ services: ccd-elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:9.0.4 + image: docker.elastic.co/elasticsearch/elasticsearch:9.1.2 container_name: ccd-elasticsearch depends_on: - "ccd-data-store-api" diff --git a/compose/logstash.yml b/compose/logstash.yml index c441e1cc..14c73dd7 100644 --- a/compose/logstash.yml +++ b/compose/logstash.yml @@ -1,7 +1,7 @@ services: ccd-logstash: - image: "logstash:9.0.4" + image: "logstash:9.1.2" container_name: ccd-logstash environment: - XPACK_MONITORING_ENABLED=false From 0147aacf7ee12e73bf184547b584a2e8100e9bdd Mon Sep 17 00:00:00 2001 From: markdathornehmcts Date: Tue, 11 Nov 2025 15:53:18 +0000 Subject: [PATCH 5/6] CCD-5849 fix dead letter config due to strict mapping in ES 9 --- .../dead_letter_indexing_pipeline.conf | 39 +++++++++++-------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/logstash/pipeline/dead_letter_indexing_pipeline.conf b/logstash/pipeline/dead_letter_indexing_pipeline.conf index e00e0a30..4553837f 100644 --- a/logstash/pipeline/dead_letter_indexing_pipeline.conf +++ b/logstash/pipeline/dead_letter_indexing_pipeline.conf @@ -6,23 +6,28 @@ input { } } filter { - # capture the entire event, and write it to a new field; we 'll call that field `failed_case` - ruby { - code => "event.set('failed_case', event.to_json())" - } - # prune every field off the event except for the one we 've just created. Note that this does not prune event metadata. - prune { - whitelist_names => ["^failed_case$"] - } - ruby { - code => "event.set('timestamp', event.get('[@metadata][dead_letter_queue][entry_time]'))" - } - # pull useful information out of the event metadata provided by the dead letter queue, and add it to the new event. - mutate { - add_field => { - "reason" => "%{[@metadata][dead_letter_queue][reason]}" - } - } + # Capture the failed event safely inside an allowed field + ruby { + code => " + # Serialize the entire failed event into the 'description' field + event.set('description', 'Failed event: ' + event.to_json) + + # Add a few safe, permitted fields + event.set('last_modified', LogStash::Timestamp.now.time) + event.set('pipeline', event.get('[@metadata][dead_letter_queue][pipeline_id]') || 'main') + event.set('username', 'logstash') + " + } + + # Remove everything else so only allowed fields remain + prune { + whitelist_names => [ + '^description$', + '^last_modified$', + '^pipeline$', + '^username$' + ] + } } output { elasticsearch { From 993f1540d0062850e2b1e83abf152efc8ef8f138 Mon Sep 17 00:00:00 2001 From: markdathornehmcts Date: Mon, 9 Feb 2026 09:17:26 +0000 Subject: [PATCH 6/6] CCD-5849 disable ECS compatibility - as it limits to ES version 8 only --- logstash/config/logstash.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/logstash/config/logstash.yml b/logstash/config/logstash.yml index 5872cf1d..ec283903 100644 --- a/logstash/config/logstash.yml +++ b/logstash/config/logstash.yml @@ -1,4 +1,5 @@ dead_letter_queue.enable: true +pipeline.ecs_compatibility: disabled queue.type: persisted xpack.monitoring.elasticsearch.hosts: ${ES_HOSTS} xpack.monitoring.enabled: ${XPACK_MONITORING_ENABLED} \ No newline at end of file