diff --git a/compose/elasticsearch.yml b/compose/elasticsearch.yml index 27c31939..a916b794 100644 --- a/compose/elasticsearch.yml +++ b/compose/elasticsearch.yml @@ -1,7 +1,6 @@ -version: '2.4' services: ccd-elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:7.17.0 + image: docker.elastic.co/elasticsearch/elasticsearch:9.1.2 container_name: ccd-elasticsearch depends_on: - "ccd-data-store-api" diff --git a/compose/logstash.yml b/compose/logstash.yml index a77e0f39..14c73dd7 100644 --- a/compose/logstash.yml +++ b/compose/logstash.yml @@ -1,8 +1,7 @@ -version: '2.4' services: ccd-logstash: - image: "logstash:7.16.3" + image: "logstash:9.1.2" container_name: ccd-logstash environment: - XPACK_MONITORING_ENABLED=false diff --git a/logstash/config/logstash.yml b/logstash/config/logstash.yml index 66cf3a87..ec283903 100644 --- a/logstash/config/logstash.yml +++ b/logstash/config/logstash.yml @@ -1,5 +1,5 @@ dead_letter_queue.enable: true -http.host: 0.0.0.0 +pipeline.ecs_compatibility: disabled queue.type: persisted -xpack.monitoring.elasticsearch.hosts: ["${ES_HOSTS}"] -xpack.monitoring.enabled: false +xpack.monitoring.elasticsearch.hosts: ${ES_HOSTS} +xpack.monitoring.enabled: ${XPACK_MONITORING_ENABLED} \ No newline at end of file diff --git a/logstash/pipeline/dead_letter_indexing_pipeline.conf b/logstash/pipeline/dead_letter_indexing_pipeline.conf index e00e0a30..4553837f 100644 --- a/logstash/pipeline/dead_letter_indexing_pipeline.conf +++ b/logstash/pipeline/dead_letter_indexing_pipeline.conf @@ -6,23 +6,28 @@ input { } } filter { - # capture the entire event, and write it to a new field; we 'll call that field `failed_case` - ruby { - code => "event.set('failed_case', event.to_json())" - } - # prune every field off the event except for the one we 've just created. Note that this does not prune event metadata. - prune { - whitelist_names => ["^failed_case$"] - } - ruby { - code => "event.set('timestamp', event.get('[@metadata][dead_letter_queue][entry_time]'))" - } - # pull useful information out of the event metadata provided by the dead letter queue, and add it to the new event. - mutate { - add_field => { - "reason" => "%{[@metadata][dead_letter_queue][reason]}" - } - } + # Capture the failed event safely inside an allowed field + ruby { + code => " + # Serialize the entire failed event into the 'description' field + event.set('description', 'Failed event: ' + event.to_json) + + # Add a few safe, permitted fields + event.set('last_modified', LogStash::Timestamp.now.time) + event.set('pipeline', event.get('[@metadata][dead_letter_queue][pipeline_id]') || 'main') + event.set('username', 'logstash') + " + } + + # Remove everything else so only allowed fields remain + prune { + whitelist_names => [ + '^description$', + '^last_modified$', + '^pipeline$', + '^username$' + ] + } } output { elasticsearch {