From c1ce60991be0cfcef43efbba2cc1240028999cb8 Mon Sep 17 00:00:00 2001 From: kcreddy Date: Fri, 15 Sep 2023 17:34:20 +0530 Subject: [PATCH 01/15] add dlm --- .../elasticsearch/ilm/default_policy.json | 23 ------------------- .../data_stream/threat/lifecycle.yml | 1 + .../data_stream/threat/manifest.yml | 1 - 3 files changed, 1 insertion(+), 24 deletions(-) delete mode 100644 packages/ti_recordedfuture/data_stream/threat/elasticsearch/ilm/default_policy.json create mode 100644 packages/ti_recordedfuture/data_stream/threat/lifecycle.yml diff --git a/packages/ti_recordedfuture/data_stream/threat/elasticsearch/ilm/default_policy.json b/packages/ti_recordedfuture/data_stream/threat/elasticsearch/ilm/default_policy.json deleted file mode 100644 index 68d2c5e57a6..00000000000 --- a/packages/ti_recordedfuture/data_stream/threat/elasticsearch/ilm/default_policy.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "policy": { - "phases": { - "hot": { - "actions": { - "rollover": { - "max_age": "2d", - "max_size": "50gb" - }, - "set_priority": { - "priority": 100 - } - } - }, - "delete": { - "min_age": "3d", - "actions": { - "delete": {} - } - } - } - } -} \ No newline at end of file diff --git a/packages/ti_recordedfuture/data_stream/threat/lifecycle.yml b/packages/ti_recordedfuture/data_stream/threat/lifecycle.yml new file mode 100644 index 00000000000..42195d391f1 --- /dev/null +++ b/packages/ti_recordedfuture/data_stream/threat/lifecycle.yml @@ -0,0 +1 @@ +data_retention: "5d" \ No newline at end of file diff --git a/packages/ti_recordedfuture/data_stream/threat/manifest.yml b/packages/ti_recordedfuture/data_stream/threat/manifest.yml index 118a0c24b43..520dfe9d505 100644 --- a/packages/ti_recordedfuture/data_stream/threat/manifest.yml +++ b/packages/ti_recordedfuture/data_stream/threat/manifest.yml @@ -1,6 +1,5 @@ type: logs title: Recorded Future -ilm_policy: logs-ti_recordedfuture.threat-default_policy streams: - input: logfile enabled: false From 79b1f02b452770f232a5f6f846064aecb46072a7 Mon Sep 17 00:00:00 2001 From: kcreddy Date: Fri, 15 Sep 2023 23:49:23 +0530 Subject: [PATCH 02/15] check what happens if both ilm and dlm are present --- .../elasticsearch/ilm/default_policy.json | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 packages/ti_recordedfuture/data_stream/threat/elasticsearch/ilm/default_policy.json diff --git a/packages/ti_recordedfuture/data_stream/threat/elasticsearch/ilm/default_policy.json b/packages/ti_recordedfuture/data_stream/threat/elasticsearch/ilm/default_policy.json new file mode 100644 index 00000000000..68d2c5e57a6 --- /dev/null +++ b/packages/ti_recordedfuture/data_stream/threat/elasticsearch/ilm/default_policy.json @@ -0,0 +1,23 @@ +{ + "policy": { + "phases": { + "hot": { + "actions": { + "rollover": { + "max_age": "2d", + "max_size": "50gb" + }, + "set_priority": { + "priority": 100 + } + } + }, + "delete": { + "min_age": "3d", + "actions": { + "delete": {} + } + } + } + } +} \ No newline at end of file From 2427fc73edbd42ece84e6c606e349bb7010324ee Mon Sep 17 00:00:00 2001 From: kcreddy Date: Fri, 15 Sep 2023 23:50:12 +0530 Subject: [PATCH 03/15] add ilm back --- packages/ti_recordedfuture/data_stream/threat/manifest.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/ti_recordedfuture/data_stream/threat/manifest.yml b/packages/ti_recordedfuture/data_stream/threat/manifest.yml index 520dfe9d505..118a0c24b43 100644 --- a/packages/ti_recordedfuture/data_stream/threat/manifest.yml +++ b/packages/ti_recordedfuture/data_stream/threat/manifest.yml @@ -1,5 +1,6 @@ type: logs title: Recorded Future +ilm_policy: logs-ti_recordedfuture.threat-default_policy streams: - input: logfile enabled: false From cf5b98a58a0bd058e35ae85cafc5a20a90f9dc2b Mon Sep 17 00:00:00 2001 From: kcreddy Date: Fri, 15 Sep 2023 23:50:36 +0530 Subject: [PATCH 04/15] update dlm --- packages/ti_recordedfuture/data_stream/threat/lifecycle.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ti_recordedfuture/data_stream/threat/lifecycle.yml b/packages/ti_recordedfuture/data_stream/threat/lifecycle.yml index 42195d391f1..a6c83b5d8c7 100644 --- a/packages/ti_recordedfuture/data_stream/threat/lifecycle.yml +++ b/packages/ti_recordedfuture/data_stream/threat/lifecycle.yml @@ -1 +1 @@ -data_retention: "5d" \ No newline at end of file +data_retention: "1h" \ No newline at end of file From 450da83f8655d2a6a70d925611aba824a7370293 Mon Sep 17 00:00:00 2001 From: kcreddy Date: Mon, 18 Sep 2023 16:54:17 +0530 Subject: [PATCH 05/15] update DLM policy --- packages/ti_recordedfuture/changelog.yml | 5 +++++ packages/ti_recordedfuture/data_stream/threat/lifecycle.yml | 2 +- packages/ti_recordedfuture/manifest.yml | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/ti_recordedfuture/changelog.yml b/packages/ti_recordedfuture/changelog.yml index a7be956a339..f7622c4e7b9 100644 --- a/packages/ti_recordedfuture/changelog.yml +++ b/packages/ti_recordedfuture/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.16.0" + changes: + - description: Add DLM policy + type: enhancement + link: https://github.com/elastic/integrations/pull/1111 #TODO - version: "1.15.1-next" changes: - description: Replace dotted YAML keys. diff --git a/packages/ti_recordedfuture/data_stream/threat/lifecycle.yml b/packages/ti_recordedfuture/data_stream/threat/lifecycle.yml index a6c83b5d8c7..42195d391f1 100644 --- a/packages/ti_recordedfuture/data_stream/threat/lifecycle.yml +++ b/packages/ti_recordedfuture/data_stream/threat/lifecycle.yml @@ -1 +1 @@ -data_retention: "1h" \ No newline at end of file +data_retention: "5d" \ No newline at end of file diff --git a/packages/ti_recordedfuture/manifest.yml b/packages/ti_recordedfuture/manifest.yml index 72ea41bdbc0..524deb8ec56 100644 --- a/packages/ti_recordedfuture/manifest.yml +++ b/packages/ti_recordedfuture/manifest.yml @@ -1,6 +1,6 @@ name: ti_recordedfuture title: Recorded Future -version: "1.15.0" +version: "1.16.0" description: Ingest threat intelligence indicators from Recorded Future risk lists with Elastic Agent. type: integration format_version: 2.10.0 From 9d9d86bf3c4d16dcabac005aee1994c5f2cd4d75 Mon Sep 17 00:00:00 2001 From: kcreddy Date: Mon, 18 Sep 2023 16:54:59 +0530 Subject: [PATCH 06/15] udpate changelog --- packages/ti_recordedfuture/changelog.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ti_recordedfuture/changelog.yml b/packages/ti_recordedfuture/changelog.yml index f7622c4e7b9..1418fac69fa 100644 --- a/packages/ti_recordedfuture/changelog.yml +++ b/packages/ti_recordedfuture/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add DLM policy type: enhancement - link: https://github.com/elastic/integrations/pull/1111 #TODO + link: https://github.com/elastic/integrations/pull/7848 - version: "1.15.1-next" changes: - description: Replace dotted YAML keys. From 395955d294f0b05eaf1f30f431a464a8b98e68e5 Mon Sep 17 00:00:00 2001 From: kcreddy Date: Mon, 18 Sep 2023 20:40:01 +0530 Subject: [PATCH 07/15] update format_version --- packages/ti_anomali/manifest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ti_anomali/manifest.yml b/packages/ti_anomali/manifest.yml index 4c9511657d3..9bc710e0d22 100644 --- a/packages/ti_anomali/manifest.yml +++ b/packages/ti_anomali/manifest.yml @@ -3,7 +3,7 @@ title: Anomali version: "1.16.0" description: Ingest threat intelligence indicators from Anomali with Elastic Agent. type: integration -format_version: 2.10.0 +format_version: 3.0.0 categories: ["security", "threat_intel"] conditions: kibana: From e21ad80d409901ed9784e4cc67d62cc7c6775352 Mon Sep 17 00:00:00 2001 From: kcreddy Date: Mon, 18 Sep 2023 20:40:49 +0530 Subject: [PATCH 08/15] revert anomali --- packages/ti_anomali/manifest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ti_anomali/manifest.yml b/packages/ti_anomali/manifest.yml index 9bc710e0d22..4c9511657d3 100644 --- a/packages/ti_anomali/manifest.yml +++ b/packages/ti_anomali/manifest.yml @@ -3,7 +3,7 @@ title: Anomali version: "1.16.0" description: Ingest threat intelligence indicators from Anomali with Elastic Agent. type: integration -format_version: 3.0.0 +format_version: 2.10.0 categories: ["security", "threat_intel"] conditions: kibana: From 88113d8d58844b480d8503e1dcd7e550f28c3fcc Mon Sep 17 00:00:00 2001 From: kcreddy Date: Mon, 18 Sep 2023 20:50:57 +0530 Subject: [PATCH 09/15] update format version --- packages/ti_recordedfuture/manifest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ti_recordedfuture/manifest.yml b/packages/ti_recordedfuture/manifest.yml index 524deb8ec56..45face87297 100644 --- a/packages/ti_recordedfuture/manifest.yml +++ b/packages/ti_recordedfuture/manifest.yml @@ -3,7 +3,7 @@ title: Recorded Future version: "1.16.0" description: Ingest threat intelligence indicators from Recorded Future risk lists with Elastic Agent. type: integration -format_version: 2.10.0 +format_version: 3.0.0 categories: ["security", "threat_intel"] conditions: kibana: From 3bb3e484a58096ee73144ffeee0fd2a9e3c8f9fa Mon Sep 17 00:00:00 2001 From: kcreddy Date: Wed, 20 Sep 2023 14:38:03 +0530 Subject: [PATCH 10/15] remove ILM --- .../elasticsearch/ilm/default_policy.json | 23 ------------------- .../data_stream/threat/lifecycle.yml | 2 +- .../data_stream/threat/manifest.yml | 1 - 3 files changed, 1 insertion(+), 25 deletions(-) delete mode 100644 packages/ti_recordedfuture/data_stream/threat/elasticsearch/ilm/default_policy.json diff --git a/packages/ti_recordedfuture/data_stream/threat/elasticsearch/ilm/default_policy.json b/packages/ti_recordedfuture/data_stream/threat/elasticsearch/ilm/default_policy.json deleted file mode 100644 index 68d2c5e57a6..00000000000 --- a/packages/ti_recordedfuture/data_stream/threat/elasticsearch/ilm/default_policy.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "policy": { - "phases": { - "hot": { - "actions": { - "rollover": { - "max_age": "2d", - "max_size": "50gb" - }, - "set_priority": { - "priority": 100 - } - } - }, - "delete": { - "min_age": "3d", - "actions": { - "delete": {} - } - } - } - } -} \ No newline at end of file diff --git a/packages/ti_recordedfuture/data_stream/threat/lifecycle.yml b/packages/ti_recordedfuture/data_stream/threat/lifecycle.yml index 42195d391f1..5a4af9095b7 100644 --- a/packages/ti_recordedfuture/data_stream/threat/lifecycle.yml +++ b/packages/ti_recordedfuture/data_stream/threat/lifecycle.yml @@ -1 +1 @@ -data_retention: "5d" \ No newline at end of file +data_retention: "5d" diff --git a/packages/ti_recordedfuture/data_stream/threat/manifest.yml b/packages/ti_recordedfuture/data_stream/threat/manifest.yml index 118a0c24b43..520dfe9d505 100644 --- a/packages/ti_recordedfuture/data_stream/threat/manifest.yml +++ b/packages/ti_recordedfuture/data_stream/threat/manifest.yml @@ -1,6 +1,5 @@ type: logs title: Recorded Future -ilm_policy: logs-ti_recordedfuture.threat-default_policy streams: - input: logfile enabled: false From a1cde9a6f8e5dea9623475e5e9ea300a2672543e Mon Sep 17 00:00:00 2001 From: kcreddy Date: Wed, 20 Sep 2023 19:40:42 +0530 Subject: [PATCH 11/15] Add ILM back --- .../elasticsearch/ilm/default_policy.json | 23 +++++++++++++++++++ .../data_stream/threat/manifest.yml | 1 + 2 files changed, 24 insertions(+) create mode 100644 packages/ti_recordedfuture/data_stream/threat/elasticsearch/ilm/default_policy.json diff --git a/packages/ti_recordedfuture/data_stream/threat/elasticsearch/ilm/default_policy.json b/packages/ti_recordedfuture/data_stream/threat/elasticsearch/ilm/default_policy.json new file mode 100644 index 00000000000..68d2c5e57a6 --- /dev/null +++ b/packages/ti_recordedfuture/data_stream/threat/elasticsearch/ilm/default_policy.json @@ -0,0 +1,23 @@ +{ + "policy": { + "phases": { + "hot": { + "actions": { + "rollover": { + "max_age": "2d", + "max_size": "50gb" + }, + "set_priority": { + "priority": 100 + } + } + }, + "delete": { + "min_age": "3d", + "actions": { + "delete": {} + } + } + } + } +} \ No newline at end of file diff --git a/packages/ti_recordedfuture/data_stream/threat/manifest.yml b/packages/ti_recordedfuture/data_stream/threat/manifest.yml index 520dfe9d505..118a0c24b43 100644 --- a/packages/ti_recordedfuture/data_stream/threat/manifest.yml +++ b/packages/ti_recordedfuture/data_stream/threat/manifest.yml @@ -1,5 +1,6 @@ type: logs title: Recorded Future +ilm_policy: logs-ti_recordedfuture.threat-default_policy streams: - input: logfile enabled: false From 4dcd364be4e5c243b2972ac41320cf87236c8dd2 Mon Sep 17 00:00:00 2001 From: kcreddy Date: Wed, 20 Sep 2023 19:42:28 +0530 Subject: [PATCH 12/15] update changelog --- packages/ti_recordedfuture/changelog.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ti_recordedfuture/changelog.yml b/packages/ti_recordedfuture/changelog.yml index 1418fac69fa..3c05f0f9d15 100644 --- a/packages/ti_recordedfuture/changelog.yml +++ b/packages/ti_recordedfuture/changelog.yml @@ -1,7 +1,7 @@ # newer versions go on top - version: "1.16.0" changes: - - description: Add DLM policy + - description: Add DLM policy and update format_version to 3.0.0 type: enhancement link: https://github.com/elastic/integrations/pull/7848 - version: "1.15.1-next" From 98493a4bb28e2ebefb0f571074f724513d3c31ac Mon Sep 17 00:00:00 2001 From: kcreddy Date: Thu, 21 Sep 2023 16:12:10 +0530 Subject: [PATCH 13/15] update transform sync delay --- .../elasticsearch/transform/latest_ioc/transform.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ti_recordedfuture/elasticsearch/transform/latest_ioc/transform.yml b/packages/ti_recordedfuture/elasticsearch/transform/latest_ioc/transform.yml index 561a8e8e238..2ea27ce7d17 100644 --- a/packages/ti_recordedfuture/elasticsearch/transform/latest_ioc/transform.yml +++ b/packages/ti_recordedfuture/elasticsearch/transform/latest_ioc/transform.yml @@ -21,7 +21,7 @@ frequency: 30s sync: time: field: event.ingested - delay: 60s + delay: 120s retention_policy: time: field: event.ingested From 0b940410e67daa0ccbf8e1160a412ccbb461498a Mon Sep 17 00:00:00 2001 From: kcreddy Date: Thu, 21 Sep 2023 16:12:33 +0530 Subject: [PATCH 14/15] remove next change --- packages/ti_recordedfuture/changelog.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/ti_recordedfuture/changelog.yml b/packages/ti_recordedfuture/changelog.yml index 7bfc7dfc84f..0d4babacefe 100644 --- a/packages/ti_recordedfuture/changelog.yml +++ b/packages/ti_recordedfuture/changelog.yml @@ -4,8 +4,6 @@ - description: Add DLM policy. Add owner.type to package manifest. Update format_version to 3.0.0 type: enhancement link: https://github.com/elastic/integrations/pull/7848 -- version: "1.15.1-next" - changes: - description: Add tags.yml file so that integration's dashboards and saved searches are tagged with "Security Solution" and displayed in the Security Solution UI. type: enhancement link: https://github.com/elastic/integrations/pull/7789 From 7082ca97ef4df1fa731d6ca9b681ea77da2a9b80 Mon Sep 17 00:00:00 2001 From: kcreddy Date: Thu, 21 Sep 2023 16:13:48 +0530 Subject: [PATCH 15/15] add comment --- .../elasticsearch/transform/latest_ioc/transform.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/ti_recordedfuture/elasticsearch/transform/latest_ioc/transform.yml b/packages/ti_recordedfuture/elasticsearch/transform/latest_ioc/transform.yml index 2ea27ce7d17..f8e79a85ec4 100644 --- a/packages/ti_recordedfuture/elasticsearch/transform/latest_ioc/transform.yml +++ b/packages/ti_recordedfuture/elasticsearch/transform/latest_ioc/transform.yml @@ -21,6 +21,7 @@ frequency: 30s sync: time: field: event.ingested + # Updated to 120s because of refresh delay in Serverless. With default 60s, sometimes transform wouldn't process all documents. delay: 120s retention_policy: time: