From 8dbbc1822b00226128de37fec35f7b2179c137ff Mon Sep 17 00:00:00 2001 From: tomr Date: Tue, 20 Aug 2019 12:53:25 +1000 Subject: [PATCH 01/12] make clones a hash so that arbitrary fields can be used --- docs/index.asciidoc | 15 ++++++++++++--- lib/logstash/filters/clone.rb | 8 ++++---- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/docs/index.asciidoc b/docs/index.asciidoc index eaa2c33..4566ce7 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -35,7 +35,7 @@ This plugin supports the following configuration options plus the <> |<>|Yes +| <> |<>|Yes |======================================================================= Also see <> for a list of options supported by all @@ -47,13 +47,22 @@ filter plugins. ===== `clones` * This is a required setting. - * Value type is <> + * Value type is <> * There is no default value for this setting. A new clone will be created with the given type for each type in this list. -Note: setting an empty array will not create any clones. A warning message is logged. +Example: +[source,ruby] + filter { + clone { + clones => { "fieldname" => "value" } + } + } + +Note: setting an empty hash will not create any clones. A warning message is logged. [id="plugins-{type}s-{plugin}-common-options"] include::{include_path}/{type}.asciidoc[] + diff --git a/lib/logstash/filters/clone.rb b/lib/logstash/filters/clone.rb index be6729c..5164312 100644 --- a/lib/logstash/filters/clone.rb +++ b/lib/logstash/filters/clone.rb @@ -13,7 +13,7 @@ class LogStash::Filters::Clone < LogStash::Filters::Base config_name "clone" # A new clone will be created with the given type for each type in this list. - config :clones, :validate => :array, :required => true + config :clones, :validate => :hash, :required => true public def register @@ -22,11 +22,11 @@ def register public def filter(event) - @clones.each do |type| + @clones.each do |field, value| clone = event.clone - clone.set("type", type) + clone.set(field, value) filter_matched(clone) - @logger.debug("Cloned event", :clone => clone, :event => event) + @logger.debug? && @logger.debug("Cloned event", :clone => clone, :event => event) # Push this new event onto the stack at the LogStash::FilterWorker yield clone From d68d04cfbc4d5333c5f2fcb59379ec96cd0940d1 Mon Sep 17 00:00:00 2001 From: tomryanx Date: Tue, 20 Aug 2019 13:11:08 +1000 Subject: [PATCH 02/12] clarify change from `type` to field + value --- docs/index.asciidoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/index.asciidoc b/docs/index.asciidoc index 4566ce7..d88642a 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -50,13 +50,13 @@ filter plugins. * Value type is <> * There is no default value for this setting. -A new clone will be created with the given type for each type in this list. +A new clone will be created with the given value for each field in this list. Example: [source,ruby] filter { clone { - clones => { "fieldname" => "value" } + clones => { "field" => "value" } } } From b0573042fd298bad8ff184f4e0048acdc57fff17 Mon Sep 17 00:00:00 2001 From: tomr Date: Tue, 20 Aug 2019 18:30:47 +1000 Subject: [PATCH 03/12] first pass at modify spec to match changes --- spec/filters/clone_spec.rb | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/spec/filters/clone_spec.rb b/spec/filters/clone_spec.rb index 9e2df98..176ae85 100644 --- a/spec/filters/clone_spec.rb +++ b/spec/filters/clone_spec.rb @@ -10,19 +10,24 @@ config <<-CONFIG filter { clone { - clones => ["clone", "clone", "clone"] + clones => { + "type" => "clone" + "clone" => "clone1" + } } } CONFIG sample("message" => "hello world", "type" => "original") do - insist { subject }.is_a? Array - insist { subject.length } == 4 + insist { subject }.is_a? Hash + insist { subject.length } == 3 subject.each_with_index do |s,i| if i == 0 # last one should be 'original' insist { s.get("type") } == "original" - else + else if i = 1 insist { s.get("type")} == "clone" + else + insist { s.get("clone")} == "clone1" end insist { s.get("message") } == "hello world" end @@ -33,7 +38,10 @@ config <<-CONFIG filter { clone { - clones => ["nginx-access-clone1", "nginx-access-clone2"] + clones => { + "clone" => "nginx-access-clone1" + "type" => "nginx-access-clone2" + } add_tag => ['RABBIT','NO_ES'] remove_tag => ["TESTLOG"] } @@ -50,7 +58,7 @@ reject { subject[0].get("tags") }.include? "RABBIT" reject { subject[0].get("tags") }.include? "NO_ES" #All clones go through filter_matched - insist { subject[1].get("type") } == "nginx-access-clone1" + insist { subject[1].get("clone") } == "nginx-access-clone1" reject { subject[1].get("tags") }.include? "TESTLOG" insist { subject[1].get("tags") }.include? "RABBIT" insist { subject[1].get("tags") }.include? "NO_ES" @@ -67,7 +75,7 @@ config <<-CONFIG filter { clone { - clones => [ 'clone1' ] + clones => { 'clone1' => 'clone1' } } } CONFIG From 25b09f6556458c5d5a23e993b82a131018ffdee4 Mon Sep 17 00:00:00 2001 From: tomr Date: Tue, 20 Aug 2019 18:41:00 +1000 Subject: [PATCH 04/12] typo in elseif --- spec/filters/clone_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/filters/clone_spec.rb b/spec/filters/clone_spec.rb index 176ae85..8f05a04 100644 --- a/spec/filters/clone_spec.rb +++ b/spec/filters/clone_spec.rb @@ -24,7 +24,7 @@ subject.each_with_index do |s,i| if i == 0 # last one should be 'original' insist { s.get("type") } == "original" - else if i = 1 + elseif i = 1 insist { s.get("type")} == "clone" else insist { s.get("clone")} == "clone1" From c4d54a91035712d6fe9a0e0f53ee3f5679ed48d8 Mon Sep 17 00:00:00 2001 From: tomr Date: Tue, 20 Aug 2019 20:58:57 +1000 Subject: [PATCH 05/12] take 3 on updating rspec --- spec/filters/clone_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/filters/clone_spec.rb b/spec/filters/clone_spec.rb index 8f05a04..befd33d 100644 --- a/spec/filters/clone_spec.rb +++ b/spec/filters/clone_spec.rb @@ -19,7 +19,7 @@ CONFIG sample("message" => "hello world", "type" => "original") do - insist { subject }.is_a? Hash + insist { subject }.is_a? Array insist { subject.length } == 3 subject.each_with_index do |s,i| if i == 0 # last one should be 'original' From ab46ccd0eb7612cdd793ea4f0679c1ea912e2879 Mon Sep 17 00:00:00 2001 From: tomr Date: Tue, 20 Aug 2019 21:08:38 +1000 Subject: [PATCH 06/12] take 4 on updating rspec --- spec/filters/clone_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/filters/clone_spec.rb b/spec/filters/clone_spec.rb index befd33d..1b269ee 100644 --- a/spec/filters/clone_spec.rb +++ b/spec/filters/clone_spec.rb @@ -24,7 +24,7 @@ subject.each_with_index do |s,i| if i == 0 # last one should be 'original' insist { s.get("type") } == "original" - elseif i = 1 + elsif i = 1 insist { s.get("type")} == "clone" else insist { s.get("clone")} == "clone1" From 7b72ba74f54e568452a386f7674578db19988275 Mon Sep 17 00:00:00 2001 From: tomr Date: Tue, 20 Aug 2019 21:19:22 +1000 Subject: [PATCH 07/12] take 5 on updating rspec - use iterations properly --- spec/filters/clone_spec.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/spec/filters/clone_spec.rb b/spec/filters/clone_spec.rb index 1b269ee..79c8712 100644 --- a/spec/filters/clone_spec.rb +++ b/spec/filters/clone_spec.rb @@ -24,8 +24,6 @@ subject.each_with_index do |s,i| if i == 0 # last one should be 'original' insist { s.get("type") } == "original" - elsif i = 1 - insist { s.get("type")} == "clone" else insist { s.get("clone")} == "clone1" end From 707d7905ee1ff793b453662e9ba3bdf9a0592a43 Mon Sep 17 00:00:00 2001 From: Tom Ryan Date: Tue, 20 Aug 2019 23:50:08 +0000 Subject: [PATCH 08/12] refactor into two configuration items --- lib/logstash/filters/clone.rb | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/logstash/filters/clone.rb b/lib/logstash/filters/clone.rb index 5164312..9c196f5 100644 --- a/lib/logstash/filters/clone.rb +++ b/lib/logstash/filters/clone.rb @@ -13,18 +13,19 @@ class LogStash::Filters::Clone < LogStash::Filters::Base config_name "clone" # A new clone will be created with the given type for each type in this list. - config :clones, :validate => :hash, :required => true + config :field, :validate => :string, :required => false, :default => "clone" + config :values, :validate => :array, :required => true public def register - logger.warn("The parameter 'clones' is empty, so no clones will be created.") if @clones.empty? + logger.warn("The parameter 'clones' is empty, so no clones will be created.") if @values.empty? end public def filter(event) - @clones.each do |field, value| + @values.each do |value| clone = event.clone - clone.set(field, value) + clone.set(field,value) filter_matched(clone) @logger.debug? && @logger.debug("Cloned event", :clone => clone, :event => event) From ca6c98ea60814fe2f79bba459ee0f975c45e2264 Mon Sep 17 00:00:00 2001 From: tomryanx Date: Wed, 21 Aug 2019 01:31:58 +0000 Subject: [PATCH 09/12] add `field` setting with default of `type` --- docs/index.asciidoc | 22 +++++++++++++++++-- lib/logstash/filters/clone.rb | 10 ++++----- spec/filters/clone_spec.rb | 40 ++++++++++++++++++++++++----------- 3 files changed, 53 insertions(+), 19 deletions(-) diff --git a/docs/index.asciidoc b/docs/index.asciidoc index d88642a..b150a3b 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -21,7 +21,8 @@ include::{include_path}/plugin_header.asciidoc[] ==== Description The clone filter is for duplicating events. -A clone will be created for each type in the clone list. +A clone will be created for each value in the clone list. +By default `clones` values overwrite the `type` field, or you can specify an optional `field`. The original event is left unchanged. Created events are inserted into the pipeline as normal events and will be processed by the remaining pipeline configuration @@ -35,6 +36,7 @@ This plugin supports the following configuration options plus the <> |<>|No | <> |<>|Yes |======================================================================= @@ -43,6 +45,22 @@ filter plugins.   +[id="plugins-{type}s-{plugin}-fields"] +===== `field` + + * Value type is <> + * Default value is `type` + +Specify the field into which `clones` values will be added. +Example: +[source,ruby] + filter { + clone { + field => "clone" + clones => [ "clone1", "clone2" ] + } + } + [id="plugins-{type}s-{plugin}-clones"] ===== `clones` @@ -56,7 +74,7 @@ Example: [source,ruby] filter { clone { - clones => { "field" => "value" } + clones => [ "clone1", "clone2" ] } } diff --git a/lib/logstash/filters/clone.rb b/lib/logstash/filters/clone.rb index 9c196f5..47b577c 100644 --- a/lib/logstash/filters/clone.rb +++ b/lib/logstash/filters/clone.rb @@ -13,21 +13,21 @@ class LogStash::Filters::Clone < LogStash::Filters::Base config_name "clone" # A new clone will be created with the given type for each type in this list. - config :field, :validate => :string, :required => false, :default => "clone" - config :values, :validate => :array, :required => true + config :field, :validate => :string, :required => false, :default => "type" + config :clones, :validate => :array, :required => true public def register - logger.warn("The parameter 'clones' is empty, so no clones will be created.") if @values.empty? + logger.warn("The parameter 'clones' is empty, so no clones will be created.") if @clones.empty? end public def filter(event) - @values.each do |value| + @clones.each do |value| clone = event.clone clone.set(field,value) filter_matched(clone) - @logger.debug? && @logger.debug("Cloned event", :clone => clone, :event => event) + @logger.debug("Cloned event", :clone => clone, :event => event) # Push this new event onto the stack at the LogStash::FilterWorker yield clone diff --git a/spec/filters/clone_spec.rb b/spec/filters/clone_spec.rb index 79c8712..969ad28 100644 --- a/spec/filters/clone_spec.rb +++ b/spec/filters/clone_spec.rb @@ -10,22 +10,19 @@ config <<-CONFIG filter { clone { - clones => { - "type" => "clone" - "clone" => "clone1" - } + clones => ["clone", "clone", "clone"] } } CONFIG sample("message" => "hello world", "type" => "original") do insist { subject }.is_a? Array - insist { subject.length } == 3 + insist { subject.length } == 4 subject.each_with_index do |s,i| if i == 0 # last one should be 'original' insist { s.get("type") } == "original" else - insist { s.get("clone")} == "clone1" + insist { s.get("type")} == "clone" end insist { s.get("message") } == "hello world" end @@ -36,10 +33,7 @@ config <<-CONFIG filter { clone { - clones => { - "clone" => "nginx-access-clone1" - "type" => "nginx-access-clone2" - } + clones => ["nginx-access-clone1", "nginx-access-clone2"] add_tag => ['RABBIT','NO_ES'] remove_tag => ["TESTLOG"] } @@ -56,7 +50,7 @@ reject { subject[0].get("tags") }.include? "RABBIT" reject { subject[0].get("tags") }.include? "NO_ES" #All clones go through filter_matched - insist { subject[1].get("clone") } == "nginx-access-clone1" + insist { subject[1].get("type") } == "nginx-access-clone1" reject { subject[1].get("tags") }.include? "TESTLOG" insist { subject[1].get("tags") }.include? "RABBIT" insist { subject[1].get("tags") }.include? "NO_ES" @@ -68,12 +62,34 @@ end end + describe "Other Field" do + config <<-CONFIG + filter { + clone { + field => "clone" + clones => ["clone1", "clone2"] + } + } + CONFIG + + sample("message" => "hello world", "type" => "original") do + insist { subject }.is_a? Array + insist { subject.length } == 3 + insist { subject[1].get("clone") } == "clone1" + insist { subject[2].get("clone") } == "clone2" + subject.each do |s| + insist { s.get("type") } == "original" + insist { s.get("message") } == "hello world" + end + end + end + describe "Bug LOGSTASH-1225" do ### LOGSTASH-1225: Cannot clone events containing numbers. config <<-CONFIG filter { clone { - clones => { 'clone1' => 'clone1' } + clones => [ 'clone1' ] } } CONFIG From 7dee2fb9ebf7f5fc50f5fab4002c57446fa53b95 Mon Sep 17 00:00:00 2001 From: tomryanx Date: Wed, 21 Aug 2019 01:35:49 +0000 Subject: [PATCH 10/12] revert hash -> array in doc --- docs/index.asciidoc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/index.asciidoc b/docs/index.asciidoc index b150a3b..5ae5774 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -37,7 +37,7 @@ This plugin supports the following configuration options plus the <> |<>|No -| <> |<>|Yes +| <> |<>|Yes |======================================================================= Also see <> for a list of options supported by all @@ -65,7 +65,7 @@ Example: ===== `clones` * This is a required setting. - * Value type is <> + * Value type is <> * There is no default value for this setting. A new clone will be created with the given value for each field in this list. @@ -78,7 +78,7 @@ Example: } } -Note: setting an empty hash will not create any clones. A warning message is logged. +Note: setting an empty array will not create any clones. A warning message is logged. [id="plugins-{type}s-{plugin}-common-options"] From ea170755d7c635cda85fb7e47c0b578df2b8ca71 Mon Sep 17 00:00:00 2001 From: tomryanx Date: Wed, 21 Aug 2019 01:41:48 +0000 Subject: [PATCH 11/12] put doc in alphabetical order --- docs/index.asciidoc | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/docs/index.asciidoc b/docs/index.asciidoc index 5ae5774..bf9bf87 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -36,8 +36,8 @@ This plugin supports the following configuration options plus the <> |<>|No | <> |<>|Yes +| <> |<>|No |======================================================================= Also see <> for a list of options supported by all @@ -45,41 +45,41 @@ filter plugins.   -[id="plugins-{type}s-{plugin}-fields"] -===== `field` +[id="plugins-{type}s-{plugin}-clones"] +===== `clones` - * Value type is <> - * Default value is `type` + * This is a required setting. + * Value type is <> + * There is no default value for this setting. + +A new clone will be created with the given value for each field in this list. -Specify the field into which `clones` values will be added. Example: [source,ruby] filter { clone { - field => "clone" clones => [ "clone1", "clone2" ] } } -[id="plugins-{type}s-{plugin}-clones"] -===== `clones` +Note: setting an empty array will not create any clones. A warning message is logged. - * This is a required setting. - * Value type is <> - * There is no default value for this setting. +[id="plugins-{type}s-{plugin}-fields"] +===== `field` -A new clone will be created with the given value for each field in this list. + * Value type is <> + * Default value is `type` +Specify the field into which `clones` values will be added. Example: [source,ruby] filter { clone { + field => "clone" clones => [ "clone1", "clone2" ] } } -Note: setting an empty array will not create any clones. A warning message is logged. - [id="plugins-{type}s-{plugin}-common-options"] include::{include_path}/{type}.asciidoc[] From 10f633d868728383e8ad08dd676fa59b86380d17 Mon Sep 17 00:00:00 2001 From: tomryanx Date: Wed, 21 Aug 2019 01:45:49 +0000 Subject: [PATCH 12/12] normalise spacing in doc --- docs/index.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/index.asciidoc b/docs/index.asciidoc index bf9bf87..26b6b99 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -71,6 +71,7 @@ Note: setting an empty array will not create any clones. A warning message is lo * Default value is `type` Specify the field into which `clones` values will be added. + Example: [source,ruby] filter {