From 40768139951250ada9e4cd9560e4f5a0eba72705 Mon Sep 17 00:00:00 2001 From: Alexey Polyakov Date: Sat, 17 Sep 2016 16:16:23 +0300 Subject: [PATCH 1/4] Set extract_tags call prior exclude_fields --- lib/logstash/outputs/influxdb.rb | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/logstash/outputs/influxdb.rb b/lib/logstash/outputs/influxdb.rb index 958dab0..9bfecea 100644 --- a/lib/logstash/outputs/influxdb.rb +++ b/lib/logstash/outputs/influxdb.rb @@ -148,9 +148,6 @@ def receive(event) end end - exclude_fields!(point) - coerce_values!(point) - tags, point = extract_tags(point) event_hash = { @@ -158,6 +155,8 @@ def receive(event) "time" => time, "fields" => point } + exclude_fields!(point) + coerce_values!(point) event_hash["tags"] = tags unless tags.empty? buffer_receive(event_hash, event.sprintf(@db)) From 98fab10b8e506950dd0efe55f1a21d15e7fe3e91 Mon Sep 17 00:00:00 2001 From: Alexey Polyakov Date: Sat, 17 Sep 2016 16:55:29 +0300 Subject: [PATCH 2/4] Add mutate to test pipeline and delete host to avoid travis-ci problem --- spec/outputs/influxdb_spec.rb | 55 ++++++++++++++++++++++++++++------- 1 file changed, 44 insertions(+), 11 deletions(-) diff --git a/spec/outputs/influxdb_spec.rb b/spec/outputs/influxdb_spec.rb index 09ca578..43c5af2 100644 --- a/spec/outputs/influxdb_spec.rb +++ b/spec/outputs/influxdb_spec.rb @@ -58,6 +58,9 @@ filter { kv { } + mutate { + remove_field => ["host"] + } } output { @@ -66,7 +69,7 @@ measurement => "my_series" allow_time_override => true use_event_fields_for_data_points => true - exclude_fields => ["@version", "@timestamp", "sequence", "message", "type", "host"] + exclude_fields => ["@version", "@timestamp", "sequence", "message", "type"] } } CONFIG @@ -93,6 +96,9 @@ filter { kv { } + mutate { + remove_field => ["host"] + } } output { @@ -101,7 +107,7 @@ measurement => "my_series" allow_time_override => true use_event_fields_for_data_points => true - exclude_fields => ["@version", "@timestamp", "sequence", "message", "type", "host"] + exclude_fields => ["@version", "@timestamp", "sequence", "message", "type"] send_as_tags => ["bar", "baz", "qux"] } } @@ -135,6 +141,9 @@ "feild space" => "pink dog" } } + mutate { + remove_field => ["host"] + } } output { @@ -143,7 +152,7 @@ measurement => "my series" allow_time_override => true use_event_fields_for_data_points => true - exclude_fields => ["@version", "@timestamp", "sequence", "message", "type", "host"] + exclude_fields => ["@version", "@timestamp", "sequence", "message", "type"] send_as_tags => ["bar", "baz", "test1", "test space"] } } @@ -177,6 +186,9 @@ "feild, space" => "pink, dog" } } + mutate { + remove_field => ["host"] + } } output { @@ -185,7 +197,7 @@ measurement => "my, series" allow_time_override => true use_event_fields_for_data_points => true - exclude_fields => ["@version", "@timestamp", "sequence", "message", "type", "host"] + exclude_fields => ["@version", "@timestamp", "sequence", "message", "type"] send_as_tags => ["bar", "baz", "test1", "test, space"] } } @@ -219,6 +231,9 @@ "feild= space" => "pink= dog" } } + mutate { + remove_field => ["host"] + } } output { @@ -227,7 +242,7 @@ measurement => "my=series" allow_time_override => true use_event_fields_for_data_points => true - exclude_fields => ["@version", "@timestamp", "sequence", "message", "type", "host"] + exclude_fields => ["@version", "@timestamp", "sequence", "message", "type"] send_as_tags => ["bar", "baz", "test1", "test=space"] } } @@ -261,6 +276,9 @@ "feildspace" => 'C:\\Griffo' } } + mutate { + remove_field => ["host"] + } } output { @@ -269,7 +287,7 @@ measurement => 'my\\series' allow_time_override => true use_event_fields_for_data_points => true - exclude_fields => ["@version", "@timestamp", "sequence", "message", "type", "host"] + exclude_fields => ["@version", "@timestamp", "sequence", "message", "type"] send_as_tags => ['bar', "baz", "test1", "test=space"] } } @@ -298,6 +316,9 @@ filter { kv { add_tag => [ "tagged" ] } + mutate { + remove_field => ["host"] + } } output { @@ -306,7 +327,7 @@ measurement => "my_series" allow_time_override => true use_event_fields_for_data_points => true - exclude_fields => ["@version", "@timestamp", "sequence", "message", "type", "host"] + exclude_fields => ["@version", "@timestamp", "sequence", "message", "type"] } } CONFIG @@ -333,6 +354,9 @@ filter { kv { } + mutate { + remove_field => ["host"] + } } output { @@ -341,7 +365,7 @@ measurement => "my_series" allow_time_override => true use_event_fields_for_data_points => true - exclude_fields => ["@version", "@timestamp", "sequence", "message", "type", "host"] + exclude_fields => ["@version", "@timestamp", "sequence", "message", "type"] coerce_values => { "foo" => "integer" "bar" => "float" } } } @@ -374,6 +398,9 @@ filter { kv { } + mutate { + remove_field => ["host"] + } } output { @@ -384,7 +411,7 @@ allow_time_override => true use_event_fields_for_data_points => true exclude_fields => [ "@version", "@timestamp", "sequence", - "message", "type", "host" ] + "message", "type"] } } CONFIG @@ -420,6 +447,9 @@ filter { kv { } + mutate { + remove_field => ["host"] + } } output { @@ -430,7 +460,7 @@ allow_time_override => true use_event_fields_for_data_points => true exclude_fields => [ "@version", "@timestamp", "sequence", - "message", "type", "host" ] + "message", "type"] } } CONFIG @@ -474,6 +504,9 @@ filter { kv { } + mutate { + remove_field => ["host"] + } } output { @@ -484,7 +517,7 @@ allow_time_override => true use_event_fields_for_data_points => true exclude_fields => [ "@version", "@timestamp", "sequence", - "message", "type", "host" ] + "message", "type"] } } CONFIG From b33821f65b5071b7eaf9234a24641d96e0a7d661 Mon Sep 17 00:00:00 2001 From: Alexey Polyakov Date: Sat, 17 Sep 2016 17:00:54 +0300 Subject: [PATCH 3/4] Add logstash-filter-mutate to dev. dep. --- logstash-output-influxdb.gemspec | 1 + 1 file changed, 1 insertion(+) diff --git a/logstash-output-influxdb.gemspec b/logstash-output-influxdb.gemspec index a19c518..1a967e9 100644 --- a/logstash-output-influxdb.gemspec +++ b/logstash-output-influxdb.gemspec @@ -27,5 +27,6 @@ Gem::Specification.new do |s| s.add_development_dependency 'logstash-devutils' s.add_development_dependency 'logstash-input-generator' s.add_development_dependency 'logstash-filter-kv' + s.add_development_dependency 'logstash-filter-mutate' end From a1b09cbd5c095a753f246d2b4697d816ed12aba1 Mon Sep 17 00:00:00 2001 From: berghauz Date: Thu, 11 May 2017 17:35:20 +0300 Subject: [PATCH 4/4] Update to new logstash API --- lib/logstash/outputs/influxdb.rb | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/logstash/outputs/influxdb.rb b/lib/logstash/outputs/influxdb.rb index 9bfecea..26b751b 100644 --- a/lib/logstash/outputs/influxdb.rb +++ b/lib/logstash/outputs/influxdb.rb @@ -148,6 +148,7 @@ def receive(event) end end + tags, point = extract_tags(point) event_hash = { @@ -155,8 +156,10 @@ def receive(event) "time" => time, "fields" => point } + exclude_fields!(point) coerce_values!(point) + event_hash["tags"] = tags unless tags.empty? buffer_receive(event_hash, event.sprintf(@db)) @@ -294,19 +297,21 @@ def exclude_fields!(event_data) # original_fields = {"foo" => 1, "bar" => 2, "tags" => ["tag"]} # tags, fields = extract_tags(original_fields) # # tags: {"bar" => 2, "tag" => "true"} and fields: {"foo" => 1} + def extract_tags(fields) remainder = fields.dup tags = if remainder.has_key?("tags") && remainder["tags"].respond_to?(:inject) - remainder.delete("tags").inject({}) { |tags, tag| tags[tag] = "true"; tags } + remainder.delete("tags").inject({}) { |tags, tag| tags[tag] = if remainder.has_key?(tag) then fields[tag] else "true" end; tags } else {} end - + @send_as_tags.each { |key| (tags[key] = remainder.delete(key)) if remainder.has_key?(key) } tags.delete_if { |key,value| value.nil? || value == "" } remainder.delete_if { |key,value| value.nil? || value == "" } + remainder.delete_if { |key,value| tags.has_key?(key) } [tags, remainder] end