diff --git a/lib/logstash/outputs/influxdb.rb b/lib/logstash/outputs/influxdb.rb index 958dab0..26b751b 100644 --- a/lib/logstash/outputs/influxdb.rb +++ b/lib/logstash/outputs/influxdb.rb @@ -148,8 +148,6 @@ def receive(event) end end - exclude_fields!(point) - coerce_values!(point) tags, point = extract_tags(point) @@ -158,6 +156,10 @@ def receive(event) "time" => time, "fields" => point } + + exclude_fields!(point) + coerce_values!(point) + event_hash["tags"] = tags unless tags.empty? buffer_receive(event_hash, event.sprintf(@db)) @@ -295,19 +297,21 @@ def exclude_fields!(event_data) # original_fields = {"foo" => 1, "bar" => 2, "tags" => ["tag"]} # tags, fields = extract_tags(original_fields) # # tags: {"bar" => 2, "tag" => "true"} and fields: {"foo" => 1} + def extract_tags(fields) remainder = fields.dup tags = if remainder.has_key?("tags") && remainder["tags"].respond_to?(:inject) - remainder.delete("tags").inject({}) { |tags, tag| tags[tag] = "true"; tags } + remainder.delete("tags").inject({}) { |tags, tag| tags[tag] = if remainder.has_key?(tag) then fields[tag] else "true" end; tags } else {} end - + @send_as_tags.each { |key| (tags[key] = remainder.delete(key)) if remainder.has_key?(key) } tags.delete_if { |key,value| value.nil? || value == "" } remainder.delete_if { |key,value| value.nil? || value == "" } + remainder.delete_if { |key,value| tags.has_key?(key) } [tags, remainder] end diff --git a/logstash-output-influxdb.gemspec b/logstash-output-influxdb.gemspec index a19c518..1a967e9 100644 --- a/logstash-output-influxdb.gemspec +++ b/logstash-output-influxdb.gemspec @@ -27,5 +27,6 @@ Gem::Specification.new do |s| s.add_development_dependency 'logstash-devutils' s.add_development_dependency 'logstash-input-generator' s.add_development_dependency 'logstash-filter-kv' + s.add_development_dependency 'logstash-filter-mutate' end diff --git a/spec/outputs/influxdb_spec.rb b/spec/outputs/influxdb_spec.rb index 09ca578..43c5af2 100644 --- a/spec/outputs/influxdb_spec.rb +++ b/spec/outputs/influxdb_spec.rb @@ -58,6 +58,9 @@ filter { kv { } + mutate { + remove_field => ["host"] + } } output { @@ -66,7 +69,7 @@ measurement => "my_series" allow_time_override => true use_event_fields_for_data_points => true - exclude_fields => ["@version", "@timestamp", "sequence", "message", "type", "host"] + exclude_fields => ["@version", "@timestamp", "sequence", "message", "type"] } } CONFIG @@ -93,6 +96,9 @@ filter { kv { } + mutate { + remove_field => ["host"] + } } output { @@ -101,7 +107,7 @@ measurement => "my_series" allow_time_override => true use_event_fields_for_data_points => true - exclude_fields => ["@version", "@timestamp", "sequence", "message", "type", "host"] + exclude_fields => ["@version", "@timestamp", "sequence", "message", "type"] send_as_tags => ["bar", "baz", "qux"] } } @@ -135,6 +141,9 @@ "feild space" => "pink dog" } } + mutate { + remove_field => ["host"] + } } output { @@ -143,7 +152,7 @@ measurement => "my series" allow_time_override => true use_event_fields_for_data_points => true - exclude_fields => ["@version", "@timestamp", "sequence", "message", "type", "host"] + exclude_fields => ["@version", "@timestamp", "sequence", "message", "type"] send_as_tags => ["bar", "baz", "test1", "test space"] } } @@ -177,6 +186,9 @@ "feild, space" => "pink, dog" } } + mutate { + remove_field => ["host"] + } } output { @@ -185,7 +197,7 @@ measurement => "my, series" allow_time_override => true use_event_fields_for_data_points => true - exclude_fields => ["@version", "@timestamp", "sequence", "message", "type", "host"] + exclude_fields => ["@version", "@timestamp", "sequence", "message", "type"] send_as_tags => ["bar", "baz", "test1", "test, space"] } } @@ -219,6 +231,9 @@ "feild= space" => "pink= dog" } } + mutate { + remove_field => ["host"] + } } output { @@ -227,7 +242,7 @@ measurement => "my=series" allow_time_override => true use_event_fields_for_data_points => true - exclude_fields => ["@version", "@timestamp", "sequence", "message", "type", "host"] + exclude_fields => ["@version", "@timestamp", "sequence", "message", "type"] send_as_tags => ["bar", "baz", "test1", "test=space"] } } @@ -261,6 +276,9 @@ "feildspace" => 'C:\\Griffo' } } + mutate { + remove_field => ["host"] + } } output { @@ -269,7 +287,7 @@ measurement => 'my\\series' allow_time_override => true use_event_fields_for_data_points => true - exclude_fields => ["@version", "@timestamp", "sequence", "message", "type", "host"] + exclude_fields => ["@version", "@timestamp", "sequence", "message", "type"] send_as_tags => ['bar', "baz", "test1", "test=space"] } } @@ -298,6 +316,9 @@ filter { kv { add_tag => [ "tagged" ] } + mutate { + remove_field => ["host"] + } } output { @@ -306,7 +327,7 @@ measurement => "my_series" allow_time_override => true use_event_fields_for_data_points => true - exclude_fields => ["@version", "@timestamp", "sequence", "message", "type", "host"] + exclude_fields => ["@version", "@timestamp", "sequence", "message", "type"] } } CONFIG @@ -333,6 +354,9 @@ filter { kv { } + mutate { + remove_field => ["host"] + } } output { @@ -341,7 +365,7 @@ measurement => "my_series" allow_time_override => true use_event_fields_for_data_points => true - exclude_fields => ["@version", "@timestamp", "sequence", "message", "type", "host"] + exclude_fields => ["@version", "@timestamp", "sequence", "message", "type"] coerce_values => { "foo" => "integer" "bar" => "float" } } } @@ -374,6 +398,9 @@ filter { kv { } + mutate { + remove_field => ["host"] + } } output { @@ -384,7 +411,7 @@ allow_time_override => true use_event_fields_for_data_points => true exclude_fields => [ "@version", "@timestamp", "sequence", - "message", "type", "host" ] + "message", "type"] } } CONFIG @@ -420,6 +447,9 @@ filter { kv { } + mutate { + remove_field => ["host"] + } } output { @@ -430,7 +460,7 @@ allow_time_override => true use_event_fields_for_data_points => true exclude_fields => [ "@version", "@timestamp", "sequence", - "message", "type", "host" ] + "message", "type"] } } CONFIG @@ -474,6 +504,9 @@ filter { kv { } + mutate { + remove_field => ["host"] + } } output { @@ -484,7 +517,7 @@ allow_time_override => true use_event_fields_for_data_points => true exclude_fields => [ "@version", "@timestamp", "sequence", - "message", "type", "host" ] + "message", "type"] } } CONFIG