diff --git a/CHANGELOG.md b/CHANGELOG.md index 237ca89..29bfdb9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 3.1.0 + - feature: Add 'metadata' option that enables inclusion of the @metadata field in the output. + ## 3.0.6 - Support flush method, see https://github.com/logstash-plugins/logstash-codec-json_lines/pull/35 diff --git a/docs/index.asciidoc b/docs/index.asciidoc index d3fc8a2..60fffac 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -36,6 +36,7 @@ Therefore this codec cannot work with line oriented inputs. |Setting |Input type|Required | <> |<>, one of `["ASCII-8BIT", "UTF-8", "US-ASCII", "Big5", "Big5-HKSCS", "Big5-UAO", "CP949", "Emacs-Mule", "EUC-JP", "EUC-KR", "EUC-TW", "GB2312", "GB18030", "GBK", "ISO-8859-1", "ISO-8859-2", "ISO-8859-3", "ISO-8859-4", "ISO-8859-5", "ISO-8859-6", "ISO-8859-7", "ISO-8859-8", "ISO-8859-9", "ISO-8859-10", "ISO-8859-11", "ISO-8859-13", "ISO-8859-14", "ISO-8859-15", "ISO-8859-16", "KOI8-R", "KOI8-U", "Shift_JIS", "UTF-16BE", "UTF-16LE", "UTF-32BE", "UTF-32LE", "Windows-31J", "Windows-1250", "Windows-1251", "Windows-1252", "IBM437", "IBM737", "IBM775", "CP850", "IBM852", "CP852", "IBM855", "CP855", "IBM857", "IBM860", "IBM861", "IBM862", "IBM863", "IBM864", "IBM865", "IBM866", "IBM869", "Windows-1258", "GB1988", "macCentEuro", "macCroatian", "macCyrillic", "macGreek", "macIceland", "macRoman", "macRomania", "macThai", "macTurkish", "macUkraine", "CP950", "CP951", "IBM037", "stateless-ISO-2022-JP", "eucJP-ms", "CP51932", "EUC-JIS-2004", "GB12345", "ISO-2022-JP", "ISO-2022-JP-2", "CP50220", "CP50221", "Windows-1256", "Windows-1253", "Windows-1255", "Windows-1254", "TIS-620", "Windows-874", "Windows-1257", "MacJapanese", "UTF-7", "UTF8-MAC", "UTF-16", "UTF-32", "UTF8-DoCoMo", "SJIS-DoCoMo", "UTF8-KDDI", "SJIS-KDDI", "ISO-2022-JP-KDDI", "stateless-ISO-2022-JP-KDDI", "UTF8-SoftBank", "SJIS-SoftBank", "BINARY", "CP437", "CP737", "CP775", "IBM850", "CP857", "CP860", "CP861", "CP862", "CP863", "CP864", "CP865", "CP866", "CP869", "CP1258", "Big5-HKSCS:2008", "ebcdic-cp-us", "eucJP", "euc-jp-ms", "EUC-JISX0213", "eucKR", "eucTW", "EUC-CN", "eucCN", "CP936", "ISO2022-JP", "ISO2022-JP2", "ISO8859-1", "ISO8859-2", "ISO8859-3", "ISO8859-4", "ISO8859-5", "ISO8859-6", "CP1256", "ISO8859-7", "CP1253", "ISO8859-8", "CP1255", "ISO8859-9", "CP1254", "ISO8859-10", "ISO8859-11", "CP874", "ISO8859-13", "CP1257", "ISO8859-14", "ISO8859-15", "ISO8859-16", "CP878", "MacJapan", "ASCII", "ANSI_X3.4-1968", "646", "CP65000", "CP65001", "UTF-8-MAC", "UTF-8-HFS", "UCS-2BE", "UCS-4BE", "UCS-4LE", "CP932", "csWindows31J", "SJIS", "PCK", "CP1250", "CP1251", "CP1252", "external", "locale"]`|No | <> |<>|No +| <> |<>|No |=======================================================================   @@ -64,4 +65,12 @@ For nxlog users, you'll want to set this to `CP1252` Change the delimiter that separates lines +[id="plugins-{type}s-{plugin}-metadata"] +===== `metadata` + * Value type is <> + * Default value is `false` + * This option is deprecated and may be removed in a future major release of the plugin. + +If true, the event's metadata (the `@metadata` field and any +subfields) will be included when used as an output codec. diff --git a/lib/logstash/codecs/json_lines.rb b/lib/logstash/codecs/json_lines.rb index 4c801ee..8373a84 100644 --- a/lib/logstash/codecs/json_lines.rb +++ b/lib/logstash/codecs/json_lines.rb @@ -28,12 +28,22 @@ class LogStash::Codecs::JSONLines < LogStash::Codecs::Base # Change the delimiter that separates lines config :delimiter, :validate => :string, :default => "\n" + # If true, the event's metadata (the `@metadata` field) will be + # included when used as an output codec. + config :metadata, :validate => :boolean, :default => false, + :deprecated => "Option may be removed in a future release." + public def register @buffer = FileWatch::BufferedTokenizer.new(@delimiter) @converter = LogStash::Util::Charset.new(@charset) @converter.logger = @logger + if @metadata + @encoder = method(:encode_with_metadata) + else + @encoder = method(:encode_default) + end end def decode(data, &block) @@ -43,9 +53,7 @@ def decode(data, &block) end def encode(event) - # Tack on a @delimiter for now because previously most of logstash's JSON - # outputs emitted one per line, and whitespace is OK in json. - @on_event.call(event, "#{event.to_json}#{@delimiter}") + @encoder.call(event) end def flush(&block) @@ -57,6 +65,18 @@ def flush(&block) private + def encode_default(event) + # Tack on a @delimiter for now because previously most of logstash's JSON + # outputs emitted one per line, and whitespace is OK in json. + @on_event.call(event, "#{event.to_json}#{@delimiter}") + end + + def encode_with_metadata(event) + # Tack on a @delimiter for now because previously most of logstash's JSON + # outputs emitted one per line, and whitespace is OK in json. + @on_event.call(event, "#{LogStash::Json.dump(event.to_hash_with_metadata)}#{@delimiter}") + end + # from_json_parse uses the Event#from_json method to deserialize and directly produce events def from_json_parse(json, &block) LogStash::Event.from_json(json).each { |event| yield event } diff --git a/logstash-codec-json_lines.gemspec b/logstash-codec-json_lines.gemspec index 8bc529f..2494995 100644 --- a/logstash-codec-json_lines.gemspec +++ b/logstash-codec-json_lines.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-codec-json_lines' - s.version = '3.0.6' + s.version = '3.1.0' s.licenses = ['Apache License (2.0)'] s.summary = "Reads and writes newline-delimited JSON" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" diff --git a/spec/codecs/json_lines_spec.rb b/spec/codecs/json_lines_spec.rb index c5b6b52..951f11e 100644 --- a/spec/codecs/json_lines_spec.rb +++ b/spec/codecs/json_lines_spec.rb @@ -120,13 +120,13 @@ end context "#encode" do - let(:data) { { LogStash::Event::TIMESTAMP => "2015-12-07T11:37:00.000Z", "foo" => "bar", "baz" => {"bah" => ["a","b","c"]}} } - let(:event) { LogStash::Event.new(data) } + let(:data) { { LogStash::Event::TIMESTAMP => "2015-12-07T11:37:00.000Z", "foo" => "bar", "baz" => {"bah" => ["a","b","c"]}, "@metadata" => {"metafield" => "metavalue"} } } + let(:event) { LogStash::Event.new(data.clone) } it "should return json data" do got_event = false subject.on_event do |e, d| - insist { d } == "#{LogStash::Event.new(data).to_json}\n" + insist { d } == "#{LogStash::Event.new(data.clone).to_json}\n" insist { LogStash::Json.load(d)["foo"] } == data["foo"] insist { LogStash::Json.load(d)["baz"] } == data["baz"] insist { LogStash::Json.load(d)["bah"] } == data["bah"] @@ -136,13 +136,35 @@ insist { got_event } end + it "should not include metadata in the output" do + got_event = false + subject.on_event do |e, d| + insist { LogStash::Json.load(d).has_key? "@metadata" } == false + got_event = true + end + subject.encode(event) + insist { got_event } + end + context "when using custom delimiter" do let(:delimiter) { "|" } let(:codec_options) { { "delimiter" => delimiter } } it "should decode multiple lines separated by the delimiter" do subject.on_event do |e, d| - insist { d } == "#{LogStash::Event.new(data).to_json}#{delimiter}" + insist { d } == "#{LogStash::Event.new(data.clone).to_json}#{delimiter}" + end + subject.encode(event) + end + end + + context "when enabling metadata output" do + let(:codec_options) { { "metadata" => true } } + + it "should include the @metadata field" do + subject.on_event do |e, d| + insist { d } == "#{LogStash::Json.dump(LogStash::Event.new(data.clone).to_hash_with_metadata)}\n" + insist { LogStash::Json.load(d)["@metadata"] } == data["@metadata"] end subject.encode(event) end