From 369018dcdb44bdc9233f693cfd0994ddb7b1caca Mon Sep 17 00:00:00 2001 From: Jon Skarpeteig Date: Thu, 17 Jan 2013 13:12:45 +0100 Subject: [PATCH 1/4] syslog.conf from https://gist.github.com/2959887 https://gist.github.com/2959887 for a better syslog parsing. Only change is syslog-relay to syslog_relay to avoid any syntax errors in Logstash 1.1.9 --- recipes/syslog-pri/syslog.conf | 95 ++++++++++++++++++++++++---------- 1 file changed, 67 insertions(+), 28 deletions(-) diff --git a/recipes/syslog-pri/syslog.conf b/recipes/syslog-pri/syslog.conf index 386410d..63265ad 100644 --- a/recipes/syslog-pri/syslog.conf +++ b/recipes/syslog-pri/syslog.conf @@ -1,41 +1,80 @@ -input { - tcp { - port => 5000 - type => syslog +filter { + # strip the syslog PRI part and create facility and severity fields. + # the original syslog message is saved in field %{syslog_raw_message}. + # the extracted PRI is available in the %{syslog_pri} field. + # + # You get %{syslog_facility_code} and %{syslog_severity_code} fields. + # You also get %{syslog_facility} and %{syslog_severity} fields if the + # use_labels option is set True (the default) on syslog_pri filter. + grok { + type => "syslog_relay" + pattern => [ "<%{POSINT:syslog_pri}>%{SPACE}%{GREEDYDATA:message_remainder}" ] + add_tag => "got_syslog_pri" + add_field => [ "syslog_raw_message", "%{@message}" ] } - udp { - port => 5000 - type => syslog + syslog_pri { + type => "syslog_relay" + tags => [ "got_syslog_pri" ] + } + mutate { + type => "syslog_relay" + tags => [ "got_syslog_pri" ] + replace => [ "@message", "%{message_remainder}" ] + } + mutate { + # XXX must not be combined with replacement which uses same field + type => "syslog_relay" + tags => [ "got_syslog_pri" ] + remove => [ "message_remainder" ] } -} -filter { + # strip the syslog timestamp and force event timestamp to be the same. + # the original string is saved in field %{syslog_timestamp}. + # the original logstash input timestamp is saved in field %{received_at}. grok { - type => "syslog" - pattern => [ "<%{POSINT:syslog_pri}>%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{PROG:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" ] - add_field => [ "received_at", "%{@timestamp}" ] - add_field => [ "received_from", "%{@source_host}" ] + type => "syslog_relay" + pattern => [ "%{SYSLOGTIMESTAMP:syslog_timestamp}%{SPACE}%{GREEDYDATA:message_remainder}" ] + add_tag => "got_syslog_timestamp" + add_field => [ "received_at", "%{@timestamp}" ] } - syslog_pri { - type => "syslog" + mutate { + type => "syslog_relay" + tags => [ "got_syslog_timestamp" ] + replace => [ "@message", "%{message_remainder}" ] + } + mutate { + # XXX must not be combined with replacement which uses same field + type => "syslog_relay" + tags => [ "got_syslog_timestamp" ] + remove => [ "message_remainder" ] } date { - type => "syslog" - syslog_timestamp => [ "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ] + type => "syslog_relay" + tags => [ "got_syslog_timestamp" ] + # season to taste for your own syslog format(s) + syslog_timestamp => [ "MMM d HH:mm:ss", "MMM dd HH:mm:ss", "ISO8601" ] + } + + # strip the host field from the syslog line. + # the extracted host field becomes the logstash %{@source_host} metadata + # and is also available in the filed %{syslog_hostname}. + # the original logstash source_host is saved in field %{logstash_source}. + grok { + type => "syslog_relay" + pattern => [ "%{SYSLOGHOST:syslog_hostname}%{SPACE}%{GREEDYDATA:message_remainder}" ] + add_tag => "got_syslog_host" + add_field => [ "logstash_source", "%{@source_host}" ] } mutate { - type => "syslog" - exclude_tags => "_grokparsefailure" - replace => [ "@source_host", "%{syslog_hostname}" ] - replace => [ "@message", "%{syslog_message}" ] + type => "syslog_relay" + tags => [ "got_syslog_host" ] + replace => [ "@source_host", "%{syslog_hostname}" ] + replace => [ "@message", "%{message_remainder}" ] } mutate { - type => "syslog" - remove => [ "syslog_hostname", "syslog_message", "syslog_timestamp" ] + # XXX must not be combined with replacement which uses same field + type => "syslog_relay" + tags => [ "got_syslog_host" ] + remove => [ "message_remainder" ] } } - -output { - # Example just to output to elasticsearch - elasticsearch { } -} From 0a49f68e722bf742202a5dc05f0ec443f1395787 Mon Sep 17 00:00:00 2001 From: Jon Skarpeteig Date: Thu, 17 Jan 2013 13:21:59 +0100 Subject: [PATCH 2/4] Include input{} and output{} --- recipes/syslog-pri/syslog.conf | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/recipes/syslog-pri/syslog.conf b/recipes/syslog-pri/syslog.conf index 63265ad..83d2e28 100644 --- a/recipes/syslog-pri/syslog.conf +++ b/recipes/syslog-pri/syslog.conf @@ -1,3 +1,25 @@ +input { + # Syslog server + tcp { + port => 5000 + type => syslog_relay + } + udp { + port => 5000 + type => syslog_relay + } + +# Default Syslog server port require root permissions due to port < 1024 +# tcp { +# port => 514 +# type => syslog_relay +# } +# udp { +# port => 514 +# type => syslog_relay +# } +} + filter { # strip the syslog PRI part and create facility and severity fields. # the original syslog message is saved in field %{syslog_raw_message}. @@ -78,3 +100,13 @@ filter { remove => [ "message_remainder" ] } } + +output { + # If your elasticsearch server is discoverable with multicast, use this: + elasticsearch { } + + # If you can't discover using multicast, set the address explicitly + #elasticsearch { + # host => "myelasticsearchserver" + #} +} From 006f5a9a7609da4fdc6df245664a0c1b11d2a27f Mon Sep 17 00:00:00 2001 From: Jon Skarpeteig Date: Mon, 21 Jan 2013 10:01:21 +0100 Subject: [PATCH 3/4] Update recipes/syslog-pri/syslog.conf Simplified mutates --- recipes/syslog-pri/syslog.conf | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/recipes/syslog-pri/syslog.conf b/recipes/syslog-pri/syslog.conf index 83d2e28..64ffbd7 100644 --- a/recipes/syslog-pri/syslog.conf +++ b/recipes/syslog-pri/syslog.conf @@ -42,11 +42,6 @@ filter { type => "syslog_relay" tags => [ "got_syslog_pri" ] replace => [ "@message", "%{message_remainder}" ] - } - mutate { - # XXX must not be combined with replacement which uses same field - type => "syslog_relay" - tags => [ "got_syslog_pri" ] remove => [ "message_remainder" ] } @@ -63,11 +58,6 @@ filter { type => "syslog_relay" tags => [ "got_syslog_timestamp" ] replace => [ "@message", "%{message_remainder}" ] - } - mutate { - # XXX must not be combined with replacement which uses same field - type => "syslog_relay" - tags => [ "got_syslog_timestamp" ] remove => [ "message_remainder" ] } date { @@ -92,11 +82,6 @@ filter { tags => [ "got_syslog_host" ] replace => [ "@source_host", "%{syslog_hostname}" ] replace => [ "@message", "%{message_remainder}" ] - } - mutate { - # XXX must not be combined with replacement which uses same field - type => "syslog_relay" - tags => [ "got_syslog_host" ] remove => [ "message_remainder" ] } } From 199afd8dce73de016bf5845758ba8244b102bcfc Mon Sep 17 00:00:00 2001 From: Jon Skarpeteig Date: Mon, 21 Jan 2013 15:48:06 +0100 Subject: [PATCH 4/4] Remove internal tags before output --- recipes/syslog-pri/syslog.conf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/recipes/syslog-pri/syslog.conf b/recipes/syslog-pri/syslog.conf index 64ffbd7..bc2b978 100644 --- a/recipes/syslog-pri/syslog.conf +++ b/recipes/syslog-pri/syslog.conf @@ -43,6 +43,7 @@ filter { tags => [ "got_syslog_pri" ] replace => [ "@message", "%{message_remainder}" ] remove => [ "message_remainder" ] + remove_tag => "got_syslog_pri" } # strip the syslog timestamp and force event timestamp to be the same. @@ -59,6 +60,7 @@ filter { tags => [ "got_syslog_timestamp" ] replace => [ "@message", "%{message_remainder}" ] remove => [ "message_remainder" ] + remove_tag => "got_syslog_timestamp" } date { type => "syslog_relay" @@ -83,6 +85,7 @@ filter { replace => [ "@source_host", "%{syslog_hostname}" ] replace => [ "@message", "%{message_remainder}" ] remove => [ "message_remainder" ] + remove_tag => "got_syslog_host" } }