From 9a63e554caa09e1fd79cc71c3bbc8d0fabf9aac0 Mon Sep 17 00:00:00 2001
From: Yanwei Guo <yanweiguo@google.com>
Date: Fri, 25 May 2018 16:16:00 -0700
Subject: [PATCH 1/2] use detect_exceptions for varlog to combine multi line
 logs

---
 pkg/controller/revision/ela_fluentd.go | 24 +++++++++++++++++-------
 1 file changed, 17 insertions(+), 7 deletions(-)

diff --git a/pkg/controller/revision/ela_fluentd.go b/pkg/controller/revision/ela_fluentd.go
index 5a21c261e975..086b42d7f699 100644
--- a/pkg/controller/revision/ela_fluentd.go
+++ b/pkg/controller/revision/ela_fluentd.go
@@ -30,7 +30,7 @@ const fluentdSidecarPreOutputConfig = `
 	@type tail
 	path /var/log/revisions/**/*.*
 	pos_file /var/log/varlog.log.pos
-	tag *
+	tag raw.*
 	<parse>
 		@type multi_format
 		<pattern>
@@ -46,18 +46,28 @@ const fluentdSidecarPreOutputConfig = `
 	read_from_head true
 </source>
 
-<filter var.log.**>
+<filter raw.var.log.**>
 	@type record_transformer
+	enable_ruby true
 	<record>
-		kubernetes.container_name "#{ENV['ELA_CONTAINER_NAME']}"
-		kubernetes.labels.elafros_dev/configuration "#{ENV['ELA_CONFIGURATION']}"
-		kubernetes.labels.elafros_dev/revision "#{ENV['ELA_REVISION']}"
-		kubernetes.namespace_name "#{ENV['ELA_NAMESPACE']}"
-		kubernetes.pod_name "#{ENV['ELA_POD_NAME']}"
+	  kubernetes ${ {"container_name": "#{ENV['ELA_CONTAINER_NAME']}", "namespace_name": "#{ENV['ELA_NAMESPACE']}", "pod_name": "#{ENV['ELA_POD_NAME']}", "labels": {"elafros_dev/configuration": "#{ENV['ELA_CONFIGURATION']}", "elafros_dev/revision": "#{ENV['ELA_REVISION']}"} } }
 		stream varlog
+		# Remove this if https://github.com/GoogleCloudPlatform/fluent-plugin-detect-exceptions/pull/10 is released
+		log ${ if record["log"].end_with?("\n") then record["log"] else record["log"] + "\n" end }
 	</record>
 </filter>
 
+<match raw.var.log.**>
+	@id raw.var.log
+	@type detect_exceptions
+	remove_tag_prefix raw
+	message log
+	stream stream
+	multiline_flush_interval 5
+	max_bytes 500000
+	max_lines 1000
+</match>
+
 `
 
 const fluentdConfigMapName = "fluentd-varlog-config"

From e2d1b860ad401816d97dc544a074d9b28917c99c Mon Sep 17 00:00:00 2001
From: Yanwei Guo <yanweiguo@google.com>
Date: Fri, 25 May 2018 16:36:48 -0700
Subject: [PATCH 2/2] add a comment

---
 pkg/controller/revision/ela_fluentd.go | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/pkg/controller/revision/ela_fluentd.go b/pkg/controller/revision/ela_fluentd.go
index 086b42d7f699..72e1ebbf6339 100644
--- a/pkg/controller/revision/ela_fluentd.go
+++ b/pkg/controller/revision/ela_fluentd.go
@@ -50,8 +50,14 @@ const fluentdSidecarPreOutputConfig = `
 	@type record_transformer
 	enable_ruby true
 	<record>
-	  kubernetes ${ {"container_name": "#{ENV['ELA_CONTAINER_NAME']}", "namespace_name": "#{ENV['ELA_NAMESPACE']}", "pod_name": "#{ENV['ELA_POD_NAME']}", "labels": {"elafros_dev/configuration": "#{ENV['ELA_CONFIGURATION']}", "elafros_dev/revision": "#{ENV['ELA_REVISION']}"} } }
+		kubernetes.container_name "#{ENV['ELA_CONTAINER_NAME']}"
+		kubernetes.labels.elafros_dev/configuration "#{ENV['ELA_CONFIGURATION']}"
+		kubernetes.labels.elafros_dev/revision "#{ENV['ELA_REVISION']}"
+		kubernetes.namespace_name "#{ENV['ELA_NAMESPACE']}"
+		kubernetes.pod_name "#{ENV['ELA_POD_NAME']}"
 		stream varlog
+		# Line breaks may be trimmed when collecting from files. Add them back so that
+		# multi line logs are still in multi line after combined by detect_exceptions.
 		# Remove this if https://github.com/GoogleCloudPlatform/fluent-plugin-detect-exceptions/pull/10 is released
 		log ${ if record["log"].end_with?("\n") then record["log"] else record["log"] + "\n" end }
 	</record>