Merge pull request #210 from whyscream/postfix-tls-features

whyscream · web-flow · commit adaed13c477d · 2025-11-29T18:07:40.000+01:00
Parse postfix TLS features
diff --git a/.github/workflows/test_config_syntax.yml b/.github/workflows/test_config_syntax.yml
@@ -5,7 +5,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        logstash-version: ['8.14.1', '7.17.22']
+        logstash-version: ['9.2.1', '8.19.7', '7.17.28']
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
diff --git a/.github/workflows/test_grok_patterns.yml b/.github/workflows/test_grok_patterns.yml
@@ -10,6 +10,6 @@ jobs:
           submodules: true
       - uses: ruby/setup-ruby@v1
         with:
-          ruby-version: '3.2'
+          ruby-version: '3.4'
       - run: gem install jls-grok minitest
       - run: ruby test/test.rb
diff --git a/50-filter-postfix.conf b/50-filter-postfix.conf
@@ -222,6 +222,14 @@ filter {
                 remove_field   => [ "postfix_delays" ]
             }
         }
+        if [postfix_tls] {
+            grok {
+                patterns_dir   => "/etc/logstash/patterns.d"
+                match          => ["postfix_tls", "^%{POSTFIX_TLS_FEATURES}$"]
+                tag_on_failure => [ "_grok_kv_postfix_tls_nomatch" ]
+                remove_field   => [ "postfix_tls" ]
+            }
+        }
     }
 
     # process command counter data if it exists
@@ -289,6 +297,12 @@ filter {
             "postfix_delay_transmission", "float",
             "postfix_postscreen_violation_time", "float"
         ]
+        gsub => [
+            # rewrite some extracted values
+            "postfix_tls_policy_undecided", "\?", "true",
+            "postfix_requiretls_policy_undecided", "\?", "true",
+            "postfix_requiretls_policy_violation", "\!", "true",
+            "postfix_requiretls", "requiretls", "true"
+        ]
     }
 }
-
diff --git a/postfix.grok b/postfix.grok
@@ -121,6 +121,10 @@ POSTFIX_VERIFY_CACHE cache %{DATA} (?<postfix_verify_cleanup_type>(full|partial)
 # local patterns
 POSTFIX_LOCAL_DELIVERY %{POSTFIX_KEYVALUE} status=%{STATUS_WORD:postfix_status}( \(%{GREEDYDATA:postfix_local_response}\))?
 
+# TLS features
+POSTFIX_TLS_FEAT_REQUIRETLS (?<postfix_requiretls_policy_violation>\!)?(?<postfix_requiretls>requiretls)(:(?<postfix_requiretls_downgrade_level>\w+))?(?<postfix_requiretls_policy_undecided>\?)?
+POSTFIX_TLS_FEATURES (?<postfix_tls_security_level>\w+)(:(?<postfix_tls_downgrade_level>\w+))?(?<postfix_tls_policy_undecided>\?)?(/%{POSTFIX_TLS_FEAT_REQUIRETLS})?
+
 # aggregate all patterns
 POSTFIX_SMTPD %{POSTFIX_SMTPD_CONNECT}|%{POSTFIX_SMTPD_DISCONNECT}|%{POSTFIX_SMTPD_LOSTCONN}|%{POSTFIX_SMTPD_NOQUEUE}|%{POSTFIX_SMTPD_PIPELINING}|%{POSTFIX_TLSCONN}|%{POSTFIX_WARNING}|%{POSTFIX_SMTPD_PROXY}|%{POSTFIX_KEYVALUE}
 POSTFIX_CLEANUP %{POSTFIX_CLEANUP_MESSAGEID}|%{POSTFIX_CLEANUP_MILTER}|%{POSTFIX_CLEANUP_PREPEND}|%{POSTFIX_WARNING}|%{POSTFIX_KEYVALUE}
diff --git a/test/tls_features_0001.yaml b/test/tls_features_0001.yaml
@@ -0,0 +1,6 @@
+pattern: "^%{POSTFIX_SMTP}$"
+# TLS features: security level only (single word)
+data: "7EE668039: to=<user@example.com>, relay=mail.example.com[1.2.3.4]:25, delay=3.6, delays=0.08/0.02/0.85/0.14, tls=dane, dsn=2.1.5, status=sent (250 2.0.0 Ok: queued as 153053D)"
+results:
+    postfix_queueid: 7EE668039
+    postfix_keyvalue_data: "to=<user@example.com>, relay=mail.example.com[1.2.3.4]:25, delay=3.6, delays=0.08/0.02/0.85/0.14, tls=dane, dsn=2.1.5,"
diff --git a/test/tls_features_0002.yaml b/test/tls_features_0002.yaml
@@ -0,0 +1,6 @@
+pattern: "^%{POSTFIX_SMTP}$"
+# TLS features: security level undecided (with question mark)
+data: "7EE668039: to=<user@example.com>, relay=mail.example.com[1.2.3.4]:25, delay=3.6, delays=0.08/0.02/0.85/0.14, tls=may?, dsn=2.1.5, status=sent (250 2.0.0 Ok: queued as 153053D)"
+results:
+    postfix_queueid: 7EE668039
+    postfix_keyvalue_data: "to=<user@example.com>, relay=mail.example.com[1.2.3.4]:25, delay=3.6, delays=0.08/0.02/0.85/0.14, tls=may?, dsn=2.1.5,"
diff --git a/test/tls_features_0003.yaml b/test/tls_features_0003.yaml
@@ -0,0 +1,6 @@
+pattern: "^%{POSTFIX_SMTP}$"
+# TLS features: with downgrade level (separated by colon)
+data: "7EE668039: to=<user@example.com>, relay=mail.example.com[1.2.3.4]:25, delay=3.6, delays=0.08/0.02/0.85/0.14, tls=may:none, dsn=2.1.5, status=sent (250 2.0.0 Ok: queued as 153053D)"
+results:
+    postfix_queueid: 7EE668039
+    postfix_keyvalue_data: "to=<user@example.com>, relay=mail.example.com[1.2.3.4]:25, delay=3.6, delays=0.08/0.02/0.85/0.14, tls=may:none, dsn=2.1.5,"
diff --git a/test/tls_features_0004.yaml b/test/tls_features_0004.yaml
@@ -0,0 +1,6 @@
+pattern: "^%{POSTFIX_SMTP}$"
+# TLS features: requiretls
+data: "7EE668039: to=<user@example.com>, relay=mail.example.com[1.2.3.4]:25, delay=3.6, delays=0.08/0.02/0.85/0.14, tls=dane/requiretls, dsn=2.1.5, status=sent (250 2.0.0 Ok: queued as 153053D)"
+results:
+    postfix_queueid: 7EE668039
+    postfix_keyvalue_data: "to=<user@example.com>, relay=mail.example.com[1.2.3.4]:25, delay=3.6, delays=0.08/0.02/0.85/0.14, tls=dane/requiretls, dsn=2.1.5,"
diff --git a/test/tls_features_0005.yaml b/test/tls_features_0005.yaml
@@ -0,0 +1,6 @@
+pattern: "^%{POSTFIX_SMTP}$"
+# TLS features: requiretls violation
+data: "7EE668039: to=<user@example.com>, relay=mail.example.com[1.2.3.4]:25, delay=3.6, delays=0.08/0.02/0.85/0.14, tls=dane/!requiretls:nocertmatch, dsn=2.1.5, status=sent (250 2.0.0 Ok: queued as 153053D)"
+results:
+    postfix_queueid: 7EE668039
+    postfix_keyvalue_data: "to=<user@example.com>, relay=mail.example.com[1.2.3.4]:25, delay=3.6, delays=0.08/0.02/0.85/0.14, tls=dane/!requiretls:nocertmatch, dsn=2.1.5,"
diff --git a/test/tls_features_0006.yaml b/test/tls_features_0006.yaml
@@ -0,0 +1,6 @@
+pattern: "^%{POSTFIX_SMTP}$"
+# TLS features: requiretls policy undecided
+data: "7EE668039: to=<user@example.com>, relay=mail.example.com[1.2.3.4]:25, delay=3.6, delays=0.08/0.02/0.85/0.14, tls=dane/requiretls?, dsn=2.1.5, status=sent (250 2.0.0 Ok: queued as 153053D)"
+results:
+    postfix_queueid: 7EE668039
+    postfix_keyvalue_data: "to=<user@example.com>, relay=mail.example.com[1.2.3.4]:25, delay=3.6, delays=0.08/0.02/0.85/0.14, tls=dane/requiretls?, dsn=2.1.5,"
diff --git a/test/tls_features_0007.yaml b/test/tls_features_0007.yaml
@@ -0,0 +1,4 @@
+pattern: "^%{POSTFIX_TLS_FEATURES}$"
+data: "dane"
+results:
+    postfix_tls_security_level: dane
diff --git a/test/tls_features_0008.yaml b/test/tls_features_0008.yaml
@@ -0,0 +1,5 @@
+pattern: "^%{POSTFIX_TLS_FEATURES}$"
+data: "dane:none"
+results:
+    postfix_tls_security_level: dane
+    postfix_tls_downgrade_level: none
diff --git a/test/tls_features_0009.yaml b/test/tls_features_0009.yaml
@@ -0,0 +1,5 @@
+pattern: "^%{POSTFIX_TLS_FEATURES}$"
+data: "may?"
+results:
+    postfix_tls_security_level: may
+    postfix_tls_policy_undecided: "?"
diff --git a/test/tls_features_0010.yaml b/test/tls_features_0010.yaml
@@ -0,0 +1,5 @@
+pattern: "^%{POSTFIX_TLS_FEATURES}$"
+data: "dane/requiretls"
+results:
+    postfix_tls_security_level: dane
+    postfix_requiretls: requiretls
diff --git a/test/tls_features_0011.yaml b/test/tls_features_0011.yaml
@@ -0,0 +1,6 @@
+pattern: "^%{POSTFIX_TLS_FEATURES}$"
+data: "dane/requiretls?"
+results:
+    postfix_tls_security_level: dane
+    postfix_requiretls: requiretls
+    postfix_requiretls_policy_undecided: "?"
diff --git a/test/tls_features_0012.yaml b/test/tls_features_0012.yaml
@@ -0,0 +1,7 @@
+pattern: "^%{POSTFIX_TLS_FEATURES}$"
+data: "dane/!requiretls:nostarttls"
+results:
+    postfix_tls_security_level: dane
+    postfix_requiretls: requiretls
+    postfix_requiretls_policy_violation: "!"
+    postfix_requiretls_downgrade_level: nostarttls
diff --git a/test_config_syntax.sh b/test_config_syntax.sh
@@ -8,7 +8,7 @@
 
 set -eu
 
-LOGSTASH_VERSION=8.14.1
+LOGSTASH_VERSION=9.2.1
 
 docker run \
   --rm \
diff --git a/test_pipeline.sh b/test_pipeline.sh
@@ -9,7 +9,7 @@
 
 set -eu
 
-LOGSTASH_VERSION=8.14.1
+LOGSTASH_VERSION=9.2.1
 
 INPUT=$(mktemp tmp.logstash.in.XXXXX)
 OUTPUT=$(mktemp tmp.logstash.out.XXXXX)
@@ -23,7 +23,7 @@ perform_cleanup() {
 trap perform_cleanup INT TERM
 
 echo Preparing input data
-echo "postfix/smtp[123]: 7EE668039: to=<admin@example.com>, relay=127.0.0.1[127.0.0.1]:2525, delay=3.6, delays=0.2/0.02/0.04/3.3, dsn=2.0.0, status=sent (250 2.0.0 Ok: queued as 153053D)" > "$INPUT"
+echo "postfix/smtp[123]: 7EE668039: to=<admin@example.com>, relay=127.0.0.1[127.0.0.1]:2525, delay=3.6, delays=0.2/0.02/0.04/3.3, dsn=2.0.0, tls=dane/!requiretls:nostarttls, status=sent (250 2.0.0 Ok: queued as 153053D)" > "$INPUT"
 
 echo Preparing pipeline config
 cat > "$PIPELINE" << EOF
@@ -63,17 +63,19 @@ CONTAINER_ID=$(docker run --rm --detach \
 
 printf "Waiting for output from logstash "
 until test -s "$OUTPUT"; do
+  # For debugging a crashing container (probably invalid configuration)
+  # docker inspect "$CONTAINER_ID" | jq '.[0].State'
   printf "."
   sleep 2
 done
 echo
 
 if test "$(jq .tags[0] "$OUTPUT")" = '"_grok_postfix_success"'; then
   echo Grok processing successful!
-  jq . "$OUTPUT"
+  jq --sort-keys . "$OUTPUT"
 else
   echo "Grok processing failed :<"
-  jq . "$OUTPUT"
+  jq --sort-keys . "$OUTPUT"
   exit 1
 fi
 

Original file line number	Diff line number	Diff line change
`@@ -222,6 +222,14 @@ filter {`
`222`	`222`	`remove_field => [ "postfix_delays" ]`
`223`	`223`	`}`
`224`	`224`	`}`
	`225`	`+ if [postfix_tls] {`
	`226`	`+ grok {`
	`227`	`+ patterns_dir => "/etc/logstash/patterns.d"`
	`228`	`+ match => ["postfix_tls", "^%{POSTFIX_TLS_FEATURES}$"]`
	`229`	`+ tag_on_failure => [ "_grok_kv_postfix_tls_nomatch" ]`
	`230`	`+ remove_field => [ "postfix_tls" ]`
	`231`	`+ }`
	`232`	`+ }`
`225`	`233`	`}`
`226`	`234`
`227`	`235`	`# process command counter data if it exists`
`@@ -289,6 +297,12 @@ filter {`
`289`	`297`	`"postfix_delay_transmission", "float",`
`290`	`298`	`"postfix_postscreen_violation_time", "float"`
`291`	`299`	`]`
	`300`	`+ gsub => [`
	`301`	`+ # rewrite some extracted values`
	`302`	`+ "postfix_tls_policy_undecided", "\?", "true",`
	`303`	`+ "postfix_requiretls_policy_undecided", "\?", "true",`
	`304`	`+ "postfix_requiretls_policy_violation", "\!", "true",`
	`305`	`+ "postfix_requiretls", "requiretls", "true"`
	`306`	`+ ]`
`292`	`307`	`}`
`293`	`308`	`}`
`294`		`-`