From 12b75dcd3c715dacda623c66af26dc2420e941ac Mon Sep 17 00:00:00 2001 From: dr4gon123 Date: Sat, 18 Apr 2026 15:23:43 -0500 Subject: [PATCH 1/2] feat(vector): split panos.yaml sinks into vector/sinks/ using automatic namespacing Move all panos sinks into separate files under vector/sinks/ per Vector's automatic namespacing feature. Each sink is now independently manageable. quickwit_panos and loki_panos are uncommented (were previously commented out). Co-Authored-By: Claude Sonnet 4.6 --- vector/panos.yaml | 115 ------------------------------- vector/sinks/elastic_panos.yaml | 28 ++++++++ vector/sinks/loki_panos.yaml | 33 +++++++++ vector/sinks/quickwit_panos.yaml | 9 +++ vector/sinks/vlogs_panos.yaml | 26 +++++++ 5 files changed, 96 insertions(+), 115 deletions(-) create mode 100644 vector/sinks/elastic_panos.yaml create mode 100644 vector/sinks/loki_panos.yaml create mode 100644 vector/sinks/quickwit_panos.yaml create mode 100644 vector/sinks/vlogs_panos.yaml diff --git a/vector/panos.yaml b/vector/panos.yaml index eb71133f..d8a0aa45 100644 --- a/vector/panos.yaml +++ b/vector/panos.yaml @@ -362,118 +362,3 @@ transforms: # delete null fields . = compact(.) - -# Print parsed logs to stdout -sinks: - # print: - # type: "console" - # inputs: ["remap_panos"] - # encoding: - # # codec: "raw_message" - # codec: "json" - # json: - # pretty: true - - vlogs_panos: - inputs: - - remap_panos - type: elasticsearch - endpoints: - - ${VICTORIA_LOGS_ENDPOINT:-http://localhost:9428}/insert/elasticsearch/ - api_version: v8 - compression: gzip - healthcheck: - enabled: false - query: - _msg_field: message - _time_field: timestamp - _stream_fields: observer.product,observer.type,observer.vendor,observer.name,log.syslog.hostname,panos.device_name,panos.vsys,panos.type,panos.subtype,network.direction - request: - headers: - AccountID: "0" - ProjectID: "0" - ### For performance optimization. Vector works really well with defaults. Don't use it unless you really need to fine-tune yor ingest. - buffer: - - type: memory - max_events: 12800 # default 500 https://www.elastic.co/docs/reference/fleet/es-output-settings#es-output-settings-performance-tuning-settings - #when_full: drop_newest #default block - batch: - #max_bytes: - max_events: 1600 # default 1000 - timeout_secs: 5 # default 1 - - elastic_panos: - type: elasticsearch - inputs: - - remap_panos - auth: - strategy: "basic" - user: "${ELASTICSEARCH_USER:-elastic}" - password: "${ELASTICSEARCH_PASS:-myelasticsearchpassword}" - endpoints: - - ${ELASTICSEARCH_ENDPOINT:-https://localhost:9200} - encoding: - except_fields: - - url.query - mode: "data_stream" - bulk: - action: "create" - data_stream: - type: "logs" - dataset: "panos.{{panos.type}}" - namespace: "default" - ### For performance optimization. Vector works really well with defaults. Don't use it unless you really need to fine-tune yor ingest. - buffer: - - type: memory - max_events: 12800 # default 500 https://www.elastic.co/docs/reference/fleet/es-output-settings#es-output-settings-performance-tuning-settings - #when_full: drop_newest #default block - batch: - #max_bytes: - max_events: 1600 # default 1000 - timeout_secs: 5 # default 1 - -# quickwit_panos: -# type: "http" -# method: "post" -# inputs: -# - remap_panos -# encoding: -# codec: "json" -# framing: -# method: "newline_delimited" -# uri: "${QUICKWIT_ENDPOINT:-http://localhost:7280}/api/v1/logs-panos.{{panos.type}}/ingest" - -# loki_panos: -# type: loki -# inputs: -# - remap_panos -# endpoint: ${LOKI_endpoint:-https://localhost:3100} -# auth: -# strategy: "basic" -# user: "${LOKI_USER:-loki_user}" -# password: "${LOKI_PASS:-mylokipassword}" -# encoding: -# codec: "json" -# compression: "snappy" -# labels: -# "syslog_appname": "{{.log.syslog.appname}}" -# "syslog_hostname": "{{.log.syslog.hostname}}" -# "panos_device_name": "{{.panos.device_name}}" -# "panos_virtual_system": "{{.panos.virtual_system}}" -# "panos_type": "{{.panos.type}}" -# "panos_threat/content_type": "{{.panos.threat/content_type}}" -# #"network_direction": "{{.network.direction}}" -# #structured_metadata: -# # "source_ip": "{{source.ip}}" -# # "destination_ip": "{{destination.ip}}" -# # "destination_port": "{{destination.port}}" -# # "network_transport_port": "{{network.transport_port}}" -# ### For performance optimization. Vector works really well with defaults. Don't use it unless you really need to fine-tune yor ingest. -# buffer: -# - type: memory -# max_events: 12800 # default 500 https://www.elastic.co/docs/reference/fleet/es-output-settings#es-output-settings-performance-tuning-settings -# #when_full: drop_newest #default block -# batch: -# #max_bytes: -# max_events: 1600 # default 1000 -# timeout_secs: 5 # default 1 diff --git a/vector/sinks/elastic_panos.yaml b/vector/sinks/elastic_panos.yaml new file mode 100644 index 00000000..879f9e5f --- /dev/null +++ b/vector/sinks/elastic_panos.yaml @@ -0,0 +1,28 @@ +type: elasticsearch +inputs: + - remap_panos +auth: + strategy: "basic" + user: "${ELASTICSEARCH_USER:-elastic}" + password: "${ELASTICSEARCH_PASS:-myelasticsearchpassword}" +endpoints: + - ${ELASTICSEARCH_ENDPOINT:-https://localhost:9200} +encoding: + except_fields: + - url.query +mode: "data_stream" +bulk: + action: "create" +data_stream: + type: "logs" + dataset: "panos.{{panos.type}}" + namespace: "default" +### For performance optimization. Vector works really well with defaults. Don't use it unless you really need to fine-tune yor ingest. +buffer: + - type: memory + max_events: 12800 # default 500 https://www.elastic.co/docs/reference/fleet/es-output-settings#es-output-settings-performance-tuning-settings + #when_full: drop_newest #default block +batch: + #max_bytes: + max_events: 1600 # default 1000 + timeout_secs: 5 # default 1 diff --git a/vector/sinks/loki_panos.yaml b/vector/sinks/loki_panos.yaml new file mode 100644 index 00000000..50aee98a --- /dev/null +++ b/vector/sinks/loki_panos.yaml @@ -0,0 +1,33 @@ +type: loki +inputs: + - remap_panos +endpoint: ${LOKI_endpoint:-https://localhost:3100} +auth: + strategy: "basic" + user: "${LOKI_USER:-loki_user}" + password: "${LOKI_PASS:-mylokipassword}" +encoding: + codec: "json" +compression: "snappy" +labels: + "syslog_appname": "{{.log.syslog.appname}}" + "syslog_hostname": "{{.log.syslog.hostname}}" + "panos_device_name": "{{.panos.device_name}}" + "panos_virtual_system": "{{.panos.virtual_system}}" + "panos_type": "{{.panos.type}}" + #"panos_threat/content_type": "{{.panos.threat/content_type}}" # invalid field path + #"network_direction": "{{.network.direction}}" +#structured_metadata: +# "source_ip": "{{source.ip}}" +# "destination_ip": "{{destination.ip}}" +# "destination_port": "{{destination.port}}" +# "network_transport_port": "{{network.transport_port}}" +### For performance optimization. Vector works really well with defaults. Don't use it unless you really need to fine-tune yor ingest. +buffer: + - type: memory + max_events: 12800 # default 500 https://www.elastic.co/docs/reference/fleet/es-output-settings#es-output-settings-performance-tuning-settings + #when_full: drop_newest #default block +batch: + #max_bytes: + max_events: 1600 # default 1000 + timeout_secs: 5 # default 1 diff --git a/vector/sinks/quickwit_panos.yaml b/vector/sinks/quickwit_panos.yaml new file mode 100644 index 00000000..20ed9e9c --- /dev/null +++ b/vector/sinks/quickwit_panos.yaml @@ -0,0 +1,9 @@ +type: "http" +method: "post" +inputs: + - remap_panos +encoding: + codec: "json" +framing: + method: "newline_delimited" +uri: "${QUICKWIT_ENDPOINT:-http://localhost:7280}/api/v1/logs-panos.{{panos.type}}/ingest" diff --git a/vector/sinks/vlogs_panos.yaml b/vector/sinks/vlogs_panos.yaml new file mode 100644 index 00000000..838c8d68 --- /dev/null +++ b/vector/sinks/vlogs_panos.yaml @@ -0,0 +1,26 @@ +inputs: + - remap_panos +type: elasticsearch +endpoints: + - ${VICTORIA_LOGS_ENDPOINT:-http://localhost:9428}/insert/elasticsearch/ +api_version: v8 +compression: gzip +healthcheck: + enabled: false +query: + _msg_field: message + _time_field: timestamp + _stream_fields: observer.product,observer.type,observer.vendor,observer.name,log.syslog.hostname,panos.device_name,panos.vsys,panos.type,panos.subtype,network.direction +request: + headers: + AccountID: "0" + ProjectID: "0" +### For performance optimization. Vector works really well with defaults. Don't use it unless you really need to fine-tune yor ingest. +buffer: + - type: memory + max_events: 12800 # default 500 https://www.elastic.co/docs/reference/fleet/es-output-settings#es-output-settings-performance-tuning-settings + #when_full: drop_newest #default block +batch: + #max_bytes: + max_events: 1600 # default 1000 + timeout_secs: 5 # default 1 From 9c47cc0014e2c6df4840539638df5f56a630a860 Mon Sep 17 00:00:00 2001 From: dr4gon123 Date: Sat, 18 Apr 2026 15:27:37 -0500 Subject: [PATCH 2/2] fix(vector): replace removed panos.threat/content_type with panos.subtype in loki labels Co-Authored-By: Claude Sonnet 4.6 --- vector/sinks/loki_panos.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vector/sinks/loki_panos.yaml b/vector/sinks/loki_panos.yaml index 50aee98a..142e6983 100644 --- a/vector/sinks/loki_panos.yaml +++ b/vector/sinks/loki_panos.yaml @@ -15,7 +15,7 @@ labels: "panos_device_name": "{{.panos.device_name}}" "panos_virtual_system": "{{.panos.virtual_system}}" "panos_type": "{{.panos.type}}" - #"panos_threat/content_type": "{{.panos.threat/content_type}}" # invalid field path + "panos_subtype": "{{panos.subtype}}" #"network_direction": "{{.network.direction}}" #structured_metadata: # "source_ip": "{{source.ip}}"