diff --git a/vector/panos.yaml b/vector/panos.yaml index eb71133f..d8a0aa45 100644 --- a/vector/panos.yaml +++ b/vector/panos.yaml @@ -362,118 +362,3 @@ transforms: # delete null fields . = compact(.) - -# Print parsed logs to stdout -sinks: - # print: - # type: "console" - # inputs: ["remap_panos"] - # encoding: - # # codec: "raw_message" - # codec: "json" - # json: - # pretty: true - - vlogs_panos: - inputs: - - remap_panos - type: elasticsearch - endpoints: - - ${VICTORIA_LOGS_ENDPOINT:-http://localhost:9428}/insert/elasticsearch/ - api_version: v8 - compression: gzip - healthcheck: - enabled: false - query: - _msg_field: message - _time_field: timestamp - _stream_fields: observer.product,observer.type,observer.vendor,observer.name,log.syslog.hostname,panos.device_name,panos.vsys,panos.type,panos.subtype,network.direction - request: - headers: - AccountID: "0" - ProjectID: "0" - ### For performance optimization. Vector works really well with defaults. Don't use it unless you really need to fine-tune yor ingest. - buffer: - - type: memory - max_events: 12800 # default 500 https://www.elastic.co/docs/reference/fleet/es-output-settings#es-output-settings-performance-tuning-settings - #when_full: drop_newest #default block - batch: - #max_bytes: - max_events: 1600 # default 1000 - timeout_secs: 5 # default 1 - - elastic_panos: - type: elasticsearch - inputs: - - remap_panos - auth: - strategy: "basic" - user: "${ELASTICSEARCH_USER:-elastic}" - password: "${ELASTICSEARCH_PASS:-myelasticsearchpassword}" - endpoints: - - ${ELASTICSEARCH_ENDPOINT:-https://localhost:9200} - encoding: - except_fields: - - url.query - mode: "data_stream" - bulk: - action: "create" - data_stream: - type: "logs" - dataset: "panos.{{panos.type}}" - namespace: "default" - ### For performance optimization. Vector works really well with defaults. Don't use it unless you really need to fine-tune yor ingest. - buffer: - - type: memory - max_events: 12800 # default 500 https://www.elastic.co/docs/reference/fleet/es-output-settings#es-output-settings-performance-tuning-settings - #when_full: drop_newest #default block - batch: - #max_bytes: - max_events: 1600 # default 1000 - timeout_secs: 5 # default 1 - -# quickwit_panos: -# type: "http" -# method: "post" -# inputs: -# - remap_panos -# encoding: -# codec: "json" -# framing: -# method: "newline_delimited" -# uri: "${QUICKWIT_ENDPOINT:-http://localhost:7280}/api/v1/logs-panos.{{panos.type}}/ingest" - -# loki_panos: -# type: loki -# inputs: -# - remap_panos -# endpoint: ${LOKI_endpoint:-https://localhost:3100} -# auth: -# strategy: "basic" -# user: "${LOKI_USER:-loki_user}" -# password: "${LOKI_PASS:-mylokipassword}" -# encoding: -# codec: "json" -# compression: "snappy" -# labels: -# "syslog_appname": "{{.log.syslog.appname}}" -# "syslog_hostname": "{{.log.syslog.hostname}}" -# "panos_device_name": "{{.panos.device_name}}" -# "panos_virtual_system": "{{.panos.virtual_system}}" -# "panos_type": "{{.panos.type}}" -# "panos_threat/content_type": "{{.panos.threat/content_type}}" -# #"network_direction": "{{.network.direction}}" -# #structured_metadata: -# # "source_ip": "{{source.ip}}" -# # "destination_ip": "{{destination.ip}}" -# # "destination_port": "{{destination.port}}" -# # "network_transport_port": "{{network.transport_port}}" -# ### For performance optimization. Vector works really well with defaults. Don't use it unless you really need to fine-tune yor ingest. -# buffer: -# - type: memory -# max_events: 12800 # default 500 https://www.elastic.co/docs/reference/fleet/es-output-settings#es-output-settings-performance-tuning-settings -# #when_full: drop_newest #default block -# batch: -# #max_bytes: -# max_events: 1600 # default 1000 -# timeout_secs: 5 # default 1 diff --git a/vector/sinks/elastic_panos.yaml b/vector/sinks/elastic_panos.yaml new file mode 100644 index 00000000..879f9e5f --- /dev/null +++ b/vector/sinks/elastic_panos.yaml @@ -0,0 +1,28 @@ +type: elasticsearch +inputs: + - remap_panos +auth: + strategy: "basic" + user: "${ELASTICSEARCH_USER:-elastic}" + password: "${ELASTICSEARCH_PASS:-myelasticsearchpassword}" +endpoints: + - ${ELASTICSEARCH_ENDPOINT:-https://localhost:9200} +encoding: + except_fields: + - url.query +mode: "data_stream" +bulk: + action: "create" +data_stream: + type: "logs" + dataset: "panos.{{panos.type}}" + namespace: "default" +### For performance optimization. Vector works really well with defaults. Don't use it unless you really need to fine-tune yor ingest. +buffer: + - type: memory + max_events: 12800 # default 500 https://www.elastic.co/docs/reference/fleet/es-output-settings#es-output-settings-performance-tuning-settings + #when_full: drop_newest #default block +batch: + #max_bytes: + max_events: 1600 # default 1000 + timeout_secs: 5 # default 1 diff --git a/vector/sinks/loki_panos.yaml b/vector/sinks/loki_panos.yaml new file mode 100644 index 00000000..142e6983 --- /dev/null +++ b/vector/sinks/loki_panos.yaml @@ -0,0 +1,33 @@ +type: loki +inputs: + - remap_panos +endpoint: ${LOKI_endpoint:-https://localhost:3100} +auth: + strategy: "basic" + user: "${LOKI_USER:-loki_user}" + password: "${LOKI_PASS:-mylokipassword}" +encoding: + codec: "json" +compression: "snappy" +labels: + "syslog_appname": "{{.log.syslog.appname}}" + "syslog_hostname": "{{.log.syslog.hostname}}" + "panos_device_name": "{{.panos.device_name}}" + "panos_virtual_system": "{{.panos.virtual_system}}" + "panos_type": "{{.panos.type}}" + "panos_subtype": "{{panos.subtype}}" + #"network_direction": "{{.network.direction}}" +#structured_metadata: +# "source_ip": "{{source.ip}}" +# "destination_ip": "{{destination.ip}}" +# "destination_port": "{{destination.port}}" +# "network_transport_port": "{{network.transport_port}}" +### For performance optimization. Vector works really well with defaults. Don't use it unless you really need to fine-tune yor ingest. +buffer: + - type: memory + max_events: 12800 # default 500 https://www.elastic.co/docs/reference/fleet/es-output-settings#es-output-settings-performance-tuning-settings + #when_full: drop_newest #default block +batch: + #max_bytes: + max_events: 1600 # default 1000 + timeout_secs: 5 # default 1 diff --git a/vector/sinks/quickwit_panos.yaml b/vector/sinks/quickwit_panos.yaml new file mode 100644 index 00000000..20ed9e9c --- /dev/null +++ b/vector/sinks/quickwit_panos.yaml @@ -0,0 +1,9 @@ +type: "http" +method: "post" +inputs: + - remap_panos +encoding: + codec: "json" +framing: + method: "newline_delimited" +uri: "${QUICKWIT_ENDPOINT:-http://localhost:7280}/api/v1/logs-panos.{{panos.type}}/ingest" diff --git a/vector/sinks/vlogs_panos.yaml b/vector/sinks/vlogs_panos.yaml new file mode 100644 index 00000000..838c8d68 --- /dev/null +++ b/vector/sinks/vlogs_panos.yaml @@ -0,0 +1,26 @@ +inputs: + - remap_panos +type: elasticsearch +endpoints: + - ${VICTORIA_LOGS_ENDPOINT:-http://localhost:9428}/insert/elasticsearch/ +api_version: v8 +compression: gzip +healthcheck: + enabled: false +query: + _msg_field: message + _time_field: timestamp + _stream_fields: observer.product,observer.type,observer.vendor,observer.name,log.syslog.hostname,panos.device_name,panos.vsys,panos.type,panos.subtype,network.direction +request: + headers: + AccountID: "0" + ProjectID: "0" +### For performance optimization. Vector works really well with defaults. Don't use it unless you really need to fine-tune yor ingest. +buffer: + - type: memory + max_events: 12800 # default 500 https://www.elastic.co/docs/reference/fleet/es-output-settings#es-output-settings-performance-tuning-settings + #when_full: drop_newest #default block +batch: + #max_bytes: + max_events: 1600 # default 1000 + timeout_secs: 5 # default 1