diff --git a/.github/workflows/vector-validate.yml b/.github/workflows/vector-validate.yml index b51a11b9..7d28addf 100644 --- a/.github/workflows/vector-validate.yml +++ b/.github/workflows/vector-validate.yml @@ -55,12 +55,13 @@ jobs: fi # Normalize: sink files (sinks/*_PIPELINE.yaml) map to their parent pipeline yaml. - # e.g. sinks/vlogs_panos.yaml → panos.yaml + # Strip the sink-type prefix (vlogs_, elastic_, etc.), then take up to the first + # remaining underscore. Handles multi-word sinks like vlogs_fortigate_traffic → fortigate. normalized="" for file in $files; do case "$file" in sinks/*) - pipeline=$(basename "$file" .yaml | sed 's/.*_//') + pipeline=$(basename "$file" .yaml | sed 's/^vlogs_//;s/^elastic_//;s/^quickwit_//;s/^loki_//;s/^prometheus_//;s/_.*$//') normalized="$normalized ${pipeline}.yaml" ;; *) diff --git a/vector/cortex.yaml b/vector/cortex.yaml index ec9ec124..68089058 100644 --- a/vector/cortex.yaml +++ b/vector/cortex.yaml @@ -70,111 +70,3 @@ transforms: # delete null fields . = compact(.) -sinks: - # print: - # type: "console" - # inputs: ["remap_fortimail"] - # encoding: - # # codec: "raw_message" - # codec: "json" - # json: - # pretty: true - - vlogs_cortex: - inputs: - - remap_cortex - type: elasticsearch - endpoints: - - ${VICTORIA_LOGS_ENDPOINT:-http://localhost:9428}/insert/elasticsearch/ - api_version: v8 - compression: gzip - healthcheck: - enabled: false - query: - _msg_field: message - _time_field: timestamp - _stream_fields: observer.product,observer.type,observer.vendor,log.syslog.hostname - request: - headers: - AccountID: "0" - ProjectID: "0" -# ### For performance optimization. Vector works really well with defaults. Don't use it unless you really need to fine-tune yor ingest. -# buffer: -# - type: memory -# max_events: 12800 # default 500 https://www.elastic.co/docs/reference/fleet/es-output-settings#es-output-settings-performance-tuning-settings -# #when_full: drop_newest #default block -# batch: -# #max_bytes: -# max_events: 1600 # default 1000 -# timeout_secs: 5 # default 1 - -# elastic_fortimail: -# type: elasticsearch -# inputs: -# - remap_fortimail -# auth: -# strategy: "basic" -# user: "${ELASTICSEARCH_USER:-elastic}" -# password: "${ELASTICSEARCH_PASS:-myelasticsearchpassword}" -# endpoints: -# - ${ELASTICSEARCH_ENDPOINT:-https://localhost:9200} -# encoding: -# except_fields: -# - url.query -# mode: "data_stream" -# bulk: -# action: "create" -# data_stream: -# type: "logs" -# dataset: "fortinet.fortimail.{{fml.type}}" -# namespace: "default" -# ### For performance optimization. Vector works really well with defaults. Don't use it unless you really need to fine-tune yor ingest. -# buffer: -# - type: memory -# max_events: 12800 # default 500 https://www.elastic.co/docs/reference/fleet/es-output-settings#es-output-settings-performance-tuning-settings -# #when_full: drop_newest #default block -# batch: -# #max_bytes: -# max_events: 1600 # default 1000 -# timeout_secs: 5 # default 1 - -# quickwit_fortimail: -# type: "http" -# method: "post" -# inputs: -# - remap_fortimail -# encoding: -# codec: "json" -# framing: -# method: "newline_delimited" -# uri: "${QUICKWIT_ENDPOINT:-http://localhost:7280}/api/v1/logs-fortinet.fortimail.{{fml.type}}/ingest" - -# loki_fortimail: -# type: loki -# inputs: -# - remap_fortimail -# endpoint: ${LOKI_ENDPOINT:-http://localhost:3100} -# auth: -# strategy: "basic" -# user: "${LOKI_USER:-loki_user}" -# password: "${LOKI_PASS:-mylokipassword}" -# encoding: -# codec: "json" -# compression: "snappy" -# labels: -# "syslog_appname": "{{.log.syslog.appname}}" -# "syslog_hostname": "{{.log.syslog.hostname}}" -# "fml_type": "{{.fml.type}}" -# #structured_metadata: -# # "source_ip": "{{source.ip}}" -# # "destination_ip": "{{destination.ip}}" -# ### For performance optimization. Vector works really well with defaults. Don't use it unless you really need to fine-tune yor ingest. -# buffer: -# - type: memory -# max_events: 12800 # default 500 https://www.elastic.co/docs/reference/fleet/es-output-settings#es-output-settings-performance-tuning-settings -# #when_full: drop_newest #default block -# batch: -# #max_bytes: -# max_events: 1600 # default 1000 -# timeout_secs: 5 # default 1 - diff --git a/vector/fortiappsec.yaml b/vector/fortiappsec.yaml index 2cd2e06e..aec2a1df 100644 --- a/vector/fortiappsec.yaml +++ b/vector/fortiappsec.yaml @@ -86,111 +86,3 @@ transforms: # delete null fields . = compact(.) -sinks: - # print: - # type: "console" - # inputs: ["remap_fortiappsec"] - # encoding: - # # codec: "raw_message" - # codec: "json" - # json: - # pretty: true - - vlogs_fortiappsec: - inputs: - - remap_fortiappsec - type: elasticsearch - endpoints: - - ${VICTORIA_LOGS_ENDPOINT:-http://localhost:9428}/insert/elasticsearch/ - api_version: v8 - compression: gzip - healthcheck: - enabled: false - query: - _msg_field: message - _time_field: timestamp - _stream_fields: observer.product,observer.type,observer.vendor,log.syslog.hostname,fwb.cat - request: - headers: - AccountID: "0" - ProjectID: "0" - -# ### For performance optimization. Vector works really well with defaults. Don't use it unless you really need to fine-tune yor ingest. -# buffer: -# - type: memory -# max_events: 12800 # default 500 https://www.elastic.co/docs/reference/fleet/es-output-settings#es-output-settings-performance-tuning-settings -# #when_full: drop_newest #default block -# batch: -# #max_bytes: -# max_events: 1600 # default 1000 -# timeout_secs: 5 # default 1 - -# elastic_fortiappsec: -# type: elasticsearch -# inputs: -# - remap_fortimail -# auth: -# strategy: "basic" -# user: "${ELASTICSEARCH_USER:-elastic}" -# password: "${ELASTICSEARCH_PASS:-myelasticsearchpassword}" -# endpoints: -# - ${ELASTICSEARCH_ENDPOINT:-https://localhost:9200} -# encoding: -# except_fields: -# - url.query -# mode: "data_stream" -# bulk: -# action: "create" -# data_stream: -# type: "logs" -# dataset: "fortinet.fortiappsec}" -# namespace: "default" -# ### For performance optimization. Vector works really well with defaults. Don't use it unless you really need to fine-tune yor ingest. -# buffer: -# - type: memory -# max_events: 12800 # default 500 https://www.elastic.co/docs/reference/fleet/es-output-settings#es-output-settings-performance-tuning-settings -# #when_full: drop_newest #default block -# batch: -# #max_bytes: -# max_events: 1600 # default 1000 -# timeout_secs: 5 # default 1 - -# quickwit_fortiappsec: -# type: "http" -# method: "post" -# inputs: -# - remap_fortimail -# encoding: -# codec: "json" -# framing: -# method: "newline_delimited" -# uri: "${QUICKWIT_ENDPOINT:-http://localhost:7280}/api/v1/logs-fortiappsec/ingest" - -# loki_fortiappsec: -# type: loki -# inputs: -# - remap_fortimail -# endpoint: ${LOKI_ENDPOINT:-http://localhost:3100} -# auth: -# strategy: "basic" -# user: "${LOKI_USER:-loki_user}" -# password: "${LOKI_PASS:-mylokipassword}" -# encoding: -# codec: "json" -# compression: "snappy" -# labels: -# "syslog_appname": "{{.log.syslog.appname}}" -# "syslog_hostname": "{{.log.syslog.hostname}}" -# "fml_type": "{{.fml.type}}" -# #structured_metadata: -# # "source_ip": "{{source.ip}}" -# # "destination_ip": "{{destination.ip}}" -# ### For performance optimization. Vector works really well with defaults. Don't use it unless you really need to fine-tune yor ingest. -# buffer: -# - type: memory -# max_events: 12800 # default 500 https://www.elastic.co/docs/reference/fleet/es-output-settings#es-output-settings-performance-tuning-settings -# #when_full: drop_newest #default block -# batch: -# #max_bytes: -# max_events: 1600 # default 1000 -# timeout_secs: 5 # default 1 diff --git a/vector/fortiedr.yaml b/vector/fortiedr.yaml index 46b2fe41..0164dfc4 100644 --- a/vector/fortiedr.yaml +++ b/vector/fortiedr.yaml @@ -88,31 +88,3 @@ transforms: # delete null fields . = compact(.) -sinks: -# print: -# type: "console" -# inputs: ["remap_fortiedr"] -# encoding: -# # codec: "raw_message" -# codec: "json" -# json: -# pretty: true - - vlogs_fortiedr: - inputs: - - remap_fortiedr - type: elasticsearch - endpoints: - - ${VICTORIA_LOGS_ENDPOINT:-http://localhost:9428}/insert/elasticsearch/ - api_version: v8 - compression: gzip - healthcheck: - enabled: false - query: - _msg_field: message - _time_field: timestamp - _stream_fields: observer.product,observer.type,observer.vendor,log.syslog.hostname,fedr.Message Type - request: - headers: - AccountID: "0" - ProjectID: "0" diff --git a/vector/fortigate.yaml b/vector/fortigate.yaml index 0e7596f1..67029c1a 100644 --- a/vector/fortigate.yaml +++ b/vector/fortigate.yaml @@ -539,155 +539,3 @@ transforms: . = compact(.) # Print parsed logs to stdout -sinks: - # print: - # type: "console" - # inputs: ["route_fortigate._unmatched"] - # encoding: - # # codec: "raw_message" - # codec: "json" - # json: - # pretty: true - - vlogs_fortigate_traffic: - inputs: - - remap_fortigate_traffic - type: elasticsearch - endpoints: - - ${VICTORIA_LOGS_ENDPOINT:-http://localhost:9428}/insert/elasticsearch/ - api_version: v8 - compression: gzip - healthcheck: - enabled: false - query: - _msg_field: message - _time_field: timestamp - _stream_fields: observer.product,observer.type,observer.vendor,observer.name,log.syslog.hostname,fgt.vd,fgt.type,fgt.subtype,fgt.policytype,network.direction,fgt.logid - request: - headers: - AccountID: "0" - ProjectID: "0" - ### For performance optimization. Vector works really well with defaults. Don't use it unless you really need to fine-tune yor ingest. - buffer: - - type: memory - max_events: 12800 # default 500 https://www.elastic.co/docs/reference/fleet/es-output-settings#es-output-settings-performance-tuning-settings - #when_full: drop_newest #default block - batch: - #max_bytes: - max_events: 1600 # default 1000 - timeout_secs: 5 # default 1 - - vlogs_fortigate: - inputs: - #- remap_fortigate_traffic - - remap_fortigate_utm - - remap_fortigate_event - - route_fortigate._unmatched - type: elasticsearch - endpoints: - - ${VICTORIA_LOGS_ENDPOINT:-http://localhost:9428}/insert/elasticsearch/ - api_version: v8 - compression: gzip - healthcheck: - enabled: false - query: - _msg_field: message - _time_field: timestamp - _stream_fields: observer.product,observer.type,observer.vendor,observer.name,log.syslog.hostname,fgt.vd,fgt.type,fgt.subtype,fgt.policytype,network.direction - request: - headers: - AccountID: "0" - ProjectID: "0" - ### For performance optimization. Vector works really well with defaults. Don't use it unless you really need to fine-tune yor ingest. - buffer: - - type: memory - max_events: 12800 # default 500 https://www.elastic.co/docs/reference/fleet/es-output-settings#es-output-settings-performance-tuning-settings - #when_full: drop_newest #default block - batch: - #max_bytes: - max_events: 1600 # default 1000 - timeout_secs: 5 # default 1 - -# elastic_fortigate: -# type: elasticsearch -# inputs: -# - remap_fortigate_traffic -# - remap_fortigate_utm -# - remap_fortigate_event -# auth: -# strategy: "basic" -# user: "${ELASTICSEARCH_USER:-elastic}" -# password: "${ELASTICSEARCH_PASS:-myelasticsearchpassword}" -# endpoints: -# - ${ELASTICSEARCH_ENDPOINT:-https://localhost:9200} -# encoding: -# except_fields: -# - url.query -# mode: "data_stream" -# bulk: -# action: "create" -# data_stream: -# type: "logs" -# dataset: "fortinet.fortigate.{{fgt.type}}" -# namespace: "default" -# ### For performance optimization. Vector works really well with defaults. Don't use it unless you really need to fine-tune yor ingest. -# buffer: -# - type: memory -# max_events: 12800 # default 500 https://www.elastic.co/docs/reference/fleet/es-output-settings#es-output-settings-performance-tuning-settings -# #when_full: drop_newest #default block -# batch: -# #max_bytes: -# max_events: 1600 # default 1000 -# timeout_secs: 5 # default 1 - -# quickwit_fortigate: -# type: "http" -# method: "post" -# inputs: -# - remap_fortigate_traffic -# - remap_fortigate_utm -# - remap_fortigate_event -# encoding: -# codec: "json" -# framing: -# method: "newline_delimited" -# uri: "${QUICKWIT_ENDPOINT:-http://localhost:7280}/api/v1/logs-fortinet.fortigate.{{fgt.type}}/ingest" - -# loki_fortigate: -# type: loki -# inputs: -# - remap_fortigate_traffic -# - remap_fortigate_utm -# - remap_fortigate_event -# - route_fortigate._unmatched -# endpoint: ${LOKI_ENDPOINT:-http://localhost:3100} -# auth: -# strategy: "basic" -# user: "${LOKI_USER:-loki_user}" -# password: "${LOKI_PASS:-mylokipassword}" -# encoding: -# codec: "json" -# compression: "snappy" -# labels: -# "syslog_appname": "{{.log.syslog.appname}}" -# "syslog_hostname": "{{.log.syslog.hostname}}" -# "fg_vdom": "{{.fgt.vd}}" -# "fgt_type": "{{.fgt.type}}" -# "fgt_subtype": "{{.fgt.subtype}}" -# #"fgt_policytype": "{{.fgt.policytype}}" -# #"fgt_eventtype": "{{.fgt.eventtype}}" -# #"network_direction": "{{.network.direction}}" -# #structured_metadata: -# # "source_ip": "{{source.ip}}" -# # "destination_ip": "{{destination.ip}}" -# # "destination_port": "{{destination.port}}" -# # "network_transport_port": "{{network.transport_port}}" -# ### For performance optimization. Vector works really well with defaults. Don't use it unless you really need to fine-tune yor ingest. -# buffer: -# - type: memory -# max_events: 12800 # default 500 https://www.elastic.co/docs/reference/fleet/es-output-settings#es-output-settings-performance-tuning-settings -# #when_full: drop_newest #default block -# batch: -# #max_bytes: -# max_events: 1600 # default 1000 -# timeout_secs: 5 # default 1 diff --git a/vector/fortimail.yaml b/vector/fortimail.yaml index 3844012a..abee19f8 100644 --- a/vector/fortimail.yaml +++ b/vector/fortimail.yaml @@ -77,112 +77,3 @@ transforms: # delete null fields . = compact(.) - -sinks: -# print: -# type: "console" -# inputs: ["remap_fortimail"] -# encoding: -# # codec: "raw_message" -# codec: "json" -# json: -# pretty: true - - vlogs_fortimail: - inputs: - - remap_fortimail - type: elasticsearch - endpoints: - - ${VICTORIA_LOGS_ENDPOINT:-http://localhost:9428}/insert/elasticsearch/ - api_version: v8 - compression: gzip - healthcheck: - enabled: false - query: - _msg_field: message - _time_field: timestamp - _stream_fields: observer.product,observer.type,observer.vendor,log.syslog.hostname,fml.type,fml.subtype - request: - headers: - AccountID: "0" - ProjectID: "0" -# ### For performance optimization. Vector works really well with defaults. Don't use it unless you really need to fine-tune yor ingest. -# buffer: -# - type: memory -# max_events: 12800 # default 500 https://www.elastic.co/docs/reference/fleet/es-output-settings#es-output-settings-performance-tuning-settings -# #when_full: drop_newest #default block -# batch: -# #max_bytes: -# max_events: 1600 # default 1000 -# timeout_secs: 5 # default 1 - -# elastic_fortimail: -# type: elasticsearch -# inputs: -# - remap_fortimail -# auth: -# strategy: "basic" -# user: "${ELASTICSEARCH_USER:-elastic}" -# password: "${ELASTICSEARCH_PASS:-myelasticsearchpassword}" -# endpoints: -# - ${ELASTICSEARCH_ENDPOINT:-https://localhost:9200} -# encoding: -# except_fields: -# - url.query -# mode: "data_stream" -# bulk: -# action: "create" -# data_stream: -# type: "logs" -# dataset: "fortinet.fortimail.{{fml.type}}" -# namespace: "default" -# ### For performance optimization. Vector works really well with defaults. Don't use it unless you really need to fine-tune yor ingest. -# buffer: -# - type: memory -# max_events: 12800 # default 500 https://www.elastic.co/docs/reference/fleet/es-output-settings#es-output-settings-performance-tuning-settings -# #when_full: drop_newest #default block -# batch: -# #max_bytes: -# max_events: 1600 # default 1000 -# timeout_secs: 5 # default 1 - -# quickwit_fortimail: -# type: "http" -# method: "post" -# inputs: -# - remap_fortimail -# encoding: -# codec: "json" -# framing: -# method: "newline_delimited" -# uri: "${QUICKWIT_ENDPOINT:-http://localhost:7280}/api/v1/logs-fortinet.fortimail.{{fml.type}}/ingest" - -# loki_fortimail: -# type: loki -# inputs: -# - remap_fortimail -# endpoint: ${LOKI_ENDPOINT:-http://localhost:3100} -# auth: -# strategy: "basic" -# user: "${LOKI_USER:-loki_user}" -# password: "${LOKI_PASS:-mylokipassword}" -# encoding: -# codec: "json" -# compression: "snappy" -# labels: -# "syslog_appname": "{{.log.syslog.appname}}" -# "syslog_hostname": "{{.log.syslog.hostname}}" -# "fml_type": "{{.fml.type}}" -# #structured_metadata: -# # "source_ip": "{{source.ip}}" -# # "destination_ip": "{{destination.ip}}" -# ### For performance optimization. Vector works really well with defaults. Don't use it unless you really need to fine-tune yor ingest. -# buffer: -# - type: memory -# max_events: 12800 # default 500 https://www.elastic.co/docs/reference/fleet/es-output-settings#es-output-settings-performance-tuning-settings -# #when_full: drop_newest #default block -# batch: -# #max_bytes: -# max_events: 1600 # default 1000 -# timeout_secs: 5 # default 1 - diff --git a/vector/fortiweb.yaml b/vector/fortiweb.yaml index 0296a82f..387e0d31 100644 --- a/vector/fortiweb.yaml +++ b/vector/fortiweb.yaml @@ -97,113 +97,3 @@ transforms: # delete null fields . = compact(.) -sinks: - # print: - # type: "console" - # inputs: ["remap_fortiweb"] - # encoding: - # # codec: "raw_message" - # codec: "json" - # json: - # pretty: true - - vlogs_fortiweb: - inputs: - - remap_fortiweb - #- syslog_fortiweb - type: elasticsearch - endpoints: - - ${VICTORIA_LOGS_ENDPOINT:-http://localhost:9428}/insert/elasticsearch/ - api_version: v8 - compression: gzip - healthcheck: - enabled: false - query: - _msg_field: message - _time_field: timestamp - _stream_fields: observer.product,observer.type,observer.vendor,log.syslog.appname,log.syslog.hostname,fwb.type,fwb.subtype - request: - headers: - AccountID: "0" - ProjectID: "0" -# ### For performance optimization. Vector works really well with defaults. Don't use it unless you really need to fine-tune yor ingest. -# buffer: -# - type: memory -# max_events: 12800 # default 500 https://www.elastic.co/docs/reference/fleet/es-output-settings#es-output-settings-performance-tuning-settings -# #when_full: drop_newest #default block -# batch: -# #max_bytes: -# max_events: 1600 # default 1000 -# timeout_secs: 5 # default 1 - -# elastic_fortiweb: -# type: elasticsearch -# inputs: -# - remap_fortiweb -# auth: -# strategy: "basic" -# user: "${ELASTICSEARCH_USER:-elastic}" -# password: "${ELASTICSEARCH_PASS:-myelasticsearchpassword}" -# endpoints: -# - ${ELASTICSEARCH_ENDPOINT:-https://localhost:9200} -# encoding: -# except_fields: -# - url.query -# mode: "data_stream" -# bulk: -# action: "create" -# data_stream: -# type: "logs" -# dataset: "fortinet.fortiweb.{{fwb.type}}" -# namespace: "default" -# ### For performance optimization. Vector works really well with defaults. Don't use it unless you really need to fine-tune yor ingest. -# buffer: -# - type: memory -# max_events: 12800 # default 500 https://www.elastic.co/docs/reference/fleet/es-output-settings#es-output-settings-performance-tuning-settings -# #when_full: drop_newest #default block -# batch: -# #max_bytes: -# max_events: 1600 # default 1000 -# timeout_secs: 5 # default 1 - -# quickwit_fortimail: -# type: "http" -# method: "post" -# inputs: -# - remap_fortimail -# encoding: -# codec: "json" -# framing: -# method: "newline_delimited" -# uri: "${QUICKWIT_ENDPOINT:-http://localhost:7280}/api/v1/logs-fortinet.fortimail.{{fml.type}}/ingest" - -# loki_fortimail: -# type: loki -# inputs: -# - remap_fortimail -# endpoint: ${LOKI_ENDPOINT:-http://localhost:3100} -# auth: -# strategy: "basic" -# user: "${LOKI_USER:-loki_user}" -# password: "${LOKI_PASS:-mylokipassword}" -# encoding: -# codec: "json" -# compression: "snappy" -# labels: -# "syslog_appname": "{{.log.syslog.appname}}" -# "syslog_hostname": "{{.log.syslog.hostname}}" -# "fml_type": "{{.fml.type}}" -# #structured_metadata: -# # "source_ip": "{{source.ip}}" -# # "destination_ip": "{{destination.ip}}" -# ### For performance optimization. Vector works really well with defaults. Don't use it unless you really need to fine-tune yor ingest. -# buffer: -# - type: memory -# max_events: 12800 # default 500 https://www.elastic.co/docs/reference/fleet/es-output-settings#es-output-settings-performance-tuning-settings -# #when_full: drop_newest #default block -# batch: -# #max_bytes: -# max_events: 1600 # default 1000 -# timeout_secs: 5 # default 1 - - diff --git a/vector/sinks/elastic_cortex.yaml.disabled b/vector/sinks/elastic_cortex.yaml.disabled new file mode 100644 index 00000000..cac9ee3a --- /dev/null +++ b/vector/sinks/elastic_cortex.yaml.disabled @@ -0,0 +1,28 @@ +type: elasticsearch +inputs: + - remap_cortex +auth: + strategy: "basic" + user: "${ELASTICSEARCH_USER:-elastic}" + password: "${ELASTICSEARCH_PASS:-myelasticsearchpassword}" +endpoints: + - ${ELASTICSEARCH_ENDPOINT:-https://localhost:9200} +encoding: + except_fields: + - url.query +mode: "data_stream" +bulk: + action: "create" +data_stream: + type: "logs" + dataset: "panos.cortex" + namespace: "default" +### For performance optimization. Vector works really well with defaults. Don't use it unless you really need to fine-tune yor ingest. +buffer: + - type: memory + max_events: 12800 # default 500 https://www.elastic.co/docs/reference/fleet/es-output-settings#es-output-settings-performance-tuning-settings + #when_full: drop_newest #default block +batch: + #max_bytes: + max_events: 1600 # default 1000 + timeout_secs: 5 # default 1 diff --git a/vector/sinks/elastic_fortiappsec.yaml.disabled b/vector/sinks/elastic_fortiappsec.yaml.disabled new file mode 100644 index 00000000..ab93baff --- /dev/null +++ b/vector/sinks/elastic_fortiappsec.yaml.disabled @@ -0,0 +1,28 @@ +type: elasticsearch +inputs: + - remap_fortiappsec +auth: + strategy: "basic" + user: "${ELASTICSEARCH_USER:-elastic}" + password: "${ELASTICSEARCH_PASS:-myelasticsearchpassword}" +endpoints: + - ${ELASTICSEARCH_ENDPOINT:-https://localhost:9200} +encoding: + except_fields: + - url.query +mode: "data_stream" +bulk: + action: "create" +data_stream: + type: "logs" + dataset: "fortinet.fortiappsec" + namespace: "default" +### For performance optimization. Vector works really well with defaults. Don't use it unless you really need to fine-tune yor ingest. +buffer: + - type: memory + max_events: 12800 # default 500 https://www.elastic.co/docs/reference/fleet/es-output-settings#es-output-settings-performance-tuning-settings + #when_full: drop_newest #default block +batch: + #max_bytes: + max_events: 1600 # default 1000 + timeout_secs: 5 # default 1 diff --git a/vector/sinks/elastic_fortigate.yaml.disabled b/vector/sinks/elastic_fortigate.yaml.disabled new file mode 100644 index 00000000..7619c415 --- /dev/null +++ b/vector/sinks/elastic_fortigate.yaml.disabled @@ -0,0 +1,30 @@ +type: elasticsearch +inputs: + - remap_fortigate_traffic + - remap_fortigate_utm + - remap_fortigate_event +auth: + strategy: "basic" + user: "${ELASTICSEARCH_USER:-elastic}" + password: "${ELASTICSEARCH_PASS:-myelasticsearchpassword}" +endpoints: + - ${ELASTICSEARCH_ENDPOINT:-https://localhost:9200} +encoding: + except_fields: + - url.query +mode: "data_stream" +bulk: + action: "create" +data_stream: + type: "logs" + dataset: "fortinet.fortigate.{{fgt.type}}" + namespace: "default" +### For performance optimization. Vector works really well with defaults. Don't use it unless you really need to fine-tune yor ingest. +buffer: + - type: memory + max_events: 12800 # default 500 https://www.elastic.co/docs/reference/fleet/es-output-settings#es-output-settings-performance-tuning-settings + #when_full: drop_newest #default block +batch: + #max_bytes: + max_events: 1600 # default 1000 + timeout_secs: 5 # default 1 diff --git a/vector/sinks/elastic_fortimail.yaml.disabled b/vector/sinks/elastic_fortimail.yaml.disabled new file mode 100644 index 00000000..42bcc46d --- /dev/null +++ b/vector/sinks/elastic_fortimail.yaml.disabled @@ -0,0 +1,28 @@ +type: elasticsearch +inputs: + - remap_fortimail +auth: + strategy: "basic" + user: "${ELASTICSEARCH_USER:-elastic}" + password: "${ELASTICSEARCH_PASS:-myelasticsearchpassword}" +endpoints: + - ${ELASTICSEARCH_ENDPOINT:-https://localhost:9200} +encoding: + except_fields: + - url.query +mode: "data_stream" +bulk: + action: "create" +data_stream: + type: "logs" + dataset: "fortinet.fortimail.{{fml.type}}" + namespace: "default" +### For performance optimization. Vector works really well with defaults. Don't use it unless you really need to fine-tune yor ingest. +buffer: + - type: memory + max_events: 12800 # default 500 https://www.elastic.co/docs/reference/fleet/es-output-settings#es-output-settings-performance-tuning-settings + #when_full: drop_newest #default block +batch: + #max_bytes: + max_events: 1600 # default 1000 + timeout_secs: 5 # default 1 diff --git a/vector/sinks/elastic_fortiweb.yaml.disabled b/vector/sinks/elastic_fortiweb.yaml.disabled new file mode 100644 index 00000000..b1c85e99 --- /dev/null +++ b/vector/sinks/elastic_fortiweb.yaml.disabled @@ -0,0 +1,28 @@ +type: elasticsearch +inputs: + - remap_fortiweb +auth: + strategy: "basic" + user: "${ELASTICSEARCH_USER:-elastic}" + password: "${ELASTICSEARCH_PASS:-myelasticsearchpassword}" +endpoints: + - ${ELASTICSEARCH_ENDPOINT:-https://localhost:9200} +encoding: + except_fields: + - url.query +mode: "data_stream" +bulk: + action: "create" +data_stream: + type: "logs" + dataset: "fortinet.fortiweb.{{fwb.type}}" + namespace: "default" +### For performance optimization. Vector works really well with defaults. Don't use it unless you really need to fine-tune yor ingest. +buffer: + - type: memory + max_events: 12800 # default 500 https://www.elastic.co/docs/reference/fleet/es-output-settings#es-output-settings-performance-tuning-settings + #when_full: drop_newest #default block +batch: + #max_bytes: + max_events: 1600 # default 1000 + timeout_secs: 5 # default 1 diff --git a/vector/sinks/elastic_panos.yaml b/vector/sinks/elastic_panos.yaml.disabled similarity index 100% rename from vector/sinks/elastic_panos.yaml rename to vector/sinks/elastic_panos.yaml.disabled diff --git a/vector/sinks/loki_cortex.yaml.disabled b/vector/sinks/loki_cortex.yaml.disabled new file mode 100644 index 00000000..b600b9db --- /dev/null +++ b/vector/sinks/loki_cortex.yaml.disabled @@ -0,0 +1,25 @@ +type: loki +inputs: + - remap_cortex +endpoint: ${LOKI_ENDPOINT:-http://localhost:3100} +auth: + strategy: "basic" + user: "${LOKI_USER:-loki_user}" + password: "${LOKI_PASS:-mylokipassword}" +encoding: + codec: "json" +compression: "snappy" +labels: + "product": "{{observer.product}}" + "type": "{{observer.type}}" + "vendor": "{{observer.vendor}}" + "syslog_hostname": "{{log.syslog.hostname}}" +### For performance optimization. Vector works really well with defaults. Don't use it unless you really need to fine-tune yor ingest. +buffer: + - type: memory + max_events: 12800 # default 500 https://www.elastic.co/docs/reference/fleet/es-output-settings#es-output-settings-performance-tuning-settings + #when_full: drop_newest #default block +batch: + #max_bytes: + max_events: 1600 # default 1000 + timeout_secs: 5 # default 1 diff --git a/vector/sinks/loki_fortiappsec.yaml.disabled b/vector/sinks/loki_fortiappsec.yaml.disabled new file mode 100644 index 00000000..d2df19d8 --- /dev/null +++ b/vector/sinks/loki_fortiappsec.yaml.disabled @@ -0,0 +1,25 @@ +type: loki +inputs: + - remap_fortiappsec +endpoint: ${LOKI_ENDPOINT:-http://localhost:3100} +auth: + strategy: "basic" + user: "${LOKI_USER:-loki_user}" + password: "${LOKI_PASS:-mylokipassword}" +encoding: + codec: "json" +compression: "snappy" +labels: + "product": "{{observer.product}}" + "type": "{{observer.type}}" + "vendor": "{{observer.vendor}}" + "syslog_hostname": "{{log.syslog.hostname}}" +### For performance optimization. Vector works really well with defaults. Don't use it unless you really need to fine-tune yor ingest. +buffer: + - type: memory + max_events: 12800 # default 500 https://www.elastic.co/docs/reference/fleet/es-output-settings#es-output-settings-performance-tuning-settings + #when_full: drop_newest #default block +batch: + #max_bytes: + max_events: 1600 # default 1000 + timeout_secs: 5 # default 1 diff --git a/vector/sinks/loki_fortigate.yaml.disabled b/vector/sinks/loki_fortigate.yaml.disabled new file mode 100644 index 00000000..287ac896 --- /dev/null +++ b/vector/sinks/loki_fortigate.yaml.disabled @@ -0,0 +1,34 @@ +type: loki +inputs: + - remap_fortigate_traffic + - remap_fortigate_utm + - remap_fortigate_event + - route_fortigate._unmatched +endpoint: ${LOKI_ENDPOINT:-http://localhost:3100} +auth: + strategy: "basic" + user: "${LOKI_USER:-loki_user}" + password: "${LOKI_PASS:-mylokipassword}" +encoding: + codec: "json" +compression: "snappy" +labels: + "product": "{{observer.product}}" + "type": "{{observer.type}}" + "vendor": "{{observer.vendor}}" + "syslog_hostname": "{{log.syslog.hostname}}" + "fgt_vdom": "{{fgt.vd}}" + "fgt_type": "{{fgt.type}}" + "fgt_subtype": "{{fgt.subtype}}" + #"fgt_policytype": "{{fgt.policytype}}" + #"fgt_eventtype": "{{fgt.eventtype}}" + #"network_direction": "{{network.direction}}" +### For performance optimization. Vector works really well with defaults. Don't use it unless you really need to fine-tune yor ingest. +buffer: + - type: memory + max_events: 12800 # default 500 https://www.elastic.co/docs/reference/fleet/es-output-settings#es-output-settings-performance-tuning-settings + #when_full: drop_newest #default block +batch: + #max_bytes: + max_events: 1600 # default 1000 + timeout_secs: 5 # default 1 diff --git a/vector/sinks/loki_fortimail.yaml.disabled b/vector/sinks/loki_fortimail.yaml.disabled new file mode 100644 index 00000000..e641f708 --- /dev/null +++ b/vector/sinks/loki_fortimail.yaml.disabled @@ -0,0 +1,25 @@ +type: loki +inputs: + - remap_fortimail +endpoint: ${LOKI_ENDPOINT:-http://localhost:3100} +auth: + strategy: "basic" + user: "${LOKI_USER:-loki_user}" + password: "${LOKI_PASS:-mylokipassword}" +encoding: + codec: "json" +compression: "snappy" +labels: + "product": "{{observer.product}}" + "type": "{{observer.type}}" + "vendor": "{{observer.vendor}}" + "syslog_hostname": "{{log.syslog.hostname}}" +### For performance optimization. Vector works really well with defaults. Don't use it unless you really need to fine-tune yor ingest. +buffer: + - type: memory + max_events: 12800 # default 500 https://www.elastic.co/docs/reference/fleet/es-output-settings#es-output-settings-performance-tuning-settings + #when_full: drop_newest #default block +batch: + #max_bytes: + max_events: 1600 # default 1000 + timeout_secs: 5 # default 1 diff --git a/vector/sinks/loki_fortiweb.yaml.disabled b/vector/sinks/loki_fortiweb.yaml.disabled new file mode 100644 index 00000000..293b1288 --- /dev/null +++ b/vector/sinks/loki_fortiweb.yaml.disabled @@ -0,0 +1,25 @@ +type: loki +inputs: + - remap_fortiweb +endpoint: ${LOKI_ENDPOINT:-http://localhost:3100} +auth: + strategy: "basic" + user: "${LOKI_USER:-loki_user}" + password: "${LOKI_PASS:-mylokipassword}" +encoding: + codec: "json" +compression: "snappy" +labels: + "product": "{{observer.product}}" + "type": "{{observer.type}}" + "vendor": "{{observer.vendor}}" + "syslog_hostname": "{{log.syslog.hostname}}" +### For performance optimization. Vector works really well with defaults. Don't use it unless you really need to fine-tune yor ingest. +buffer: + - type: memory + max_events: 12800 # default 500 https://www.elastic.co/docs/reference/fleet/es-output-settings#es-output-settings-performance-tuning-settings + #when_full: drop_newest #default block +batch: + #max_bytes: + max_events: 1600 # default 1000 + timeout_secs: 5 # default 1 diff --git a/vector/sinks/loki_panos.yaml b/vector/sinks/loki_panos.yaml.disabled similarity index 100% rename from vector/sinks/loki_panos.yaml rename to vector/sinks/loki_panos.yaml.disabled diff --git a/vector/sinks/quickwit_cortex.yaml.disabled b/vector/sinks/quickwit_cortex.yaml.disabled new file mode 100644 index 00000000..fc12c8f3 --- /dev/null +++ b/vector/sinks/quickwit_cortex.yaml.disabled @@ -0,0 +1,9 @@ +type: "http" +method: "post" +inputs: + - remap_cortex +encoding: + codec: "json" +framing: + method: "newline_delimited" +uri: "${QUICKWIT_ENDPOINT:-http://localhost:7280}/api/v1/panos.cortex/ingest" diff --git a/vector/sinks/quickwit_fortiappsec.yaml.disabled b/vector/sinks/quickwit_fortiappsec.yaml.disabled new file mode 100644 index 00000000..a6c7688b --- /dev/null +++ b/vector/sinks/quickwit_fortiappsec.yaml.disabled @@ -0,0 +1,9 @@ +type: "http" +method: "post" +inputs: + - remap_fortiappsec +encoding: + codec: "json" +framing: + method: "newline_delimited" +uri: "${QUICKWIT_ENDPOINT:-http://localhost:7280}/api/v1/logs-fortiappsec/ingest" diff --git a/vector/sinks/quickwit_fortigate.yaml.disabled b/vector/sinks/quickwit_fortigate.yaml.disabled new file mode 100644 index 00000000..c4bc6cb1 --- /dev/null +++ b/vector/sinks/quickwit_fortigate.yaml.disabled @@ -0,0 +1,11 @@ +type: "http" +method: "post" +inputs: + - remap_fortigate_traffic + - remap_fortigate_utm + - remap_fortigate_event +encoding: + codec: "json" +framing: + method: "newline_delimited" +uri: "${QUICKWIT_ENDPOINT:-http://localhost:7280}/api/v1/logs-fortinet.fortigate.{{fgt.type}}/ingest" diff --git a/vector/sinks/quickwit_fortimail.yaml.disabled b/vector/sinks/quickwit_fortimail.yaml.disabled new file mode 100644 index 00000000..8ed1cd8d --- /dev/null +++ b/vector/sinks/quickwit_fortimail.yaml.disabled @@ -0,0 +1,9 @@ +type: "http" +method: "post" +inputs: + - remap_fortimail +encoding: + codec: "json" +framing: + method: "newline_delimited" +uri: "${QUICKWIT_ENDPOINT:-http://localhost:7280}/api/v1/logs-fortinet.fortimail.{{fml.type}}/ingest" diff --git a/vector/sinks/quickwit_fortiweb.yaml.disabled b/vector/sinks/quickwit_fortiweb.yaml.disabled new file mode 100644 index 00000000..75d1671e --- /dev/null +++ b/vector/sinks/quickwit_fortiweb.yaml.disabled @@ -0,0 +1,9 @@ +type: "http" +method: "post" +inputs: + - remap_fortiweb +encoding: + codec: "json" +framing: + method: "newline_delimited" +uri: "${QUICKWIT_ENDPOINT:-http://localhost:7280}/api/v1/logs-fortinet.fortiweb.{{fwb.type}}/ingest" diff --git a/vector/sinks/quickwit_panos.yaml b/vector/sinks/quickwit_panos.yaml.disabled similarity index 100% rename from vector/sinks/quickwit_panos.yaml rename to vector/sinks/quickwit_panos.yaml.disabled diff --git a/vector/sinks/vlogs_cortex.yaml b/vector/sinks/vlogs_cortex.yaml new file mode 100644 index 00000000..8f3baa14 --- /dev/null +++ b/vector/sinks/vlogs_cortex.yaml @@ -0,0 +1,17 @@ +inputs: + - remap_cortex +type: elasticsearch +endpoints: + - ${VICTORIA_LOGS_ENDPOINT:-http://localhost:9428}/insert/elasticsearch/ +api_version: v8 +compression: gzip +healthcheck: + enabled: false +query: + _msg_field: message + _time_field: timestamp + _stream_fields: observer.product,observer.type,observer.vendor,log.syslog.hostname +request: + headers: + AccountID: "0" + ProjectID: "0" diff --git a/vector/sinks/vlogs_fortiappsec.yaml b/vector/sinks/vlogs_fortiappsec.yaml new file mode 100644 index 00000000..f7580d96 --- /dev/null +++ b/vector/sinks/vlogs_fortiappsec.yaml @@ -0,0 +1,17 @@ +inputs: + - remap_fortiappsec +type: elasticsearch +endpoints: + - ${VICTORIA_LOGS_ENDPOINT:-http://localhost:9428}/insert/elasticsearch/ +api_version: v8 +compression: gzip +healthcheck: + enabled: false +query: + _msg_field: message + _time_field: timestamp + _stream_fields: observer.product,observer.type,observer.vendor,log.syslog.hostname,fwb.cat +request: + headers: + AccountID: "0" + ProjectID: "0" diff --git a/vector/sinks/vlogs_fortiedr.yaml b/vector/sinks/vlogs_fortiedr.yaml new file mode 100644 index 00000000..c195e188 --- /dev/null +++ b/vector/sinks/vlogs_fortiedr.yaml @@ -0,0 +1,17 @@ +inputs: + - remap_fortiedr +type: elasticsearch +endpoints: + - ${VICTORIA_LOGS_ENDPOINT:-http://localhost:9428}/insert/elasticsearch/ +api_version: v8 +compression: gzip +healthcheck: + enabled: false +query: + _msg_field: message + _time_field: timestamp + _stream_fields: observer.product,observer.type,observer.vendor,log.syslog.hostname,fedr.Message Type +request: + headers: + AccountID: "0" + ProjectID: "0" diff --git a/vector/sinks/vlogs_fortigate.yaml b/vector/sinks/vlogs_fortigate.yaml new file mode 100644 index 00000000..1d2b1d33 --- /dev/null +++ b/vector/sinks/vlogs_fortigate.yaml @@ -0,0 +1,29 @@ +inputs: + #- remap_fortigate_traffic + - remap_fortigate_utm + - remap_fortigate_event + - route_fortigate._unmatched +type: elasticsearch +endpoints: + - ${VICTORIA_LOGS_ENDPOINT:-http://localhost:9428}/insert/elasticsearch/ +api_version: v8 +compression: gzip +healthcheck: + enabled: false +query: + _msg_field: message + _time_field: timestamp + _stream_fields: observer.product,observer.type,observer.vendor,observer.name,log.syslog.hostname,fgt.vd,fgt.type,fgt.subtype,fgt.policytype,network.direction +request: + headers: + AccountID: "0" + ProjectID: "0" +### For performance optimization. Vector works really well with defaults. Don't use it unless you really need to fine-tune yor ingest. +buffer: + - type: memory + max_events: 12800 # default 500 https://www.elastic.co/docs/reference/fleet/es-output-settings#es-output-settings-performance-tuning-settings + #when_full: drop_newest #default block +batch: + #max_bytes: + max_events: 1600 # default 1000 + timeout_secs: 5 # default 1 diff --git a/vector/sinks/vlogs_fortigate_traffic.yaml b/vector/sinks/vlogs_fortigate_traffic.yaml new file mode 100644 index 00000000..22aff2b8 --- /dev/null +++ b/vector/sinks/vlogs_fortigate_traffic.yaml @@ -0,0 +1,26 @@ +inputs: + - remap_fortigate_traffic +type: elasticsearch +endpoints: + - ${VICTORIA_LOGS_ENDPOINT:-http://localhost:9428}/insert/elasticsearch/ +api_version: v8 +compression: gzip +healthcheck: + enabled: false +query: + _msg_field: message + _time_field: timestamp + _stream_fields: observer.product,observer.type,observer.vendor,observer.name,log.syslog.hostname,fgt.vd,fgt.type,fgt.subtype,fgt.policytype,network.direction,fgt.logid +request: + headers: + AccountID: "0" + ProjectID: "0" +### For performance optimization. Vector works really well with defaults. Don't use it unless you really need to fine-tune yor ingest. +buffer: + - type: memory + max_events: 12800 # default 500 https://www.elastic.co/docs/reference/fleet/es-output-settings#es-output-settings-performance-tuning-settings + #when_full: drop_newest #default block +batch: + #max_bytes: + max_events: 1600 # default 1000 + timeout_secs: 5 # default 1 diff --git a/vector/sinks/vlogs_fortimail.yaml b/vector/sinks/vlogs_fortimail.yaml new file mode 100644 index 00000000..7da628cf --- /dev/null +++ b/vector/sinks/vlogs_fortimail.yaml @@ -0,0 +1,17 @@ +inputs: + - remap_fortimail +type: elasticsearch +endpoints: + - ${VICTORIA_LOGS_ENDPOINT:-http://localhost:9428}/insert/elasticsearch/ +api_version: v8 +compression: gzip +healthcheck: + enabled: false +query: + _msg_field: message + _time_field: timestamp + _stream_fields: observer.product,observer.type,observer.vendor,log.syslog.hostname,fml.type,fml.subtype +request: + headers: + AccountID: "0" + ProjectID: "0" diff --git a/vector/sinks/vlogs_fortiweb.yaml b/vector/sinks/vlogs_fortiweb.yaml new file mode 100644 index 00000000..5a255aa5 --- /dev/null +++ b/vector/sinks/vlogs_fortiweb.yaml @@ -0,0 +1,17 @@ +inputs: + - remap_fortiweb +type: elasticsearch +endpoints: + - ${VICTORIA_LOGS_ENDPOINT:-http://localhost:9428}/insert/elasticsearch/ +api_version: v8 +compression: gzip +healthcheck: + enabled: false +query: + _msg_field: message + _time_field: timestamp + _stream_fields: observer.product,observer.type,observer.vendor,log.syslog.appname,log.syslog.hostname,fwb.type,fwb.subtype +request: + headers: + AccountID: "0" + ProjectID: "0"