feat(vector): split sinks for all remaining pipelines into vector/sinks/

.github/workflows/vector-validate.yml

@@ -55,12 +55,13 @@ jobs:
           fi
 
           # Normalize: sink files (sinks/*_PIPELINE.yaml) map to their parent pipeline yaml.
-          # e.g. sinks/vlogs_panos.yaml → panos.yaml
+          # Strip the sink-type prefix (vlogs_, elastic_, etc.), then take up to the first
+          # remaining underscore. Handles multi-word sinks like vlogs_fortigate_traffic → fortigate.
           normalized=""
           for file in $files; do
             case "$file" in
               sinks/*)
-                pipeline=$(basename "$file" .yaml | sed 's/.*_//')
+                pipeline=$(basename "$file" .yaml | sed 's/^vlogs_//;s/^elastic_//;s/^quickwit_//;s/^loki_//;s/^prometheus_//;s/_.*$//')
                 normalized="$normalized ${pipeline}.yaml"
                 ;;
               *)

vector/cortex.yaml

@@ -70,111 +70,3 @@ transforms:
       # delete null fields
       . = compact(.)
 
-sinks:
-  #  print:
-  #   type: "console"
-  #   inputs: ["remap_fortimail"]
-  #   encoding:
-  #        #      codec: "raw_message"
-  #     codec: "json"
-  #     json:
-  #        pretty: true
-
-  vlogs_cortex:
-    inputs:
-      - remap_cortex
-    type: elasticsearch
-    endpoints:
-      - ${VICTORIA_LOGS_ENDPOINT:-http://localhost:9428}/insert/elasticsearch/
-    api_version: v8
-    compression: gzip
-    healthcheck:
-      enabled: false
-    query:
-      _msg_field: message
-      _time_field: timestamp
-      _stream_fields: observer.product,observer.type,observer.vendor,log.syslog.hostname
-    request:
-      headers:
-        AccountID: "0"
-        ProjectID: "0"
-#    ### For performance optimization. Vector works really well with defaults. Don't use it unless you really need to fine-tune yor ingest.
-#    buffer:
-#    - type: memory
-#      max_events: 12800 # default 500 https://www.elastic.co/docs/reference/fleet/es-output-settings#es-output-settings-performance-tuning-settings
-#      #when_full: drop_newest #default block
-#    batch:
-#      #max_bytes:
-#      max_events: 1600 # default 1000
-#      timeout_secs: 5 # default 1
-
-#  elastic_fortimail:
-#    type: elasticsearch
-#    inputs:
-#      - remap_fortimail
-#    auth:
-#      strategy: "basic"
-#      user: "${ELASTICSEARCH_USER:-elastic}"
-#      password: "${ELASTICSEARCH_PASS:-myelasticsearchpassword}"
-#    endpoints:
-#      - ${ELASTICSEARCH_ENDPOINT:-https://localhost:9200}
-#    encoding:
-#      except_fields:
-#        - url.query
-#    mode: "data_stream"
-#    bulk:
-#      action: "create"
-#    data_stream:
-#      type: "logs"
-#      dataset: "fortinet.fortimail.{{fml.type}}"
-#      namespace: "default"
-#    ### For performance optimization. Vector works really well with defaults. Don't use it unless you really need to fine-tune yor ingest.
-#    buffer:
-#    - type: memory
-#      max_events: 12800 # default 500 https://www.elastic.co/docs/reference/fleet/es-output-settings#es-output-settings-performance-tuning-settings
-#      #when_full: drop_newest #default block
-#    batch:
-#      #max_bytes:
-#      max_events: 1600 # default 1000
-#      timeout_secs: 5 # default 1
-
-#  quickwit_fortimail:
-#    type: "http"
-#    method: "post"
-#    inputs:
-#      - remap_fortimail
-#    encoding:
-#      codec: "json"
-#    framing:
-#      method: "newline_delimited"
-#    uri: "${QUICKWIT_ENDPOINT:-http://localhost:7280}/api/v1/logs-fortinet.fortimail.{{fml.type}}/ingest"
-
-#  loki_fortimail:
-#    type: loki
-#    inputs:
-#      - remap_fortimail
-#    endpoint: ${LOKI_ENDPOINT:-http://localhost:3100}
-#    auth:
-#      strategy: "basic"
-#      user: "${LOKI_USER:-loki_user}"
-#      password: "${LOKI_PASS:-mylokipassword}"
-#    encoding:
-#      codec: "json"
-#    compression: "snappy"
-#    labels:
-#      "syslog_appname": "{{.log.syslog.appname}}"
-#      "syslog_hostname": "{{.log.syslog.hostname}}"
-#      "fml_type": "{{.fml.type}}"
-#    #structured_metadata:
-#     # "source_ip": "{{source.ip}}"
-#     # "destination_ip": "{{destination.ip}}"
-#    ### For performance optimization. Vector works really well with defaults. Don't use it unless you really need to fine-tune yor ingest.
-#    buffer:
-#    - type: memory
-#      max_events: 12800 # default 500 https://www.elastic.co/docs/reference/fleet/es-output-settings#es-output-settings-performance-tuning-settings
-#      #when_full: drop_newest #default block
-#    batch:
-#      #max_bytes:
-#      max_events: 1600 # default 1000
-#      timeout_secs: 5 # default 1
-

vector/fortiappsec.yaml

@@ -86,111 +86,3 @@ transforms:
       # delete null fields
       . = compact(.)
 
-sinks:
-  #  print:
-  #   type: "console"
-  #   inputs: ["remap_fortiappsec"]
-  #   encoding:
-  #        #      codec: "raw_message"
-  #     codec: "json"
-  #     json:
-  #        pretty: true
-
-  vlogs_fortiappsec:
-    inputs:
-      - remap_fortiappsec
-    type: elasticsearch
-    endpoints:
-      - ${VICTORIA_LOGS_ENDPOINT:-http://localhost:9428}/insert/elasticsearch/
-    api_version: v8
-    compression: gzip
-    healthcheck:
-      enabled: false
-    query:
-      _msg_field: message
-      _time_field: timestamp
-      _stream_fields: observer.product,observer.type,observer.vendor,log.syslog.hostname,fwb.cat
-    request:
-      headers:
-        AccountID: "0"
-        ProjectID: "0"
-
-#    ### For performance optimization. Vector works really well with defaults. Don't use it unless you really need to fine-tune yor ingest. 
-#    buffer:
-#    - type: memory
-#      max_events: 12800 # default 500 https://www.elastic.co/docs/reference/fleet/es-output-settings#es-output-settings-performance-tuning-settings
-#      #when_full: drop_newest #default block
-#    batch:
-#      #max_bytes:
-#      max_events: 1600 # default 1000
-#      timeout_secs: 5 # default 1
-
-#  elastic_fortiappsec:
-#    type: elasticsearch
-#    inputs:
-#      - remap_fortimail
-#    auth:
-#      strategy: "basic"
-#      user: "${ELASTICSEARCH_USER:-elastic}"
-#      password: "${ELASTICSEARCH_PASS:-myelasticsearchpassword}"
-#    endpoints:
-#      - ${ELASTICSEARCH_ENDPOINT:-https://localhost:9200}
-#    encoding:
-#      except_fields:
-#        - url.query
-#    mode: "data_stream"
-#    bulk:
-#      action: "create"
-#    data_stream:
-#      type: "logs"
-#      dataset: "fortinet.fortiappsec}"
-#      namespace: "default"
-#    ### For performance optimization. Vector works really well with defaults. Don't use it unless you really need to fine-tune yor ingest.
-#    buffer:
-#    - type: memory
-#      max_events: 12800 # default 500 https://www.elastic.co/docs/reference/fleet/es-output-settings#es-output-settings-performance-tuning-settings
-#      #when_full: drop_newest #default block
-#    batch:
-#      #max_bytes:
-#      max_events: 1600 # default 1000
-#      timeout_secs: 5 # default 1
-
-#  quickwit_fortiappsec:
-#    type: "http"
-#    method: "post"
-#    inputs:
-#      - remap_fortimail
-#    encoding:
-#      codec: "json"
-#    framing:
-#      method: "newline_delimited"
-#    uri: "${QUICKWIT_ENDPOINT:-http://localhost:7280}/api/v1/logs-fortiappsec/ingest"
-
-#  loki_fortiappsec:
-#    type: loki
-#    inputs:
-#      - remap_fortimail
-#    endpoint: ${LOKI_ENDPOINT:-http://localhost:3100}
-#    auth:
-#      strategy: "basic"
-#      user: "${LOKI_USER:-loki_user}"
-#      password: "${LOKI_PASS:-mylokipassword}"
-#    encoding:
-#      codec: "json"
-#    compression: "snappy"
-#    labels:
-#      "syslog_appname": "{{.log.syslog.appname}}"
-#      "syslog_hostname": "{{.log.syslog.hostname}}"
-#      "fml_type": "{{.fml.type}}"
-#    #structured_metadata:
-#     # "source_ip": "{{source.ip}}"
-#     # "destination_ip": "{{destination.ip}}"
-#    ### For performance optimization. Vector works really well with defaults. Don't use it unless you really need to fine-tune yor ingest. 
-#    buffer:
-#    - type: memory
-#      max_events: 12800 # default 500 https://www.elastic.co/docs/reference/fleet/es-output-settings#es-output-settings-performance-tuning-settings
-#      #when_full: drop_newest #default block
-#    batch:
-#      #max_bytes:
-#      max_events: 1600 # default 1000
-#      timeout_secs: 5 # default 1

vector/fortiedr.yaml

@@ -88,31 +88,3 @@ transforms:
       # delete null fields
       . = compact(.)
 
-sinks:
-#  print:
-#   type: "console"
-#   inputs: ["remap_fortiedr"]
-#   encoding:
-#        #      codec: "raw_message"
-#     codec: "json"
-#     json:
-#        pretty: true
-
-  vlogs_fortiedr:
-    inputs:
-      - remap_fortiedr
-    type: elasticsearch
-    endpoints:
-      - ${VICTORIA_LOGS_ENDPOINT:-http://localhost:9428}/insert/elasticsearch/
-    api_version: v8
-    compression: gzip
-    healthcheck:
-      enabled: false
-    query:
-      _msg_field: message
-      _time_field: timestamp
-      _stream_fields: observer.product,observer.type,observer.vendor,log.syslog.hostname,fedr.Message Type
-    request:
-      headers:
-        AccountID: "0"
-        ProjectID: "0"