From 0819c3112c5059eefb88a73e03072fd7df96b7a1 Mon Sep 17 00:00:00 2001
From: jmorascalyr <42879226+jmorascalyr@users.noreply.github.com>
Date: Wed, 18 Feb 2026 06:59:42 -0700
Subject: [PATCH] feat: Refactor parser sync to use correct SIEM paths and add
auto-sync support
- Changed parser paths from `/parsers/` to `/logParsers/` with `.json` extension to match SIEM config tree structure
- Added `LOCAL_PARSER_ALIASES` mapping to handle marketplace parser names that don't match local directory names (e.g., marketplace-paloaltonetworksfirewall-latest -> paloalto_firewall-latest)
- Updated Palo Alto firewall sourcetype mapping from `paloalto_logs-latest` to `paloalto_firewall-latest`
- Ref
---
Backend/api/app/routers/parser_sync.py | 16 +-
.../api/app/services/parser_sync_service.py | 128 +-
.../network_security/paloalto_firewall.py | 48 +-
Backend/event_generators/shared/hec_sender.py | 80 +-
.../event_generators/shared/randomization.py | 387 +++++
.../tests/test_randomization.py | 328 ++++
.../paloalto_alternate_logs.json | 1531 +++++++++++++----
.../paloalto_firewall.json | 1436 +++++++++++++---
Frontend/log_generator_ui.py | 27 +
Frontend/templates/log_generator.html | 12 +-
10 files changed, 3273 insertions(+), 720 deletions(-)
create mode 100644 Backend/event_generators/shared/randomization.py
create mode 100644 Backend/event_generators/tests/test_randomization.py
diff --git a/Backend/api/app/routers/parser_sync.py b/Backend/api/app/routers/parser_sync.py
index 3dc163f..d68c8a3 100644
--- a/Backend/api/app/routers/parser_sync.py
+++ b/Backend/api/app/routers/parser_sync.py
@@ -135,19 +135,15 @@ async def sync_single_parser(
service = ParserSyncService(config_api_url=request.config_api_url)
logger.info(f"Syncing single parser for sourcetype: {request.sourcetype}")
-
- # Use the sourcetype as the source name for lookup
- results = service.ensure_parsers_for_sources(
- sources=[request.sourcetype],
+
+ result = service.ensure_parser_for_sourcetype(
+ sourcetype=request.sourcetype,
config_write_token=request.config_write_token,
github_repo_urls=request.github_repo_urls,
- github_token=request.github_token
+ github_token=request.github_token,
)
-
- # Get result for the sourcetype
- result = results.get(request.sourcetype, {})
- status = result.get('status', 'no_parser')
- message = result.get('message', 'Unknown status')
+ status = result.get("status", "no_parser")
+ message = result.get("message", "Unknown status")
logger.info(f"Single parser sync for {request.sourcetype}: {status} - {message}")
diff --git a/Backend/api/app/services/parser_sync_service.py b/Backend/api/app/services/parser_sync_service.py
index c8dd217..dd7d429 100644
--- a/Backend/api/app/services/parser_sync_service.py
+++ b/Backend/api/app/services/parser_sync_service.py
@@ -18,6 +18,15 @@
logger = logging.getLogger(__name__)
+
+# Some generator sourcetypes correspond to marketplace parser names that don't exist
+# as local parser directories in this repo. Provide aliases so we can still upload
+# a reasonable local parser when asked to sync those names.
+LOCAL_PARSER_ALIASES: Dict[str, str] = {
+ # Palo Alto Networks Firewall marketplace parser name -> local parser folder name
+ "marketplace-paloaltonetworksfirewall-latest": "paloalto_firewall-latest",
+}
+
# Mapping from generator/source names to parser sourcetypes
# This maps scenario sources to their corresponding parser directory names
SCENARIO_SOURCE_TO_PARSER = {
@@ -49,7 +58,7 @@
# Network Security
"darktrace": "darktrace_darktrace_logs-latest",
- "paloalto_firewall": "paloalto_logs-latest",
+ "paloalto_firewall": "paloalto_firewall-latest",
"f5_networks": "f5_networks_logs-latest",
"fortinet_fortigate": "fortinet_fortigate_candidate_logs-latest",
"zscaler": "zscaler_logs-latest",
@@ -125,9 +134,36 @@ def get_parser_path_in_siem(self, sourcetype: str) -> str:
sourcetype: The parser sourcetype (e.g., 'okta_authentication-latest')
Returns:
- The parser path in SIEM (e.g., '/parsers/okta_authentication-latest')
+ The parser path in SIEM (e.g., '/logParsers/okta_authentication-latest')
"""
- return f"/parsers/{sourcetype}"
+ # In the Scalyr/SentinelOne config tree, log parsers are stored as JSON files
+ # under /logParsers.
+ leaf = sourcetype
+ if not leaf.endswith(".json"):
+ leaf = f"{leaf}.json"
+ return f"/logParsers/{leaf}"
+
+ def _local_parser_directories_for_sourcetype(self, sourcetype: str) -> List[Path]:
+ local_name = LOCAL_PARSER_ALIASES.get(sourcetype, sourcetype)
+
+ # Handle prefixed sourcetypes produced by generator tooling (e.g., community-foo-latest)
+ if local_name.startswith("community-"):
+ leaf = local_name[len("community-"):]
+ return [
+ self.parsers_dir / "community" / leaf,
+ self.parsers_dir / "community_new" / leaf,
+ ]
+ if local_name.startswith("marketplace-"):
+ leaf = local_name[len("marketplace-"):]
+ return [
+ self.parsers_dir / "marketplace" / leaf,
+ ]
+
+ return [
+ self.parsers_dir / "community" / local_name,
+ self.parsers_dir / "community_new" / local_name,
+ self.parsers_dir / "sentinelone" / local_name,
+ ]
def load_local_parser(self, sourcetype: str) -> Optional[str]:
"""
@@ -139,12 +175,7 @@ def load_local_parser(self, sourcetype: str) -> Optional[str]:
Returns:
The parser JSON content as string, or None if not found
"""
- # Try community directory first
- parser_dirs = [
- self.parsers_dir / "community" / sourcetype,
- self.parsers_dir / "community_new" / sourcetype,
- self.parsers_dir / "sentinelone" / sourcetype,
- ]
+ parser_dirs = self._local_parser_directories_for_sourcetype(sourcetype)
for parser_dir in parser_dirs:
if parser_dir.exists():
@@ -168,6 +199,54 @@ def load_local_parser(self, sourcetype: str) -> Optional[str]:
logger.warning(f"Parser not found locally: {sourcetype}")
return None
+
+ def ensure_parser_for_sourcetype(
+ self,
+ sourcetype: str,
+ config_write_token: str,
+ github_repo_urls: Optional[List[str]] = None,
+ github_token: Optional[str] = None,
+ selected_parser: Optional[Dict] = None,
+ ) -> Dict[str, str]:
+ parser_path = self.get_parser_path_in_siem(sourcetype)
+
+ exists, _ = self.check_parser_exists(config_write_token, parser_path)
+ if exists:
+ return {
+ "status": "exists",
+ "message": f"Parser already exists: {parser_path}",
+ }
+
+ parser_content = self.load_local_parser(sourcetype)
+ from_github = False
+
+ if not parser_content and github_repo_urls:
+ parser_content = self.load_parser_from_github(
+ sourcetype=sourcetype,
+ repo_urls=github_repo_urls,
+ selected_parser=selected_parser,
+ github_token=github_token,
+ )
+ from_github = parser_content is not None
+
+ if not parser_content:
+ return {
+ "status": "no_parser",
+ "message": f"Parser not found locally or in GitHub repos: {sourcetype}",
+ }
+
+ success = self.upload_parser(config_write_token, parser_path, parser_content)
+ ok, detail = success
+ if not ok:
+ return {
+ "status": "failed",
+ "message": f"Failed to upload parser: {parser_path} ({detail})",
+ }
+
+ return {
+ "status": "uploaded_from_github" if from_github else "uploaded",
+ "message": f"Parser uploaded successfully: {parser_path}",
+ }
def load_parser_from_github(
self,
@@ -298,7 +377,7 @@ def upload_parser(
parser_path: str,
content: str,
timeout: int = 30
- ) -> bool:
+ ) -> Tuple[bool, str]:
"""
Upload a parser to the destination SIEM using putFile API
@@ -309,7 +388,7 @@ def upload_parser(
timeout: Request timeout in seconds
Returns:
- True if upload succeeded, False otherwise
+ Tuple of (success, message)
"""
try:
url = f"{self.api_base_url}/putFile"
@@ -332,25 +411,22 @@ def upload_parser(
result = response.json()
if result.get("status") == "success":
logger.info(f"Parser uploaded successfully: {parser_path}")
- return True
+ return True, "success"
else:
- logger.error(
- f"Failed to upload parser {parser_path}: {result.get('message', 'Unknown error')}"
- )
- return False
+ msg = result.get('message', 'Unknown error')
+ logger.error(f"Failed to upload parser {parser_path}: {msg}")
+ return False, msg
else:
- logger.error(
- f"Failed to upload parser {parser_path}: "
- f"{response.status_code} - {response.text}"
- )
- return False
+ msg = f"{response.status_code} - {response.text}"
+ logger.error(f"Failed to upload parser {parser_path}: {msg}")
+ return False, msg
except requests.exceptions.Timeout:
logger.error(f"Timeout uploading parser: {parser_path}")
- return False
+ return False, "timeout"
except Exception as e:
logger.error(f"Error uploading parser {parser_path}: {e}")
- return False
+ return False, str(e)
def ensure_parsers_for_sources(
self,
@@ -457,9 +533,9 @@ def ensure_parsers_for_sources(
continue
# Upload the parser
- success = self.upload_parser(config_write_token, parser_path, parser_content)
+ ok, detail = self.upload_parser(config_write_token, parser_path, parser_content)
- if success:
+ if ok:
status = "uploaded_from_github" if from_github else "uploaded"
source_label = "GitHub" if from_github else "local"
results[source] = {
@@ -471,7 +547,7 @@ def ensure_parsers_for_sources(
results[source] = {
"status": "failed",
"sourcetype": actual_sourcetype,
- "message": f"Failed to upload parser: {parser_path}"
+ "message": f"Failed to upload parser: {parser_path} ({detail})"
}
return results
diff --git a/Backend/event_generators/network_security/paloalto_firewall.py b/Backend/event_generators/network_security/paloalto_firewall.py
index a062ae8..77f2dcd 100644
--- a/Backend/event_generators/network_security/paloalto_firewall.py
+++ b/Backend/event_generators/network_security/paloalto_firewall.py
@@ -5,6 +5,12 @@
from datetime import datetime, timezone, timedelta
import time
+import os
+import sys
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'shared'))
+from randomization import Randomizer
+
# Palo Alto log types
LOG_TYPES = ["TRAFFIC", "THREAT", "SYSTEM", "CONFIG", "HIP-MATCH", "GLOBALPROTECT", "USERID", "URL"]
@@ -19,14 +25,18 @@
def get_random_ip(internal_probability=0.5):
"""Generate a random IP address."""
- if random.random() < internal_probability:
- return random.choice([
- f"10.{random.randint(0, 255)}.{random.randint(0, 255)}.{random.randint(1, 254)}",
- f"172.{random.randint(16, 31)}.{random.randint(0, 255)}.{random.randint(1, 254)}",
- f"192.168.{random.randint(0, 255)}.{random.randint(1, 254)}"
- ])
- else:
- return f"{random.randint(1, 223)}.{random.randint(0, 255)}.{random.randint(0, 255)}.{random.randint(1, 254)}"
+ internal = random.random() < internal_probability
+ return _R.ip(internal=internal)
+
+
+def get_random_username(domain_probability: float = 0.7, empty_probability: float = 0.2) -> str:
+ if random.random() < empty_probability:
+ return ""
+
+ username = _R.person(domain="corp.local").username
+ if random.random() < domain_probability:
+ return f"corp\\{username}"
+ return username
def generate_serial_number():
"""Generate a firewall serial number."""
@@ -36,6 +46,9 @@ def generate_session_id():
"""Generate a session ID."""
return str(random.randint(10000, 999999))
+
+_R = Randomizer()
+
def generate_traffic_log():
"""Generate a TRAFFIC log entry."""
now = datetime.now(timezone.utc)
@@ -89,7 +102,7 @@ def generate_traffic_log():
src_ip, # natsrc
dst_ip, # natdst
f"allow-{app}" if action == "allow" else f"block-{random.choice(['threats', 'malware', 'default'])}", # rule
- random.choice([f"domain\\user{random.randint(1, 100)}", ""]), # srcuser
+ get_random_username(), # srcuser
"", # dstuser
app, # app
"vsys1", # vsys
@@ -125,7 +138,14 @@ def generate_traffic_log():
str(int(packets * 0.4)), # pkts_received
random.choice(["aged-out", "tcp-fin", "tcp-rst", "policy-deny", ""]) if action != "allow" else "aged-out", # session_end_reason
]
-
+
+ # The marketplace Palo Alto firewall parser expects a fixed number of CSV columns.
+ # If we stop emitting fields early, the line will not match even if the earlier
+ # fields are correct (because required delimiters/columns are missing).
+ expected_fields = 115
+ if len(fields) < expected_fields:
+ fields.extend([""] * (expected_fields - len(fields)))
+
return ",".join(fields)
def generate_threat_log():
@@ -155,7 +175,7 @@ def generate_threat_log():
src_ip, # natsrc
dst_ip, # natdst
"block-threats", # rule
- "", # srcuser
+ get_random_username(), # srcuser
"", # dstuser
random.choice(["web-browsing", "ssl", "ftp", "smtp"]), # app
"vsys1", # vsys
@@ -197,7 +217,11 @@ def generate_threat_log():
"", # recipient
"", # reportid
]
-
+
+ expected_fields = 120
+ if len(fields) < expected_fields:
+ fields.extend([""] * (expected_fields - len(fields)))
+
return ",".join(fields)
def paloalto_firewall_log(overrides: dict | None = None) -> str:
diff --git a/Backend/event_generators/shared/hec_sender.py b/Backend/event_generators/shared/hec_sender.py
index e429c49..1cbc99b 100644
--- a/Backend/event_generators/shared/hec_sender.py
+++ b/Backend/event_generators/shared/hec_sender.py
@@ -826,22 +826,19 @@ def _send_batch(lines: list, is_json: bool, product: str):
"aws_route53": "aws_route53-latest",
"aws_vpc_dns": "aws_vpc_dns_logs-latest",
"aws_vpcflow": "aws_vpcflow_logs-latest",
-
# Network security - actual directory names
"fortinet_fortigate": "fortinet_fortigate_candidate_logs-latest",
"fortimanager": "fortinet_fortigate_fortimanager_logs-latest",
"checkpoint": "checkpoint_checkpoint_logs-latest",
- "paloalto_firewall": "paloalto_logs-latest",
- "paloalto_prismasase": "paloalto_prismasase_logs-latest",
+ "paloalto_firewall": "marketplace-paloaltonetworksfirewall-latest",
+ "paloalto_prismasase": "community-paloalto_prismasase_logs-latest",
"cisco_firewall_threat_defense": "cisco_firewall_threat_defense-latest",
"infoblox_ddi": "infoblox_ddi-latest",
-
# Zscaler products
"zscaler": "zscaler_logs-latest",
"zscaler_private_access": "zscaler_private_access-latest",
"zscaler_firewall": "zscaler_firewall_logs-latest",
"zscaler_dns_firewall": "zscaler_dns_firewall-latest",
-
# Netskope
"netskope": "netskope_netskope_logs-latest",
@@ -959,6 +956,77 @@ def _send_batch(lines: list, is_json: bool, product: str):
ENV_HOST = os.getenv("S1_HEC_HOST")
ENV_INDEX = os.getenv("S1_HEC_INDEX")
+# Optional: ensure parsers exist in the destination SIEM before sending events
+_ENSURE_PARSER = os.getenv("JARVIS_ENSURE_PARSER", "false").lower() == "true"
+_JARVIS_API_BASE_URL = os.getenv("JARVIS_API_BASE_URL", "http://localhost:8000").rstrip("/")
+_JARVIS_API_KEY = os.getenv("JARVIS_API_KEY")
+_S1_CONFIG_API_URL = os.getenv("S1_CONFIG_API_URL")
+_S1_CONFIG_WRITE_TOKEN = os.getenv("S1_CONFIG_WRITE_TOKEN")
+
+_ENSURED_PARSERS: set[str] = set()
+
+
+def _ensure_parser_in_destination(product: str) -> None:
+ if not _ENSURE_PARSER:
+ return
+
+ sourcetype = SOURCETYPE_MAP.get(product, product)
+
+ sync_sourcetype = sourcetype
+ if sync_sourcetype.startswith("community-"):
+ sync_sourcetype = sync_sourcetype[len("community-"):]
+
+
+ if sync_sourcetype in _ENSURED_PARSERS:
+ return
+
+ # If the sourcetype isn't one that maps to a parser bundle name, nothing to do.
+ if not sourcetype:
+ return
+
+ # Require config API info to actually sync to the destination SIEM
+ if not _S1_CONFIG_API_URL or not _S1_CONFIG_WRITE_TOKEN:
+ if _VERBOSITY in ('info', 'verbose', 'debug'):
+ print(
+ f"[PARSER] Skipping ensure-parser for {product}: missing S1_CONFIG_API_URL or S1_CONFIG_WRITE_TOKEN",
+ flush=True,
+ )
+ return
+
+ headers = {}
+ if _JARVIS_API_KEY:
+ headers["X-API-Key"] = _JARVIS_API_KEY
+
+ url = f"{_JARVIS_API_BASE_URL}/api/v1/parser-sync/sync-single"
+ payload = {
+ "sourcetype": sync_sourcetype,
+ "config_api_url": _S1_CONFIG_API_URL,
+ "config_write_token": _S1_CONFIG_WRITE_TOKEN,
+ }
+
+ try:
+ resp = requests.post(url, json=payload, headers=headers, timeout=30)
+ if resp.status_code >= 400:
+ if _VERBOSITY in ('info', 'verbose', 'debug'):
+ print(
+ f"[PARSER] Ensure-parser failed for {sync_sourcetype}: {resp.status_code} {resp.text[:200]}",
+ flush=True,
+ )
+ return
+
+ data = resp.json() if resp.content else {}
+ if _VERBOSITY in ('info', 'verbose', 'debug'):
+ print(
+ f"[PARSER] Ensure-parser {sync_sourcetype}: {data.get('status', 'unknown')} - {data.get('message', '')}",
+ flush=True,
+ )
+
+ _ENSURED_PARSERS.add(sync_sourcetype)
+ except Exception as e:
+ if _VERBOSITY in ('info', 'verbose', 'debug'):
+ print(f"[PARSER] Ensure-parser error for {sync_sourcetype}: {e}", flush=True)
+ return
+
def _build_qs(product: str) -> str:
parts = [f"sourcetype={SOURCETYPE_MAP.get(product, product)}"]
if ENV_SOURCE:
@@ -1103,8 +1171,8 @@ def send_one(line, product: str, attr_fields: dict, event_time: float | None = N
"""
Route JSON‑structured products to the /event endpoint and all
raw / CSV / syslog products to the /raw endpoint.
- Uses cached connection config after first successful send for performance.
"""
+ _ensure_parser_in_destination(product)
# Build endpoint bases to try (env override → us1 → usea1 → global)
env_event = os.getenv("S1_HEC_EVENT_URL_BASE")
env_raw = os.getenv("S1_HEC_RAW_URL_BASE")
diff --git a/Backend/event_generators/shared/randomization.py b/Backend/event_generators/shared/randomization.py
new file mode 100644
index 0000000..64f2ab8
--- /dev/null
+++ b/Backend/event_generators/shared/randomization.py
@@ -0,0 +1,387 @@
+#!/usr/bin/env python3
+"""
+Randomization Service for Event Generators & Scenarios
+=======================================================
+
+Centralized randomization for IPs, names, and other fields.
+Provides a RandomizationContext that caches assignments so the same
+entity (e.g., a username) gets consistent values across all events
+within a single run.
+
+Usage:
+ from shared.randomization import Randomizer
+
+ r = Randomizer(seed=42) # reproducible
+ r = Randomizer() # fully random
+
+ # IPs
+ r.internal_ip() # random from default private ranges
+ r.internal_ip(cidr="10.50.0.0/16") # constrained to a subnet
+ r.external_ip() # random public IP (no reserved)
+
+ # Names
+ r.person() # -> {"first": "Sara", "last": "Mitchell", ...}
+ r.person(domain="starfleet.corp") # email uses that domain
+
+ # Consistent assignments via context
+ r.assign("jeanluc", "ip", r.internal_ip) # first call generates, subsequent return cached
+ r.get("jeanluc", "ip") # retrieve without generating
+"""
+
+from __future__ import annotations
+
+import ipaddress
+import random
+from dataclasses import dataclass, field
+from typing import Any, Callable, Optional
+
+
+# ---------------------------------------------------------------------------
+# Default CIDR ranges
+# ---------------------------------------------------------------------------
+
+DEFAULT_INTERNAL_CIDRS = [
+ "10.0.0.0/8",
+ "172.16.0.0/12",
+ "192.168.0.0/16",
+]
+
+# Public ranges to sample from (major allocations, avoids reserved blocks)
+_PUBLIC_RANGES = [
+ ("100.128.0.0", "126.255.255.255"),
+ ("198.20.0.0", "198.51.99.255"),
+ ("198.51.101.0", "203.0.112.255"),
+ ("203.0.114.0", "223.255.255.255"),
+]
+
+# ---------------------------------------------------------------------------
+# Name pools
+# ---------------------------------------------------------------------------
+
+FIRST_NAMES = [
+ "James", "Mary", "Robert", "Patricia", "John", "Jennifer", "Michael",
+ "Linda", "David", "Elizabeth", "William", "Barbara", "Richard", "Susan",
+ "Joseph", "Jessica", "Thomas", "Sarah", "Christopher", "Karen",
+ "Charles", "Lisa", "Daniel", "Nancy", "Matthew", "Betty", "Anthony",
+ "Margaret", "Mark", "Sandra", "Donald", "Ashley", "Steven", "Kimberly",
+ "Paul", "Emily", "Andrew", "Donna", "Joshua", "Michelle", "Kenneth",
+ "Carol", "Kevin", "Amanda", "Brian", "Dorothy", "George", "Melissa",
+ "Timothy", "Deborah", "Ronald", "Stephanie", "Edward", "Rebecca",
+ "Jason", "Sharon", "Jeffrey", "Laura", "Ryan", "Cynthia",
+ "Jacob", "Kathleen", "Gary", "Amy", "Nicholas", "Angela", "Eric",
+ "Shirley", "Jonathan", "Anna", "Stephen", "Brenda", "Larry", "Pamela",
+ "Justin", "Emma", "Scott", "Nicole", "Brandon", "Helen", "Benjamin",
+ "Samantha", "Samuel", "Katherine", "Raymond", "Christine", "Gregory",
+ "Debra", "Frank", "Rachel", "Alexander", "Carolyn", "Patrick", "Janet",
+ "Jack", "Catherine", "Dennis", "Maria", "Jerry", "Heather", "Tyler",
+ "Diane",
+]
+
+LAST_NAMES = [
+ "Smith", "Johnson", "Williams", "Brown", "Jones", "Garcia", "Miller",
+ "Davis", "Rodriguez", "Martinez", "Hernandez", "Lopez", "Gonzalez",
+ "Wilson", "Anderson", "Thomas", "Taylor", "Moore", "Jackson", "Martin",
+ "Lee", "Perez", "Thompson", "White", "Harris", "Sanchez", "Clark",
+ "Ramirez", "Lewis", "Robinson", "Walker", "Young", "Allen", "King",
+ "Wright", "Scott", "Torres", "Nguyen", "Hill", "Flores", "Green",
+ "Adams", "Nelson", "Baker", "Hall", "Rivera", "Campbell", "Mitchell",
+ "Carter", "Roberts", "Gomez", "Phillips", "Evans", "Turner", "Diaz",
+ "Parker", "Cruz", "Edwards", "Collins", "Reyes", "Stewart", "Morris",
+ "Morales", "Murphy", "Cook", "Rogers", "Gutierrez", "Ortiz", "Morgan",
+ "Cooper", "Peterson", "Bailey", "Reed", "Kelly", "Howard", "Ramos",
+ "Kim", "Cox", "Ward", "Richardson", "Watson", "Brooks", "Chavez",
+ "Wood", "James", "Bennett", "Gray", "Mendoza", "Ruiz", "Hughes",
+ "Price", "Alvarez", "Castillo", "Sanders", "Patel", "Myers", "Long",
+ "Ross", "Foster", "Jimenez",
+]
+
+DEPARTMENTS = [
+ "Engineering", "Security", "Finance", "Human Resources", "IT",
+ "Operations", "Marketing", "Sales", "Legal", "Executive",
+ "Research", "Support", "Product", "Data Science", "DevOps",
+]
+
+ROLES = [
+ "Analyst", "Engineer", "Manager", "Director", "Specialist",
+ "Administrator", "Coordinator", "Lead", "Architect", "Consultant",
+]
+
+LOCATIONS = [
+ ("New York", "New York", "US"),
+ ("San Francisco", "California", "US"),
+ ("Austin", "Texas", "US"),
+ ("Chicago", "Illinois", "US"),
+ ("Denver", "Colorado", "US"),
+ ("Seattle", "Washington", "US"),
+ ("Boston", "Massachusetts", "US"),
+ ("Los Angeles", "California", "US"),
+ ("Atlanta", "Georgia", "US"),
+ ("Dallas", "Texas", "US"),
+ ("London", "England", "GB"),
+ ("Berlin", "Brandenburg", "DE"),
+ ("Paris", "Île-de-France", "FR"),
+ ("Tokyo", "Tokyo", "JP"),
+ ("Sydney", "New South Wales", "AU"),
+ ("Toronto", "Ontario", "CA"),
+ ("Mumbai", "Maharashtra", "IN"),
+]
+
+
+# ---------------------------------------------------------------------------
+# Data classes
+# ---------------------------------------------------------------------------
+
+@dataclass
+class PersonInfo:
+ """Generated person identity."""
+ first_name: str
+ last_name: str
+ email: str
+ username: str
+ display_name: str
+ department: str
+ role: str
+ location: tuple[str, str, str] # (city, state, country_code)
+
+ def to_dict(self) -> dict[str, Any]:
+ city, state, country = self.location
+ return {
+ "first_name": self.first_name,
+ "last_name": self.last_name,
+ "email": self.email,
+ "username": self.username,
+ "display_name": self.display_name,
+ "department": self.department,
+ "role": self.role,
+ "city": city,
+ "state": state,
+ "country": country,
+ }
+
+
+# ---------------------------------------------------------------------------
+# Randomizer
+# ---------------------------------------------------------------------------
+
+class Randomizer:
+ """Centralized randomization service with optional seed and context caching."""
+
+ def __init__(
+ self,
+ seed: Optional[int] = None,
+ ip_uniqueness_rate: float = 0.10,
+ ip_pool_max_size: int = 10_000,
+ ):
+ self._rng = random.Random(seed)
+ self._context: dict[str, dict[str, Any]] = {}
+ self._internal_networks: list[ipaddress.IPv4Network] = [
+ ipaddress.IPv4Network(cidr) for cidr in DEFAULT_INTERNAL_CIDRS
+ ]
+ self._used_names: set[str] = set()
+ self._ip_uniqueness_rate = max(0.0, min(1.0, float(ip_uniqueness_rate)))
+ self._ip_pool_max_size = max(0, int(ip_pool_max_size))
+ self._ip_pools: dict[tuple[str, str], list[str]] = {}
+
+ # ------------------------------------------------------------------
+ # IP generation
+ # ------------------------------------------------------------------
+
+ def internal_ip(self, cidr: Optional[str] = None) -> str:
+ """Generate a random private/internal IP address.
+
+ Args:
+ cidr: Optional CIDR string to constrain the range.
+ e.g. "10.50.0.0/16", "192.168.1.0/24"
+ If None, picks randomly from the three standard
+ private ranges.
+ """
+ network = ipaddress.IPv4Network(cidr, strict=False) if cidr else self._rng.choice(self._internal_networks)
+ pool_key = ("internal", str(network))
+ pool = self._ip_pools.get(pool_key, [])
+
+ if pool and self._rng.random() >= self._ip_uniqueness_rate:
+ return self._rng.choice(pool)
+
+ num = network.num_addresses
+ if num <= 2:
+ ip_str = str(network.network_address)
+ else:
+ offset = self._rng.randint(1, num - 2)
+ ip_str = str(network.network_address + offset)
+
+ if self._ip_pool_max_size > 0:
+ if len(pool) < self._ip_pool_max_size:
+ if ip_str not in pool:
+ pool.append(ip_str)
+ self._ip_pools[pool_key] = pool
+ return ip_str
+
+ def external_ip(self) -> str:
+ """Generate a random public/external IP address.
+
+ Avoids all private, reserved, loopback, link-local,
+ and documentation ranges.
+ """
+ pool_key = ("external", "default")
+ pool = self._ip_pools.get(pool_key, [])
+
+ if pool and self._rng.random() >= self._ip_uniqueness_rate:
+ return self._rng.choice(pool)
+
+ while True:
+ start_str, end_str = self._rng.choice(_PUBLIC_RANGES)
+ start_int = int(ipaddress.IPv4Address(start_str))
+ end_int = int(ipaddress.IPv4Address(end_str))
+ ip_int = self._rng.randint(start_int, end_int)
+ addr = ipaddress.IPv4Address(ip_int)
+ if not (addr.is_private or addr.is_reserved or addr.is_loopback):
+ ip_str = str(addr)
+ break
+
+ if self._ip_pool_max_size > 0:
+ if len(pool) < self._ip_pool_max_size:
+ if ip_str not in pool:
+ pool.append(ip_str)
+ self._ip_pools[pool_key] = pool
+
+ return ip_str
+
+ def ip(self, internal: bool = True, cidr: Optional[str] = None) -> str:
+ """Generate an IP — convenience wrapper.
+
+ Args:
+ internal: True for private, False for public.
+ cidr: Only used when internal=True.
+ """
+ if internal:
+ return self.internal_ip(cidr=cidr)
+ return self.external_ip()
+
+ # ------------------------------------------------------------------
+ # Name generation
+ # ------------------------------------------------------------------
+
+ def person(
+ self,
+ domain: str = "company.com",
+ department: Optional[str] = None,
+ role: Optional[str] = None,
+ location: Optional[tuple[str, str, str]] = None,
+ unique: bool = True,
+ ) -> PersonInfo:
+ """Generate a random person identity.
+
+ Args:
+ domain: Email domain.
+ department: Force a specific department, or random.
+ role: Force a specific role, or random.
+ location: Force (city, state, country) tuple, or random.
+ unique: If True, avoids repeating the same first+last combo
+ within this Randomizer instance.
+ """
+ max_attempts = 200
+ for _ in range(max_attempts):
+ first = self._rng.choice(FIRST_NAMES)
+ last = self._rng.choice(LAST_NAMES)
+ key = f"{first.lower()}.{last.lower()}"
+ if not unique or key not in self._used_names:
+ break
+ else:
+ # Exhausted attempts — allow duplicates
+ first = self._rng.choice(FIRST_NAMES)
+ last = self._rng.choice(LAST_NAMES)
+ key = f"{first.lower()}.{last.lower()}"
+
+ self._used_names.add(key)
+
+ username = f"{first.lower()}.{last.lower()}"
+ email = f"{username}@{domain}"
+ display_name = f"{first} {last}"
+ dept = department or self._rng.choice(DEPARTMENTS)
+ r = role or self._rng.choice(ROLES)
+ loc = location or self._rng.choice(LOCATIONS)
+
+ return PersonInfo(
+ first_name=first,
+ last_name=last,
+ email=email,
+ username=username,
+ display_name=display_name,
+ department=dept,
+ role=r,
+ location=loc,
+ )
+
+ def first_name(self) -> str:
+ """Generate a random first name."""
+ return self._rng.choice(FIRST_NAMES)
+
+ def last_name(self) -> str:
+ """Generate a random last name."""
+ return self._rng.choice(LAST_NAMES)
+
+ def email(self, first: Optional[str] = None, last: Optional[str] = None,
+ domain: str = "company.com") -> str:
+ """Generate an email address from name parts."""
+ f = (first or self._rng.choice(FIRST_NAMES)).lower()
+ l = (last or self._rng.choice(LAST_NAMES)).lower()
+ return f"{f}.{l}@{domain}"
+
+ def username(self, first: Optional[str] = None,
+ last: Optional[str] = None) -> str:
+ """Generate a username from name parts."""
+ f = (first or self._rng.choice(FIRST_NAMES)).lower()
+ l = (last or self._rng.choice(LAST_NAMES)).lower()
+ return f"{f}.{l}"
+
+ # ------------------------------------------------------------------
+ # Context / assignment caching
+ # ------------------------------------------------------------------
+
+ def assign(self, entity: str, field_name: str,
+ generator: Callable[..., Any], *args: Any,
+ **kwargs: Any) -> Any:
+ """Assign a value to an entity+field, caching the result.
+
+ If the entity+field already has a cached value, return it.
+ Otherwise call generator(*args, **kwargs), cache, and return.
+
+ Args:
+ entity: Identifier for the entity (e.g. "jeanluc", "victim").
+ field_name: Name of the field (e.g. "ip", "name").
+ generator: Callable that produces the value.
+ *args, **kwargs: Passed to generator on first call.
+
+ Returns:
+ The cached or newly generated value.
+ """
+ if entity not in self._context:
+ self._context[entity] = {}
+
+ if field_name not in self._context[entity]:
+ self._context[entity][field_name] = generator(*args, **kwargs)
+
+ return self._context[entity][field_name]
+
+ def get(self, entity: str, field_name: str,
+ default: Any = None) -> Any:
+ """Retrieve a previously assigned value.
+
+ Returns default if not found.
+ """
+ return self._context.get(entity, {}).get(field_name, default)
+
+ def set(self, entity: str, field_name: str, value: Any) -> None:
+ """Manually set a cached value for an entity+field."""
+ if entity not in self._context:
+ self._context[entity] = {}
+ self._context[entity][field_name] = value
+
+ def context_snapshot(self) -> dict[str, dict[str, Any]]:
+ """Return a copy of the full context for debugging/logging."""
+ return {k: dict(v) for k, v in self._context.items()}
+
+ def reset(self) -> None:
+ """Clear all cached assignments and used names."""
+ self._context.clear()
+ self._used_names.clear()
diff --git a/Backend/event_generators/tests/test_randomization.py b/Backend/event_generators/tests/test_randomization.py
new file mode 100644
index 0000000..c953fea
--- /dev/null
+++ b/Backend/event_generators/tests/test_randomization.py
@@ -0,0 +1,328 @@
+"""
+Tests for the randomization service
+"""
+import ipaddress
+import pytest
+import sys
+import os
+
+# Add the parent directory to the path
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+try:
+ from shared.randomization import Randomizer, PersonInfo
+except ImportError as e:
+ pytest.skip(f"Cannot import randomization: {e}", allow_module_level=True)
+
+
+class TestInternalIP:
+ """Test internal/private IP generation"""
+
+ def test_default_returns_private_ip(self):
+ r = Randomizer(seed=1)
+ ip = r.internal_ip()
+ addr = ipaddress.IPv4Address(ip)
+ assert addr.is_private
+
+ def test_many_ips_all_private(self):
+ r = Randomizer(seed=42)
+ for _ in range(200):
+ addr = ipaddress.IPv4Address(r.internal_ip())
+ assert addr.is_private, f"{addr} is not private"
+
+ def test_cidr_constraint(self):
+ r = Randomizer(seed=10)
+ network = ipaddress.IPv4Network("10.50.0.0/16")
+ for _ in range(100):
+ ip = r.internal_ip(cidr="10.50.0.0/16")
+ addr = ipaddress.IPv4Address(ip)
+ assert addr in network, f"{addr} not in 10.50.0.0/16"
+
+ def test_cidr_small_subnet(self):
+ r = Randomizer(seed=5)
+ network = ipaddress.IPv4Network("192.168.1.0/24")
+ for _ in range(50):
+ ip = r.internal_ip(cidr="192.168.1.0/24")
+ addr = ipaddress.IPv4Address(ip)
+ assert addr in network
+
+ def test_cidr_slash_30(self):
+ """A /30 has 4 addresses, 2 usable (excluding network + broadcast)."""
+ r = Randomizer(seed=7)
+ network = ipaddress.IPv4Network("10.0.0.0/30")
+ for _ in range(20):
+ ip = r.internal_ip(cidr="10.0.0.0/30")
+ addr = ipaddress.IPv4Address(ip)
+ assert addr in network
+ # Should not be network or broadcast
+ assert addr != network.network_address
+ assert addr != network.broadcast_address
+
+ def test_returns_string(self):
+ r = Randomizer()
+ ip = r.internal_ip()
+ assert isinstance(ip, str)
+ # Parseable
+ ipaddress.IPv4Address(ip)
+
+
+class TestExternalIP:
+ """Test external/public IP generation"""
+
+ def test_returns_public_ip(self):
+ r = Randomizer(seed=1)
+ ip = r.external_ip()
+ addr = ipaddress.IPv4Address(ip)
+ assert not addr.is_private
+ assert not addr.is_reserved
+ assert not addr.is_loopback
+
+ def test_many_ips_all_public(self):
+ r = Randomizer(seed=42)
+ for _ in range(200):
+ ip = r.external_ip()
+ addr = ipaddress.IPv4Address(ip)
+ assert not addr.is_private, f"{addr} is private"
+ assert not addr.is_loopback, f"{addr} is loopback"
+
+ def test_returns_string(self):
+ r = Randomizer()
+ ip = r.external_ip()
+ assert isinstance(ip, str)
+ ipaddress.IPv4Address(ip)
+
+
+class TestIPConvenience:
+ """Test the ip() convenience wrapper"""
+
+ def test_internal_flag(self):
+ r = Randomizer(seed=1)
+ ip = r.ip(internal=True)
+ assert ipaddress.IPv4Address(ip).is_private
+
+ def test_external_flag(self):
+ r = Randomizer(seed=1)
+ ip = r.ip(internal=False)
+ assert not ipaddress.IPv4Address(ip).is_private
+
+ def test_cidr_passthrough(self):
+ r = Randomizer(seed=1)
+ network = ipaddress.IPv4Network("172.16.5.0/24")
+ ip = r.ip(internal=True, cidr="172.16.5.0/24")
+ assert ipaddress.IPv4Address(ip) in network
+
+
+class TestPerson:
+ """Test person/name generation"""
+
+ def test_returns_person_info(self):
+ r = Randomizer(seed=1)
+ p = r.person()
+ assert isinstance(p, PersonInfo)
+
+ def test_person_fields(self):
+ r = Randomizer(seed=1)
+ p = r.person(domain="starfleet.corp")
+ assert p.first_name
+ assert p.last_name
+ assert p.email.endswith("@starfleet.corp")
+ assert p.username == f"{p.first_name.lower()}.{p.last_name.lower()}"
+ assert p.display_name == f"{p.first_name} {p.last_name}"
+ assert p.department
+ assert p.role
+ assert len(p.location) == 3
+
+ def test_custom_department_and_role(self):
+ r = Randomizer(seed=1)
+ p = r.person(department="Security", role="Analyst")
+ assert p.department == "Security"
+ assert p.role == "Analyst"
+
+ def test_unique_names(self):
+ r = Randomizer(seed=42)
+ people = [r.person() for _ in range(50)]
+ usernames = [p.username for p in people]
+ assert len(set(usernames)) == len(usernames), "Duplicate usernames generated"
+
+ def test_to_dict(self):
+ r = Randomizer(seed=1)
+ p = r.person()
+ d = p.to_dict()
+ assert isinstance(d, dict)
+ assert "first_name" in d
+ assert "last_name" in d
+ assert "email" in d
+ assert "username" in d
+ assert "display_name" in d
+ assert "department" in d
+ assert "role" in d
+ assert "city" in d
+ assert "state" in d
+ assert "country" in d
+
+
+class TestNameHelpers:
+ """Test individual name helper methods"""
+
+ def test_first_name(self):
+ r = Randomizer(seed=1)
+ name = r.first_name()
+ assert isinstance(name, str)
+ assert len(name) > 0
+
+ def test_last_name(self):
+ r = Randomizer(seed=1)
+ name = r.last_name()
+ assert isinstance(name, str)
+ assert len(name) > 0
+
+ def test_email(self):
+ r = Randomizer(seed=1)
+ email = r.email(first="Jean", last="Picard", domain="starfleet.corp")
+ assert email == "jean.picard@starfleet.corp"
+
+ def test_email_random(self):
+ r = Randomizer(seed=1)
+ email = r.email()
+ assert "@company.com" in email
+ assert "." in email.split("@")[0]
+
+ def test_username(self):
+ r = Randomizer(seed=1)
+ u = r.username(first="Jean", last="Picard")
+ assert u == "jean.picard"
+
+ def test_username_random(self):
+ r = Randomizer(seed=1)
+ u = r.username()
+ assert isinstance(u, str)
+ assert "." in u
+
+
+class TestContext:
+ """Test assignment caching / context"""
+
+ def test_assign_caches_value(self):
+ r = Randomizer(seed=1)
+ ip1 = r.assign("victim", "ip", r.internal_ip)
+ ip2 = r.assign("victim", "ip", r.internal_ip)
+ assert ip1 == ip2
+
+ def test_assign_different_entities(self):
+ r = Randomizer(seed=1)
+ ip1 = r.assign("victim", "ip", r.internal_ip)
+ ip2 = r.assign("attacker", "ip", r.external_ip)
+ assert ip1 != ip2
+
+ def test_assign_different_fields(self):
+ r = Randomizer(seed=1)
+ ip = r.assign("victim", "ip", r.internal_ip)
+ name = r.assign("victim", "name", r.first_name)
+ assert isinstance(ip, str)
+ assert isinstance(name, str)
+ assert "." in ip # IP has dots
+ assert "." not in name # Name doesn't
+
+ def test_get_returns_cached(self):
+ r = Randomizer(seed=1)
+ ip = r.assign("victim", "ip", r.internal_ip)
+ assert r.get("victim", "ip") == ip
+
+ def test_get_returns_default_when_missing(self):
+ r = Randomizer(seed=1)
+ assert r.get("nobody", "ip") is None
+ assert r.get("nobody", "ip", "fallback") == "fallback"
+
+ def test_set_manual(self):
+ r = Randomizer(seed=1)
+ r.set("c2", "ip", "185.234.72.156")
+ assert r.get("c2", "ip") == "185.234.72.156"
+
+ def test_assign_with_kwargs(self):
+ r = Randomizer(seed=1)
+ ip = r.assign("server", "ip", r.internal_ip, cidr="10.50.0.0/16")
+ addr = ipaddress.IPv4Address(ip)
+ assert addr in ipaddress.IPv4Network("10.50.0.0/16")
+
+ def test_context_snapshot(self):
+ r = Randomizer(seed=1)
+ r.assign("victim", "ip", r.internal_ip)
+ r.assign("victim", "name", r.first_name)
+ snap = r.context_snapshot()
+ assert "victim" in snap
+ assert "ip" in snap["victim"]
+ assert "name" in snap["victim"]
+
+ def test_reset_clears_context(self):
+ r = Randomizer(seed=1)
+ r.assign("victim", "ip", r.internal_ip)
+ r.reset()
+ assert r.get("victim", "ip") is None
+
+
+class TestSeedReproducibility:
+ """Test that seeded Randomizer produces deterministic results"""
+
+ def test_same_seed_same_ips(self):
+ r1 = Randomizer(seed=99)
+ r2 = Randomizer(seed=99)
+ ips1 = [r1.internal_ip() for _ in range(10)]
+ ips2 = [r2.internal_ip() for _ in range(10)]
+ assert ips1 == ips2
+
+ def test_same_seed_same_external_ips(self):
+ r1 = Randomizer(seed=99)
+ r2 = Randomizer(seed=99)
+ ips1 = [r1.external_ip() for _ in range(10)]
+ ips2 = [r2.external_ip() for _ in range(10)]
+ assert ips1 == ips2
+
+ def test_same_seed_same_persons(self):
+ r1 = Randomizer(seed=99)
+ r2 = Randomizer(seed=99)
+ p1 = r1.person()
+ p2 = r2.person()
+ assert p1.email == p2.email
+ assert p1.department == p2.department
+
+ def test_different_seeds_different_results(self):
+ r1 = Randomizer(seed=1)
+ r2 = Randomizer(seed=2)
+ ips1 = [r1.internal_ip() for _ in range(5)]
+ ips2 = [r2.internal_ip() for _ in range(5)]
+ assert ips1 != ips2
+
+ def test_no_seed_is_random(self):
+ r1 = Randomizer()
+ r2 = Randomizer()
+ # Very unlikely to be the same with no seed
+ ips1 = [r1.internal_ip() for _ in range(10)]
+ ips2 = [r2.internal_ip() for _ in range(10)]
+ assert ips1 != ips2
+
+
+class TestPerformance:
+ """Test generation performance"""
+
+ def test_batch_ip_generation(self):
+ import time
+ r = Randomizer(seed=1)
+ start = time.time()
+ for _ in range(1000):
+ r.internal_ip()
+ r.external_ip()
+ elapsed = time.time() - start
+ assert elapsed < 2.0, f"1000 IP pairs took {elapsed:.2f}s"
+
+ def test_batch_person_generation(self):
+ import time
+ r = Randomizer(seed=1)
+ start = time.time()
+ for _ in range(100):
+ r.person(unique=False)
+ elapsed = time.time() - start
+ assert elapsed < 2.0, f"100 persons took {elapsed:.2f}s"
+
+
+if __name__ == "__main__":
+ pytest.main([__file__, "-v"])
diff --git a/Backend/parsers/community/paloalto_alternate_logs-latest/paloalto_alternate_logs.json b/Backend/parsers/community/paloalto_alternate_logs-latest/paloalto_alternate_logs.json
index 344ac6f..9080daf 100644
--- a/Backend/parsers/community/paloalto_alternate_logs-latest/paloalto_alternate_logs.json
+++ b/Backend/parsers/community/paloalto_alternate_logs-latest/paloalto_alternate_logs.json
@@ -1,400 +1,1169 @@
{
- // Specify time zone if logs are not UTC
- //timezone: "America/Detroit"
-
- attributes: {
- "dataSource.category": "security",
- "dataSource.name": "Palo Alto Networks",
- "dataSource.vendor": "Palo Alto Networks"
- }
- patterns: {
- timestamp_pat: "\\d{4}\\/\\d{2}\\/\\d{2}.\\d{2}:\\d{2}:\\d{2}", //YYYY/MM/DD HH:MM:SS
- value_pat: "\".*\"|[^,]*", //field value can be a comma-delimited list wrapped in quotes, a single value containing words and spaces, or empty
- }
- formats: [
- // TRAFFIC
- {
- id: "traffic-11-0", //PAN-OS 11.0 TRAFFIC, adds Flow Type and Cluster Name
- attributes: {
- "event.type": "traffic",
- "format": "traffic-v11"
- },
- format: "$network_activity.future_use_1$,$network_activity.receive_time$,$firewall.serial_number$,TRAFFIC,$network_activity.sub_type$,$network_activity.future_use_2$,$timestamp=timestamp_pat$,$src.ip.address$,$dst.ip.address$,$network_endpoint.nat_src_ip$,$network_endpoint.nat_dst_ip$,$rule.name$,$user.src_name$,$user.dst_name$,$network_activity.app_name$,$network_traffic.virtual_system_name$,$source_zone$,$destination_zone$,$network_interface.inbound_name$,$network_interface.outbound_name$,$network_activity.log_action$,$network_activity.future_use_3$,$session.uid$,$network_activity.repeat_count$,$network_endpoint.src_port$,$network_endpoint.dst_port$,$nenetwork_endpoint.nat_src_port$,$nenetwork_endpoint.nat_dst_port$,$network_connection_info.flag$,$network_connection_info.protocol_name$,$network_activity.action$,$network_traffic.bytes$,$network_traffic.bytes_out$,$network_traffic.bytes_in$,$network_traffic.packets$,$network_activity.start_time_dt$,$network_activity.elapsed_time$,$network_activity.category_name$,$network_activity.future_use_4$,$network_activity.sequence_number$,$network_activity.action_flags$,$location.src_country$,$location.dst_country$,$network_activity.future_use_5$,$network_traffic.packets_out$,$network_traffic.packets_in$,$session.expiration_reason$,$device.group_hierarchy.level_1$,$device.group_hierarchy.level_2$,$device.group_hierarchy.level_3$,$device.group_hierarchy.level_4$,$firewall.virtual_system_name$,$device.name$,$network_activity.action_source$,$virtual_machine.src_vm_uuid$,$virtual_machine.dst_vm_uuid$,$device.imsi$,$device.imei$,$session.parent_uid$,$network_activity.parent_start_time_dt$,$network_connection_info.tunnel_type$,$network_connection_info.sctp_id$,$network_connection_info.sctp_chunks$,$network_connection_info.sctp_chunks_out$,$network_connection_info.sctp_chunks_in$,$rule.uid$,$network_activity.http_connection$,$network_connection_info.app_flap_count$,$policy.uid$,$network_connection_info.link_switches$,$network_connection_info.sd_wan_cluster$,$network_connection_info.sd_wan_device_type$,$network_connection_info.sd_wan_cluster_type$,$network_connection_info.sd_wan_site$,$user.groups$,$http_request.x_forwarded_for$,$device.src_type$,$device.src_profile$,$device.src_model$,$device.src_vendor_name$,$device.src_os_edition$,$device.src_os_version$,$network_connection_info.src_hostname$,$device.src_mac$,$device.dst_type$,$device.dst_profile$,$device.dst_model$,$device.dst_vendor_name$,$device.src_os_edition$,$device.src_os_version$,$network_connection_info.dst_hostname$,$network_connection_info.dst_mac$,$container.id$,$container.pod_namespace$,$container.pod_name$,$network_endpoint.src_host_list$,$network_endpoint.dst_host_list$,$network_endpoint.host_id$,$device_hardware_info.serial_number$,$policy.src_group$,$policy.dst_group$,$session.owner$,$network_activity.time$,$network_activity.a_slice.service_type$,$network_activity.a_slice.differentiator$,$network_activity.sub_category$,$network_activity.category_name$,$network_activity.app_model$,$network_activity.severity$,$network_activity.app_characteristic=value_pat$,$network_activity.container.id$,$network_activity.app_tunnel_type$,$network_activity.is_saas$,$network_activity.is_sanctioned$,$network_activity.is_offloaded$,$network_activity.flow_type$,$network_activity.cluster.name$",
- rewrites: [
- {
- input: "device.name",
- output: "event.source",
- match: ".*",
- replace: "$0"
- }
- ],
- halt: true
- }, {
- id: "traffic-10-2", //PAN-OS 10.2 TRAFFIC, adds numerous fields after Dynamic User Group Name
- attributes: {
- "event.type": "traffic",
- "format": "traffic-v10.2"
- },
- format: "$network_activity.future_use_1$,$network_activity.receive_time$,$firewall.serial_number$,TRAFFIC,$network_activity.sub_type$,$network_activity.future_use_2$,$timestamp=timestamp_pat$,$src.ip.address$,$dst.ip.address$,$network_endpoint.nat_src_ip$,$network_endpoint.nat_dst_ip$,$rule.name$,$user.src_name$,$user.dst_name$,$network_activity.app_name$,$network_traffic.virtual_system_name$,$source_zone$,$destination_zone$,$network_interface.inbound_name$,$network_interface.outbound_name$,$network_activity.log_action$,$network_activity.future_use_3$,$session.uid$,$network_activity.repeat_count$,$network_endpoint.src_port$,$network_endpoint.dst_port$,$nenetwork_endpoint.nat_src_port$,$nenetwork_endpoint.nat_dst_port$,$network_connection_info.flag$,$network_connection_info.protocol_name$,$network_activity.action$,$network_traffic.bytes$,$network_traffic.bytes_out$,$network_traffic.bytes_in$,$network_traffic.packets$,$network_activity.start_time_dt$,$network_activity.elapsed_time$,$network_activity.category_name$,$network_activity.future_use_4$,$network_activity.sequence_number$,$network_activity.action_flags$,$location.src_country$,$location.dst_country$,$network_activity.future_use_5$,$network_traffic.packets_out$,$network_traffic.packets_in$,$session.expiration_reason$,$device.group_hierarchy.level_1$,$device.group_hierarchy.level_2$,$device.group_hierarchy.level_3$,$device.group_hierarchy.level_4$,$firewall.virtual_system_name$,$device.name$,$network_activity.action_source$,$virtual_machine.src_vm_uuid$,$virtual_machine.dst_vm_uuid$,$device.imsi$,$device.imei$,$session.parent_uid$,$network_activity.parent_start_time_dt$,$network_connection_info.tunnel_type$,$network_connection_info.sctp_id$,$network_connection_info.sctp_chunks$,$network_connection_info.sctp_chunks_out$,$network_connection_info.sctp_chunks_in$,$rule.uid$,$network_activity.http_connection$,$network_connection_info.app_flap_count$,$policy.uid$,$network_connection_info.link_switches$,$network_connection_info.sd_wan_cluster$,$network_connection_info.sd_wan_device_type$,$network_connection_info.sd_wan_cluster_type$,$network_connection_info.sd_wan_site$,$user.groups$,$http_request.x_forwarded_for$,$device.src_type$,$device.src_profile$,$device.src_model$,$device.src_vendor_name$,$device.src_os_edition$,$device.src_os_version$,$network_connection_info.src_hostname$,$device.src_mac$,$device.dst_type$,$device.dst_profile$,$device.dst_model$,$device.dst_vendor_name$,$device.src_os_edition$,$device.src_os_version$,$network_connection_info.dst_hostname$,$network_connection_info.dst_mac$,$container.id$,$container.pod_namespace$,$container.pod_name$,$network_endpoint.src_host_list$,$network_endpoint.dst_host_list$,$network_endpoint.host_id$,$device_hardware_info.serial_number$,$policy.src_group$,$policy.dst_group$,$session.owner$,$network_activity.time$,$network_activity.a_slice.service_type$,$network_activity.a_slice.differentiator$,$network_activity.sub_category$,$network_activity.category_name$,$network_activity.app_model$,$network_activity.severity$,$network_activity.app_characteristic=value_pat$,$network_activity.container.id$,$network_activity.app_tunnel_type$,$network_activity.is_saas$,$network_activity.is_sanctioned$,$network_activity.is_offloaded$",
- rewrites: [
- {
- input: "device.name",
- output: "event.source",
- match: ".*",
- replace: "$0"
- }
- ],
- halt: true
- }, {
- id: "traffic-9-1", //PAN-OS 9.1 TRAFFIC
- attribute: {
- "event.type": "traffic",
- "format": "traffic-v9.1"
- },
- format: "$network_activity.future_use_1$,$network_activity.receive_time$,$firewall.serial_number$,TRAFFIC,$network_activity.sub_type$,$network_activity.future_use_2$,$timestamp=timestamp_pat$,$src.ip.address$,$dst.ip.address$,$network_endpoint.nat_src_ip$,$network_endpoint.nat_dst_ip$,$rule.name$,$user.src_name$,$user.dst_name$,$network_activity.app_name$,$network_traffic.virtual_system_name$,$source_zone$,$destination_zone$,$network_interface.inbound_name$,$network_interface.outbound_name$,$network_activity.log_action$,$network_activity.future_use_3$,$session.uid$,$network_activity.repeat_count$,$network_endpoint.src_port$,$network_endpoint.dst_port$,$nenetwork_endpoint.nat_src_port$,$nenetwork_endpoint.nat_dst_port$,$network_connection_info.flag$,$network_connection_info.protocol_name$,$network_activity.action$,$network_traffic.bytes$,$network_traffic.bytes_out$,$network_traffic.bytes_in$,$network_traffic.packets$,$network_activity.start_time_dt$,$network_activity.elapsed_time$,$network_activity.category_name$,$network_activity.future_use_4$,$network_activity.sequence_number$,$network_activity.action_flags$,$location.src_country$,$location.dst_country$,$network_activity.future_use_5$,$network_traffic.packets_out$,$network_traffic.packets_in$,$session.expiration_reason$,$device.group_hierarchy.level_1$,$device.group_hierarchy.level_2$,$device.group_hierarchy.level_3$,$device.group_hierarchy.level_4$,$firewall.virtual_system_name$,$device.name$,$network_activity.action_source$,$virtual_machine.src_vm_uuid$,$virtual_machine.dst_vm_uuid$,$device.imsi$,$device.imei$,$session.parent_uid$,$network_activity.parent_start_time_dt$,$network_connection_info.tunnel_type$,$network_connection_info.sctp_id$,$network_connection_info.sctp_chunks$,$network_connection_info.sctp_chunks_out$,$network_connection_info.sctp_chunks_in$,$rule.uid$,$network_activity.http_connection$,$network_connection_info.app_flap_count$,$policy.uid$,$network_connection_info.link_switches$,$network_connection_info.sd_wan_cluster$,$network_connection_info.sd_wan_device_type$,$network_connection_info.sd_wan_cluster_type$,$network_connection_info.sd_wan_site$,$user.groups$",
- rewrites: [
- {
- input: "device.name",
- output: "event.source",
- match: ".*",
- replace: "$0"
- }
- ],
- halt: true
- },
- // THREAT
- {
- id: "threat-11-0", //PAN-OS 11.0 THREAT, adds Cluster Name and Flow Type
- attributes: {
- "event.type": "threat",
- "format": "threat-v11"
- },
- format: "$network_activity.future_use_1$,$network_activity.receive_time$,$firewall.serial_number$,THREAT,$network_activity.sub_type$,$network_activity.future_use_2$,$timestamp=timestamp_pat$,$src.ip.address$,$dst.ip.address$,$network_endpoint.nat_src_ip$,$network_endpoint.nat_dst_ip$,$rule.name$,$user.src_name$,$user.dst_name$,$network_activity.app_name$,$network_traffic.virtual_system_name$,$source_zone$,$destination_zone$,$network_interface.inbound_name$,$network_interface.outbound_name$,$network_activity.log_action$,$network_activity.future_use_3$,$session.uid$,$network_activity.repeat_count$,$network_endpoint.src_port$,$network_endpoint.dst_port$,$nenetwork_endpoint.nat_src_port$,$nenetwork_endpoint.nat_dst_port$,$network_connection_info.flag$,$network_connection_info.protocol_name$,$network_activity.action$,$url.filename=value_pat$,$cwe.uid$,$network_activity.category_name$,$severity$,$direction$,$network_activity.sequence_number$,$network_activity.action_flags$,$location.src_location$,$location.dst_location$,$misc$,$Content_type$,$pcap.id$,$file_digest$,$cloud$,$url.index$,$user_agent$,$file_type$,$x-forwarded-for$,$referer$,$sender$,$subject$,$recipient$,$report.id$,$device.group_hierarchy.level_1$,$device.group_hierarchy.level_2$,$device.group_hierarchy.level_3$,$device.group_hierarchy.level_4$,$firewall.virtual_system_name$,$device.name$,$network_activity.future_use_4$,$virtual_machine.src_vm_uuid$,$virtual_machine.dst_vm_uuid$,$http_method$,$device.imsi$,$device.imei$,$session.parent_uid$,$network_activity.parent_start_time_dt$,$network_connection_info.tunnel_type$,$cwe.caption$,$content_version$,$network_activity.future_use_5$,$network_activity.sctp_association_id$,$network_connection_info.protocol_name$,$http_header$,$url.category_list=value_pat$,$rule.uid$,$network_activity.http_connection$,$user.groups$,$http_request.x_forwarded_for$,$device.src_type$,$device.src_profile$,$device.src_model$,$device.src_vendor_name$,$device.src_os_edition$,$device.src_os_version$,$network_connection_info.src_hostname$,$device.src_mac$,$device.dst_type$,$device.dst_profile$,$device.dst_model$,$device.dst_vendor_name$,$device.src_os_edition$,$device.src_os_version$,$network_connection_info.dst_hostname$,$network_connection_info.dst_mac$,$container.id$,$container.pod_namespace$,$container.pod_name$,$network_endpoint.src_host_list$,$network_endpoint.dst_host_list$,$network_endpoint.host_id$,$device_hardware_info.serial_number$,$domain_edl$,$policy.src_group$,$policy.dst_group$,$partial_hash$,$network_activity.time$,$reason$,$justification$,$network_activity.a_slice.service_type$,$network_activity.sub_category$,$network_activity.category_name$,$network_activity.app_model$,$network_activity.severity$,$network_activity.app_characteristic=value_pat$,$network_activity.container.id$,$network_activity.app_tunnel_type$,$network_activity.is_saas$,$network_activity.is_sanctioned$,$cloud_report_id$,$network_activity.cluster.name$,$network_activity.flow_type$",
- rewrites: [
- {
- input: "device.name",
- output: "event.source",
- match: ".*",
- replace: "$0"
- }
- ],
- halt: true
- }, {
- id: "threat-10-2", //PAN-OS 10.2 THREAT, adds Cloud Report ID
- attributes: {
- "event.type": "threat",
- "format": "threat-v10.2"
- },
- format: "$network_activity.future_use_1$,$network_activity.receive_time$,$firewall.serial_number$,THREAT,$network_activity.sub_type$,$network_activity.future_use_2$,$timestamp=timestamp_pat$,$src.ip.address$,$dst.ip.address$,$network_endpoint.nat_src_ip$,$network_endpoint.nat_dst_ip$,$rule.name$,$user.src_name$,$user.dst_name$,$network_activity.app_name$,$network_traffic.virtual_system_name$,$source_zone$,$destination_zone$,$network_interface.inbound_name$,$network_interface.outbound_name$,$network_activity.log_action$,$network_activity.future_use_3$,$session.uid$,$network_activity.repeat_count$,$network_endpoint.src_port$,$network_endpoint.dst_port$,$nenetwork_endpoint.nat_src_port$,$nenetwork_endpoint.nat_dst_port$,$network_connection_info.flag$,$network_connection_info.protocol_name$,$network_activity.action$,$url.filename=value_pat$,$cwe.uid$,$network_activity.category_name$,$severity$,$direction$,$network_activity.sequence_number$,$network_activity.action_flags$,$location.src_location$,$location.dst_location$,$misc$,$Content_type$,$pcap.id$,$file_digest$,$cloud$,$url.index$,$user_agent$,$file_type$,$x-forwarded-for$,$referer$,$sender$,$subject$,$recipient$,$report.id$,$device.group_hierarchy.level_1$,$device.group_hierarchy.level_2$,$device.group_hierarchy.level_3$,$device.group_hierarchy.level_4$,$firewall.virtual_system_name$,$device.name$,$network_activity.future_use_4$,$virtual_machine.src_vm_uuid$,$virtual_machine.dst_vm_uuid$,$http_method$,$device.imsi$,$device.imei$,$session.parent_uid$,$network_activity.parent_start_time_dt$,$network_connection_info.tunnel_type$,$cwe.caption$,$content_version$,$network_activity.future_use_5$,$network_activity.sctp_association_id$,$network_connection_info.protocol_name$,$http_header$,$url.category_list=value_pat$,$rule.uid$,$network_activity.http_connection$,$user.groups$,$http_request.x_forwarded_for$,$device.src_type$,$device.src_profile$,$device.src_model$,$device.src_vendor_name$,$device.src_os_edition$,$device.src_os_version$,$network_connection_info.src_hostname$,$device.src_mac$,$device.dst_type$,$device.dst_profile$,$device.dst_model$,$device.dst_vendor_name$,$device.src_os_edition$,$device.src_os_version$,$network_connection_info.dst_hostname$,$network_connection_info.dst_mac$,$container.id$,$container.pod_namespace$,$container.pod_name$,$network_endpoint.src_host_list$,$network_endpoint.dst_host_list$,$network_endpoint.host_id$,$device_hardware_info.serial_number$,$domain_edl$,$policy.src_group$,$policy.dst_group$,$partial_hash$,$network_activity.time$,$reason$,$justification$,$network_activity.a_slice.service_type$,$network_activity.sub_category$,$network_activity.category_name$,$network_activity.app_model$,$network_activity.severity$,$network_activity.app_characteristic=value_pat$,$network_activity.container.id$,$network_activity.app_tunnel_type$,$network_activity.is_saas$,$network_activity.is_sanctioned$,$cloud_report_id$",
- rewrites: [
- {
- input: "device.name",
- output: "event.source",
- match: ".*",
- replace: "$0"
- }
- ],
- halt: true
- }, {
- id: "threat-10-1", //PAN-OS 10.1 THREAT, adds numerous fields after XFF Address
- attributes: {
- "event.type": "threat",
- "format": "threat-v10.1"
- },
- format: "$network_activity.future_use_1$,$network_activity.receive_time$,$firewall.serial_number$,THREAT,$network_activity.sub_type$,$network_activity.future_use_2$,$timestamp=timestamp_pat$,$src.ip.address$,$dst.ip.address$,$network_endpoint.nat_src_ip$,$network_endpoint.nat_dst_ip$,$rule.name$,$user.src_name$,$user.dst_name$,$network_activity.app_name$,$network_traffic.virtual_system_name$,$source_zone$,$destination_zone$,$network_interface.inbound_name$,$network_interface.outbound_name$,$network_activity.log_action$,$network_activity.future_use_3$,$session.uid$,$network_activity.repeat_count$,$network_endpoint.src_port$,$network_endpoint.dst_port$,$nenetwork_endpoint.nat_src_port$,$nenetwork_endpoint.nat_dst_port$,$network_connection_info.flag$,$network_connection_info.protocol_name$,$network_activity.action$,$url.filename=value_pat$,$cwe.uid$,$network_activity.category_name$,$severity$,$direction$,$network_activity.sequence_number$,$network_activity.action_flags$,$location.src_location$,$location.dst_location$,$misc$,$Content_type$,$pcap.id$,$file_digest$,$cloud$,$url.index$,$user_agent$,$file_type$,$x-forwarded-for$,$referer$,$sender$,$subject$,$recipient$,$report.id$,$device.group_hierarchy.level_1$,$device.group_hierarchy.level_2$,$device.group_hierarchy.level_3$,$device.group_hierarchy.level_4$,$firewall.virtual_system_name$,$device.name$,$network_activity.future_use_4$,$virtual_machine.src_vm_uuid$,$virtual_machine.dst_vm_uuid$,$http_method$,$device.imsi$,$device.imei$,$session.parent_uid$,$network_activity.parent_start_time_dt$,$network_connection_info.tunnel_type$,$cwe.caption$,$content_version$,$network_activity.future_use_5$,$network_activity.sctp_association_id$,$network_connection_info.protocol_name$,$http_header$,$url.category_list=value_pat$,$rule.uid$,$network_activity.http_connection$,$user.groups$,$http_request.x_forwarded_for$,$device.src_type$,$device.src_profile$,$device.src_model$,$device.src_vendor_name$,$device.src_os_edition$,$device.src_os_version$,$network_connection_info.src_hostname$,$device.src_mac$,$device.dst_type$,$device.dst_profile$,$device.dst_model$,$device.dst_vendor_name$,$device.src_os_edition$,$device.src_os_version$,$network_connection_info.dst_hostname$,$network_connection_info.dst_mac$,$container.id$,$container.pod_namespace$,$container.pod_name$,$network_endpoint.src_host_list$,$network_endpoint.dst_host_list$,$network_endpoint.host_id$,$device_hardware_info.serial_number$,$domain_edl$,$policy.src_group$,$policy.dst_group$,$partial_hash$,$network_activity.time$,$reason$,$justification$,$network_activity.a_slice.service_type$,$network_activity.sub_category$,$network_activity.category_name$,$network_activity.app_model$,$network_activity.severity$,$network_activity.app_characteristic=value_pat$,$network_activity.container.id$,$network_activity.app_tunnel_type$,$network_activity.is_saas$,$network_activity.is_sanctioned$"
- rewrites: [
+ attributes: {
+ "dataSource.category": "security",
+ "dataSource.name": "Palo Alto Networks Firewall",
+ "dataSource.vendor": "Palo Alto Networks",
+ },
+ patterns: {
+ //maps to high_resolution_timestamp:
+ //timestamp: "\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}.\\d{3}(\\+|-)\\d{2}:\\d{2}",
+ //application_characteristic can be a single value, a comma delimited list in quotes, or blank. Null value is handled by format: traffic-2, not by this pattern.
+ app_characteristic: "(\".*\")|[^,]+",
+ //description field from system log is wrapped in quotes and may contain commas
+ desc: "(\".*\")",
+ userid_log_type: "USERID",
+ logout_sub_type: "logout",
+ login_sub_type: "login",
+ hipmatch_log_type: "HIPMATCH",
+ config_log_type: "CONFIG",
+ wildfire_sub_type: "wildfire",
+ data_filtering_sub_type: "file",
+ globalprotect_log_type: "GLOBALPROTECT",
+ iptag_log_type: "IPTAG",
+ gtp_log_type: "GTP",
+ tunnel_log_type: "\\b(?:START|END|start|end)\\b",
+ sctp_log_type: "SCTP",
+ system_log_type: "SYSTEM"
+ },
+
+ formats: [
+ // {
+ // format: ".*$timestamp=timestamp$(\\,)*",
+ //},
+ {
+ //match all fields. application_characteristic can be a single value, or a comma delimited list in quotes.
+ attributes: {
+ "class_uid": "4001",
+ "category_uid": "4",
+ "severity_id": "0",
+ "class_name": "Network Activity",
+ "category_name": "Network Activity",
+ "metadata.product.name": "Palo Alto Networks Firewall",
+ "metadata.product.vendor_name": "Palo Alto Networks",
+ "metadata.version":"1.0.0-rc.3",
+ "metadata.log_name": "TRAFFIC",
+ },
+ format: ".*,$metadata.logged_time_dt$,$device.hw_info.serial_number$,TRAFFIC,$unmapped.sub_type$,.*,$metadata.original_time$,$src_endpoint.ip$,$dst_endpoint.ip$,$src_endpoint.intermediate_ips$,$dst_endpoint.intermediate_ips$,$unmapped.rule_matched$,$actor.user.name$,$unmapped.dst_user$,$app_name$,$unmapped.vsys$,$unmapped.from_zone$,$unmapped.to_zone$,$unmapped.inbound_if$,$unmapped.outbound_if$,$actor.session.issuer$,$metadata.original_time$,$actor.session.uid$,$unmapped.repeat_count$,$src_endpoint.port$,$dst_endpoint.port$,$unmapped.nat_src_port$,$unmapped.nat_dst_port$,$unmapped.flags$,$connection_info.protocol_name$,$unmapped.action_value$,$traffic.bytes$,$traffic.bytes_in$,$traffic.bytes_out$,$traffic.packets$,$actor.session.created_time_dt$,$duration$,$unmapped.url_category_value$,.*,$metadata.sequence$,$unmapped.action_flags$,$src_endpoint.location.region$,$dst_endpoint.location.region$,.*,$traffic.packets_out$,$traffic.packets_in$,$unmapped.session_end_reason_value$,$unmapped.dg_hier_level_1$,$unmapped.dg_hier_level_2$,$unmapped.dg_hier_level_3$,$unmapped.dg_hier_level_4$,$unmapped.vsys_name$,$device.hostname$,$unmapped.action_source$,$unmapped.src_uuid$,$unmapped.dst_uuid$,$device.imsi$,$device.imei$,$unmapped.parent_session_id$,$unmapped.parent_start_time$,$unmapped.tunnel_type$,$unmapped.ep_assoc_id$,$unmapped.chunks_total$,$unmapped.chunks_sent$,$unmapped.chunks_received$,$unmapped.rule_matched_uuid$,$unmapped.http2_connection$,$unmapped.link_change_count$,$unmapped.policy_id$,$unmapped.link_switches$,$unmapped.sdwan_cluster$,$unmapped.sdwan_device_type$,$unmapped.sdwan_cluster_type$,$unmapped.sdwan_site$,$actor.user.groups$,$src_endpoint.intermediate_ips$,$unmapped.src_category$,$unmapped.src_profile$,$unmapped.src_model$,$unmapped.src_vendor$,$unmapped.src_osfamily$,$unmapped.src_osversion$,$src_endpoint.hostname$,$src_endpoint.mac$,$unmapped.dst_category$,$unmapped.dst_profile$,$unmapped.dst_model$,$unmapped.dst_vendor$,$unmapped.dst_osfamily$,$unmapped.dst_osversion$,$dst_endpoint.hostname$,$dst_endpoint.mac$,$unmapped.container_id$,$unmapped.pod_namespace$,$unmapped.pod_name$,$unmapped.src_edl$,$unmapped.dst_edl$,$device.uid$,$unmapped.serial_number$,$unmapped.src_dag$,$unmapped.dst_dag$,$unmapped.session_owner$,$unmapped.high_res_timestamp$,$unmapped.nsdsai_sst$,$unmapped.nsdsai_sd$,$unmapped.subcategory_of_app$,$unmapped.category_of_app$,$unmapped.technology_of_app$,$unmapped.risk_of_app$,$unmapped.characteristic_of_app=app_characteristic$,$unmapped.container_of_app$,$unmapped.tunneled_app$,$unmapped.is_saas_of_app$,$unmapped.sanctioned_state_of_app$,$unmapped.offloaded$",
+ halt: true,
+ rewrites: [
+ {
+ input: "unmapped.sub_type",
+ output: "activity_id",
+ match: "^start$",
+ replace: "1"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "activity_id",
+ match: "^end$",
+ replace: "2"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "activity_id",
+ match: "^drop$",
+ replace: "4"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "activity_id",
+ match: "^deny$",
+ replace: "5"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "activity_name",
+ match: "^start$",
+ replace: "Open"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "activity_name",
+ match: "^end$",
+ replace: "Close"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "activity_name",
+ match: "^drop$",
+ replace: "Fail"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "activity_name",
+ match: "^deny$",
+ replace: "Refuse"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "event.type",
+ match: "^start$",
+ replace: "Open"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "event.type",
+ match: "^end$",
+ replace: "Close"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "event.type",
+ match: "^drop$",
+ replace: "Fail"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "event.type",
+ match: "^deny$",
+ replace: "Refuse"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "type_uid",
+ match: "^start$",
+ replace: "400101"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "type_uid",
+ match: "^end$",
+ replace: "400102"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "type_uid",
+ match: "^drop$",
+ replace: "400104"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "type_uid",
+ match: "^deny$",
+ replace: "400105"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "type_name",
+ match: "^start$",
+ replace: "Network Activity: Open"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "type_name",
+ match: "^end$",
+ replace: "Network Activity: Close"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "type_name",
+ match: "^drop$",
+ replace: "Network Activity: Fail"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "type_name",
+ match: "^deny$",
+ replace: "Network Activity: Refuse"
+ },
+ {
+ input: "unmapped.action_value",
+ output: "status_id",
+ match: "^allow$",
+ replace: "1"
+ },
+ {
+ input: "unmapped.action_value",
+ output: "status_id",
+ match: "^deny$",
+ replace: "2"
+ },
+ {
+ input: "unmapped.action_value",
+ output: "status",
+ match: "^allow$",
+ replace: "Success"
+ },
+ {
+ input: "unmapped.action_value",
+ output: "status",
+ match: "^deny$",
+ replace: "Failure"
+ },
+ {
+ input: "unmapped.action_value",
+ output: "status_id",
+ match: "^(?!allow|deny$).*",
+ replace: "99"
+ },
+ {
+ input: "unmapped.action_value",
+ output: "status",
+ match: "^(?!allow|deny$).*",
+ replace: "Other"
+ },
+ {
+ input: "dst_endpoint.intermediate_ips",
+ output: "dst_endpoint.intermediate_ips",
+ match: ".*",
+ replace: "\\[\"$0\"\\]"
+ },
+ {
+ input: "message",
+ output: "src_endpoint.intermediate_ips",
+ match: "(?:[^,]*,){9}([^,]*){1},(?:[^,]*,){65}([^,]*){1},(?:[^,]*,){38}.*",
+ replace: "\\[\"$1\"\\, \"$2\"\\]"
+ },
+ {
+ input: "message",
+ output: "observables",
+ match: "(?:[^,]*,){7}([^,]*),([^,]*),(?:[^,]*,){3}([^,]*),(?:[^,]*,){69}([^,]*),([^,]*),(?:[^,]*,){6}([^,]*),([^,]*),.*",
+ replace: "\\[\\{\"type_id\"\\: \"1\"\\, \"type\"\\: \"Hostname\"\\, \"name\"\\: \"src_endpoint.hostname\"\\, \"value\"\\: \"$4\"\\}\\, \\{\"type_id\"\\: \"1\"\\, \"type\"\\: \"Hostname\"\\, \"name\"\\: \"dst_endpoint.hostname\"\\, \"value\"\\: \"$6\"\\}\\, \\{\"type_id\"\\: \"2\"\\, \"type\"\\: \"IP Address\"\\, \"name\"\\: \"src_endpoint.ip\"\\, \"value\"\\: \"$1\"\\}\\, \\{\"type_id\"\\: \"2\"\\, \"type\"\\: \"IP Address\"\\, \"name\"\\: \"dst_endpoint.ip\"\\, \"value\"\\: \"$2\"\\}\\, \\{\"type_id\"\\: \"4\"\\, \"type\"\\: \"User Name\"\\, \"name\"\\: \"actor.user.name\"\\, \"value\"\\: \"$3\"\\}\\, \\{\"type_id\"\\: \"3\"\\, \"type\"\\: \"MAC Address\"\\, \"name\"\\: \"src_endpoint.mac\"\\, \"value\"\\: \"$5\"\\}\\, \\{\"type_id\"\\: \"3\"\\, \"type\"\\: \"MAC Address\"\\, \"name\"\\: \"dst_endpoint.mac\"\\, \"value\"\\: \"$7\"\\}\\]"
+ },
+ ]
+ },
+ {
+ //dont match on application_characteristic for cases where is it blank.
+ attributes: {
+ "class_uid": "4001",
+ "category_uid": "4",
+ "severity_id": "0",
+ "class_name": "Network Activity",
+ "category_name": "Network Activity",
+ "metadata.product.name": "Palo Alto Networks Firewall",
+ "metadata.product.vendor_name": "Palo Alto Networks",
+ "metadata.version":"1.0.0-rc.3",
+ "metadata.log_name": "TRAFFIC",
+ },
+ format: ".*,$metadata.logged_time_dt$,$device.hw_info.serial_number$,TRAFFIC,$unmapped.sub_type$,.*,$metadata.original_time$,$src_endpoint.ip$,$dst_endpoint.ip$,$src_endpoint.intermediate_ips$,$dst_endpoint.intermediate_ips$,$unmapped.rule_matched$,$actor.user.name$,$unmapped.dst_user$,$app_name$,$unmapped.vsys$,$unmapped.from_zone$,$unmapped.to_zone$,$unmapped.inbound_if$,$unmapped.outbound_if$,$actor.session.issuer$,$metadata.original_time$,$actor.session.uid$,$unmapped.repeat_count$,$src_endpoint.port$,$dst_endpoint.port$,$unmapped.nat_src_port$,$unmapped.nat_dst_port$,$unmapped.flags$,$connection_info.protocol_name$,$unmapped.action_value$,$traffic.bytes$,$traffic.bytes_in$,$traffic.bytes_out$,$traffic.packets$,$actor.session.created_time_dt$,$duration$,$unmapped.url_category_value$,.*,$metadata.sequence$,$unmapped.action_flags$,$src_endpoint.location.region$,$dst_endpoint.location.region$,.*,$traffic.packets_out$,$traffic.packets_in$,$unmapped.session_end_reason_value$,$unmapped.dg_hier_level_1$,$unmapped.dg_hier_level_2$,$unmapped.dg_hier_level_3$,$unmapped.dg_hier_level_4$,$unmapped.vsys_name$,$device.hostname$,$unmapped.action_source$,$unmapped.src_uuid$,$unmapped.dst_uuid$,$device.imsi$,$device.imei$,$unmapped.parent_session_id$,$unmapped.parent_start_time$,$unmapped.tunnel_type$,$unmapped.ep_assoc_id$,$unmapped.chunks_total$,$unmapped.chunks_sent$,$unmapped.chunks_received$,$unmapped.rule_matched_uuid$,$unmapped.http2_connection$,$unmapped.link_change_count$,$unmapped.policy_id$,$unmapped.link_switches$,$unmapped.sdwan_cluster$,$unmapped.sdwan_device_type$,$unmapped.sdwan_cluster_type$,$unmapped.sdwan_site$,$actor.user.groups$,$src_endpoint.intermediate_ips$,$unmapped.src_category$,$unmapped.src_profile$,$unmapped.src_model$,$unmapped.src_vendor$,$unmapped.src_osfamily$,$unmapped.src_osversion$,$src_endpoint.hostname$,$src_endpoint.mac$,$unmapped.dst_category$,$unmapped.dst_profile$,$unmapped.dst_model$,$unmapped.dst_vendor$,$unmapped.dst_osfamily$,$unmapped.dst_osversion$,$dst_endpoint.hostname$,$dst_endpoint.mac$,$unmapped.container_id$,$unmapped.pod_namespace$,$unmapped.pod_name$,$unmapped.src_edl$,$unmapped.dst_edl$,$device.uid$,$unmapped.serial_number$,$unmapped.src_dag$,$unmapped.dst_dag$,$unmapped.session_owner$,$unmapped.high_res_timestamp$,$unmapped.nsdsai_sst$,$unmapped.nsdsai_sd$,$unmapped.subcategory_of_app$,$unmapped.category_of_app$,$unmapped.technology_of_app$,$unmapped.risk_of_app$,$unmapped.characteristic_of_app$,$unmapped.container_of_app$,$unmapped.tunneled_app$,$unmapped.is_saas_of_app$,$unmapped.sanctioned_state_of_app$,$unmapped.offloaded$",
+ halt: true,
+ rewrites: [
+ {
+ input: "unmapped.sub_type",
+ output: "event.type",
+ match: "^start$",
+ replace: "Open"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "event.type",
+ match: "^end$",
+ replace: "Close"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "event.type",
+ match: "^drop$",
+ replace: "Fail"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "event.type",
+ match: "^deny$",
+ replace: "Refuse"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "activity_id",
+ match: "^start$",
+ replace: "1"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "activity_id",
+ match: "^end$",
+ replace: "2"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "activity_id",
+ match: "^drop$",
+ replace: "4"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "activity_id",
+ match: "^deny$",
+ replace: "5"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "activity_name",
+ match: "^start$",
+ replace: "Open"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "activity_name",
+ match: "^end$",
+ replace: "Close"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "activity_name",
+ match: "^drop$",
+ replace: "Fail"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "activity_name",
+ match: "^deny$",
+ replace: "Refuse"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "type_uid",
+ match: "^start$",
+ replace: "400101"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "type_uid",
+ match: "^end$",
+ replace: "400102"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "type_uid",
+ match: "^drop$",
+ replace: "400104"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "type_uid",
+ match: "^deny$",
+ replace: "400105"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "type_name",
+ match: "^start$",
+ replace: "Network Activity: Open"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "type_name",
+ match: "^end$",
+ replace: "Network Activity: Close"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "type_name",
+ match: "^drop$",
+ replace: "Network Activity: Fail"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "type_name",
+ match: "^deny$",
+ replace: "Network Activity: Refuse"
+ },
+ {
+ input: "unmapped.action_value",
+ output: "status_id",
+ match: "^allow$",
+ replace: "1"
+ },
+ {
+ input: "unmapped.action_value",
+ output: "status_id",
+ match: "^deny$",
+ replace: "2"
+ },
+ {
+ input: "unmapped.action_value",
+ output: "status",
+ match: "^allow$",
+ replace: "Success"
+ },
+ {
+ input: "unmapped.action_value",
+ output: "status",
+ match: "^deny$",
+ replace: "Failure"
+ },
+ {
+ input: "unmapped.action_value",
+ output: "status_id",
+ match: "^(?!allow|deny$).*",
+ replace: "99"
+ },
+ {
+ input: "unmapped.action_value",
+ output: "status",
+ match: "^(?!allow|deny$).*",
+ replace: "Other"
+ },
+ {
+ input: "dst_endpoint.intermediate_ips",
+ output: "dst_endpoint.intermediate_ips",
+ match: ".*",
+ replace: "\\[\"$0\"\\]"
+ },
+ {
+ input: "message",
+ output: "src_endpoint.intermediate_ips",
+ match: "(?:[^,]*,){9}([^,]*){1},(?:[^,]*,){65}([^,]*){1},(?:[^,]*,){38}.*",
+ replace: "\\[\"$1\"\\, \"$2\"\\]"
+ },
+ {
+ input: "message",
+ output: "observables",
+ match: "(?:[^,]*,){7}([^,]*),([^,]*),(?:[^,]*,){3}([^,]*),(?:[^,]*,){69}([^,]*),([^,]*),(?:[^,]*,){6}([^,]*),([^,]*),.*",
+ replace: "\\[\\{\"type_id\"\\: \"1\"\\, \"type\"\\: \"Hostname\"\\, \"name\"\\: \"src_endpoint.hostname\"\\, \"value\"\\: \"$4\"\\}\\, \\{\"type_id\"\\: \"1\"\\, \"type\"\\: \"Hostname\"\\, \"name\"\\: \"dst_endpoint.hostname\"\\, \"value\"\\: \"$6\"\\}\\, \\{\"type_id\"\\: \"2\"\\, \"type\"\\: \"IP Address\"\\, \"name\"\\: \"src_endpoint.ip\"\\, \"value\"\\: \"$1\"\\}\\, \\{\"type_id\"\\: \"2\"\\, \"type\"\\: \"IP Address\"\\, \"name\"\\: \"dst_endpoint.ip\"\\, \"value\"\\: \"$2\"\\}\\, \\{\"type_id\"\\: \"4\"\\, \"type\"\\: \"User Name\"\\, \"name\"\\: \"actor.user.name\"\\, \"value\"\\: \"$3\"\\}\\, \\{\"type_id\"\\: \"3\"\\, \"type\"\\: \"MAC Address\"\\, \"name\"\\: \"src_endpoint.mac\"\\, \"value\"\\: \"$5\"\\}\\, \\{\"type_id\"\\: \"3\"\\, \"type\"\\: \"MAC Address\"\\, \"name\"\\: \"dst_endpoint.mac\"\\, \"value\"\\: \"$7\"\\}\\]"
+ },
+ ]
+ },
+ {
+ attributes: {
+ "class_uid": "0",
+ "activity_id": "99",
+ "category_uid": "0",
+ "type_uid": "99",
+ "type_name": "Base Event: Other",
+ "class_name": "Base Event",
+ "category_name": "Uncategorized",
+ "metadata.product.name": "Palo Alto Networks Firewall",
+ "metadata.product.vendor_name": "Palo Alto Networks",
+ "metadata.version":"1.0.0-rc.3",
+ "metadata.log_name": "SYSTEM",
+ },
+ format: ".*,$metadata.logged_time_dt$,$unmapped.serial$,SYSTEM,$unmapped.sub_type$,.*,$metadata.original_time$,$unmapped.vsys$,$unmapped.event_id$,$unmapped.object$,.*,.*,$unmapped.module$,$unmapped.severity$,$unmapped.description=desc$,$metadata.sequence$,$unmapped.action_flags$,$unmapped.dg_hier_level_1$,$unmapped.dg_hier_level_2$,$unmapped.dg_hier_level_3$,$unmapped.dg_hier_level_4$,$unmapped.vsys_name$,$unmapped.device_name$,.*,.*,$unmapped.high_res_timestamp$",
+ halt: true,
+ rewrites: [
+ {
+ input: "unmapped.sub_type",
+ output: "activity_name",
+ match: ".*",
+ replace: "$0"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "event.type",
+ match: ".*",
+ replace: "$0"
+ },
+ {
+ input: "unmapped.severity",
+ output: "severity_id",
+ match: "^informational$",
+ replace: "1"
+ },
+ {
+ input: "unmapped.severity",
+ output: "severity_id",
+ match: "^low$",
+ replace: "2"
+ },
+ {
+ input: "unmapped.severity",
+ output: "severity_id",
+ match: "^medium$",
+ replace: "3"
+ },
+ {
+ input: "unmapped.severity",
+ output: "severity_id",
+ match: "^high$",
+ replace: "4"
+ },
+ {
+ input: "unmapped.severity",
+ output: "severity_id",
+ match: "^critical$",
+ replace: "5"
+ },
+ {
+ input: "message",
+ output: "observables",
+ match: "(?:[^,]*,){14}(\".*\"),(?:[^,]*,){7}([^,]*),.*",
+ replace: "\\[\\{\"type_id\"\\: \"1\"\\, \"type\"\\: \"Hostname\"\\, \"name\"\\: \"device.hostname\"\\, \"value\"\\: \"$2\"\\}\\]"
+ },
+ ]
+ },
+ {
+ //matches THREAT logs with comma surround lists in application_characteristic and url_category_list.
+ attributes: {
+ "activity_name": "THREAT",
+ "class_uid": "4001",
+ "activity_id": "99",
+ "category_uid": "4",
+ "type_uid": "400199",
+ "type_name": "Network Activity: Other",
+ "class_name": "Network Activity",
+ "category_name": "Network Activity",
+ "metadata.product.name": "Palo Alto Networks Firewall",
+ "metadata.product.vendor_name": "Palo Alto Networks",
+ "metadata.version":"1.0.0-rc.3",
+ "metadata.log_name": "THREAT",
+ "event.type": "THREAT"
+ },
+ format: ".*,$metadata.logged_time_dt$,$device.hw_info.serial_number$,THREAT,$unmapped.sub_type$,.*,$metadata.original_time$,$src_endpoint.ip$,$dst_endpoint.ip$,$src_endpoint.intermediate_ips$,$dst_endpoint.intermediate_ips$,$unmapped.rule_matched$,$actor.user.name$,$unmapped.dst_user$,$app_name$,$unmapped.vsys$,$unmapped.from_zone$,$unmapped.to_zone$,$unmapped.inbound_if$,$unmapped.outbound_if$,$actor.session.issuer$,$metadata.original_time$,$actor.session.uid$,$unmapped.repeat_count$,$src_endpoint.port$,$dst_endpoint.port$,$unmapped.nat_src_port$,$unmapped.nat_dst_port$,$unmapped.flags$,$connection_info.protocol_name$,$unmapped.action_value$,$unmapped.file$,$unmapped.threat_id$,$unmapped.url_category_value$,$unmapped.severity$,$unmapped.direction_of_attack$,$metadata.sequence$,$unmapped.action_flags$,$src_endpoint.location.region$,$dst_endpoint.location.region$,$metadata.product.version$,$unmapped.pcap_id$,$unmapped.file_digest$,.*,$cloud.account_uid$,$unmapped.url_idx$,$unmapped.user_agent$,$unmapped.file_type$,$src_endpoint.intermediate_ips$,$unmapped.referrer$,$unmapped.sender_of_email$,$unmapped.subject_of_email$,$unmapped.receipent_of_email$,$unmapped.report_id$,$unmapped.dg_hier_level_1$,$unmapped.dg_hier_level_2$,$unmapped.dg_hier_level_3$,$unmapped.dg_hier_level_4$,$unmapped.vsys_name$,$device.hostname$,.*,$unmapped.src_uuid$,$unmapped.dst_uuid$,$unmapped.http_method$,$device.imsi$,$device.imei$,$unmapped.parent_session_id$,$unmapped.parent_start_time$,$unmapped.tunnel_type$,$unmapped.threat_category$,$unmapped.content_version$,.*,$unmapped.ep_assoc_id$,$unmapped.ppid$,$unmapped.http_headers$,\"$unmapped.url_category_list$\",$unmapped.rule_matched_uuid$,$unmapped.http2_connection$,$actor.user.groups$,$src_endpoint.intermediate_ips$,$unmapped.src_category$,$unmapped.src_profile$,$unmapped.src_model$,$unmapped.src_vendor$,$unmapped.src_osfamily$,$unmapped.src_osversion$,$src_endpoint.hostname$,$src_endpoint.mac$,$unmapped.dst_category$,$unmapped.dst_profile$,$unmapped.dst_model$,$unmapped.dst_vendor$,$unmapped.dst_osfamily$,$unmapped.dst_osversion$,$dst_endpoint.hostname$,$dst_endpoint.mac$,$unmapped.container_id$,$unmapped.pod_namespace$,$unmapped.pod_name$,$unmapped.src_edl$,$unmapped.dst_edl$,$device.uid$,$unmapped.serial_number$,$unmapped.src_dag$,$unmapped.dst_dag$,$unmapped.partial_hash$,.*,$unmapped.high_res_timestamp$,$unmapped.reason$,$unmapped.justification$,$unmapped.nssai_sst$,$unmapped.subcategory_of_app$,$unmapped.category_of_app$,$unmapped.technology_of_app$,$unmapped.risk_of_app$,\"$unmapped.characteristic_of_app$\",$unmapped.container_of_app$,$unmapped.tunneled_app$,$unmapped.is_saas_of_app$,$unmapped.sanctioned_state_of_app$",
+ halt: true,
+ rewrites: [
{
- input: "device.name",
- output: "event.source",
- match: ".*",
- replace: "$0"
- }
- ],
- halt: true
- }, {
- id: "threat-9-1-a", //PAN-OS 9.1 THREAT
- attributes: {
- "event.type": "threat",
- "format": "threat-v9.1-a"
- },
- format: "$network_activity.future_use_1$,$network_activity.receive_time$,$firewall.serial_number$,THREAT,$network_activity.sub_type$,$network_activity.future_use_2$,$timestamp=timestamp_pat$,$src.ip.address$,$dst.ip.address$,$network_endpoint.nat_src_ip$,$network_endpoint.nat_dst_ip$,$rule.name$,$user.src_name$,$user.dst_name$,$network_activity.app_name$,$network_traffic.virtual_system_name$,$source_zone$,$destination_zone$,$network_interface.inbound_name$,$network_interface.outbound_name$,$network_activity.log_action$,$network_activity.future_use_3$,$session.uid$,$network_activity.repeat_count$,$network_endpoint.src_port$,$network_endpoint.dst_port$,$nenetwork_endpoint.nat_src_port$,$nenetwork_endpoint.nat_dst_port$,$network_connection_info.flag$,$network_connection_info.protocol_name$,$network_activity.action$,$url.filename=value_pat$,$cwe.uid$,$network_activity.category_name$,$severity$,$direction$,$network_activity.sequence_number$,$network_activity.action_flags$,$location.src_location$,$location.dst_location$,$misc$,$Content_type$,$pcap.id$,$file_digest$,$cloud$,$url.index$,$user_agent$,$file_type$,$x-forwarded-for$,$referer$,$sender$,$subject$,$recipient$,$report.id$,$device.group_hierarchy.level_1$,$device.group_hierarchy.level_2$,$device.group_hierarchy.level_3$,$device.group_hierarchy.level_4$,$firewall.virtual_system_name$,$device.name$,$network_activity.future_use_4$,$virtual_machine.src_vm_uuid$,$virtual_machine.dst_vm_uuid$,$http_method$,$device.imsi$,$device.imei$,$session.parent_uid$,$network_activity.parent_start_time_dt$,$network_connection_info.tunnel_type$,$cwe.caption$,$content_version$,$network_activity.future_use_5$,$network_activity.sctp_association_id$,$network_connection_info.protocol_name$,$http_header$,$url.category_list=value_pat$,$rule.uid$,$network_activity.http_connection$,$user.groups$"
- rewrites: [
+ input: "unmapped.severity",
+ output: "severity_id",
+ match: "^informational$",
+ replace: "1"
+ },
+ {
+ input: "unmapped.severity",
+ output: "severity_id",
+ match: "^low$",
+ replace: "2"
+ },
+ {
+ input: "unmapped.severity",
+ output: "severity_id",
+ match: "^medium$",
+ replace: "3"
+ },
+ {
+ input: "unmapped.severity",
+ output: "severity_id",
+ match: "^high$",
+ replace: "4"
+ },
+ {
+ input: "unmapped.severity",
+ output: "severity_id",
+ match: "^critical$",
+ replace: "5"
+ },
+ {
+ input: "unmapped.action_value",
+ output: "status_id",
+ match: "^allow$",
+ replace: "1"
+ },
+ {
+ input: "unmapped.action_value",
+ output: "status_id",
+ match: "^deny$",
+ replace: "2"
+ },
+ {
+ input: "unmapped.action_value",
+ output: "status",
+ match: "^allow$",
+ replace: "Success"
+ },
+ {
+ input: "unmapped.action_value",
+ output: "status",
+ match: "^deny$",
+ replace: "Failure"
+ },
+ {
+ input: "unmapped.action_value",
+ output: "status_id",
+ match: "^(?!allow|deny$).*",
+ replace: "99"
+ },
+ {
+ input: "unmapped.action_value",
+ output: "status",
+ match: "^(?!allow|deny$).*",
+ replace: "Other"
+ },
+ {
+ input: "dst_endpoint.intermediate_ips",
+ output: "dst_endpoint.intermediate_ips",
+ match: ".*",
+ replace: "\\[\"$0\"\\]"
+ },
+ {
+ input: "message",
+ output: "src_endpoint.intermediate_ips",
+ match: "(?:[^,]*,){9}([^,]*),(?:[^,]*,){21}(\".*\"),(?:[^,]*,){16}([^,]*),(?:[^,]*,){26}(\".*\"),(?:[^,]*,){3}([^,]*),.*",
+ replace: "\\[\"$1\"\\, \"$3\"\\, \"$5\"\\]"
+ },
+ {
+ input: "message",
+ output: "observables",
+ match: "(?:[^,]*,){7}([^,]*),([^,]*),(?:[^,]*,){3}([^,]*),(?:[^,]*,){18}(\".*\"),(?:[^,]*,){43}(\".*\"),(?:[^,]*,){10}([^,]*),([^,]*),(?:[^,]*,){6}([^,]*),([^,]*),.*",
+ replace: "\\[\\{\"type_id\"\\: \"1\"\\, \"type\"\\: \"Hostname\"\\, \"name\"\\: \"src_endpoint.hostname\"\\, \"value\"\\: \"$6\"\\}\\, \\{\"type_id\"\\: \"1\"\\, \"type\"\\: \"Hostname\"\\, \"name\"\\: \"dst_endpoint.hostname\"\\, \"value\"\\: \"$8\"\\}\\, \\{\"type_id\"\\: \"2\"\\, \"type\"\\: \"IP Address\"\\, \"name\"\\: \"src_endpoint.ip\"\\, \"value\"\\: \"$1\"\\}\\, \\{\"type_id\"\\: \"2\"\\, \"type\"\\: \"IP Address\"\\, \"name\"\\: \"dst_endpoint.ip\"\\, \"value\"\\: \"$2\"\\}\\, \\{\"type_id\"\\: \"4\"\\, \"type\"\\: \"User Name\"\\, \"name\"\\: \"actor.user.name\"\\, \"value\"\\: \"$3\"\\}\\, \\{\"type_id\"\\: \"3\"\\, \"type\"\\: \"MAC Address\"\\, \"name\"\\: \"src_endpoint.mac\"\\, \"value\"\\: \"$7\"\\}\\, \\{\"type_id\"\\: \"3\"\\, \"type\"\\: \"MAC Address\"\\, \"name\"\\: \"dst_endpoint.mac\"\\, \"value\"\\: \"$9\"\\}\\]"
+ },
+ ]
+ },
+ {
+ //matches THREAT logs with comma surround lists in application_characteristic and url_category_list.
+ attributes: {
+ "activity_name": "THREAT",
+ "class_uid": "4001",
+ "activity_id": "99",
+ "category_uid": "4",
+ "type_uid": "400199",
+ "type_name": "Network Activity: Other",
+ "class_name": "Network Activity",
+ "category_name": "Network Activity",
+ "metadata.product.name": "Palo Alto Networks Firewall",
+ "metadata.product.vendor_name": "Palo Alto Networks",
+ "metadata.version":"1.0.0-rc.3",
+ "metadata.log_name": "THREAT",
+ "event.type": "THREAT"
+ },
+ format: ".*,$metadata.logged_time_dt$,$device.hw_info.serial_number$,THREAT,$unmapped.sub_type$,.*,$metadata.original_time$,$src_endpoint.ip$,$dst_endpoint.ip$,$src_endpoint.intermediate_ips$,$dst_endpoint.intermediate_ips$,$unmapped.rule_matched$,$actor.user.name$,$unmapped.dst_user$,$app_name$,$unmapped.vsys$,$unmapped.from_zone$,$unmapped.to_zone$,$unmapped.inbound_if$,$unmapped.outbound_if$,$actor.session.issuer$,$metadata.original_time$,$actor.session.uid$,$unmapped.repeat_count$,$src_endpoint.port$,$dst_endpoint.port$,$unmapped.nat_src_port$,$unmapped.nat_dst_port$,$unmapped.flags$,$connection_info.protocol_name$,$unmapped.action_value$,$unmapped.file$,$unmapped.threat_id$,$unmapped.url_category_value$,$unmapped.severity$,$unmapped.direction_of_attack$,$metadata.sequence$,$unmapped.action_flags$,$src_endpoint.location.region$,$dst_endpoint.location.region$,$metadata.product.version$,$unmapped.pcap_id$,$unmapped.file_digest$,.*,$cloud.account_uid$,$unmapped.url_idx$,$unmapped.user_agent$,$unmapped.file_type$,$src_endpoint.intermediate_ips$,$unmapped.referrer$,$unmapped.sender_of_email$,$unmapped.subject_of_email$,$unmapped.receipent_of_email$,$unmapped.report_id$,$unmapped.dg_hier_level_1$,$unmapped.dg_hier_level_2$,$unmapped.dg_hier_level_3$,$unmapped.dg_hier_level_4$,$unmapped.vsys_name$,$device.hostname$,.*,$unmapped.src_uuid$,$unmapped.dst_uuid$,$unmapped.http_method$,$device.imsi$,$device.imei$,$unmapped.parent_session_id$,$unmapped.parent_start_time$,$unmapped.tunnel_type$,$unmapped.threat_category$,$unmapped.content_version$,.*,$unmapped.ep_assoc_id$,$unmapped.ppid$,$unmapped.http_headers$,$unmapped.url_category_list$,$unmapped.rule_matched_uuid$,$unmapped.http2_connection$,$actor.user.groups$,$src_endpoint.intermediate_ips$,$unmapped.src_category$,$unmapped.src_profile$,$unmapped.src_model$,$unmapped.src_vendor$,$unmapped.src_osfamily$,$unmapped.src_osversion$,$src_endpoint.hostname$,$src_endpoint.mac$,$unmapped.dst_category$,$unmapped.dst_profile$,$unmapped.dst_model$,$unmapped.dst_vendor$,$unmapped.dst_osfamily$,$unmapped.dst_osversion$,$dst_endpoint.hostname$,$dst_endpoint.mac$,$unmapped.container_id$,$unmapped.pod_namespace$,$unmapped.pod_name$,$unmapped.src_edl$,$unmapped.dst_edl$,$device.uid$,$unmapped.serial_number$,$unmapped.src_dag$,$unmapped.dst_dag$,$unmapped.partial_hash$,.*,$unmapped.high_res_timestamp$,$unmapped.reason$,$unmapped.justification$,$unmapped.nssai_sst$,$unmapped.subcategory_of_app$,$unmapped.category_of_app$,$unmapped.technology_of_app$,$unmapped.risk_of_app$,$unmapped.characteristic_of_app$,$unmapped.container_of_app$,$unmapped.tunneled_app$,$unmapped.is_saas_of_app$,$unmapped.sanctioned_state_of_app$",
+ halt: true,
+ rewrites: [
{
- input: "device.name",
- output: "event.source",
- match: ".*",
- replace: "$0"
- }
- ],
- halt: true
- }, {
- id: "threat-9-1-b", //PAN-OS 9.1 THREAT without Dynamic User Group Name
- attributes: {
- "event.type": "threat",
- "format": "threat-v9.1-b"
+ input: "unmapped.severity",
+ output: "severity_id",
+ match: "^informational$",
+ replace: "1"
+ },
+ {
+ input: "unmapped.severity",
+ output: "severity_id",
+ match: "^low$",
+ replace: "2"
+ },
+ {
+ input: "unmapped.severity",
+ output: "severity_id",
+ match: "^medium$",
+ replace: "3"
+ },
+ {
+ input: "unmapped.severity",
+ output: "severity_id",
+ match: "^high$",
+ replace: "4"
+ },
+ {
+ input: "unmapped.severity",
+ output: "severity_id",
+ match: "^critical$",
+ replace: "5"
+ },
+ {
+ input: "unmapped.action_value",
+ output: "status_id",
+ match: "^allow$",
+ replace: "1"
+ },
+ {
+ input: "unmapped.action_value",
+ output: "status_id",
+ match: "^deny$",
+ replace: "2"
+ },
+ {
+ input: "unmapped.action_value",
+ output: "status",
+ match: "^allow$",
+ replace: "Success"
+ },
+ {
+ input: "unmapped.action_value",
+ output: "status",
+ match: "^deny$",
+ replace: "Failure"
+ },
+ {
+ input: "unmapped.action_value",
+ output: "status_id",
+ match: "^(?!allow|deny$).*",
+ replace: "99"
+ },
+ {
+ input: "unmapped.action_value",
+ output: "status",
+ match: "^(?!allow|deny$).*",
+ replace: "Other"
+ },
+ {
+ input: "dst_endpoint.intermediate_ips",
+ output: "dst_endpoint.intermediate_ips",
+ match: ".*",
+ replace: "\\[\"$0\"\\]"
+ },
+ {
+ input: "message",
+ output: "src_endpoint.intermediate_ips",
+ match: "(?:[^,]*,){9}([^,]*),(?:[^,]*,){38}([^,]*),(?:[^,]*,){30}([^,]*),.*",
+ replace: "\\[\"$1\"\\, \"$2\"\\, \"$3\"\\]"
+ },
+ {
+ input: "message",
+ output: "observables",
+ match: "(?:[^,]*,){7}([^,]*),([^,]*),(?:[^,]*,){3}([^,]*),(?:[^,]*,){73}([^,]*),([^,]*),(?:[^,]*,){6}([^,]*),([^,]*),.*",
+ replace: "\\[\\{\"type_id\"\\: \"1\"\\, \"type\"\\: \"Hostname\"\\, \"name\"\\: \"src_endpoint.hostname\"\\, \"value\"\\: \"$4\"\\}\\, \\{\"type_id\"\\: \"1\"\\, \"type\"\\: \"Hostname\"\\, \"name\"\\: \"dst_endpoint.hostname\"\\, \"value\"\\: \"$6\"\\}\\, \\{\"type_id\"\\: \"2\"\\, \"type\"\\: \"IP Address\"\\, \"name\"\\: \"src_endpoint.ip\"\\, \"value\"\\: \"$1\"\\}\\, \\{\"type_id\"\\: \"2\"\\, \"type\"\\: \"IP Address\"\\, \"name\"\\: \"dst_endpoint.ip\"\\, \"value\"\\: \"$2\"\\}\\, \\{\"type_id\"\\: \"4\"\\, \"type\"\\: \"User Name\"\\, \"name\"\\: \"actor.user.name\"\\, \"value\"\\: \"$3\"\\}\\, \\{\"type_id\"\\: \"3\"\\, \"type\"\\: \"MAC Address\"\\, \"name\"\\: \"src_endpoint.mac\"\\, \"value\"\\: \"$5\"\\}\\, \\{\"type_id\"\\: \"3\"\\, \"type\"\\: \"MAC Address\"\\, \"name\"\\: \"dst_endpoint.mac\"\\, \"value\"\\: \"$7\"\\}\\]"
+ },
+ ]
+ },
+ {
+ attributes: {
+ "activity_name": "Logoff",
+ "activity_id": "2",
+ "category_name": "Identity & Access Management",
+ "category_uid": "3",
+ "class_name": "Authentication",
+ "class_uid": "3002",
+ "cloud.provider": "Palo Alto Networks"
+ "metadata.product.name": "Palo Alto Networks Firewall",
+ "metadata.product.vendor_name": "Palo Alto Networks",
+ "metadata.version":"1.1.0",
+ "type_uid": "300202",
+ "type_name": "Authentication: Logoff",
+ "event.type": "Logoff",
+ "severity_id": "99"
+ },
+ format: "$metadata.original_time$,$metadata.product.uid$,$unmapped.type=userid_log_type$,$unmapped.subtype=logout_sub_type$,.*,$start_time_dt$,$unmapped.vsys$,$src_endpoint.ip$,$user.name$,$user.uid$,$metadata.event_code$,$unmapped.repeatcnt$,$unmapped.timeout$,$src_endpoint.port$,$dst_endpoint.port$,$unmapped.datasource$,$unmapped.datasourcetype$,$unmapped.seqno$,$unmapped.actionflags$,$unmapped.dg_hier_level_1$,$unmapped.dg_hier_level_2$,$unmapped.dg_hier_level_3$,$unmapped.dg_hier_level_4$,$unmapped.vsys_name$,$src_endpoint.hostname$,$unmapped.vsys_id$,$unmapped.factortype$,$unmapped.factorcompletiontime$,$unmapped.factorno$,$unmapped.ugflags$,$unmapped.userbysource$,$unmapped.tag_name$,$unmapped.high_res_timestamp$",
+ halt: true,
+ rewrites: [
+ {
+ input: "message",
+ output: "observables",
+ match: "(?:[^,]*,){7}([^,]*)(?:,[^,]*){0},([^,]*)(?:,[^,]*){15},([^,]*).*",
+ replace: "\\[\\{\"type_id\"\\: 1\\, \"type\"\\: \"Hostname\"\\, \"name\"\\: \"src_endpoint.hostname\"\\, \"value\"\\: $3\\}\\, \\{\"type_id\"\\: 2\\, \"type\"\\: \"IP Address\"\\, \"name\"\\: \"src_endpoint.ip\"\\, \"value\"\\: $1\\}\\, \\{\"type_id\"\\: 4\\, \"type\"\\: \"User Name\"\\, \"name\"\\: \"user.name\"\\, \"value\"\\: $2\\}\\]"
},
- format: "$network_activity.future_use_1$,$network_activity.receive_time$,$firewall.serial_number$,THREAT,$network_activity.sub_type$,$network_activity.future_use_2$,$timestamp=timestamp_pat$,$src.ip.address$,$dst.ip.address$,$network_endpoint.nat_src_ip$,$network_endpoint.nat_dst_ip$,$rule.name$,$user.src_name$,$user.dst_name$,$network_activity.app_name$,$network_traffic.virtual_system_name$,$source_zone$,$destination_zone$,$network_interface.inbound_name$,$network_interface.outbound_name$,$network_activity.log_action$,$network_activity.future_use_3$,$session.uid$,$network_activity.repeat_count$,$network_endpoint.src_port$,$network_endpoint.dst_port$,$nenetwork_endpoint.nat_src_port$,$nenetwork_endpoint.nat_dst_port$,$network_connection_info.flag$,$network_connection_info.protocol_name$,$network_activity.action$,$url.filename=value_pat$,$cwe.uid$,$network_activity.category_name$,$severity$,$direction$,$network_activity.sequence_number$,$network_activity.action_flags$,$location.src_location$,$location.dst_location$,$misc$,$Content_type$,$pcap.id$,$file_digest$,$cloud$,$url.index$,$user_agent$,$file_type$,$x-forwarded-for$,$referer$,$sender$,$subject$,$recipient$,$report.id$,$device.group_hierarchy.level_1$,$device.group_hierarchy.level_2$,$device.group_hierarchy.level_3$,$device.group_hierarchy.level_4$,$firewall.virtual_system_name$,$device.name$,$network_activity.future_use_4$,$virtual_machine.src_vm_uuid$,$virtual_machine.dst_vm_uuid$,$http_method$,$device.imsi$,$device.imei$,$session.parent_uid$,$network_activity.parent_start_time_dt$,$network_connection_info.tunnel_type$,$cwe.caption$,$content_version$,$network_activity.future_use_5$,$network_activity.sctp_association_id$,$network_connection_info.protocol_name$,$http_header$,$url.category_list=value_pat$,$rule.uid$,$network_activity.http_connection$"
- rewrites: [
- {
- input: "device.name",
- output: "event.source",
- match: ".*",
- replace: "$0"
- }
- ],
- halt: true
- },
- // CONFIG
- {
- id: "config-10-1", //PAN-OS v10.1-v11.0 CONFIG, adds Future Use 3 and High Resolution Timestamp
- attributes: {
- "event.type": "config",
- "format": "config-v10.1+"
+ {
+ input: "activity_name",
+ output: "event.type",
+ match: ".*",
+ replace: "$0"
+ }
+ ]
+ },
+ {
+ attributes: {
+ "activity_name": "Logon",
+ "activity_id": "1",
+ "category_name": "Identity & Access Management",
+ "category_uid": "3",
+ "class_name": "Authentication",
+ "class_uid": "3002",
+ "cloud.provider": "Palo Alto Networks"
+ "metadata.product.name": "Palo Alto Networks Firewall",
+ "metadata.product.vendor_name": "Palo Alto Networks",
+ "metadata.version":"1.1.0",
+ "type_uid": "300201",
+ "type_name": "Authentication: Logon",
+ "event.type": "Logon",
+ "severity_id": "99"
+ },
+ format: "$metadata.original_time$,$metadata.product.uid$,$unmapped.type=userid_log_type$,$unmapped.subtype=login_sub_type$,.*,$start_time_dt$,$unmapped.vsys$,$src_endpoint.ip$,$user.name$,$user.uid$,$metadata.event_code$,$unmapped.repeatcnt$,$unmapped.timeout$,$src_endpoint.port$,$dst_endpoint.port$,$unmapped.datasource$,$unmapped.datasourcetype$,$unmapped.seqno$,$unmapped.actionflags$,$unmapped.dg_hier_level_1$,$unmapped.dg_hier_level_2$,$unmapped.dg_hier_level_3$,$unmapped.dg_hier_level_4$,$unmapped.vsys_name$,$src_endpoint.hostname$,$unmapped.vsys_id$,$unmapped.factortype$,$unmapped.factorcompletiontime$,$unmapped.factorno$,$unmapped.ugflags$,$unmapped.userbysource$,$unmapped.tag_name$,$unmapped.high_res_timestamp$",
+ halt: true,
+ rewrites: [
+ {
+ input: "message",
+ output: "observables",
+ match: "(?:[^,]*,){7}([^,]*)(?:,[^,]*){0},([^,]*)(?:,[^,]*){15},([^,]*).*",
+ replace: "\\[\\{\"type_id\"\\: 1\\, \"type\"\\: \"Hostname\"\\, \"name\"\\: \"src_endpoint.hostname\"\\, \"value\"\\: $3\\}\\, \\{\"type_id\"\\: 2\\, \"type\"\\: \"IP Address\"\\, \"name\"\\: \"src_endpoint.ip\"\\, \"value\"\\: $1\\}\\, \\{\"type_id\"\\: 4\\, \"type\"\\: \"User Name\"\\, \"name\"\\: \"user.name\"\\, \"value\"\\: $2\\}\\]"
},
- format: "$network_activity.future_use_1$,$network_activity.receive_time$,$firewall.serial_number$,CONFIG,$network_activity.sub_type$,$network_activity.future_use_2$,$timestamp=timestamp_pat$,$host_name$,$network_traffic.virtual_system_name$,$command=value_pat$,$admin$,$client$,$result$,$configuration_path$,$before_change_detail$,$after_change_detail$,$device.group_hierarchy.level_1$,$device.group_hierarchy.level_2$,$device.group_hierarchy.level_3$,$device.group_hierarchy.level_4$,$firewall.virtual_system_name$,$device.name$,$device.group$,$audit_comment=value_pat$,$network_activity.future_use_3$,$network_activity.time$",
- rewrites: [
- {
- input: "device.name",
- output: "event.source",
- match: ".*",
- replace: "$0"
- }
- ],
- halt: true
- }, {
- id: "config-9-1", //PAN-OS v9.1 CONFIG
- attributes: {
- "event.type": "config",
- "format": "config-v9.1"
+ {
+ input: "activity_name",
+ output: "event.type",
+ match: ".*",
+ replace: "$0"
+ }
+ ]
+ },
+ {
+ attributes: {
+ "action": "Other",
+ "action_id": "99",
+ "activity_name": "Other",
+ "activity_id": "99",
+ "category_name": "Findings",
+ "category_uid": "2",
+ "class_name": "Detection Finding",
+ "class_uid": "2004",
+ "cloud.provider": "Palo Alto Networks",
+ "metadata.product.name": "Palo Alto Networks Firewall",
+ "metadata.product.vendor_name": "Palo Alto Networks",
+ "metadata.version":"1.1.0",
+ "type_uid": "200499",
+ "type_name": "Detection Finding: Other",
+ "severity_id": "99"
+ },
+ format: "$metadata.original_time$,$metadata.product.uid$,$finding_info.title=hipmatch_log_type$,$unmapped.subtype$,.*,$start_time_dt$,$actor.user.name$,$unmapped.vsys$,$device.name$,$device.os.name$,$device.ip$,$unmapped.matchname$,$unmapped.repeatcnt$,$unmapped.matchtype$,.*,.*,$unmapped.seqno$,$unmapped.actionflags$,$unmapped.dg_hier_level_1$,$unmapped.dg_hier_level_2$,$unmapped.dg_hier_level_3$,$unmapped.dg_hier_level_4$,$unmapped.vsys_name$,$device.hostname$,$unmapped.vsys_id$,$unmapped.srcipv6$,$unmapped.uid_alt$,$device.uid$,$device.mac$,$unmapped.high_res_timestamp$",
+ halt: true,
+ rewrites: [
+ {
+ input: "message",
+ output: "observables",
+ match: "(?:[^,]*,){10}([^,]*)(?:,[^,]*){12},([^,]*)(?:,[^,]*){4},([^,]*).*",
+ replace: "\\[\\{\"type_id\"\\: 1\\, \"type\"\\: \"Hostname\"\\, \"name\"\\: \"device.hostname\"\\, \"value\"\\: $2\\}\\, \\{\"type_id\"\\: 2\\, \"type\"\\: \"IP Address\"\\, \"name\"\\: \"device.ip\"\\, \"value\"\\: $1\\}\\, \\{\"type_id\"\\: 3\\, \"type\"\\: \"MAC Address\"\\, \"name\"\\: \"device.mac\"\\, \"value\"\\: $3\\}\\]"
},
- format: "$network_activity.future_use_1$,$network_activity.receive_time$,$firewall.serial_number$,CONFIG,$network_activity.sub_type$,$network_activity.future_use_2$,$timestamp=timestamp_pat$,$host_name$,$network_traffic.virtual_system_name$,$command=value_pat$,$admin$,$client$,$result$,$configuration_path$,$before_change_detail$,$after_change_detail$,$device.group_hierarchy.level_1$,$device.group_hierarchy.level_2$,$device.group_hierarchy.level_3$,$device.group_hierarchy.level_4$,$firewall.virtual_system_name$,$device.name$,$device.group$,$audit_comment=value_pat$",
- rewrites: [
- {
- input: "device.name",
- output: "event.source",
- match: ".*",
- replace: "$0"
- }
- ],
- halt: true
- },
- // SYSTEM
- {
- id: "config-10-1", //PAN-OS v10.1-v11.0 SYSTEM, adds Future Use 5 & 6 and High Resolution Timestamp
- attributes: {
- "event.type": "system",
- "format": "system-v11.0"
+ {
+ input: "activity_name",
+ output: "event.type",
+ match: ".*",
+ replace: "$0"
+ }
+ ]
+ },
+ {
+ attributes: {
+ "activity_name": "Log",
+ "activity_id": "1",
+ "category_name": "Discovery",
+ "category_uid": "5",
+ "class_name": "Device Config State",
+ "class_uid": "5002",
+ "cloud.provider": "Palo Alto Networks",
+ "metadata.product.name": "Palo Alto Networks Firewall",
+ "metadata.product.vendor_name": "Palo Alto Networks",
+ "metadata.version":"1.1.0",
+ "type_uid": "500201",
+ "type_name": "Device Config State: Log",
+ "severity_id": "99"
+ },
+ format: "$metadata.original_time$,$metadata.product.uid$,$unmapped.type=config_log_type$,$unmapped.subtype$,.*,$start_time_dt$,$device.hostname$,$unmapped.vsys$,$actor.process.cmd_line$,$actor.user.name$,$unmapped.client$,$unmapped.result$,$metadata.product.path$,$unmapped.before-change-detail$,$unmapped.after-change-detail$,$unmapped.seqno$,$unmapped.actionflags$,$unmapped.dg_hier_level_1$,$unmapped.dg_hier_level_2$,$unmapped.dg_hier_level_3$,$unmapped.dg_hier_level_4$,$unmapped.vsys_name$,$device.name$,$device.groups$,$unmapped.comment$",
+ halt: true,
+ rewrites: [
+ {
+ input: "message",
+ output: "observables",
+ match: "(?:[^,]*,){6}([^,]*)(?:,[^,]*){2},([^,]*).*",
+ replace: "\\[\\{\"type_id\"\\: 1\\, \"type\"\\: \"Hostname\"\\, \"name\"\\: \"device.hostname\"\\, \"value\"\\: $1\\}\\, \\{\"type_id\"\\: 4\\, \"type\"\\: \"User Name\"\\, \"name\"\\: \"actor.user.name\"\\, \"value\"\\: $2\\}\\]"
},
- format: "$network_activity.future_use_1$,$network_activity.receive_time$,$firewall.serial_number$,SYSTEM,$network_activity.sub_type$,$network_activity.future_use_2$,$timestamp=timestamp_pat$,$network_traffic.virtual_system_name$,$session.parent_uid$,$object$,$network_activity.future_use_3$,$network_activity.future_use_4$,$module$,$severity$,$description$,$network_activity.sequence_number$,$network_activity.action_flags$,$device.group_hierarchy.level_1$,$device.group_hierarchy.level_2$,$device.group_hierarchy.level_3$,$device.group_hierarchy.level_4$,$firewall.virtual_system_name$,$device.name$,$network_activity.future_use_5$,$network_activity.future_use_6$,$network_activity.time$",
- rewrites: [
- {
- input: "device.name",
- output: "event.source",
- match: ".*",
- replace: "$0"
- }
- ],
- halt: true
- }, {
- id: "config-10-1", //PAN-OS v9.1 SYSTEM
- attributes: {
- "event.type": "system",
- "format": "system-v9.1"
+ {
+ input: "activity_name",
+ output: "event.type",
+ match: ".*",
+ replace: "$0"
+ }
+ ]
+ },
+ {
+ attributes: {
+ "action": "Other",
+ "action_id": "99",
+ "activity_name": "Other",
+ "activity_id": "99",
+ "category_name": "Findings",
+ "category_uid": "2",
+ "class_name": "Detection Finding",
+ "class_uid": "2004",
+ "cloud.provider": "Palo Alto Networks",
+ "metadata.product.name": "Palo Alto Networks Firewall",
+ "metadata.product.vendor_name": "Palo Alto Networks",
+ "metadata.version":"1.1.0",
+ "type_uid": "200499",
+ "type_name": "Detection Finding: Other",
+ "severity_id": "99"
+ },
+ format: "$metadata.original_time$,$device.hw_info.serial_number$,$unmapped.type$,$unmapped.subtype=wildfire_sub_type$,.*,$finding_info.created_time_dt$,$source_address$,$destination_address$,$nat_source_ip$,$nat_destination_ip$,$firewall_rule.name$,$actor.user.name$,$unmapped.dstuser$,$unmapped.app$,$unmapped.vsys$,$source_zone$,$destination_zone$,$inbound_interface$,$outbound_interface$,$unmapped.logset$,.*,$actor.session.uid$,$count$,$source_port$,$destination_port$,$unmapped.natsport$,$unmapped.natdport$,$unmapped.flags$,$ip_protocol$,$action$,$filename$,$finding_info.uid$,$unmapped.category$,$unmapped.severity$,$unmapped.direction$,$metadata.sequence$,$unmapped.actionflags$,$source_location$,$destination_location$,.*,$unmapped.contenttype$,$unmapped.pcap_id$,$unmapped.filedigest$,$unmapped.cloud$,$unmapped.url_idx$,$unmapped.user_agent$,$file_type$,$unmapped.xff$,$unmapped.referer$,$unmapped.sender$,$unmapped.subject$,$unmapped.recipient$,$unmapped.reportid$,$unmapped.dg_hier_level_1$,$unmapped.dg_hier_level_2$,$unmapped.dg_hier_level_3$,$unmapped.dg_hier_level_4$,$unmapped.vsys_name$,$device.name$,.*,$source_vm_uuid$,$destination_vm_uuid$,$unmapped.http_method$,$unmapped.imsi$,$device.imei$,$parent_session_id$,$parent_start_time$,$unmapped.tunnel$,$unmapped.thr_category$,$unmapped.contentver$,.*,$unmapped.assoc_id$,$unmapped.ppid$,$unmapped.http_headers$,$unmapped.url_category_list$,$unmapped.rule_uuid$,$unmapped.http2_connection$,$unmapped.dynusergroup_name$,$unmapped.xff_ip$,$unmapped.src_category$,$unmapped.src_profile$,$unmapped.src_model$,$unmapped.src_vendor$,$unmapped.src_osfamily$,$unmapped.src_osversion$,$source_hostname$,$source_mac_address$,$unmapped.dst_category$,$unmapped.dst_profile$,$unmapped.dst_model$,$unmapped.dst_vendor$,$unmapped.dst_osfamily$,$unmapped.dst_osversion$,$destination_hostname$,$destination_mac_address$,$unmapped.container_id$,$unmapped.pod_namespace$,$unmapped.pod_name$,$unmapped.src_edl$,$unmapped.dst_edl$,$unmapped.hostid$,$unmapped.serialnumber$,$unmapped.domain_edl$,$unmapped.src_dag$,$unmapped.dst_dag$,$unmapped.partial_hash$,$unmapped.high_res_timestamp$,$unmapped.reason$,$unmapped.justification$,$unmapped.nssai_sst$,$unmapped.subcategory_of_app$,$unmapped.category_of_app$,$unmapped.technology_of_app$,$risk_level$,$unmapped.characteristic_of_app=app_characteristic$,$unmapped.container_of_app$,$unmapped.tunneled_app$,$unmapped.is_saas_of_app$,$unmapped.sanctioned_state_of_app$,$unmapped.cloud_reportid$",
+ halt: true,
+ rewrites: [
+ {
+ input: "message",
+ output: "evidences",
+ match: "^(?:[^,]*,){6}([^,]*),([^,]*),([^,]*),([^,]*),(?:[^,]*,){5}([^,]*),([^,]*),([^,]*),([^,]*),(?:[^,]*,){4}([^,]*),([^,]*),(?:[^,]*,){3}([^,]*),(?:[^,]*,){1}([^,]*),(?:[^,]*,){6}([^,]*),([^,]*),(?:[^,]*,){7}([^,]*),([^,]*),(?:[^,]*,){13}([^,]*),(?:[^,]*,){1}([^,]*),(?:[^,]*,){18}([^,]*),([^,]*),(?:[^,]*,){6}([^,]*),([^,]*),(?:[^,]*,){7}([^,]*),([^,]*),(?:[^,]*,){6}([^,]*),([^,]*),(?:[^,]*,){6}([^,]*),([^,]*),(?:[^,]*,){1}([^,]*),([^,]*).*",
+ replace: "\\[\"src_endpoint\"\\:\\{\"ip\"\\: $1\\, \"intermediate_ips\"\\:\\[$3\\]\\, \"zone\"\\: $5\\, \"interface_name\"\\: $7\\, \"port\"\\: $9\\, \"location\"\\: \\{\"country\"\\: $13\\}\\, \"uid\"\\: $16\\, \"hostname\"\\: $20\\, \"mac\"\\: $21\\}\\, \"dst_endpoint\"\\:\\{\"ip\"\\: $2\\, \"intermediate_ips\"\\:\\[$4\\]\\, \"zone\"\\: $6\\, \"interface_name\"\\: $8\\, \"port\"\\: $10\\, \"location\"\\: \\{\"country\"\\: $14\\}\\, \"uid\"\\: $17\\, \"hostname\"\\: $22\\, \"mac\"\\: $23\\}\\, \"connection_info\"\\: \\{ \"protocol_name\"\\: $11\\}\\, \"process\"\\: \\{\"file\"\\: \\{\"name\"\\: $12\\, \"type\"\\: $15\\}\\, \"parent_process\"\\: \\{\"session\": \\{\"uid\": $18\\}\\, \"created_time\"\\: $19\\}\\} \\]"
},
- format: "$network_activity.future_use_1$,$network_activity.receive_time$,$firewall.serial_number$,SYSTEM,$network_activity.sub_type$,$network_activity.future_use_2$,$timestamp=timestamp_pat$,$network_traffic.virtual_system_name$,$session.parent_uid$,$object$,$network_activity.future_use_3$,$network_activity.future_use_4$,$module$,$severity$,$description$,$network_activity.sequence_number$,$network_activity.action_flags$,$device.group_hierarchy.level_1$,$device.group_hierarchy.level_2$,$device.group_hierarchy.level_3$,$device.group_hierarchy.level_4$,$firewall.virtual_system_name$,$device.name$",
- rewrites: [
- {
- input: "device.name",
- output: "event.source",
- match: ".*",
- replace: "$0"
- }
- ],
- halt: true
- },
- // HIP-MATCH
- {
- id: "hipmatch-11-0", //PAN-OS v11.0 HIPMATCH, adds Cluster Name
- attributes: {
- "event.type": "hipmatch",
- "format": "hiptmatch-v11.0"
+ {
+ input: "message",
+ output: "observables",
+ match: "^(?:[^,]*,){6}([^,]*),(?:[^,]*,){0}([^,]*),(?:[^,]*,){3}([^,]*),(?:[^,]*,){73}([^,]*),(?:[^,]*,){7}([^,]*).*",
+ replace: "\\[\\{\"type_id\"\\: 1\\, \"type\"\\: \"Hostname\"\\, \"name\"\\: \"evidences.src_endpoint.hostname\"\\, \"value\"\\: $4\\}\\, \\{\"type_id\"\\: 1\\, \"type\"\\: \"Hostname\"\\, \"name\"\\: \"evidences.dst_endpoint.hostname\"\\, \"value\"\\: $5\\}\\, \\{\"type_id\"\\: 2\\, \"type\"\\: \"IP Address\"\\, \"name\"\\: \"evidences.src_endpoint.ip\"\\, \"value\"\\: $1\\}\\, \\{\"type_id\"\\: 2\\, \"type\"\\: \"IP Address\"\\, \"name\"\\: \"evidences.dst_endpoint.ip\"\\, \"value\"\\: $2\\}\\, \\{\"type_id\"\\: 4\\, \"type\"\\: \"User Name\"\\, \"name\"\\: \"actor.user.name\"\\, \"value\"\\: $3\\}\\]"
},
- format: "$network_activity.future_use_1$,$network_activity.receive_time$,$firewall.serial_number$,HIP-MATCH,$network_activity.sub_type$,$network_activity.future_use_2$,$timestamp=timestamp_pat$,$user.src_name$,$network_traffic.virtual_system_name$,$machine_name$,$os$,$src.ip.address$,$hip$,$network_activity.repeat_count$,$hip_type$,$network_activity.future_use_2$,$network_activity.future_use_3$,$network_activity.sequence_number$,$network_activity.action_flags$,$device.group_hierarchy.level_1$,$device.group_hierarchy.level_2$,$device.group_hierarchy.level_3$,$device.group_hierarchy.level_4$,$firewall.virtual_system_name$,$device.name$,$firewall.virtual_system_id$,$ipv6_source_address$,$network_endpoint.host_id$,$user_device_serial_number$,$device_mac_address$,$network_activity.time$,$network_activity.cluster.name$",
- rewrites: [
- {
- input: "device.name",
- output: "event.source",
- match: ".*",
- replace: "$0"
- }
- ],
- halt: true
- }, {
- id: "hipmatch-10-1", //PAN-OS v10.1-10.2 HIPMATCH, adds Device MAC Address and High Resolution Timestamp
- attributes: {
- "event.type": "hipmatch",
- "format": "hiptmatch-v9.1"
+ {
+ input: "activity_name",
+ output: "event.type",
+ match: ".*",
+ replace: "$0"
},
- format: "$network_activity.future_use_1$,$network_activity.receive_time$,$firewall.serial_number$,HIP-MATCH,$network_activity.sub_type$,$network_activity.future_use_2$,$timestamp=timestamp_pat$,$user.src_name$,$network_traffic.virtual_system_name$,$machine_name$,$os$,$src.ip.address$,$hip$,$network_activity.repeat_count$,$hip_type$,$network_activity.future_use_2$,$network_activity.future_use_3$,$network_activity.sequence_number$,$network_activity.action_flags$,$device.group_hierarchy.level_1$,$device.group_hierarchy.level_2$,$device.group_hierarchy.level_3$,$device.group_hierarchy.level_4$,$firewall.virtual_system_name$,$device.name$,$firewall.virtual_system_id$,$ipv6_source_address$,$network_endpoint.host_id$,$user_device_serial_number$,$device_mac_address$,$network_activity.time$",
- rewrites: [
- {
- input: "device.name",
- output: "event.source",
- match: ".*",
- replace: "$0"
- }
- ],
- halt: true
- }, {
- id: "hipmatch-9-1", //PAN-OS v9.1 HIPMATCH
- attributes: {
- "event.type": "hipmatch",
- "format": "hiptmatch-v9.1"
+ {
+ action: "removeFields",
+ fields: [
+ "source_address",
+ "destination_address",
+ "nat_source_ip",
+ "nat_destination_ip",
+ "source_zone",
+ "destination_zone",
+ "inbound_interface",
+ "outbound_interface",
+ "source_port",
+ "destination_port",
+ "ip_protocol",
+ "filename",
+ "source_location",
+ "destination_location",
+ "file_type",
+ "source_vm_uuid",
+ "destination_vm_uuid",
+ "parent_session_id",
+ "parent_start_time",
+ "source_hostname",
+ "source_mac_address",
+ "destination_hostname",
+ "destination_mac_address"
+ ]
+ }
+ ]
+ },
+ {
+ attributes: {
+ "action": "Other",
+ "action_id": "99",
+ "activity_name": "Other",
+ "activity_id": "99",
+ "category_name": "Findings",
+ "category_uid": "2",
+ "class_name": "Detection Finding",
+ "class_uid": "2004",
+ "cloud.provider": "Palo Alto Networks",
+ "metadata.product.name": "Palo Alto Networks Firewall",
+ "metadata.product.vendor_name": "Palo Alto Networks",
+ "metadata.version":"1.1.0",
+ "type_uid": "200499",
+ "type_name": "Detection Finding: Other",
+ "severity_id": "99"
+ },
+ format: "$metadata.original_time$,$device.hw_info.serial_number$,$unmapped.type$,$unmapped.subtype=data_filtering_sub_type$,.*,$finding_info.created_time_dt$,$source_address$,$destination_address$,$nat_source_ip$,$nat_destination_ip$,$firewall_rule.name$,$actor.user.name$,$unmapped.dstuser$,$unmapped.app$,$unmapped.vsys$,$source_zone$,$destination_zone$,$inbound_interface$,$outbound_interface$,$unmapped.logset$,.*,$actor.session.uid$,$count$,$source_port$,$destination_port$,$unmapped.natsport$,$unmapped.natdport$,$unmapped.flags$,$ip_protocol$,$action$,$filename$,$finding_info.uid$,$unmapped.category$,$unmapped.severity$,$unmapped.direction$,$metadata.sequence$,$unmapped.actionflags$,$source_location$,$destination_location$,.*,$unmapped.contenttype$,$unmapped.pcap_id$,$unmapped.filedigest$,$unmapped.cloud$,$unmapped.url_idx$,$unmapped.user_agent$,$file_type$,$unmapped.xff$,$unmapped.referer$,$unmapped.sender$,$unmapped.subject$,$unmapped.recipient$,$unmapped.reportid$,$unmapped.dg_hier_level_1$,$unmapped.dg_hier_level_2$,$unmapped.dg_hier_level_3$,$unmapped.dg_hier_level_4$,$unmapped.vsys_name$,$device.name$,.*,$source_vm_uuid$,$destination_vm_uuid$,$unmapped.http_method$,$unmapped.imsi$,$device.imei$,$parent_session_id$,$parent_start_time$,$unmapped.tunnel$,$unmapped.thr_category$,$unmapped.contentver$,.*,$unmapped.assoc_id$,$unmapped.ppid$,$unmapped.http_headers$,$unmapped.url_category_list$,$unmapped.rule_uuid$,$unmapped.http2_connection$,$unmapped.dynusergroup_name$,$unmapped.xff_ip$,$unmapped.src_category$,$unmapped.src_profile$,$unmapped.src_model$,$unmapped.src_vendor$,$unmapped.src_osfamily$,$unmapped.src_osversion$,$source_hostname$,$source_mac_address$,$unmapped.dst_category$,$unmapped.dst_profile$,$unmapped.dst_model$,$unmapped.dst_vendor$,$unmapped.dst_osfamily$,$unmapped.dst_osversion$,$destination_hostname$,$destination_mac_address$,$unmapped.container_id$,$unmapped.pod_namespace$,$unmapped.pod_name$,$unmapped.src_edl$,$unmapped.dst_edl$,$unmapped.hostid$,$unmapped.serialnumber$,$unmapped.domain_edl$,$unmapped.src_dag$,$unmapped.dst_dag$,$unmapped.partial_hash$,$unmapped.high_res_timestamp$,$unmapped.reason$,$unmapped.justification$,$unmapped.nssai_sst$,$unmapped.subcategory_of_app$,$unmapped.category_of_app$,$unmapped.technology_of_app$,$risk_level$,$unmapped.characteristic_of_app=app_characteristic$,$unmapped.container_of_app$,$unmapped.tunneled_app$,$unmapped.is_saas_of_app$,$unmapped.sanctioned_state_of_app$,$unmapped.cloud_reportid$",
+ halt: true,
+ rewrites: [
+ {
+ input: "message",
+ output: "evidences",
+ match: "^(?:[^,]*,){6}([^,]*),([^,]*),([^,]*),([^,]*),(?:[^,]*,){5}([^,]*),([^,]*),([^,]*),([^,]*),(?:[^,]*,){4}([^,]*),([^,]*),(?:[^,]*,){3}([^,]*),(?:[^,]*,){1}([^,]*),(?:[^,]*,){6}([^,]*),([^,]*),(?:[^,]*,){7}([^,]*),([^,]*),(?:[^,]*,){13}([^,]*),(?:[^,]*,){1}([^,]*),(?:[^,]*,){18}([^,]*),([^,]*),(?:[^,]*,){6}([^,]*),([^,]*),(?:[^,]*,){7}([^,]*),([^,]*),(?:[^,]*,){6}([^,]*),([^,]*),(?:[^,]*,){6}([^,]*),([^,]*),(?:[^,]*,){1}([^,]*),([^,]*).*",
+ replace: "\\[\"src_endpoint\"\\:\\{\"ip\"\\: $1\\, \"intermediate_ips\"\\:\\[$3\\]\\, \"zone\"\\: $5\\, \"interface_name\"\\: $7\\, \"port\"\\: $9\\, \"location\"\\: \\{\"country\"\\: $13\\}\\, \"uid\"\\: $16\\, \"hostname\"\\: $20\\, \"mac\"\\: $21\\}\\, \"dst_endpoint\"\\:\\{\"ip\"\\: $2\\, \"intermediate_ips\"\\:\\[$4\\]\\, \"zone\"\\: $6\\, \"interface_name\"\\: $8\\, \"port\"\\: $10\\, \"location\"\\: \\{\"country\"\\: $14\\}\\, \"uid\"\\: $17\\, \"hostname\"\\: $22\\, \"mac\"\\: $23\\}\\, \"connection_info\"\\: \\{ \"protocol_name\"\\: $11\\}\\, \"process\"\\: \\{\"file\"\\: \\{\"name\"\\: $12\\, \"type\"\\: $15\\}\\, \"parent_process\"\\: \\{\"session\": \\{\"uid\": $18\\}\\, \"created_time\"\\: $19\\}\\} \\]"
},
- format: "$network_activity.future_use_1$,$network_activity.receive_time$,$firewall.serial_number$,HIP-MATCH,$network_activity.sub_type$,$network_activity.future_use_2$,$timestamp=timestamp_pat$,$user.src_name$,$network_traffic.virtual_system_name$,$machine_name$,$os$,$src.ip.address$,$hip$,$network_activity.repeat_count$,$hip_type$,$network_activity.future_use_2$,$network_activity.future_use_3$,$network_activity.sequence_number$,$network_activity.action_flags$,$device.group_hierarchy.level_1$,$device.group_hierarchy.level_2$,$device.group_hierarchy.level_3$,$device.group_hierarchy.level_4$,$firewall.virtual_system_name$,$device.name$,$firewall.virtual_system_id$,$ipv6_source_address$,$network_endpoint.host_id$,$user_device_serial_number$",
- rewrites: [
- {
- input: "device.name",
- output: "event.source",
- match: ".*",
- replace: "$0"
- }
- ],
- halt: true
- },
- // CORRELATION
- {
- id: "correlation", //PAN-OS v9.1-11.0 CORRELATION
- attributes: {
- "event.type": "correlation",
- "format": "correlation"
+ {
+ input: "message",
+ output: "observables",
+ match: "^(?:[^,]*,){6}([^,]*),(?:[^,]*,){0}([^,]*),(?:[^,]*,){3}([^,]*),(?:[^,]*,){73}([^,]*),(?:[^,]*,){7}([^,]*).*",
+ replace: "\\[\\{\"type_id\"\\: 1\\, \"type\"\\: \"Hostname\"\\, \"name\"\\: \"evidences.src_endpoint.hostname\"\\, \"value\"\\: $4\\}\\, \\{\"type_id\"\\: 1\\, \"type\"\\: \"Hostname\"\\, \"name\"\\: \"evidences.dst_endpoint.hostname\"\\, \"value\"\\: $5\\}\\, \\{\"type_id\"\\: 2\\, \"type\"\\: \"IP Address\"\\, \"name\"\\: \"evidences.src_endpoint.ip\"\\, \"value\"\\: $1\\}\\, \\{\"type_id\"\\: 2\\, \"type\"\\: \"IP Address\"\\, \"name\"\\: \"evidences.dst_endpoint.ip\"\\, \"value\"\\: $2\\}\\, \\{\"type_id\"\\: 4\\, \"type\"\\: \"User Name\"\\, \"name\"\\: \"actor.user.name\"\\, \"value\"\\: $3\\}\\]"
},
- format: "$network_activity.future_use_1$,$network_activity.receive_time$,$firewall.serial_number$,CORRELATION,$network_activity.sub_type$,$network_activity.future_use_2$,$timestamp=timestamp_pat$,$src.ip.address$,$user.src_name$,$network_traffic.virtual_system_name$,$network_activity.category_name$,$severity$,$device.group_hierarchy.level_1$,$device.group_hierarchy.level_2$,$device.group_hierarchy.level_3$,$device.group_hierarchy.level_4$,$firewall.virtual_system_name$,$device.name$,$firewall.virtual_system_id$,$object_name$,$object_id$,$evidence$",
- rewrites: [
- {
- input: "device.name",
- output: "event.source",
- match: ".*",
- replace: "$0"
- }
- ],
- halt: true
- },
- // USER ID
- {
- id: "userid-11-0", //PAN-OS v11.0 USERID, adds Origin Data Source, Future Use 3 and Cluster Name
- attributes: {
- "event.type": "userid",
- "format": "userid-v11.0"
+ {
+ input: "activity_name",
+ output: "event.type",
+ match: ".*",
+ replace: "$0"
},
- format: "$network_activity.future_use_1$,$network_activity.receive_time$,$firewall.serial_number$,USERID,$network_activity.sub_type$,$network_activity.future_use_2$,$timestamp=timestamp_pat$,$network_traffic.virtual_system_name$,$src.ip.address$,$user$,$data_source_name$,$session.parent_uid$,$network_activity.repeat_count$,$time_out_threshold$,$network_endpoint.src_port$,$network_endpoint.dst_port$,$data_source$,$data_source_type$,$network_activity.sequence_number$,$network_activity.action_flags$,$device.group_hierarchy.level_1$,$device.group_hierarchy.level_2$,$device.group_hierarchy.level_3$,$device.group_hierarchy.level_4$,$firewall.virtual_system_name$,$device.name$,$firewall.virtual_system_id$,$factor_type$,$factor_completion_time$,$factor_number$,$user.groups$,$user_by_source$,$network_activity.tag$,$network_activity.time$,$origin_data_source$,$future_use3$,$network_activity.cluster.name$",
- rewrites: [
- {
- input: "device.name",
- output: "event.source",
- match: ".*",
- replace: "$0"
- }
- ],
- halt: true
- }, {
- id: "userid-10-1", //PAN-OS v10.1-10.2 USERID, adds Tag Name and High Resolution Timestamp
- attributes: {
- "event.type": "userid",
- "format": "userid-v10.1"
+ {
+ action: "removeFields",
+ fields: [
+ "source_address",
+ "destination_address",
+ "nat_source_ip",
+ "nat_destination_ip",
+ "source_zone",
+ "destination_zone",
+ "inbound_interface",
+ "outbound_interface",
+ "source_port",
+ "destination_port",
+ "ip_protocol",
+ "filename",
+ "source_location",
+ "destination_location",
+ "file_type",
+ "source_vm_uuid",
+ "destination_vm_uuid",
+ "parent_session_id",
+ "parent_start_time",
+ "source_hostname",
+ "source_mac_address",
+ "destination_hostname",
+ "destination_mac_address"
+ ]
+ }
+ ]
+ },
+ {
+ attributes: {
+ "activity_name": "Other",
+ "activity_id": "99",
+ "category_name": "Identity & Access Management",
+ "category_uid": "3",
+ "class_name": "Authentication",
+ "class_uid": "3002",
+ "cloud.provider": "Palo Alto Networks",
+ "metadata.product.name": "Palo Alto Networks Firewall",
+ "metadata.product.vendor_name": "Palo Alto Networks",
+ "metadata.version":"1.1.0",
+ "type_uid": "300299",
+ "type_name": "Authentication: Other",
+ "severity_id": "99"
+ },
+ format: "$metadata.original_time$,$metadata.product.uid$,$unmapped.type=globalprotect_log_type$,$unmapped.subtype$,.*,$start_time_dt$,$unmapped.vsys$,$metadata.event_code$,$unmapped.stage$,$auth_protocol$,$unmapped.tunnel_type$,$actor.user.name$,$src_endpoint.location.region$,$device.name$,$device.ip$,$unmapped.public_ipv6$,$unmapped.private_ip$,$unmapped.private_ipv6$,$unmapped.hostid$,$src_endpoint.hw_info.serial_number$,$metadata.product.version$,$src_endpoint.os.name$,$src_endpoint.os.version$,$unmapped.repeatcnt$,$unmapped.reason$,$unmapped.error$,$unmapped.description$,$status$,$unmapped.location$,$unmapped.login_duration$,$unmapped.connect_method$,$unmapped.error_code$,$unmapped.portal$,$unmapped.seqno$,$unmapped.actionflags$,$unmapped.high_res_timestamp$,$unmapped.selection_type$,$unmapped.response_time$,$unmapped.priority$,$unmapped.attempted_gateways$,$unmapped.gateway$,$unmapped.dg_hier_level_1$,$unmapped.dg_hier_level_2$,$unmapped.dg_hier_level_3$,$unmapped.dg_hier_level_4$,$unmapped.vsys_name$,$src_endpoint.hostname$,$unmapped.vsys_id$",
+ halt: true,
+ rewrites: [
+ {
+ input: "message",
+ output: "observables",
+ match: "(?:[^,]*,){11}([^,]*)(?:,[^,]*){2},([^,]*).*",
+ replace: "\\[\\{\"type_id\"\\: 2\\, \"type\"\\: \"IP Address\"\\, \"name\"\\: \"device.ip\"\\, \"value\"\\: $2\\}\\, \\{\"type_id\"\\: 4\\, \"type\"\\: \"User Name\"\\, \"name\"\\: \"actor.user.name\"\\, \"value\"\\: $1\\}\\]"
},
- format: "$network_activity.future_use_1$,$network_activity.receive_time$,$firewall.serial_number$,USERID,$network_activity.sub_type$,$network_activity.future_use_2$,$timestamp=timestamp_pat$,$network_traffic.virtual_system_name$,$src.ip.address$,$user$,$data_source_name$,$session.parent_uid$,$network_activity.repeat_count$,$time_out_threshold$,$network_endpoint.src_port$,$network_endpoint.dst_port$,$data_source$,$data_source_type$,$network_activity.sequence_number$,$network_activity.action_flags$,$device.group_hierarchy.level_1$,$device.group_hierarchy.level_2$,$device.group_hierarchy.level_3$,$device.group_hierarchy.level_4$,$firewall.virtual_system_name$,$device.name$,$firewall.virtual_system_id$,$factor_type$,$factor_completion_time$,$factor_number$,$user.groups$,$user_by_source$,$network_activity.tag$,$network_activity.time$",
- rewrites: [
- {
- input: "device.name",
- output: "event.source",
- match: ".*",
- replace: "$0"
- }
- ],
- halt: true
- }, {
- id: "userid-10-1", //PAN-OS v9.1 USERID
- attributes: {
- "event.type": "userid",
- "format": "userid-v9.1"
+ {
+ input: "activity_name",
+ output: "event.type",
+ match: ".*",
+ replace: "$0"
+ }
+ ]
+ },
+ {
+ attributes: {
+ "activity_name": "Update",
+ "activity_id": "3",
+ "category_name": "Identity & Access Management",
+ "category_uid": "3",
+ "class_name": "Entity Management",
+ "class_uid": "3004",
+ "cloud.provider": "Palo Alto Networks",
+ "metadata.product.name": "Palo Alto Networks Firewall",
+ "metadata.product.vendor_name": "Palo Alto Networks",
+ "metadata.version":"1.1.0",
+ "type_uid": "300403",
+ "type_name": "Entity Management: Update",
+ "severity_id": "99"
+ },
+ format: "$metadata.original_time$,$metadata.product.uid$,$unmapped.type=iptag_log_type$,$unmapped.subtype$,.*,$start_time_dt$,$unmapped.vsys$,$device.ip$,$unmapped.tag_name$,$metadata.event_code$,$unmapped.repeatcnt$,$unmapped.timeout$,$unmapped.datasource$,$unmapped.datasourcetype$,$unmapped.datasource_subtype$,$unmapped.seqno$,$unmapped.actionflags$,$unmapped.dg_hier_level_1$,$unmapped.dg_hier_level_2$,$unmapped.dg_hier_level_3$,$unmapped.dg_hier_level_4$,$unmapped.vsys_name$,$device.name$,$unmapped.vsys_id$",
+ halt: true,
+ rewrites: [
+ {
+ input: "message",
+ output: "observables",
+ match: "(?:[^,]*,){7}([^,]*)(?:,[^,]*){14},([^,]*).*",
+ replace: "\\[\\{\"type_id\"\\: 1\\, \"type\"\\: \"Hostname\"\\, \"name\"\\: \"device.name\"\\, \"value\"\\: $2\\}\\, \\{\"type_id\"\\: 2\\, \"type\"\\: \"IP Address\"\\, \"name\"\\: \"device.ip\"\\, \"value\"\\: $1\\}\\]"
},
- format: "$network_activity.future_use_1$,$network_activity.receive_time$,$firewall.serial_number$,USERID,$network_activity.sub_type$,$network_activity.future_use_2$,$timestamp=timestamp_pat$,$network_traffic.virtual_system_name$,$src.ip.address$,$user$,$data_source_name$,$session.parent_uid$,$network_activity.repeat_count$,$time_out_threshold$,$network_endpoint.src_port$,$network_endpoint.dst_port$,$data_source$,$data_source_type$,$network_activity.sequence_number$,$network_activity.action_flags$,$device.group_hierarchy.level_1$,$device.group_hierarchy.level_2$,$device.group_hierarchy.level_3$,$device.group_hierarchy.level_4$,$firewall.virtual_system_name$,$device.name$,$firewall.virtual_system_id$,$factor_type$,$factor_completion_time$,$factor_number$,$user.groups$,$user_by_source$,$network_activity.tag$,$network_activity.time$",
- rewrites: [
- {
- input: "device.name",
- output: "event.source",
- match: ".*",
- replace: "$0"
- }
- ],
- halt: true
- },
- // GLOBALPROTECT
- {
- id: "globalprotect-9-1-3", //PAN-OS v9.1.3+ GLOBALPROTECT, different format
- attributes: {
- "event.type": "globalprotect",
- "format": "globalprotect-v9.1.3"
+ {
+ input: "activity_name",
+ output: "event.type",
+ match: ".*",
+ replace: "$0"
+ }
+ ]
+ },
+ {
+ attributes: {
+ "activity_name": "Open",
+ "activity_id": "1",
+ "category_name": "Network Activity",
+ "category_uid": "4",
+ "class_name": "Network Activity",
+ "class_uid": "4001",
+ "cloud.provider": "Palo Alto Networks",
+ "metadata.product.name": "Palo Alto Networks Firewall",
+ "metadata.product.vendor_name": "Palo Alto Networks",
+ "metadata.version":"1.1.0",
+ "type_uid": "400101",
+ "type_name": "Network Activity: Open",
+ "severity_id": "99"
+ },
+ format: "$metadata.original_time$,$metadata.product.uid$,$unmapped.type=gtp_log_type$,$unmapped.subtype$,.*,$start_time_dt$,$src_endpoint.ip$,$dst_endpoint.ip$,.*,.*,$firewall_rule.name$,.*,.*,$unmapped.app$,$unmapped.vsys$,$src_endpoint.zone$,$dst_endpoint.zone$,$src_endpoint.interface_name$,$dst_endpoint.interface_name$,$unmapped.logset$,.*,$actor.session.uid$,.*,$src_endpoint.port$,$dst_endpoint.port$,.*,.*,.*,$connection_info.protocol_name$,$action$,$unmapped.event_type$,$unmapped.msisdn$,$unmapped.apn$,$unmapped.rat$,$unmapped.msg_type$,$device.ip$,$unmapped.teid1$,$unmapped.teid2$,$unmapped.gtp_interface$,$unmapped.cause_code$,$unmapped.severity$,$unmapped.mcc$,$unmapped.mnc$,$unmapped.area_code$,$unmapped.cell_id$,$unmapped.event_code$,.*,.*,$src_endpoint.location.country$,$dst_endpoint.location.country$,.*,.*,.*,.*,.*,.*,.*,$unmapped.imsi$,$device.imei$,.*,.*,.*,.*,.*,.*,.*,.*,.*,.*,.*,.*,.*,.*,.*,.*,$start_time$,$unmapped.elapsed$,$unmapped.tunnel_insp_rule$,$unmapped.tunnel_insp_rule$,$unmapped.tunnel_insp_rule$,$unmapped.rule_uuid$,$unmapped.pcap_id$",
+ halt: true,
+ rewrites: [
+ {
+ input: "message",
+ output: "observables",
+ match: "(?:[^,]*,){6}([^,]*)(?:,[^,]*){0},([^,]*)(?:,[^,]*){27},([^,]*).*",
+ replace: "\\[\\{\"type_id\"\\: 2\\, \"type\"\\: \"IP Address\"\\, \"name\"\\: \"src_endpoint.ip\"\\, \"value\"\\: $1\\}\\, \\{\"type_id\"\\: 2\\, \"type\"\\: \"IP Address\"\\, \"name\"\\: \"dst_endpoint.ip\"\\, \"value\"\\: $2\\}, \\{\"type_id\"\\: 2\\, \"type\"\\: \"IP Address\"\\, \"name\"\\: \"device.ip\"\\, \"value\"\\: $3\\}\\]"
},
- format: "$network_activity.future_use_1$,$network_activity.receive_time$,$firewall.serial_number$,GLOBALPROTECT,$network_activity.sub_type$,$network_activity.future_use_2$,$timestamp=timestamp_pat$,$network_traffic.virtual_system_name$,$session.parent_uid$,$stage$,$authentication_method$,$network_connection_info.tunnel_type$,$user.src_name$,$source_region$,$machine_name$,$public_ip$,$public_ipv6$,$private_ip$,$private_ipv6$,$network_endpoint.host_id$,$device_hardware_info.serial_number$,$client_version$,$client_os$,$client_os_version$,$network_activity.repeat_count$,$reason$,$error$,$description$,$status$,$location$,$login_duration$,$connect_method$,$error_code$,$portal$,$network_activity.sequence_number$,$network_activity.action_flags$",
- rewrites: [
- {
- input: "device.name",
- output: "event.source",
- match: ".*",
- replace: "$0"
- }
- ],
- halt: true
- }, {
- id: "globalprotect-9-1-0", //PAN-OS v9.1.0-9.1.2 GLOBALPROTECT
- attributes: {
- "event.type": "globalprotect",
- "format": "globalprotect-v9.1.3"
+ {
+ input: "activity_name",
+ output: "event.type",
+ match: ".*",
+ replace: "$0"
+ }
+ ]
+ },
+ {
+ attributes: {
+ "activity_name": "Open",
+ "activity_id": "1",
+ "category_name": "Network Activity",
+ "category_uid": "4",
+ "class_name": "Network Activity",
+ "class_uid": "4001",
+ "cloud.provider": "Palo Alto Networks",
+ "metadata.product.name": "Palo Alto Networks Firewall",
+ "metadata.product.vendor_name": "Palo Alto Networks",
+ "metadata.version":"1.1.0",
+ "type_uid": "400101",
+ "type_name": "Network Activity: Open",
+ "severity_id": "99"
+ },
+ format: "$metadata.original_time$,$metadata.product.uid$,$unmapped.type=tunnel_log_type$,$unmapped.subtype$,.*,$start_time_dt$,$src_endpoint.ip$,$dst_endpoint.ip$,$src_endpoint.proxy_endpoint.ip$,$dst_endpoint.proxy_endpoint.ip$,$firewall_rule.name$,$actor.user.name$,$user.name$,$unmapped.app$,$unmapped.vsys$,$src_endpoint.zone$,$dst_endpoint.zone$,$src_endpoint.interface_name$,$dst_endpoint.interface_name$,$unmapped.logset$,.*,$actor.session.uid$,$unmapped.repeatcnt$,$src_endpoint.port$,$dst_endpoint.port$,$src_endpoint.proxy_endpoint.port$,$dst_endpoint.proxy_endpoint.port$,$unmapped.flags$,$connection_info.protocol_name$,$action$,$unmapped.severity$,$unmapped.seqno$,$unmapped.actionflags$,$src_endpoint.location.country$,$dst_endpoint.location.country$,$unmapped.dg_hier_level_1$,$unmapped.dg_hier_level_2$,$unmapped.dg_hier_level_3$,$unmapped.dg_hier_level_4$,$unmapped.vsys_name$,$device.name$,$unmapped.imsi$,$device.imei$,$session.uid$,$start_time$,$tunnel_type$,$traffic.bytes$,$traffic.bytes_out$,$traffic.bytes_in$,$traffic.packets$,$traffic.packets_out$,$traffic.packets_in$,$unmapped.max_encap$,$unmapped.unknown_proto$,$unmapped.strict_check$,$unmapped.tunnel_fragment$,$session.count$,$unmapped.sessions_closed$,$session.expiration_reason$,$unmapped.action_source$,$session.created_time$,$session.expiration_time$,$unmapped.tunnel_insp_rule$,$device.ip$,$user.uid$,$unmapped.rule_uuid$,$unmapped.pcap_id$,$unmapped.dynusergroup_name$,$unmapped.src_edl$,$unmapped.dst_edl$,$unmapped.high_res_timestamp$,$unmapped.nssai_sd$,$unmapped.nssai_sd$,$unmapped.pdu_session_id$,$unmapped.subcategory_of_app$,$unmapped.category_of_app$,$unmapped.technology_of_app$,$unmapped.risk_of_app$,$unmapped.characteristic_of_app$,$unmapped.container_of_app$,$unmapped.is_saas_of_app$,$unmapped.sanctioned_state_of_app$,$unmapped.cluster_name$",
+ halt: true,
+ rewrites: [
+ {
+ input: "message",
+ output: "observables",
+ match: "(?:[^,]*,){6}([^,]*)(?:,[^,]*){0},([^,]*)(?:,[^,]*){57},([^,]*).*",
+ replace: "\\[\\{\"type_id\"\\: 2\\, \"type\"\\: \"IP Address\"\\, \"name\"\\: \"src_endpoint.ip\"\\, \"value\"\\: $1\\}\\, \\{\"type_id\"\\: 2\\, \"type\"\\: \"IP Address\"\\, \"name\"\\: \"dst_endpoint.ip\"\\, \"value\"\\: $2\\}, \\{\"type_id\"\\: 2\\, \"type\"\\: \"IP Address\"\\, \"name\"\\: \"device.ip\"\\, \"value\"\\: $3\\}\\]"
},
- format: "$network_activity.future_use_1$,$network_activity.receive_time$,$device_hardware_info.serial_number$,$network_activity.sequence_number$,$network_activity.action_flags$,GLOBALPROTECT,$network_activity.future_use_2$,$network_activity.future_use_3$,$timestamp=timestamp_pat$,$network_traffic.virtual_system_name$,$session.parent_uid$,$stage$,$authentication_method$,$network_connection_info.tunnel_type$,$user.src_name$,$source_region$,$machine_name$,$public_ip$,$public_ipv6$,$private_ip$,$private_ipv6$,$network_endpoint.host_id$,$device_hardware_info.serial_number$,$client_version$,$client_os$,$client_os_version$,$network_activity.repeat_count$,$reason$,$error$,$description$,$status$,$location$,$login_duration$,$connect_method$,$error_code$,$portal$",
- rewrites: [
- {
- input: "device.name",
- output: "event.source",
- match: ".*",
- replace: "$0"
- }
- ],
- halt: true
- }, {
- id: "decryption-11-0", //PAN-OS v11.0, adds Cluster Name
- attributes: {
- "event.type": "decryption",
- "format": "decryption-v11.0"
+ {
+ input: "activity_name",
+ output: "event.type",
+ match: ".*",
+ replace: "$0"
+ }
+ ]
+ },
+ {
+ attributes: {
+ "activity_name": "Open",
+ "activity_id": "1",
+ "category_name": "Network Activity",
+ "category_uid": "4",
+ "class_name": "Network Activity",
+ "class_uid": "4001",
+ "cloud.provider": "Palo Alto Networks",
+ "metadata.product.name": "Palo Alto Networks Firewall",
+ "metadata.product.vendor_name": "Palo Alto Networks",
+ "metadata.version":"1.1.0",
+ "type_uid": "400101",
+ "type_name": "Network Activity: Open",
+ "severity_id": "99"
+ },
+ format: "$metadata.original_time$,$metadata.product.uid$,$unmapped.type=sctp_log_type$,.*,.*,$start_time_dt$,$src_endpoint.ip$,$dst_endpoint.ip$,.*,.*,$firewall_rule.name$,.*,.*,.*,$unmapped.vsys$,$src_endpoint.zone$,$dst_endpoint.zone$,$src_endpoint.interface_name$,$dst_endpoint.interface_name$,$unmapped.logset$,.*,$actor.session.uid$,$unmapped.repeatcnt$,$src_endpoint.port$,$dst_endpoint.port$,.*,.*,.*,.*,$connection_info.protocol_name$,$action$,$unmapped.dg_hier_level_1$,$unmapped.dg_hier_level_2$,$unmapped.dg_hier_level_3$,$unmapped.dg_hier_level_4$,$unmapped.vsys_name$,$device.name$,$unmapped.seqno$,.*,$unmapped.assoc_id$,$unmapped.ppid$,$unmapped.severity$,$unmappedsctp_chunk_type$,.*,$unmapped.verif_tag_1$,$unmapped.verif_tag_2$,$unmapped.sctp_cause_code$,$unmapped.diam_app_id$,$unmapped.diam_cmd_code$,$unmapped.diam_avp_code$,$unmapped.stream_id$,$unmapped.assoc_end_reason$,$unmapped.op_code$,$unmapped.sccp_calling_ssn$,$unmapped.sccp_calling_gt$,$unmapped.sctp_filter$,$unmapped.chunks$,$unmapped.chunks_sent$,$unmapped.chunks_received$,$traffic.packets$,$traffic.packets_out$,$traffic.packets_in$,$unmapped.rule_uuid$",
+ halt: true,
+ rewrites: [
+ {
+ input: "message",
+ output: "observables",
+ match: "(?:[^,]*,){6}([^,]*)(?:,[^,]*){0},([^,]*).*",
+ replace: "\\[\\{\"type_id\"\\: 2\\, \"type\"\\: \"IP Address\"\\, \"name\"\\: \"src_endpoint.ip\"\\, \"value\"\\: $1\\}\\, \\{\"type_id\"\\: 2\\, \"type\"\\: \"IP Address\"\\, \"name\"\\: \"dst_endpoint.ip\"\\, \"value\"\\: $2\\}\\]"
},
- format: "$network_activity.future_use_1$,$network_activity.receive_time$,$firewall.serial_number$,DECRYPTION,$network_activity.sub_type$,$config_version$,$timestamp=timestamp_pat$,$src.ip.address$,$dst.ip.address$,$network_endpoint.nat_src_ip$,$network_endpoint.nat_dst_ip$,$rule.name$,$user.src_name$,$user.dst_name$,$network_activity.app_name$,$network_traffic.virtual_system_name$,$source_zone$,$destination_zone$,$network_interface.inbound_name$,$network_interface.outbound_name$,$network_activity.log_action$,$time_logged$,$session.uid$,$network_activity.repeat_count$,$network_endpoint.src_port$,$network_endpoint.dst_port$,$nenetwork_endpoint.nat_src_port$,$nenetwork_endpoint.nat_dst_port$,$network_connection_info.flag$,$network_connection_info.protocol_name$,$network_activity.action$,$tunnel$,$network_activity.future_use_2$,$network_activity.future_use_3$,$virtual_machine.src_vm_uuid$,$dest_vm_uuid$,$rule.uid$,$stage_client_firewall$,$stage_firewall_server$,$tls_version$,$key_exchange_algorithm$,$encryption_algorithm$,$hash_algorithm$,$rule$,$elliptic_curve$,$error_index$,$root_status$,$chain_status$,$proxy_type$,$cert_serial_number$,$fingerprint$,$certificate.created_time$,$certificate.expiration_time$,$certificate.version$,$certificate.size$,$cn_length$,$issuer_cn_length$,$root_cn_length$,$sni_length$,$cert_flags=value_pat$,$subject_cn$,$issuer_subject_cn$,$root_subject_cn$,$server_name$,$error$,$container.id$,$container.pod_namespace$,$container.pod_name$,$src_edl$,$dest_edl$,$src_dag$,$dest_dag$,$network_activity.time$,$device.src_type$,$device.src_profile$,$device.src_model$,$device.dst_vendor_name$,$device.src_os_edition$,$device.src_os_version$,$network_connection_info.dst_hostname$,$network_connection_info.dst_mac$,$network_activity.sequence_number$,$network_activity.action_flags$,$device.group_hierarchy.level_1$,$device.group_hierarchy.level_2$,$device.group_hierarchy.level_3$,$device.group_hierarchy.level_4$,$firewall.virtual_system_name$,$device.name$,$firewall.virtual_system_id$,$network_activity.sub_category$,$network_activity.category_name$,$network_activity.app_model$,$network_activity.severity$,$network_activity.app_characteristic=value_pat$,$network_activity.container.id$,$network_activity.is_saas$,$network_activity.is_sanctioned$,$network_activity.cluster.name$",
- rewrites: [
- {
- input: "device.name",
- output: "event.source",
- match: ".*",
- replace: "$0"
- }
- ],
- halt: true
- }, {
- id: "decryption-10-1", //PAN-OS v10.1-10.2
- attributes: {
- "event.type": "decryption",
- "format": "decryption-v10.1"
+ {
+ input: "activity_name",
+ output: "event.type",
+ match: ".*",
+ replace: "$0"
+ }
+ ]
+ },
+ {
+ attributes: {
+ "activity_name": "Create",
+ "activity_id": "1",
+ "category_name": "Findings",
+ "category_uid": "2",
+ "class_name": "Detection Finding",
+ "class_uid": "2004",
+ "cloud.provider": "Palo Alto Networks",
+ "metadata.product.name": "Palo Alto Networks Firewall",
+ "metadata.product.vendor_name": "Palo Alto Networks",
+ "metadata.version":"1.1.0",
+ "type_uid": "200401",
+ "type_name": "Detection Finding: Create",
+ "severity_id": "99"
+ },
+ format: "$metadata.original_time$,$metadata.product.uid$,$unmapped.type=system_log_type$,$unmapped.subtype$,.*,$start_time_dt$,$unmapped.vsys$,$metadata.event_code$,$unmapped.object$,.*,.*,$unmapped.module$,$unmapped.severity$,$unmapped.description$,$unmapped.seqno$,$unmapped.actionflags$,$unmapped.dg_hier_level_1$,$unmapped.dg_hier_level_2$,$unmapped.dg_hier_level_3$,$unmapped.dg_hier_level_4$,$unmapped.vsys_name$,$device.hostname$",
+ halt: true,
+ rewrites: [
+ {
+ input: "message",
+ output: "observables",
+ match: "(?:[^,]*,){21}([^,]*).*",
+ replace: "\\[\\{\"type_id\"\\: 1\\, \"type\"\\: \"Hostname\"\\, \"name\"\\: \"device.hostname\"\\, \"value\"\\: $1\\}\\}\\]"
},
- format: "$network_activity.future_use_1$,$network_activity.receive_time$,$firewall.serial_number$,DECRYPTION,$network_activity.sub_type$,$config_version$,$timestamp=timestamp_pat$,$src.ip.address$,$dst.ip.address$,$network_endpoint.nat_src_ip$,$network_endpoint.nat_dst_ip$,$rule.name$,$user.src_name$,$user.dst_name$,$network_activity.app_name$,$network_traffic.virtual_system_name$,$source_zone$,$destination_zone$,$network_interface.inbound_name$,$network_interface.outbound_name$,$network_activity.log_action$,$time_logged$,$session.uid$,$network_activity.repeat_count$,$network_endpoint.src_port$,$network_endpoint.dst_port$,$nenetwork_endpoint.nat_src_port$,$nenetwork_endpoint.nat_dst_port$,$network_connection_info.flag$,$network_connection_info.protocol_name$,$network_activity.action$,$tunnel$,$network_activity.future_use_2$,$network_activity.future_use_3$,$virtual_machine.src_vm_uuid$,$dest_vm_uuid$,$rule.uid$,$stage_client_firewall$,$stage_firewall_server$,$tls_version$,$key_exchange_algorithm$,$encryption_algorithm$,$hash_algorithm$,$rule$,$elliptic_curve$,$error_index$,$root_status$,$chain_status$,$proxy_type$,$cert_serial_number$,$fingerprint$,$certificate.created_time$,$certificate.expiration_time$,$certificate.version$,$certificate.size$,$cn_length$,$issuer_cn_length$,$root_cn_length$,$sni_length$,$cert_flags=value_pat$,$subject_cn$,$issuer_subject_cn$,$root_subject_cn$,$server_name$,$error$,$container.id$,$container.pod_namespace$,$container.pod_name$,$src_edl$,$dest_edl$,$src_dag$,$dest_dag$,$network_activity.time$,$device.src_type$,$device.src_profile$,$device.src_model$,$device.dst_vendor_name$,$device.src_os_edition$,$device.src_os_version$,$network_connection_info.dst_hostname$,$network_connection_info.dst_mac$,$network_activity.sequence_number$,$network_activity.action_flags$,$device.group_hierarchy.level_1$,$device.group_hierarchy.level_2$,$device.group_hierarchy.level_3$,$device.group_hierarchy.level_4$,$firewall.virtual_system_name$,$device.name$,$firewall.virtual_system_id$,$network_activity.sub_category$,$network_activity.category_name$,$network_activity.app_model$,$network_activity.severity$,$network_activity.app_characteristic=value_pat$,$network_activity.container.id$,$network_activity.is_saas$,$network_activity.is_sanctioned$",
- rewrites: [
- {
- input: "device.name",
- output: "event.source",
- match: ".*",
- replace: "$0"
- }
- ],
- halt: true
- }
- ]
- }
\ No newline at end of file
+ {
+ input: "activity_name",
+ output: "event.type",
+ match: ".*",
+ replace: "$0"
+ }
+ ]
+ },
+ ]
+}
diff --git a/Backend/parsers/community/paloalto_firewall-latest/paloalto_firewall.json b/Backend/parsers/community/paloalto_firewall-latest/paloalto_firewall.json
index a85f745..9080daf 100644
--- a/Backend/parsers/community/paloalto_firewall-latest/paloalto_firewall.json
+++ b/Backend/parsers/community/paloalto_firewall-latest/paloalto_firewall.json
@@ -1,301 +1,1169 @@
{
- "attributes": {
- "dataSource.vendor": "Palo Alto Networks",
- "dataSource.name": "Palo Alto Firewall",
+ attributes: {
"dataSource.category": "security",
- "metadata.product.vendor_name": "Palo Alto Networks",
- "metadata.product.name": "Palo Alto Firewall",
- "metadata.version": "1.0.0"
+ "dataSource.name": "Palo Alto Networks Firewall",
+ "dataSource.vendor": "Palo Alto Networks",
+ },
+ patterns: {
+ //maps to high_resolution_timestamp:
+ //timestamp: "\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}.\\d{3}(\\+|-)\\d{2}:\\d{2}",
+ //application_characteristic can be a single value, a comma delimited list in quotes, or blank. Null value is handled by format: traffic-2, not by this pattern.
+ app_characteristic: "(\".*\")|[^,]+",
+ //description field from system log is wrapped in quotes and may contain commas
+ desc: "(\".*\")",
+ userid_log_type: "USERID",
+ logout_sub_type: "logout",
+ login_sub_type: "login",
+ hipmatch_log_type: "HIPMATCH",
+ config_log_type: "CONFIG",
+ wildfire_sub_type: "wildfire",
+ data_filtering_sub_type: "file",
+ globalprotect_log_type: "GLOBALPROTECT",
+ iptag_log_type: "IPTAG",
+ gtp_log_type: "GTP",
+ tunnel_log_type: "\\b(?:START|END|start|end)\\b",
+ sctp_log_type: "SCTP",
+ system_log_type: "SYSTEM"
},
- "formats": [
+
+ formats: [
+ // {
+ // format: ".*$timestamp=timestamp$(\\,)*",
+ //},
{
- "format": "$unmapped.{parse=csv}$",
- "rewrites": [
+ //match all fields. application_characteristic can be a single value, or a comma delimited list in quotes.
+ attributes: {
+ "class_uid": "4001",
+ "category_uid": "4",
+ "severity_id": "0",
+ "class_name": "Network Activity",
+ "category_name": "Network Activity",
+ "metadata.product.name": "Palo Alto Networks Firewall",
+ "metadata.product.vendor_name": "Palo Alto Networks",
+ "metadata.version":"1.0.0-rc.3",
+ "metadata.log_name": "TRAFFIC",
+ },
+ format: ".*,$metadata.logged_time_dt$,$device.hw_info.serial_number$,TRAFFIC,$unmapped.sub_type$,.*,$metadata.original_time$,$src_endpoint.ip$,$dst_endpoint.ip$,$src_endpoint.intermediate_ips$,$dst_endpoint.intermediate_ips$,$unmapped.rule_matched$,$actor.user.name$,$unmapped.dst_user$,$app_name$,$unmapped.vsys$,$unmapped.from_zone$,$unmapped.to_zone$,$unmapped.inbound_if$,$unmapped.outbound_if$,$actor.session.issuer$,$metadata.original_time$,$actor.session.uid$,$unmapped.repeat_count$,$src_endpoint.port$,$dst_endpoint.port$,$unmapped.nat_src_port$,$unmapped.nat_dst_port$,$unmapped.flags$,$connection_info.protocol_name$,$unmapped.action_value$,$traffic.bytes$,$traffic.bytes_in$,$traffic.bytes_out$,$traffic.packets$,$actor.session.created_time_dt$,$duration$,$unmapped.url_category_value$,.*,$metadata.sequence$,$unmapped.action_flags$,$src_endpoint.location.region$,$dst_endpoint.location.region$,.*,$traffic.packets_out$,$traffic.packets_in$,$unmapped.session_end_reason_value$,$unmapped.dg_hier_level_1$,$unmapped.dg_hier_level_2$,$unmapped.dg_hier_level_3$,$unmapped.dg_hier_level_4$,$unmapped.vsys_name$,$device.hostname$,$unmapped.action_source$,$unmapped.src_uuid$,$unmapped.dst_uuid$,$device.imsi$,$device.imei$,$unmapped.parent_session_id$,$unmapped.parent_start_time$,$unmapped.tunnel_type$,$unmapped.ep_assoc_id$,$unmapped.chunks_total$,$unmapped.chunks_sent$,$unmapped.chunks_received$,$unmapped.rule_matched_uuid$,$unmapped.http2_connection$,$unmapped.link_change_count$,$unmapped.policy_id$,$unmapped.link_switches$,$unmapped.sdwan_cluster$,$unmapped.sdwan_device_type$,$unmapped.sdwan_cluster_type$,$unmapped.sdwan_site$,$actor.user.groups$,$src_endpoint.intermediate_ips$,$unmapped.src_category$,$unmapped.src_profile$,$unmapped.src_model$,$unmapped.src_vendor$,$unmapped.src_osfamily$,$unmapped.src_osversion$,$src_endpoint.hostname$,$src_endpoint.mac$,$unmapped.dst_category$,$unmapped.dst_profile$,$unmapped.dst_model$,$unmapped.dst_vendor$,$unmapped.dst_osfamily$,$unmapped.dst_osversion$,$dst_endpoint.hostname$,$dst_endpoint.mac$,$unmapped.container_id$,$unmapped.pod_namespace$,$unmapped.pod_name$,$unmapped.src_edl$,$unmapped.dst_edl$,$device.uid$,$unmapped.serial_number$,$unmapped.src_dag$,$unmapped.dst_dag$,$unmapped.session_owner$,$unmapped.high_res_timestamp$,$unmapped.nsdsai_sst$,$unmapped.nsdsai_sd$,$unmapped.subcategory_of_app$,$unmapped.category_of_app$,$unmapped.technology_of_app$,$unmapped.risk_of_app$,$unmapped.characteristic_of_app=app_characteristic$,$unmapped.container_of_app$,$unmapped.tunneled_app$,$unmapped.is_saas_of_app$,$unmapped.sanctioned_state_of_app$,$unmapped.offloaded$",
+ halt: true,
+ rewrites: [
+ {
+ input: "unmapped.sub_type",
+ output: "activity_id",
+ match: "^start$",
+ replace: "1"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "activity_id",
+ match: "^end$",
+ replace: "2"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "activity_id",
+ match: "^drop$",
+ replace: "4"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "activity_id",
+ match: "^deny$",
+ replace: "5"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "activity_name",
+ match: "^start$",
+ replace: "Open"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "activity_name",
+ match: "^end$",
+ replace: "Close"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "activity_name",
+ match: "^drop$",
+ replace: "Fail"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "activity_name",
+ match: "^deny$",
+ replace: "Refuse"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "event.type",
+ match: "^start$",
+ replace: "Open"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "event.type",
+ match: "^end$",
+ replace: "Close"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "event.type",
+ match: "^drop$",
+ replace: "Fail"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "event.type",
+ match: "^deny$",
+ replace: "Refuse"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "type_uid",
+ match: "^start$",
+ replace: "400101"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "type_uid",
+ match: "^end$",
+ replace: "400102"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "type_uid",
+ match: "^drop$",
+ replace: "400104"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "type_uid",
+ match: "^deny$",
+ replace: "400105"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "type_name",
+ match: "^start$",
+ replace: "Network Activity: Open"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "type_name",
+ match: "^end$",
+ replace: "Network Activity: Close"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "type_name",
+ match: "^drop$",
+ replace: "Network Activity: Fail"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "type_name",
+ match: "^deny$",
+ replace: "Network Activity: Refuse"
+ },
+ {
+ input: "unmapped.action_value",
+ output: "status_id",
+ match: "^allow$",
+ replace: "1"
+ },
+ {
+ input: "unmapped.action_value",
+ output: "status_id",
+ match: "^deny$",
+ replace: "2"
+ },
+ {
+ input: "unmapped.action_value",
+ output: "status",
+ match: "^allow$",
+ replace: "Success"
+ },
+ {
+ input: "unmapped.action_value",
+ output: "status",
+ match: "^deny$",
+ replace: "Failure"
+ },
+ {
+ input: "unmapped.action_value",
+ output: "status_id",
+ match: "^(?!allow|deny$).*",
+ replace: "99"
+ },
+ {
+ input: "unmapped.action_value",
+ output: "status",
+ match: "^(?!allow|deny$).*",
+ replace: "Other"
+ },
{
- "input": "unmapped.receive_time",
- "output": "timestamp",
- "match": ".*",
- "replace": "$0"
- }
+ input: "dst_endpoint.intermediate_ips",
+ output: "dst_endpoint.intermediate_ips",
+ match: ".*",
+ replace: "\\[\"$0\"\\]"
+ },
+ {
+ input: "message",
+ output: "src_endpoint.intermediate_ips",
+ match: "(?:[^,]*,){9}([^,]*){1},(?:[^,]*,){65}([^,]*){1},(?:[^,]*,){38}.*",
+ replace: "\\[\"$1\"\\, \"$2\"\\]"
+ },
+ {
+ input: "message",
+ output: "observables",
+ match: "(?:[^,]*,){7}([^,]*),([^,]*),(?:[^,]*,){3}([^,]*),(?:[^,]*,){69}([^,]*),([^,]*),(?:[^,]*,){6}([^,]*),([^,]*),.*",
+ replace: "\\[\\{\"type_id\"\\: \"1\"\\, \"type\"\\: \"Hostname\"\\, \"name\"\\: \"src_endpoint.hostname\"\\, \"value\"\\: \"$4\"\\}\\, \\{\"type_id\"\\: \"1\"\\, \"type\"\\: \"Hostname\"\\, \"name\"\\: \"dst_endpoint.hostname\"\\, \"value\"\\: \"$6\"\\}\\, \\{\"type_id\"\\: \"2\"\\, \"type\"\\: \"IP Address\"\\, \"name\"\\: \"src_endpoint.ip\"\\, \"value\"\\: \"$1\"\\}\\, \\{\"type_id\"\\: \"2\"\\, \"type\"\\: \"IP Address\"\\, \"name\"\\: \"dst_endpoint.ip\"\\, \"value\"\\: \"$2\"\\}\\, \\{\"type_id\"\\: \"4\"\\, \"type\"\\: \"User Name\"\\, \"name\"\\: \"actor.user.name\"\\, \"value\"\\: \"$3\"\\}\\, \\{\"type_id\"\\: \"3\"\\, \"type\"\\: \"MAC Address\"\\, \"name\"\\: \"src_endpoint.mac\"\\, \"value\"\\: \"$5\"\\}\\, \\{\"type_id\"\\: \"3\"\\, \"type\"\\: \"MAC Address\"\\, \"name\"\\: \"dst_endpoint.mac\"\\, \"value\"\\: \"$7\"\\}\\]"
+ },
]
- }
- ],
- "mappings": {
- "version": 1,
- "mappings": [
- {
- "predicate": "true",
- "transformations": [
- {
- "constant": {
- "value": 4001,
- "field": "class_uid"
- }
- },
- {
- "constant": {
- "value": "Network Activity",
- "field": "class_name"
- }
- },
- {
- "constant": {
- "value": 4,
- "field": "category_uid"
- }
- },
- {
- "constant": {
- "value": "Network Activity",
- "field": "category_name"
- }
- },
- {
- "copy": {
- "from": "unmapped.receive_time",
- "to": "time"
- }
- },
- {
- "cast": {
- "field": "time",
- "type": "iso8601TimestampToEpochSec"
- }
- },
- {
- "copy": {
- "from": "unmapped.serial",
- "to": "metadata.uid"
- }
- },
- {
- "copy": {
- "from": "unmapped.action",
- "to": "message"
- }
- },
- {
- "copy": {
- "from": "unmapped.action",
- "to": "activity_name"
- }
- },
- {
- "copy": {
- "from": "unmapped.src",
- "to": "src_endpoint.ip"
- }
- },
- {
- "copy": {
- "from": "unmapped.dst",
- "to": "dst_endpoint.ip"
- }
- },
- {
- "copy": {
- "from": "unmapped.sport",
- "to": "src_endpoint.port"
- }
- },
- {
- "copy": {
- "from": "unmapped.dport",
- "to": "dst_endpoint.port"
- }
- },
- {
- "copy": {
- "from": "unmapped.proto",
- "to": "connection_info.protocol_name"
- }
- },
- {
- "copy": {
- "from": "unmapped.app",
- "to": "app_name"
- }
- },
- {
- "copy": {
- "from": "unmapped.rule",
- "to": "metadata.extensions.rule_name"
- }
- },
- {
- "copy": {
- "from": "unmapped.srcuser",
- "to": "user.name"
- }
- },
- {
- "copy": {
- "from": "unmapped.dstuser",
- "to": "dst_endpoint.name"
- }
- },
- {
- "copy": {
- "from": "unmapped.category",
- "to": "metadata.extensions.category"
- }
- },
- {
- "copy": {
- "from": "unmapped.subcategory",
- "to": "metadata.extensions.subcategory"
- }
- },
- {
- "copy": {
- "from": "unmapped.severity",
- "to": "severity"
- }
- },
- {
- "copy": {
- "from": "unmapped.direction",
- "to": "connection_info.direction"
- }
- },
- {
- "copy": {
- "from": "unmapped.actionflags",
- "to": "metadata.extensions.action_flags"
- }
- },
- {
- "copy": {
- "from": "unmapped.srczone",
- "to": "src_endpoint.location.region"
- }
- },
- {
- "copy": {
- "from": "unmapped.dstzone",
- "to": "dst_endpoint.location.region"
- }
- },
- {
- "copy": {
- "from": "unmapped.inbound_if",
- "to": "src_endpoint.interface_name"
- }
- },
- {
- "copy": {
- "from": "unmapped.outbound_if",
- "to": "dst_endpoint.interface_name"
- }
- },
- {
- "copy": {
- "from": "unmapped.logset",
- "to": "metadata.extensions.logset"
- }
- },
- {
- "copy": {
- "from": "unmapped.bytes",
- "to": "traffic.bytes"
- }
- },
- {
- "copy": {
- "from": "unmapped.bytes_sent",
- "to": "traffic.bytes_out"
- }
- },
- {
- "copy": {
- "from": "unmapped.bytes_received",
- "to": "traffic.bytes_in"
- }
- },
- {
- "copy": {
- "from": "unmapped.packets",
- "to": "traffic.packets"
- }
- },
- {
- "copy": {
- "from": "unmapped.start",
- "to": "start_time"
- }
- },
- {
- "copy": {
- "from": "unmapped.elapsed",
- "to": "connection_info.duration"
- }
- },
- {
- "copy": {
- "from": "unmapped.device_name",
- "to": "device.hostname"
- }
- },
- {
- "constant": {
- "value": 1,
- "field": "activity_id",
- "predicate": "unmapped.action = 'allow'"
- }
- },
- {
- "constant": {
- "value": 2,
- "field": "activity_id",
- "predicate": "unmapped.action = 'deny'"
- }
- },
- {
- "constant": {
- "value": 1,
- "field": "severity_id",
- "predicate": "unmapped.action = 'allow'"
- }
- },
+ },
+ {
+ //dont match on application_characteristic for cases where is it blank.
+ attributes: {
+ "class_uid": "4001",
+ "category_uid": "4",
+ "severity_id": "0",
+ "class_name": "Network Activity",
+ "category_name": "Network Activity",
+ "metadata.product.name": "Palo Alto Networks Firewall",
+ "metadata.product.vendor_name": "Palo Alto Networks",
+ "metadata.version":"1.0.0-rc.3",
+ "metadata.log_name": "TRAFFIC",
+ },
+ format: ".*,$metadata.logged_time_dt$,$device.hw_info.serial_number$,TRAFFIC,$unmapped.sub_type$,.*,$metadata.original_time$,$src_endpoint.ip$,$dst_endpoint.ip$,$src_endpoint.intermediate_ips$,$dst_endpoint.intermediate_ips$,$unmapped.rule_matched$,$actor.user.name$,$unmapped.dst_user$,$app_name$,$unmapped.vsys$,$unmapped.from_zone$,$unmapped.to_zone$,$unmapped.inbound_if$,$unmapped.outbound_if$,$actor.session.issuer$,$metadata.original_time$,$actor.session.uid$,$unmapped.repeat_count$,$src_endpoint.port$,$dst_endpoint.port$,$unmapped.nat_src_port$,$unmapped.nat_dst_port$,$unmapped.flags$,$connection_info.protocol_name$,$unmapped.action_value$,$traffic.bytes$,$traffic.bytes_in$,$traffic.bytes_out$,$traffic.packets$,$actor.session.created_time_dt$,$duration$,$unmapped.url_category_value$,.*,$metadata.sequence$,$unmapped.action_flags$,$src_endpoint.location.region$,$dst_endpoint.location.region$,.*,$traffic.packets_out$,$traffic.packets_in$,$unmapped.session_end_reason_value$,$unmapped.dg_hier_level_1$,$unmapped.dg_hier_level_2$,$unmapped.dg_hier_level_3$,$unmapped.dg_hier_level_4$,$unmapped.vsys_name$,$device.hostname$,$unmapped.action_source$,$unmapped.src_uuid$,$unmapped.dst_uuid$,$device.imsi$,$device.imei$,$unmapped.parent_session_id$,$unmapped.parent_start_time$,$unmapped.tunnel_type$,$unmapped.ep_assoc_id$,$unmapped.chunks_total$,$unmapped.chunks_sent$,$unmapped.chunks_received$,$unmapped.rule_matched_uuid$,$unmapped.http2_connection$,$unmapped.link_change_count$,$unmapped.policy_id$,$unmapped.link_switches$,$unmapped.sdwan_cluster$,$unmapped.sdwan_device_type$,$unmapped.sdwan_cluster_type$,$unmapped.sdwan_site$,$actor.user.groups$,$src_endpoint.intermediate_ips$,$unmapped.src_category$,$unmapped.src_profile$,$unmapped.src_model$,$unmapped.src_vendor$,$unmapped.src_osfamily$,$unmapped.src_osversion$,$src_endpoint.hostname$,$src_endpoint.mac$,$unmapped.dst_category$,$unmapped.dst_profile$,$unmapped.dst_model$,$unmapped.dst_vendor$,$unmapped.dst_osfamily$,$unmapped.dst_osversion$,$dst_endpoint.hostname$,$dst_endpoint.mac$,$unmapped.container_id$,$unmapped.pod_namespace$,$unmapped.pod_name$,$unmapped.src_edl$,$unmapped.dst_edl$,$device.uid$,$unmapped.serial_number$,$unmapped.src_dag$,$unmapped.dst_dag$,$unmapped.session_owner$,$unmapped.high_res_timestamp$,$unmapped.nsdsai_sst$,$unmapped.nsdsai_sd$,$unmapped.subcategory_of_app$,$unmapped.category_of_app$,$unmapped.technology_of_app$,$unmapped.risk_of_app$,$unmapped.characteristic_of_app$,$unmapped.container_of_app$,$unmapped.tunneled_app$,$unmapped.is_saas_of_app$,$unmapped.sanctioned_state_of_app$,$unmapped.offloaded$",
+ halt: true,
+ rewrites: [
+ {
+ input: "unmapped.sub_type",
+ output: "event.type",
+ match: "^start$",
+ replace: "Open"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "event.type",
+ match: "^end$",
+ replace: "Close"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "event.type",
+ match: "^drop$",
+ replace: "Fail"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "event.type",
+ match: "^deny$",
+ replace: "Refuse"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "activity_id",
+ match: "^start$",
+ replace: "1"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "activity_id",
+ match: "^end$",
+ replace: "2"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "activity_id",
+ match: "^drop$",
+ replace: "4"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "activity_id",
+ match: "^deny$",
+ replace: "5"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "activity_name",
+ match: "^start$",
+ replace: "Open"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "activity_name",
+ match: "^end$",
+ replace: "Close"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "activity_name",
+ match: "^drop$",
+ replace: "Fail"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "activity_name",
+ match: "^deny$",
+ replace: "Refuse"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "type_uid",
+ match: "^start$",
+ replace: "400101"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "type_uid",
+ match: "^end$",
+ replace: "400102"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "type_uid",
+ match: "^drop$",
+ replace: "400104"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "type_uid",
+ match: "^deny$",
+ replace: "400105"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "type_name",
+ match: "^start$",
+ replace: "Network Activity: Open"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "type_name",
+ match: "^end$",
+ replace: "Network Activity: Close"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "type_name",
+ match: "^drop$",
+ replace: "Network Activity: Fail"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "type_name",
+ match: "^deny$",
+ replace: "Network Activity: Refuse"
+ },
+ {
+ input: "unmapped.action_value",
+ output: "status_id",
+ match: "^allow$",
+ replace: "1"
+ },
+ {
+ input: "unmapped.action_value",
+ output: "status_id",
+ match: "^deny$",
+ replace: "2"
+ },
+ {
+ input: "unmapped.action_value",
+ output: "status",
+ match: "^allow$",
+ replace: "Success"
+ },
+ {
+ input: "unmapped.action_value",
+ output: "status",
+ match: "^deny$",
+ replace: "Failure"
+ },
+ {
+ input: "unmapped.action_value",
+ output: "status_id",
+ match: "^(?!allow|deny$).*",
+ replace: "99"
+ },
+ {
+ input: "unmapped.action_value",
+ output: "status",
+ match: "^(?!allow|deny$).*",
+ replace: "Other"
+ },
+ {
+ input: "dst_endpoint.intermediate_ips",
+ output: "dst_endpoint.intermediate_ips",
+ match: ".*",
+ replace: "\\[\"$0\"\\]"
+ },
+ {
+ input: "message",
+ output: "src_endpoint.intermediate_ips",
+ match: "(?:[^,]*,){9}([^,]*){1},(?:[^,]*,){65}([^,]*){1},(?:[^,]*,){38}.*",
+ replace: "\\[\"$1\"\\, \"$2\"\\]"
+ },
+ {
+ input: "message",
+ output: "observables",
+ match: "(?:[^,]*,){7}([^,]*),([^,]*),(?:[^,]*,){3}([^,]*),(?:[^,]*,){69}([^,]*),([^,]*),(?:[^,]*,){6}([^,]*),([^,]*),.*",
+ replace: "\\[\\{\"type_id\"\\: \"1\"\\, \"type\"\\: \"Hostname\"\\, \"name\"\\: \"src_endpoint.hostname\"\\, \"value\"\\: \"$4\"\\}\\, \\{\"type_id\"\\: \"1\"\\, \"type\"\\: \"Hostname\"\\, \"name\"\\: \"dst_endpoint.hostname\"\\, \"value\"\\: \"$6\"\\}\\, \\{\"type_id\"\\: \"2\"\\, \"type\"\\: \"IP Address\"\\, \"name\"\\: \"src_endpoint.ip\"\\, \"value\"\\: \"$1\"\\}\\, \\{\"type_id\"\\: \"2\"\\, \"type\"\\: \"IP Address\"\\, \"name\"\\: \"dst_endpoint.ip\"\\, \"value\"\\: \"$2\"\\}\\, \\{\"type_id\"\\: \"4\"\\, \"type\"\\: \"User Name\"\\, \"name\"\\: \"actor.user.name\"\\, \"value\"\\: \"$3\"\\}\\, \\{\"type_id\"\\: \"3\"\\, \"type\"\\: \"MAC Address\"\\, \"name\"\\: \"src_endpoint.mac\"\\, \"value\"\\: \"$5\"\\}\\, \\{\"type_id\"\\: \"3\"\\, \"type\"\\: \"MAC Address\"\\, \"name\"\\: \"dst_endpoint.mac\"\\, \"value\"\\: \"$7\"\\}\\]"
+ },
+ ]
+ },
+ {
+ attributes: {
+ "class_uid": "0",
+ "activity_id": "99",
+ "category_uid": "0",
+ "type_uid": "99",
+ "type_name": "Base Event: Other",
+ "class_name": "Base Event",
+ "category_name": "Uncategorized",
+ "metadata.product.name": "Palo Alto Networks Firewall",
+ "metadata.product.vendor_name": "Palo Alto Networks",
+ "metadata.version":"1.0.0-rc.3",
+ "metadata.log_name": "SYSTEM",
+ },
+ format: ".*,$metadata.logged_time_dt$,$unmapped.serial$,SYSTEM,$unmapped.sub_type$,.*,$metadata.original_time$,$unmapped.vsys$,$unmapped.event_id$,$unmapped.object$,.*,.*,$unmapped.module$,$unmapped.severity$,$unmapped.description=desc$,$metadata.sequence$,$unmapped.action_flags$,$unmapped.dg_hier_level_1$,$unmapped.dg_hier_level_2$,$unmapped.dg_hier_level_3$,$unmapped.dg_hier_level_4$,$unmapped.vsys_name$,$unmapped.device_name$,.*,.*,$unmapped.high_res_timestamp$",
+ halt: true,
+ rewrites: [
+ {
+ input: "unmapped.sub_type",
+ output: "activity_name",
+ match: ".*",
+ replace: "$0"
+ },
+ {
+ input: "unmapped.sub_type",
+ output: "event.type",
+ match: ".*",
+ replace: "$0"
+ },
+ {
+ input: "unmapped.severity",
+ output: "severity_id",
+ match: "^informational$",
+ replace: "1"
+ },
+ {
+ input: "unmapped.severity",
+ output: "severity_id",
+ match: "^low$",
+ replace: "2"
+ },
+ {
+ input: "unmapped.severity",
+ output: "severity_id",
+ match: "^medium$",
+ replace: "3"
+ },
+ {
+ input: "unmapped.severity",
+ output: "severity_id",
+ match: "^high$",
+ replace: "4"
+ },
+ {
+ input: "unmapped.severity",
+ output: "severity_id",
+ match: "^critical$",
+ replace: "5"
+ },
+ {
+ input: "message",
+ output: "observables",
+ match: "(?:[^,]*,){14}(\".*\"),(?:[^,]*,){7}([^,]*),.*",
+ replace: "\\[\\{\"type_id\"\\: \"1\"\\, \"type\"\\: \"Hostname\"\\, \"name\"\\: \"device.hostname\"\\, \"value\"\\: \"$2\"\\}\\]"
+ },
+ ]
+ },
+ {
+ //matches THREAT logs with comma surround lists in application_characteristic and url_category_list.
+ attributes: {
+ "activity_name": "THREAT",
+ "class_uid": "4001",
+ "activity_id": "99",
+ "category_uid": "4",
+ "type_uid": "400199",
+ "type_name": "Network Activity: Other",
+ "class_name": "Network Activity",
+ "category_name": "Network Activity",
+ "metadata.product.name": "Palo Alto Networks Firewall",
+ "metadata.product.vendor_name": "Palo Alto Networks",
+ "metadata.version":"1.0.0-rc.3",
+ "metadata.log_name": "THREAT",
+ "event.type": "THREAT"
+ },
+ format: ".*,$metadata.logged_time_dt$,$device.hw_info.serial_number$,THREAT,$unmapped.sub_type$,.*,$metadata.original_time$,$src_endpoint.ip$,$dst_endpoint.ip$,$src_endpoint.intermediate_ips$,$dst_endpoint.intermediate_ips$,$unmapped.rule_matched$,$actor.user.name$,$unmapped.dst_user$,$app_name$,$unmapped.vsys$,$unmapped.from_zone$,$unmapped.to_zone$,$unmapped.inbound_if$,$unmapped.outbound_if$,$actor.session.issuer$,$metadata.original_time$,$actor.session.uid$,$unmapped.repeat_count$,$src_endpoint.port$,$dst_endpoint.port$,$unmapped.nat_src_port$,$unmapped.nat_dst_port$,$unmapped.flags$,$connection_info.protocol_name$,$unmapped.action_value$,$unmapped.file$,$unmapped.threat_id$,$unmapped.url_category_value$,$unmapped.severity$,$unmapped.direction_of_attack$,$metadata.sequence$,$unmapped.action_flags$,$src_endpoint.location.region$,$dst_endpoint.location.region$,$metadata.product.version$,$unmapped.pcap_id$,$unmapped.file_digest$,.*,$cloud.account_uid$,$unmapped.url_idx$,$unmapped.user_agent$,$unmapped.file_type$,$src_endpoint.intermediate_ips$,$unmapped.referrer$,$unmapped.sender_of_email$,$unmapped.subject_of_email$,$unmapped.receipent_of_email$,$unmapped.report_id$,$unmapped.dg_hier_level_1$,$unmapped.dg_hier_level_2$,$unmapped.dg_hier_level_3$,$unmapped.dg_hier_level_4$,$unmapped.vsys_name$,$device.hostname$,.*,$unmapped.src_uuid$,$unmapped.dst_uuid$,$unmapped.http_method$,$device.imsi$,$device.imei$,$unmapped.parent_session_id$,$unmapped.parent_start_time$,$unmapped.tunnel_type$,$unmapped.threat_category$,$unmapped.content_version$,.*,$unmapped.ep_assoc_id$,$unmapped.ppid$,$unmapped.http_headers$,\"$unmapped.url_category_list$\",$unmapped.rule_matched_uuid$,$unmapped.http2_connection$,$actor.user.groups$,$src_endpoint.intermediate_ips$,$unmapped.src_category$,$unmapped.src_profile$,$unmapped.src_model$,$unmapped.src_vendor$,$unmapped.src_osfamily$,$unmapped.src_osversion$,$src_endpoint.hostname$,$src_endpoint.mac$,$unmapped.dst_category$,$unmapped.dst_profile$,$unmapped.dst_model$,$unmapped.dst_vendor$,$unmapped.dst_osfamily$,$unmapped.dst_osversion$,$dst_endpoint.hostname$,$dst_endpoint.mac$,$unmapped.container_id$,$unmapped.pod_namespace$,$unmapped.pod_name$,$unmapped.src_edl$,$unmapped.dst_edl$,$device.uid$,$unmapped.serial_number$,$unmapped.src_dag$,$unmapped.dst_dag$,$unmapped.partial_hash$,.*,$unmapped.high_res_timestamp$,$unmapped.reason$,$unmapped.justification$,$unmapped.nssai_sst$,$unmapped.subcategory_of_app$,$unmapped.category_of_app$,$unmapped.technology_of_app$,$unmapped.risk_of_app$,\"$unmapped.characteristic_of_app$\",$unmapped.container_of_app$,$unmapped.tunneled_app$,$unmapped.is_saas_of_app$,$unmapped.sanctioned_state_of_app$",
+ halt: true,
+ rewrites: [
{
- "constant": {
- "value": 3,
- "field": "severity_id",
- "predicate": "unmapped.action = 'deny'"
- }
- },
+ input: "unmapped.severity",
+ output: "severity_id",
+ match: "^informational$",
+ replace: "1"
+ },
+ {
+ input: "unmapped.severity",
+ output: "severity_id",
+ match: "^low$",
+ replace: "2"
+ },
+ {
+ input: "unmapped.severity",
+ output: "severity_id",
+ match: "^medium$",
+ replace: "3"
+ },
+ {
+ input: "unmapped.severity",
+ output: "severity_id",
+ match: "^high$",
+ replace: "4"
+ },
+ {
+ input: "unmapped.severity",
+ output: "severity_id",
+ match: "^critical$",
+ replace: "5"
+ },
+ {
+ input: "unmapped.action_value",
+ output: "status_id",
+ match: "^allow$",
+ replace: "1"
+ },
+ {
+ input: "unmapped.action_value",
+ output: "status_id",
+ match: "^deny$",
+ replace: "2"
+ },
+ {
+ input: "unmapped.action_value",
+ output: "status",
+ match: "^allow$",
+ replace: "Success"
+ },
+ {
+ input: "unmapped.action_value",
+ output: "status",
+ match: "^deny$",
+ replace: "Failure"
+ },
+ {
+ input: "unmapped.action_value",
+ output: "status_id",
+ match: "^(?!allow|deny$).*",
+ replace: "99"
+ },
+ {
+ input: "unmapped.action_value",
+ output: "status",
+ match: "^(?!allow|deny$).*",
+ replace: "Other"
+ },
+ {
+ input: "dst_endpoint.intermediate_ips",
+ output: "dst_endpoint.intermediate_ips",
+ match: ".*",
+ replace: "\\[\"$0\"\\]"
+ },
+ {
+ input: "message",
+ output: "src_endpoint.intermediate_ips",
+ match: "(?:[^,]*,){9}([^,]*),(?:[^,]*,){21}(\".*\"),(?:[^,]*,){16}([^,]*),(?:[^,]*,){26}(\".*\"),(?:[^,]*,){3}([^,]*),.*",
+ replace: "\\[\"$1\"\\, \"$3\"\\, \"$5\"\\]"
+ },
+ {
+ input: "message",
+ output: "observables",
+ match: "(?:[^,]*,){7}([^,]*),([^,]*),(?:[^,]*,){3}([^,]*),(?:[^,]*,){18}(\".*\"),(?:[^,]*,){43}(\".*\"),(?:[^,]*,){10}([^,]*),([^,]*),(?:[^,]*,){6}([^,]*),([^,]*),.*",
+ replace: "\\[\\{\"type_id\"\\: \"1\"\\, \"type\"\\: \"Hostname\"\\, \"name\"\\: \"src_endpoint.hostname\"\\, \"value\"\\: \"$6\"\\}\\, \\{\"type_id\"\\: \"1\"\\, \"type\"\\: \"Hostname\"\\, \"name\"\\: \"dst_endpoint.hostname\"\\, \"value\"\\: \"$8\"\\}\\, \\{\"type_id\"\\: \"2\"\\, \"type\"\\: \"IP Address\"\\, \"name\"\\: \"src_endpoint.ip\"\\, \"value\"\\: \"$1\"\\}\\, \\{\"type_id\"\\: \"2\"\\, \"type\"\\: \"IP Address\"\\, \"name\"\\: \"dst_endpoint.ip\"\\, \"value\"\\: \"$2\"\\}\\, \\{\"type_id\"\\: \"4\"\\, \"type\"\\: \"User Name\"\\, \"name\"\\: \"actor.user.name\"\\, \"value\"\\: \"$3\"\\}\\, \\{\"type_id\"\\: \"3\"\\, \"type\"\\: \"MAC Address\"\\, \"name\"\\: \"src_endpoint.mac\"\\, \"value\"\\: \"$7\"\\}\\, \\{\"type_id\"\\: \"3\"\\, \"type\"\\: \"MAC Address\"\\, \"name\"\\: \"dst_endpoint.mac\"\\, \"value\"\\: \"$9\"\\}\\]"
+ },
+ ]
+ },
+ {
+ //matches THREAT logs with comma surround lists in application_characteristic and url_category_list.
+ attributes: {
+ "activity_name": "THREAT",
+ "class_uid": "4001",
+ "activity_id": "99",
+ "category_uid": "4",
+ "type_uid": "400199",
+ "type_name": "Network Activity: Other",
+ "class_name": "Network Activity",
+ "category_name": "Network Activity",
+ "metadata.product.name": "Palo Alto Networks Firewall",
+ "metadata.product.vendor_name": "Palo Alto Networks",
+ "metadata.version":"1.0.0-rc.3",
+ "metadata.log_name": "THREAT",
+ "event.type": "THREAT"
+ },
+ format: ".*,$metadata.logged_time_dt$,$device.hw_info.serial_number$,THREAT,$unmapped.sub_type$,.*,$metadata.original_time$,$src_endpoint.ip$,$dst_endpoint.ip$,$src_endpoint.intermediate_ips$,$dst_endpoint.intermediate_ips$,$unmapped.rule_matched$,$actor.user.name$,$unmapped.dst_user$,$app_name$,$unmapped.vsys$,$unmapped.from_zone$,$unmapped.to_zone$,$unmapped.inbound_if$,$unmapped.outbound_if$,$actor.session.issuer$,$metadata.original_time$,$actor.session.uid$,$unmapped.repeat_count$,$src_endpoint.port$,$dst_endpoint.port$,$unmapped.nat_src_port$,$unmapped.nat_dst_port$,$unmapped.flags$,$connection_info.protocol_name$,$unmapped.action_value$,$unmapped.file$,$unmapped.threat_id$,$unmapped.url_category_value$,$unmapped.severity$,$unmapped.direction_of_attack$,$metadata.sequence$,$unmapped.action_flags$,$src_endpoint.location.region$,$dst_endpoint.location.region$,$metadata.product.version$,$unmapped.pcap_id$,$unmapped.file_digest$,.*,$cloud.account_uid$,$unmapped.url_idx$,$unmapped.user_agent$,$unmapped.file_type$,$src_endpoint.intermediate_ips$,$unmapped.referrer$,$unmapped.sender_of_email$,$unmapped.subject_of_email$,$unmapped.receipent_of_email$,$unmapped.report_id$,$unmapped.dg_hier_level_1$,$unmapped.dg_hier_level_2$,$unmapped.dg_hier_level_3$,$unmapped.dg_hier_level_4$,$unmapped.vsys_name$,$device.hostname$,.*,$unmapped.src_uuid$,$unmapped.dst_uuid$,$unmapped.http_method$,$device.imsi$,$device.imei$,$unmapped.parent_session_id$,$unmapped.parent_start_time$,$unmapped.tunnel_type$,$unmapped.threat_category$,$unmapped.content_version$,.*,$unmapped.ep_assoc_id$,$unmapped.ppid$,$unmapped.http_headers$,$unmapped.url_category_list$,$unmapped.rule_matched_uuid$,$unmapped.http2_connection$,$actor.user.groups$,$src_endpoint.intermediate_ips$,$unmapped.src_category$,$unmapped.src_profile$,$unmapped.src_model$,$unmapped.src_vendor$,$unmapped.src_osfamily$,$unmapped.src_osversion$,$src_endpoint.hostname$,$src_endpoint.mac$,$unmapped.dst_category$,$unmapped.dst_profile$,$unmapped.dst_model$,$unmapped.dst_vendor$,$unmapped.dst_osfamily$,$unmapped.dst_osversion$,$dst_endpoint.hostname$,$dst_endpoint.mac$,$unmapped.container_id$,$unmapped.pod_namespace$,$unmapped.pod_name$,$unmapped.src_edl$,$unmapped.dst_edl$,$device.uid$,$unmapped.serial_number$,$unmapped.src_dag$,$unmapped.dst_dag$,$unmapped.partial_hash$,.*,$unmapped.high_res_timestamp$,$unmapped.reason$,$unmapped.justification$,$unmapped.nssai_sst$,$unmapped.subcategory_of_app$,$unmapped.category_of_app$,$unmapped.technology_of_app$,$unmapped.risk_of_app$,$unmapped.characteristic_of_app$,$unmapped.container_of_app$,$unmapped.tunneled_app$,$unmapped.is_saas_of_app$,$unmapped.sanctioned_state_of_app$",
+ halt: true,
+ rewrites: [
{
- "constant": {
- "value": 1,
- "field": "status_id"
- }
- }
- ]
- }
- ]
- },
- "observables": {
- "fields": [
+ input: "unmapped.severity",
+ output: "severity_id",
+ match: "^informational$",
+ replace: "1"
+ },
+ {
+ input: "unmapped.severity",
+ output: "severity_id",
+ match: "^low$",
+ replace: "2"
+ },
+ {
+ input: "unmapped.severity",
+ output: "severity_id",
+ match: "^medium$",
+ replace: "3"
+ },
+ {
+ input: "unmapped.severity",
+ output: "severity_id",
+ match: "^high$",
+ replace: "4"
+ },
+ {
+ input: "unmapped.severity",
+ output: "severity_id",
+ match: "^critical$",
+ replace: "5"
+ },
+ {
+ input: "unmapped.action_value",
+ output: "status_id",
+ match: "^allow$",
+ replace: "1"
+ },
{
- "name": "src_endpoint.ip",
- "type": "IP Address"
+ input: "unmapped.action_value",
+ output: "status_id",
+ match: "^deny$",
+ replace: "2"
},
{
- "name": "dst_endpoint.ip",
- "type": "IP Address"
+ input: "unmapped.action_value",
+ output: "status",
+ match: "^allow$",
+ replace: "Success"
},
{
- "name": "user.name",
- "type": "User"
+ input: "unmapped.action_value",
+ output: "status",
+ match: "^deny$",
+ replace: "Failure"
},
{
- "name": "app_name",
- "type": "Other"
+ input: "unmapped.action_value",
+ output: "status_id",
+ match: "^(?!allow|deny$).*",
+ replace: "99"
},
{
- "name": "device.hostname",
- "type": "Hostname"
- }
- ]
- }
-}
\ No newline at end of file
+ input: "unmapped.action_value",
+ output: "status",
+ match: "^(?!allow|deny$).*",
+ replace: "Other"
+ },
+ {
+ input: "dst_endpoint.intermediate_ips",
+ output: "dst_endpoint.intermediate_ips",
+ match: ".*",
+ replace: "\\[\"$0\"\\]"
+ },
+ {
+ input: "message",
+ output: "src_endpoint.intermediate_ips",
+ match: "(?:[^,]*,){9}([^,]*),(?:[^,]*,){38}([^,]*),(?:[^,]*,){30}([^,]*),.*",
+ replace: "\\[\"$1\"\\, \"$2\"\\, \"$3\"\\]"
+ },
+ {
+ input: "message",
+ output: "observables",
+ match: "(?:[^,]*,){7}([^,]*),([^,]*),(?:[^,]*,){3}([^,]*),(?:[^,]*,){73}([^,]*),([^,]*),(?:[^,]*,){6}([^,]*),([^,]*),.*",
+ replace: "\\[\\{\"type_id\"\\: \"1\"\\, \"type\"\\: \"Hostname\"\\, \"name\"\\: \"src_endpoint.hostname\"\\, \"value\"\\: \"$4\"\\}\\, \\{\"type_id\"\\: \"1\"\\, \"type\"\\: \"Hostname\"\\, \"name\"\\: \"dst_endpoint.hostname\"\\, \"value\"\\: \"$6\"\\}\\, \\{\"type_id\"\\: \"2\"\\, \"type\"\\: \"IP Address\"\\, \"name\"\\: \"src_endpoint.ip\"\\, \"value\"\\: \"$1\"\\}\\, \\{\"type_id\"\\: \"2\"\\, \"type\"\\: \"IP Address\"\\, \"name\"\\: \"dst_endpoint.ip\"\\, \"value\"\\: \"$2\"\\}\\, \\{\"type_id\"\\: \"4\"\\, \"type\"\\: \"User Name\"\\, \"name\"\\: \"actor.user.name\"\\, \"value\"\\: \"$3\"\\}\\, \\{\"type_id\"\\: \"3\"\\, \"type\"\\: \"MAC Address\"\\, \"name\"\\: \"src_endpoint.mac\"\\, \"value\"\\: \"$5\"\\}\\, \\{\"type_id\"\\: \"3\"\\, \"type\"\\: \"MAC Address\"\\, \"name\"\\: \"dst_endpoint.mac\"\\, \"value\"\\: \"$7\"\\}\\]"
+ },
+ ]
+ },
+ {
+ attributes: {
+ "activity_name": "Logoff",
+ "activity_id": "2",
+ "category_name": "Identity & Access Management",
+ "category_uid": "3",
+ "class_name": "Authentication",
+ "class_uid": "3002",
+ "cloud.provider": "Palo Alto Networks"
+ "metadata.product.name": "Palo Alto Networks Firewall",
+ "metadata.product.vendor_name": "Palo Alto Networks",
+ "metadata.version":"1.1.0",
+ "type_uid": "300202",
+ "type_name": "Authentication: Logoff",
+ "event.type": "Logoff",
+ "severity_id": "99"
+ },
+ format: "$metadata.original_time$,$metadata.product.uid$,$unmapped.type=userid_log_type$,$unmapped.subtype=logout_sub_type$,.*,$start_time_dt$,$unmapped.vsys$,$src_endpoint.ip$,$user.name$,$user.uid$,$metadata.event_code$,$unmapped.repeatcnt$,$unmapped.timeout$,$src_endpoint.port$,$dst_endpoint.port$,$unmapped.datasource$,$unmapped.datasourcetype$,$unmapped.seqno$,$unmapped.actionflags$,$unmapped.dg_hier_level_1$,$unmapped.dg_hier_level_2$,$unmapped.dg_hier_level_3$,$unmapped.dg_hier_level_4$,$unmapped.vsys_name$,$src_endpoint.hostname$,$unmapped.vsys_id$,$unmapped.factortype$,$unmapped.factorcompletiontime$,$unmapped.factorno$,$unmapped.ugflags$,$unmapped.userbysource$,$unmapped.tag_name$,$unmapped.high_res_timestamp$",
+ halt: true,
+ rewrites: [
+ {
+ input: "message",
+ output: "observables",
+ match: "(?:[^,]*,){7}([^,]*)(?:,[^,]*){0},([^,]*)(?:,[^,]*){15},([^,]*).*",
+ replace: "\\[\\{\"type_id\"\\: 1\\, \"type\"\\: \"Hostname\"\\, \"name\"\\: \"src_endpoint.hostname\"\\, \"value\"\\: $3\\}\\, \\{\"type_id\"\\: 2\\, \"type\"\\: \"IP Address\"\\, \"name\"\\: \"src_endpoint.ip\"\\, \"value\"\\: $1\\}\\, \\{\"type_id\"\\: 4\\, \"type\"\\: \"User Name\"\\, \"name\"\\: \"user.name\"\\, \"value\"\\: $2\\}\\]"
+ },
+ {
+ input: "activity_name",
+ output: "event.type",
+ match: ".*",
+ replace: "$0"
+ }
+ ]
+ },
+ {
+ attributes: {
+ "activity_name": "Logon",
+ "activity_id": "1",
+ "category_name": "Identity & Access Management",
+ "category_uid": "3",
+ "class_name": "Authentication",
+ "class_uid": "3002",
+ "cloud.provider": "Palo Alto Networks"
+ "metadata.product.name": "Palo Alto Networks Firewall",
+ "metadata.product.vendor_name": "Palo Alto Networks",
+ "metadata.version":"1.1.0",
+ "type_uid": "300201",
+ "type_name": "Authentication: Logon",
+ "event.type": "Logon",
+ "severity_id": "99"
+ },
+ format: "$metadata.original_time$,$metadata.product.uid$,$unmapped.type=userid_log_type$,$unmapped.subtype=login_sub_type$,.*,$start_time_dt$,$unmapped.vsys$,$src_endpoint.ip$,$user.name$,$user.uid$,$metadata.event_code$,$unmapped.repeatcnt$,$unmapped.timeout$,$src_endpoint.port$,$dst_endpoint.port$,$unmapped.datasource$,$unmapped.datasourcetype$,$unmapped.seqno$,$unmapped.actionflags$,$unmapped.dg_hier_level_1$,$unmapped.dg_hier_level_2$,$unmapped.dg_hier_level_3$,$unmapped.dg_hier_level_4$,$unmapped.vsys_name$,$src_endpoint.hostname$,$unmapped.vsys_id$,$unmapped.factortype$,$unmapped.factorcompletiontime$,$unmapped.factorno$,$unmapped.ugflags$,$unmapped.userbysource$,$unmapped.tag_name$,$unmapped.high_res_timestamp$",
+ halt: true,
+ rewrites: [
+ {
+ input: "message",
+ output: "observables",
+ match: "(?:[^,]*,){7}([^,]*)(?:,[^,]*){0},([^,]*)(?:,[^,]*){15},([^,]*).*",
+ replace: "\\[\\{\"type_id\"\\: 1\\, \"type\"\\: \"Hostname\"\\, \"name\"\\: \"src_endpoint.hostname\"\\, \"value\"\\: $3\\}\\, \\{\"type_id\"\\: 2\\, \"type\"\\: \"IP Address\"\\, \"name\"\\: \"src_endpoint.ip\"\\, \"value\"\\: $1\\}\\, \\{\"type_id\"\\: 4\\, \"type\"\\: \"User Name\"\\, \"name\"\\: \"user.name\"\\, \"value\"\\: $2\\}\\]"
+ },
+ {
+ input: "activity_name",
+ output: "event.type",
+ match: ".*",
+ replace: "$0"
+ }
+ ]
+ },
+ {
+ attributes: {
+ "action": "Other",
+ "action_id": "99",
+ "activity_name": "Other",
+ "activity_id": "99",
+ "category_name": "Findings",
+ "category_uid": "2",
+ "class_name": "Detection Finding",
+ "class_uid": "2004",
+ "cloud.provider": "Palo Alto Networks",
+ "metadata.product.name": "Palo Alto Networks Firewall",
+ "metadata.product.vendor_name": "Palo Alto Networks",
+ "metadata.version":"1.1.0",
+ "type_uid": "200499",
+ "type_name": "Detection Finding: Other",
+ "severity_id": "99"
+ },
+ format: "$metadata.original_time$,$metadata.product.uid$,$finding_info.title=hipmatch_log_type$,$unmapped.subtype$,.*,$start_time_dt$,$actor.user.name$,$unmapped.vsys$,$device.name$,$device.os.name$,$device.ip$,$unmapped.matchname$,$unmapped.repeatcnt$,$unmapped.matchtype$,.*,.*,$unmapped.seqno$,$unmapped.actionflags$,$unmapped.dg_hier_level_1$,$unmapped.dg_hier_level_2$,$unmapped.dg_hier_level_3$,$unmapped.dg_hier_level_4$,$unmapped.vsys_name$,$device.hostname$,$unmapped.vsys_id$,$unmapped.srcipv6$,$unmapped.uid_alt$,$device.uid$,$device.mac$,$unmapped.high_res_timestamp$",
+ halt: true,
+ rewrites: [
+ {
+ input: "message",
+ output: "observables",
+ match: "(?:[^,]*,){10}([^,]*)(?:,[^,]*){12},([^,]*)(?:,[^,]*){4},([^,]*).*",
+ replace: "\\[\\{\"type_id\"\\: 1\\, \"type\"\\: \"Hostname\"\\, \"name\"\\: \"device.hostname\"\\, \"value\"\\: $2\\}\\, \\{\"type_id\"\\: 2\\, \"type\"\\: \"IP Address\"\\, \"name\"\\: \"device.ip\"\\, \"value\"\\: $1\\}\\, \\{\"type_id\"\\: 3\\, \"type\"\\: \"MAC Address\"\\, \"name\"\\: \"device.mac\"\\, \"value\"\\: $3\\}\\]"
+ },
+ {
+ input: "activity_name",
+ output: "event.type",
+ match: ".*",
+ replace: "$0"
+ }
+ ]
+ },
+ {
+ attributes: {
+ "activity_name": "Log",
+ "activity_id": "1",
+ "category_name": "Discovery",
+ "category_uid": "5",
+ "class_name": "Device Config State",
+ "class_uid": "5002",
+ "cloud.provider": "Palo Alto Networks",
+ "metadata.product.name": "Palo Alto Networks Firewall",
+ "metadata.product.vendor_name": "Palo Alto Networks",
+ "metadata.version":"1.1.0",
+ "type_uid": "500201",
+ "type_name": "Device Config State: Log",
+ "severity_id": "99"
+ },
+ format: "$metadata.original_time$,$metadata.product.uid$,$unmapped.type=config_log_type$,$unmapped.subtype$,.*,$start_time_dt$,$device.hostname$,$unmapped.vsys$,$actor.process.cmd_line$,$actor.user.name$,$unmapped.client$,$unmapped.result$,$metadata.product.path$,$unmapped.before-change-detail$,$unmapped.after-change-detail$,$unmapped.seqno$,$unmapped.actionflags$,$unmapped.dg_hier_level_1$,$unmapped.dg_hier_level_2$,$unmapped.dg_hier_level_3$,$unmapped.dg_hier_level_4$,$unmapped.vsys_name$,$device.name$,$device.groups$,$unmapped.comment$",
+ halt: true,
+ rewrites: [
+ {
+ input: "message",
+ output: "observables",
+ match: "(?:[^,]*,){6}([^,]*)(?:,[^,]*){2},([^,]*).*",
+ replace: "\\[\\{\"type_id\"\\: 1\\, \"type\"\\: \"Hostname\"\\, \"name\"\\: \"device.hostname\"\\, \"value\"\\: $1\\}\\, \\{\"type_id\"\\: 4\\, \"type\"\\: \"User Name\"\\, \"name\"\\: \"actor.user.name\"\\, \"value\"\\: $2\\}\\]"
+ },
+ {
+ input: "activity_name",
+ output: "event.type",
+ match: ".*",
+ replace: "$0"
+ }
+ ]
+ },
+ {
+ attributes: {
+ "action": "Other",
+ "action_id": "99",
+ "activity_name": "Other",
+ "activity_id": "99",
+ "category_name": "Findings",
+ "category_uid": "2",
+ "class_name": "Detection Finding",
+ "class_uid": "2004",
+ "cloud.provider": "Palo Alto Networks",
+ "metadata.product.name": "Palo Alto Networks Firewall",
+ "metadata.product.vendor_name": "Palo Alto Networks",
+ "metadata.version":"1.1.0",
+ "type_uid": "200499",
+ "type_name": "Detection Finding: Other",
+ "severity_id": "99"
+ },
+ format: "$metadata.original_time$,$device.hw_info.serial_number$,$unmapped.type$,$unmapped.subtype=wildfire_sub_type$,.*,$finding_info.created_time_dt$,$source_address$,$destination_address$,$nat_source_ip$,$nat_destination_ip$,$firewall_rule.name$,$actor.user.name$,$unmapped.dstuser$,$unmapped.app$,$unmapped.vsys$,$source_zone$,$destination_zone$,$inbound_interface$,$outbound_interface$,$unmapped.logset$,.*,$actor.session.uid$,$count$,$source_port$,$destination_port$,$unmapped.natsport$,$unmapped.natdport$,$unmapped.flags$,$ip_protocol$,$action$,$filename$,$finding_info.uid$,$unmapped.category$,$unmapped.severity$,$unmapped.direction$,$metadata.sequence$,$unmapped.actionflags$,$source_location$,$destination_location$,.*,$unmapped.contenttype$,$unmapped.pcap_id$,$unmapped.filedigest$,$unmapped.cloud$,$unmapped.url_idx$,$unmapped.user_agent$,$file_type$,$unmapped.xff$,$unmapped.referer$,$unmapped.sender$,$unmapped.subject$,$unmapped.recipient$,$unmapped.reportid$,$unmapped.dg_hier_level_1$,$unmapped.dg_hier_level_2$,$unmapped.dg_hier_level_3$,$unmapped.dg_hier_level_4$,$unmapped.vsys_name$,$device.name$,.*,$source_vm_uuid$,$destination_vm_uuid$,$unmapped.http_method$,$unmapped.imsi$,$device.imei$,$parent_session_id$,$parent_start_time$,$unmapped.tunnel$,$unmapped.thr_category$,$unmapped.contentver$,.*,$unmapped.assoc_id$,$unmapped.ppid$,$unmapped.http_headers$,$unmapped.url_category_list$,$unmapped.rule_uuid$,$unmapped.http2_connection$,$unmapped.dynusergroup_name$,$unmapped.xff_ip$,$unmapped.src_category$,$unmapped.src_profile$,$unmapped.src_model$,$unmapped.src_vendor$,$unmapped.src_osfamily$,$unmapped.src_osversion$,$source_hostname$,$source_mac_address$,$unmapped.dst_category$,$unmapped.dst_profile$,$unmapped.dst_model$,$unmapped.dst_vendor$,$unmapped.dst_osfamily$,$unmapped.dst_osversion$,$destination_hostname$,$destination_mac_address$,$unmapped.container_id$,$unmapped.pod_namespace$,$unmapped.pod_name$,$unmapped.src_edl$,$unmapped.dst_edl$,$unmapped.hostid$,$unmapped.serialnumber$,$unmapped.domain_edl$,$unmapped.src_dag$,$unmapped.dst_dag$,$unmapped.partial_hash$,$unmapped.high_res_timestamp$,$unmapped.reason$,$unmapped.justification$,$unmapped.nssai_sst$,$unmapped.subcategory_of_app$,$unmapped.category_of_app$,$unmapped.technology_of_app$,$risk_level$,$unmapped.characteristic_of_app=app_characteristic$,$unmapped.container_of_app$,$unmapped.tunneled_app$,$unmapped.is_saas_of_app$,$unmapped.sanctioned_state_of_app$,$unmapped.cloud_reportid$",
+ halt: true,
+ rewrites: [
+ {
+ input: "message",
+ output: "evidences",
+ match: "^(?:[^,]*,){6}([^,]*),([^,]*),([^,]*),([^,]*),(?:[^,]*,){5}([^,]*),([^,]*),([^,]*),([^,]*),(?:[^,]*,){4}([^,]*),([^,]*),(?:[^,]*,){3}([^,]*),(?:[^,]*,){1}([^,]*),(?:[^,]*,){6}([^,]*),([^,]*),(?:[^,]*,){7}([^,]*),([^,]*),(?:[^,]*,){13}([^,]*),(?:[^,]*,){1}([^,]*),(?:[^,]*,){18}([^,]*),([^,]*),(?:[^,]*,){6}([^,]*),([^,]*),(?:[^,]*,){7}([^,]*),([^,]*),(?:[^,]*,){6}([^,]*),([^,]*),(?:[^,]*,){6}([^,]*),([^,]*),(?:[^,]*,){1}([^,]*),([^,]*).*",
+ replace: "\\[\"src_endpoint\"\\:\\{\"ip\"\\: $1\\, \"intermediate_ips\"\\:\\[$3\\]\\, \"zone\"\\: $5\\, \"interface_name\"\\: $7\\, \"port\"\\: $9\\, \"location\"\\: \\{\"country\"\\: $13\\}\\, \"uid\"\\: $16\\, \"hostname\"\\: $20\\, \"mac\"\\: $21\\}\\, \"dst_endpoint\"\\:\\{\"ip\"\\: $2\\, \"intermediate_ips\"\\:\\[$4\\]\\, \"zone\"\\: $6\\, \"interface_name\"\\: $8\\, \"port\"\\: $10\\, \"location\"\\: \\{\"country\"\\: $14\\}\\, \"uid\"\\: $17\\, \"hostname\"\\: $22\\, \"mac\"\\: $23\\}\\, \"connection_info\"\\: \\{ \"protocol_name\"\\: $11\\}\\, \"process\"\\: \\{\"file\"\\: \\{\"name\"\\: $12\\, \"type\"\\: $15\\}\\, \"parent_process\"\\: \\{\"session\": \\{\"uid\": $18\\}\\, \"created_time\"\\: $19\\}\\} \\]"
+ },
+ {
+ input: "message",
+ output: "observables",
+ match: "^(?:[^,]*,){6}([^,]*),(?:[^,]*,){0}([^,]*),(?:[^,]*,){3}([^,]*),(?:[^,]*,){73}([^,]*),(?:[^,]*,){7}([^,]*).*",
+ replace: "\\[\\{\"type_id\"\\: 1\\, \"type\"\\: \"Hostname\"\\, \"name\"\\: \"evidences.src_endpoint.hostname\"\\, \"value\"\\: $4\\}\\, \\{\"type_id\"\\: 1\\, \"type\"\\: \"Hostname\"\\, \"name\"\\: \"evidences.dst_endpoint.hostname\"\\, \"value\"\\: $5\\}\\, \\{\"type_id\"\\: 2\\, \"type\"\\: \"IP Address\"\\, \"name\"\\: \"evidences.src_endpoint.ip\"\\, \"value\"\\: $1\\}\\, \\{\"type_id\"\\: 2\\, \"type\"\\: \"IP Address\"\\, \"name\"\\: \"evidences.dst_endpoint.ip\"\\, \"value\"\\: $2\\}\\, \\{\"type_id\"\\: 4\\, \"type\"\\: \"User Name\"\\, \"name\"\\: \"actor.user.name\"\\, \"value\"\\: $3\\}\\]"
+ },
+ {
+ input: "activity_name",
+ output: "event.type",
+ match: ".*",
+ replace: "$0"
+ },
+ {
+ action: "removeFields",
+ fields: [
+ "source_address",
+ "destination_address",
+ "nat_source_ip",
+ "nat_destination_ip",
+ "source_zone",
+ "destination_zone",
+ "inbound_interface",
+ "outbound_interface",
+ "source_port",
+ "destination_port",
+ "ip_protocol",
+ "filename",
+ "source_location",
+ "destination_location",
+ "file_type",
+ "source_vm_uuid",
+ "destination_vm_uuid",
+ "parent_session_id",
+ "parent_start_time",
+ "source_hostname",
+ "source_mac_address",
+ "destination_hostname",
+ "destination_mac_address"
+ ]
+ }
+ ]
+ },
+ {
+ attributes: {
+ "action": "Other",
+ "action_id": "99",
+ "activity_name": "Other",
+ "activity_id": "99",
+ "category_name": "Findings",
+ "category_uid": "2",
+ "class_name": "Detection Finding",
+ "class_uid": "2004",
+ "cloud.provider": "Palo Alto Networks",
+ "metadata.product.name": "Palo Alto Networks Firewall",
+ "metadata.product.vendor_name": "Palo Alto Networks",
+ "metadata.version":"1.1.0",
+ "type_uid": "200499",
+ "type_name": "Detection Finding: Other",
+ "severity_id": "99"
+ },
+ format: "$metadata.original_time$,$device.hw_info.serial_number$,$unmapped.type$,$unmapped.subtype=data_filtering_sub_type$,.*,$finding_info.created_time_dt$,$source_address$,$destination_address$,$nat_source_ip$,$nat_destination_ip$,$firewall_rule.name$,$actor.user.name$,$unmapped.dstuser$,$unmapped.app$,$unmapped.vsys$,$source_zone$,$destination_zone$,$inbound_interface$,$outbound_interface$,$unmapped.logset$,.*,$actor.session.uid$,$count$,$source_port$,$destination_port$,$unmapped.natsport$,$unmapped.natdport$,$unmapped.flags$,$ip_protocol$,$action$,$filename$,$finding_info.uid$,$unmapped.category$,$unmapped.severity$,$unmapped.direction$,$metadata.sequence$,$unmapped.actionflags$,$source_location$,$destination_location$,.*,$unmapped.contenttype$,$unmapped.pcap_id$,$unmapped.filedigest$,$unmapped.cloud$,$unmapped.url_idx$,$unmapped.user_agent$,$file_type$,$unmapped.xff$,$unmapped.referer$,$unmapped.sender$,$unmapped.subject$,$unmapped.recipient$,$unmapped.reportid$,$unmapped.dg_hier_level_1$,$unmapped.dg_hier_level_2$,$unmapped.dg_hier_level_3$,$unmapped.dg_hier_level_4$,$unmapped.vsys_name$,$device.name$,.*,$source_vm_uuid$,$destination_vm_uuid$,$unmapped.http_method$,$unmapped.imsi$,$device.imei$,$parent_session_id$,$parent_start_time$,$unmapped.tunnel$,$unmapped.thr_category$,$unmapped.contentver$,.*,$unmapped.assoc_id$,$unmapped.ppid$,$unmapped.http_headers$,$unmapped.url_category_list$,$unmapped.rule_uuid$,$unmapped.http2_connection$,$unmapped.dynusergroup_name$,$unmapped.xff_ip$,$unmapped.src_category$,$unmapped.src_profile$,$unmapped.src_model$,$unmapped.src_vendor$,$unmapped.src_osfamily$,$unmapped.src_osversion$,$source_hostname$,$source_mac_address$,$unmapped.dst_category$,$unmapped.dst_profile$,$unmapped.dst_model$,$unmapped.dst_vendor$,$unmapped.dst_osfamily$,$unmapped.dst_osversion$,$destination_hostname$,$destination_mac_address$,$unmapped.container_id$,$unmapped.pod_namespace$,$unmapped.pod_name$,$unmapped.src_edl$,$unmapped.dst_edl$,$unmapped.hostid$,$unmapped.serialnumber$,$unmapped.domain_edl$,$unmapped.src_dag$,$unmapped.dst_dag$,$unmapped.partial_hash$,$unmapped.high_res_timestamp$,$unmapped.reason$,$unmapped.justification$,$unmapped.nssai_sst$,$unmapped.subcategory_of_app$,$unmapped.category_of_app$,$unmapped.technology_of_app$,$risk_level$,$unmapped.characteristic_of_app=app_characteristic$,$unmapped.container_of_app$,$unmapped.tunneled_app$,$unmapped.is_saas_of_app$,$unmapped.sanctioned_state_of_app$,$unmapped.cloud_reportid$",
+ halt: true,
+ rewrites: [
+ {
+ input: "message",
+ output: "evidences",
+ match: "^(?:[^,]*,){6}([^,]*),([^,]*),([^,]*),([^,]*),(?:[^,]*,){5}([^,]*),([^,]*),([^,]*),([^,]*),(?:[^,]*,){4}([^,]*),([^,]*),(?:[^,]*,){3}([^,]*),(?:[^,]*,){1}([^,]*),(?:[^,]*,){6}([^,]*),([^,]*),(?:[^,]*,){7}([^,]*),([^,]*),(?:[^,]*,){13}([^,]*),(?:[^,]*,){1}([^,]*),(?:[^,]*,){18}([^,]*),([^,]*),(?:[^,]*,){6}([^,]*),([^,]*),(?:[^,]*,){7}([^,]*),([^,]*),(?:[^,]*,){6}([^,]*),([^,]*),(?:[^,]*,){6}([^,]*),([^,]*),(?:[^,]*,){1}([^,]*),([^,]*).*",
+ replace: "\\[\"src_endpoint\"\\:\\{\"ip\"\\: $1\\, \"intermediate_ips\"\\:\\[$3\\]\\, \"zone\"\\: $5\\, \"interface_name\"\\: $7\\, \"port\"\\: $9\\, \"location\"\\: \\{\"country\"\\: $13\\}\\, \"uid\"\\: $16\\, \"hostname\"\\: $20\\, \"mac\"\\: $21\\}\\, \"dst_endpoint\"\\:\\{\"ip\"\\: $2\\, \"intermediate_ips\"\\:\\[$4\\]\\, \"zone\"\\: $6\\, \"interface_name\"\\: $8\\, \"port\"\\: $10\\, \"location\"\\: \\{\"country\"\\: $14\\}\\, \"uid\"\\: $17\\, \"hostname\"\\: $22\\, \"mac\"\\: $23\\}\\, \"connection_info\"\\: \\{ \"protocol_name\"\\: $11\\}\\, \"process\"\\: \\{\"file\"\\: \\{\"name\"\\: $12\\, \"type\"\\: $15\\}\\, \"parent_process\"\\: \\{\"session\": \\{\"uid\": $18\\}\\, \"created_time\"\\: $19\\}\\} \\]"
+ },
+ {
+ input: "message",
+ output: "observables",
+ match: "^(?:[^,]*,){6}([^,]*),(?:[^,]*,){0}([^,]*),(?:[^,]*,){3}([^,]*),(?:[^,]*,){73}([^,]*),(?:[^,]*,){7}([^,]*).*",
+ replace: "\\[\\{\"type_id\"\\: 1\\, \"type\"\\: \"Hostname\"\\, \"name\"\\: \"evidences.src_endpoint.hostname\"\\, \"value\"\\: $4\\}\\, \\{\"type_id\"\\: 1\\, \"type\"\\: \"Hostname\"\\, \"name\"\\: \"evidences.dst_endpoint.hostname\"\\, \"value\"\\: $5\\}\\, \\{\"type_id\"\\: 2\\, \"type\"\\: \"IP Address\"\\, \"name\"\\: \"evidences.src_endpoint.ip\"\\, \"value\"\\: $1\\}\\, \\{\"type_id\"\\: 2\\, \"type\"\\: \"IP Address\"\\, \"name\"\\: \"evidences.dst_endpoint.ip\"\\, \"value\"\\: $2\\}\\, \\{\"type_id\"\\: 4\\, \"type\"\\: \"User Name\"\\, \"name\"\\: \"actor.user.name\"\\, \"value\"\\: $3\\}\\]"
+ },
+ {
+ input: "activity_name",
+ output: "event.type",
+ match: ".*",
+ replace: "$0"
+ },
+ {
+ action: "removeFields",
+ fields: [
+ "source_address",
+ "destination_address",
+ "nat_source_ip",
+ "nat_destination_ip",
+ "source_zone",
+ "destination_zone",
+ "inbound_interface",
+ "outbound_interface",
+ "source_port",
+ "destination_port",
+ "ip_protocol",
+ "filename",
+ "source_location",
+ "destination_location",
+ "file_type",
+ "source_vm_uuid",
+ "destination_vm_uuid",
+ "parent_session_id",
+ "parent_start_time",
+ "source_hostname",
+ "source_mac_address",
+ "destination_hostname",
+ "destination_mac_address"
+ ]
+ }
+ ]
+ },
+ {
+ attributes: {
+ "activity_name": "Other",
+ "activity_id": "99",
+ "category_name": "Identity & Access Management",
+ "category_uid": "3",
+ "class_name": "Authentication",
+ "class_uid": "3002",
+ "cloud.provider": "Palo Alto Networks",
+ "metadata.product.name": "Palo Alto Networks Firewall",
+ "metadata.product.vendor_name": "Palo Alto Networks",
+ "metadata.version":"1.1.0",
+ "type_uid": "300299",
+ "type_name": "Authentication: Other",
+ "severity_id": "99"
+ },
+ format: "$metadata.original_time$,$metadata.product.uid$,$unmapped.type=globalprotect_log_type$,$unmapped.subtype$,.*,$start_time_dt$,$unmapped.vsys$,$metadata.event_code$,$unmapped.stage$,$auth_protocol$,$unmapped.tunnel_type$,$actor.user.name$,$src_endpoint.location.region$,$device.name$,$device.ip$,$unmapped.public_ipv6$,$unmapped.private_ip$,$unmapped.private_ipv6$,$unmapped.hostid$,$src_endpoint.hw_info.serial_number$,$metadata.product.version$,$src_endpoint.os.name$,$src_endpoint.os.version$,$unmapped.repeatcnt$,$unmapped.reason$,$unmapped.error$,$unmapped.description$,$status$,$unmapped.location$,$unmapped.login_duration$,$unmapped.connect_method$,$unmapped.error_code$,$unmapped.portal$,$unmapped.seqno$,$unmapped.actionflags$,$unmapped.high_res_timestamp$,$unmapped.selection_type$,$unmapped.response_time$,$unmapped.priority$,$unmapped.attempted_gateways$,$unmapped.gateway$,$unmapped.dg_hier_level_1$,$unmapped.dg_hier_level_2$,$unmapped.dg_hier_level_3$,$unmapped.dg_hier_level_4$,$unmapped.vsys_name$,$src_endpoint.hostname$,$unmapped.vsys_id$",
+ halt: true,
+ rewrites: [
+ {
+ input: "message",
+ output: "observables",
+ match: "(?:[^,]*,){11}([^,]*)(?:,[^,]*){2},([^,]*).*",
+ replace: "\\[\\{\"type_id\"\\: 2\\, \"type\"\\: \"IP Address\"\\, \"name\"\\: \"device.ip\"\\, \"value\"\\: $2\\}\\, \\{\"type_id\"\\: 4\\, \"type\"\\: \"User Name\"\\, \"name\"\\: \"actor.user.name\"\\, \"value\"\\: $1\\}\\]"
+ },
+ {
+ input: "activity_name",
+ output: "event.type",
+ match: ".*",
+ replace: "$0"
+ }
+ ]
+ },
+ {
+ attributes: {
+ "activity_name": "Update",
+ "activity_id": "3",
+ "category_name": "Identity & Access Management",
+ "category_uid": "3",
+ "class_name": "Entity Management",
+ "class_uid": "3004",
+ "cloud.provider": "Palo Alto Networks",
+ "metadata.product.name": "Palo Alto Networks Firewall",
+ "metadata.product.vendor_name": "Palo Alto Networks",
+ "metadata.version":"1.1.0",
+ "type_uid": "300403",
+ "type_name": "Entity Management: Update",
+ "severity_id": "99"
+ },
+ format: "$metadata.original_time$,$metadata.product.uid$,$unmapped.type=iptag_log_type$,$unmapped.subtype$,.*,$start_time_dt$,$unmapped.vsys$,$device.ip$,$unmapped.tag_name$,$metadata.event_code$,$unmapped.repeatcnt$,$unmapped.timeout$,$unmapped.datasource$,$unmapped.datasourcetype$,$unmapped.datasource_subtype$,$unmapped.seqno$,$unmapped.actionflags$,$unmapped.dg_hier_level_1$,$unmapped.dg_hier_level_2$,$unmapped.dg_hier_level_3$,$unmapped.dg_hier_level_4$,$unmapped.vsys_name$,$device.name$,$unmapped.vsys_id$",
+ halt: true,
+ rewrites: [
+ {
+ input: "message",
+ output: "observables",
+ match: "(?:[^,]*,){7}([^,]*)(?:,[^,]*){14},([^,]*).*",
+ replace: "\\[\\{\"type_id\"\\: 1\\, \"type\"\\: \"Hostname\"\\, \"name\"\\: \"device.name\"\\, \"value\"\\: $2\\}\\, \\{\"type_id\"\\: 2\\, \"type\"\\: \"IP Address\"\\, \"name\"\\: \"device.ip\"\\, \"value\"\\: $1\\}\\]"
+ },
+ {
+ input: "activity_name",
+ output: "event.type",
+ match: ".*",
+ replace: "$0"
+ }
+ ]
+ },
+ {
+ attributes: {
+ "activity_name": "Open",
+ "activity_id": "1",
+ "category_name": "Network Activity",
+ "category_uid": "4",
+ "class_name": "Network Activity",
+ "class_uid": "4001",
+ "cloud.provider": "Palo Alto Networks",
+ "metadata.product.name": "Palo Alto Networks Firewall",
+ "metadata.product.vendor_name": "Palo Alto Networks",
+ "metadata.version":"1.1.0",
+ "type_uid": "400101",
+ "type_name": "Network Activity: Open",
+ "severity_id": "99"
+ },
+ format: "$metadata.original_time$,$metadata.product.uid$,$unmapped.type=gtp_log_type$,$unmapped.subtype$,.*,$start_time_dt$,$src_endpoint.ip$,$dst_endpoint.ip$,.*,.*,$firewall_rule.name$,.*,.*,$unmapped.app$,$unmapped.vsys$,$src_endpoint.zone$,$dst_endpoint.zone$,$src_endpoint.interface_name$,$dst_endpoint.interface_name$,$unmapped.logset$,.*,$actor.session.uid$,.*,$src_endpoint.port$,$dst_endpoint.port$,.*,.*,.*,$connection_info.protocol_name$,$action$,$unmapped.event_type$,$unmapped.msisdn$,$unmapped.apn$,$unmapped.rat$,$unmapped.msg_type$,$device.ip$,$unmapped.teid1$,$unmapped.teid2$,$unmapped.gtp_interface$,$unmapped.cause_code$,$unmapped.severity$,$unmapped.mcc$,$unmapped.mnc$,$unmapped.area_code$,$unmapped.cell_id$,$unmapped.event_code$,.*,.*,$src_endpoint.location.country$,$dst_endpoint.location.country$,.*,.*,.*,.*,.*,.*,.*,$unmapped.imsi$,$device.imei$,.*,.*,.*,.*,.*,.*,.*,.*,.*,.*,.*,.*,.*,.*,.*,.*,$start_time$,$unmapped.elapsed$,$unmapped.tunnel_insp_rule$,$unmapped.tunnel_insp_rule$,$unmapped.tunnel_insp_rule$,$unmapped.rule_uuid$,$unmapped.pcap_id$",
+ halt: true,
+ rewrites: [
+ {
+ input: "message",
+ output: "observables",
+ match: "(?:[^,]*,){6}([^,]*)(?:,[^,]*){0},([^,]*)(?:,[^,]*){27},([^,]*).*",
+ replace: "\\[\\{\"type_id\"\\: 2\\, \"type\"\\: \"IP Address\"\\, \"name\"\\: \"src_endpoint.ip\"\\, \"value\"\\: $1\\}\\, \\{\"type_id\"\\: 2\\, \"type\"\\: \"IP Address\"\\, \"name\"\\: \"dst_endpoint.ip\"\\, \"value\"\\: $2\\}, \\{\"type_id\"\\: 2\\, \"type\"\\: \"IP Address\"\\, \"name\"\\: \"device.ip\"\\, \"value\"\\: $3\\}\\]"
+ },
+ {
+ input: "activity_name",
+ output: "event.type",
+ match: ".*",
+ replace: "$0"
+ }
+ ]
+ },
+ {
+ attributes: {
+ "activity_name": "Open",
+ "activity_id": "1",
+ "category_name": "Network Activity",
+ "category_uid": "4",
+ "class_name": "Network Activity",
+ "class_uid": "4001",
+ "cloud.provider": "Palo Alto Networks",
+ "metadata.product.name": "Palo Alto Networks Firewall",
+ "metadata.product.vendor_name": "Palo Alto Networks",
+ "metadata.version":"1.1.0",
+ "type_uid": "400101",
+ "type_name": "Network Activity: Open",
+ "severity_id": "99"
+ },
+ format: "$metadata.original_time$,$metadata.product.uid$,$unmapped.type=tunnel_log_type$,$unmapped.subtype$,.*,$start_time_dt$,$src_endpoint.ip$,$dst_endpoint.ip$,$src_endpoint.proxy_endpoint.ip$,$dst_endpoint.proxy_endpoint.ip$,$firewall_rule.name$,$actor.user.name$,$user.name$,$unmapped.app$,$unmapped.vsys$,$src_endpoint.zone$,$dst_endpoint.zone$,$src_endpoint.interface_name$,$dst_endpoint.interface_name$,$unmapped.logset$,.*,$actor.session.uid$,$unmapped.repeatcnt$,$src_endpoint.port$,$dst_endpoint.port$,$src_endpoint.proxy_endpoint.port$,$dst_endpoint.proxy_endpoint.port$,$unmapped.flags$,$connection_info.protocol_name$,$action$,$unmapped.severity$,$unmapped.seqno$,$unmapped.actionflags$,$src_endpoint.location.country$,$dst_endpoint.location.country$,$unmapped.dg_hier_level_1$,$unmapped.dg_hier_level_2$,$unmapped.dg_hier_level_3$,$unmapped.dg_hier_level_4$,$unmapped.vsys_name$,$device.name$,$unmapped.imsi$,$device.imei$,$session.uid$,$start_time$,$tunnel_type$,$traffic.bytes$,$traffic.bytes_out$,$traffic.bytes_in$,$traffic.packets$,$traffic.packets_out$,$traffic.packets_in$,$unmapped.max_encap$,$unmapped.unknown_proto$,$unmapped.strict_check$,$unmapped.tunnel_fragment$,$session.count$,$unmapped.sessions_closed$,$session.expiration_reason$,$unmapped.action_source$,$session.created_time$,$session.expiration_time$,$unmapped.tunnel_insp_rule$,$device.ip$,$user.uid$,$unmapped.rule_uuid$,$unmapped.pcap_id$,$unmapped.dynusergroup_name$,$unmapped.src_edl$,$unmapped.dst_edl$,$unmapped.high_res_timestamp$,$unmapped.nssai_sd$,$unmapped.nssai_sd$,$unmapped.pdu_session_id$,$unmapped.subcategory_of_app$,$unmapped.category_of_app$,$unmapped.technology_of_app$,$unmapped.risk_of_app$,$unmapped.characteristic_of_app$,$unmapped.container_of_app$,$unmapped.is_saas_of_app$,$unmapped.sanctioned_state_of_app$,$unmapped.cluster_name$",
+ halt: true,
+ rewrites: [
+ {
+ input: "message",
+ output: "observables",
+ match: "(?:[^,]*,){6}([^,]*)(?:,[^,]*){0},([^,]*)(?:,[^,]*){57},([^,]*).*",
+ replace: "\\[\\{\"type_id\"\\: 2\\, \"type\"\\: \"IP Address\"\\, \"name\"\\: \"src_endpoint.ip\"\\, \"value\"\\: $1\\}\\, \\{\"type_id\"\\: 2\\, \"type\"\\: \"IP Address\"\\, \"name\"\\: \"dst_endpoint.ip\"\\, \"value\"\\: $2\\}, \\{\"type_id\"\\: 2\\, \"type\"\\: \"IP Address\"\\, \"name\"\\: \"device.ip\"\\, \"value\"\\: $3\\}\\]"
+ },
+ {
+ input: "activity_name",
+ output: "event.type",
+ match: ".*",
+ replace: "$0"
+ }
+ ]
+ },
+ {
+ attributes: {
+ "activity_name": "Open",
+ "activity_id": "1",
+ "category_name": "Network Activity",
+ "category_uid": "4",
+ "class_name": "Network Activity",
+ "class_uid": "4001",
+ "cloud.provider": "Palo Alto Networks",
+ "metadata.product.name": "Palo Alto Networks Firewall",
+ "metadata.product.vendor_name": "Palo Alto Networks",
+ "metadata.version":"1.1.0",
+ "type_uid": "400101",
+ "type_name": "Network Activity: Open",
+ "severity_id": "99"
+ },
+ format: "$metadata.original_time$,$metadata.product.uid$,$unmapped.type=sctp_log_type$,.*,.*,$start_time_dt$,$src_endpoint.ip$,$dst_endpoint.ip$,.*,.*,$firewall_rule.name$,.*,.*,.*,$unmapped.vsys$,$src_endpoint.zone$,$dst_endpoint.zone$,$src_endpoint.interface_name$,$dst_endpoint.interface_name$,$unmapped.logset$,.*,$actor.session.uid$,$unmapped.repeatcnt$,$src_endpoint.port$,$dst_endpoint.port$,.*,.*,.*,.*,$connection_info.protocol_name$,$action$,$unmapped.dg_hier_level_1$,$unmapped.dg_hier_level_2$,$unmapped.dg_hier_level_3$,$unmapped.dg_hier_level_4$,$unmapped.vsys_name$,$device.name$,$unmapped.seqno$,.*,$unmapped.assoc_id$,$unmapped.ppid$,$unmapped.severity$,$unmappedsctp_chunk_type$,.*,$unmapped.verif_tag_1$,$unmapped.verif_tag_2$,$unmapped.sctp_cause_code$,$unmapped.diam_app_id$,$unmapped.diam_cmd_code$,$unmapped.diam_avp_code$,$unmapped.stream_id$,$unmapped.assoc_end_reason$,$unmapped.op_code$,$unmapped.sccp_calling_ssn$,$unmapped.sccp_calling_gt$,$unmapped.sctp_filter$,$unmapped.chunks$,$unmapped.chunks_sent$,$unmapped.chunks_received$,$traffic.packets$,$traffic.packets_out$,$traffic.packets_in$,$unmapped.rule_uuid$",
+ halt: true,
+ rewrites: [
+ {
+ input: "message",
+ output: "observables",
+ match: "(?:[^,]*,){6}([^,]*)(?:,[^,]*){0},([^,]*).*",
+ replace: "\\[\\{\"type_id\"\\: 2\\, \"type\"\\: \"IP Address\"\\, \"name\"\\: \"src_endpoint.ip\"\\, \"value\"\\: $1\\}\\, \\{\"type_id\"\\: 2\\, \"type\"\\: \"IP Address\"\\, \"name\"\\: \"dst_endpoint.ip\"\\, \"value\"\\: $2\\}\\]"
+ },
+ {
+ input: "activity_name",
+ output: "event.type",
+ match: ".*",
+ replace: "$0"
+ }
+ ]
+ },
+ {
+ attributes: {
+ "activity_name": "Create",
+ "activity_id": "1",
+ "category_name": "Findings",
+ "category_uid": "2",
+ "class_name": "Detection Finding",
+ "class_uid": "2004",
+ "cloud.provider": "Palo Alto Networks",
+ "metadata.product.name": "Palo Alto Networks Firewall",
+ "metadata.product.vendor_name": "Palo Alto Networks",
+ "metadata.version":"1.1.0",
+ "type_uid": "200401",
+ "type_name": "Detection Finding: Create",
+ "severity_id": "99"
+ },
+ format: "$metadata.original_time$,$metadata.product.uid$,$unmapped.type=system_log_type$,$unmapped.subtype$,.*,$start_time_dt$,$unmapped.vsys$,$metadata.event_code$,$unmapped.object$,.*,.*,$unmapped.module$,$unmapped.severity$,$unmapped.description$,$unmapped.seqno$,$unmapped.actionflags$,$unmapped.dg_hier_level_1$,$unmapped.dg_hier_level_2$,$unmapped.dg_hier_level_3$,$unmapped.dg_hier_level_4$,$unmapped.vsys_name$,$device.hostname$",
+ halt: true,
+ rewrites: [
+ {
+ input: "message",
+ output: "observables",
+ match: "(?:[^,]*,){21}([^,]*).*",
+ replace: "\\[\\{\"type_id\"\\: 1\\, \"type\"\\: \"Hostname\"\\, \"name\"\\: \"device.hostname\"\\, \"value\"\\: $1\\}\\}\\]"
+ },
+ {
+ input: "activity_name",
+ output: "event.type",
+ match: ".*",
+ replace: "$0"
+ }
+ ]
+ },
+ ]
+}
diff --git a/Frontend/log_generator_ui.py b/Frontend/log_generator_ui.py
index bae4a28..fc7bc91 100644
--- a/Frontend/log_generator_ui.py
+++ b/Frontend/log_generator_ui.py
@@ -1686,6 +1686,7 @@ def generate_logs():
eps = float(data.get('eps', 1.0))
continuous = data.get('continuous', False)
speed_mode = data.get('speed_mode', False)
+ ensure_parser = bool(data.get('ensure_parser', False))
syslog_ip = data.get('ip')
syslog_port = int(data.get('port')) if data.get('port') is not None else None
syslog_protocol = data.get('protocol')
@@ -1828,6 +1829,21 @@ def generate_and_stream():
hec_url = chosen.get('url')
dest_id = chosen.get('id')
+
+ # Resolve config API URL + config write token (for optional parser sync)
+ config_api_url = chosen.get('config_api_url')
+ config_write_token = None
+ if ensure_parser:
+ try:
+ token_res = requests.get(
+ f"{API_BASE_URL}/api/v1/destinations/{dest_id}/config-tokens",
+ headers=_get_api_headers(),
+ timeout=10,
+ )
+ if token_res.status_code == 200:
+ config_write_token = token_res.json().get('config_write_token')
+ except Exception as e:
+ logger.warning(f"Failed to resolve config tokens for destination {dest_id}: {e}")
# Use local token if provided, otherwise fetch from backend
if local_hec_token:
@@ -1897,6 +1913,17 @@ def _normalize_hec_url(u: str) -> str:
env['S1_HEC_TLS_LOW'] = '1'
# Enable automatic insecure fallback as last resort
env['S1_HEC_AUTO_INSECURE'] = 'true'
+
+ if ensure_parser:
+ env['JARVIS_ENSURE_PARSER'] = 'true'
+ env['JARVIS_API_BASE_URL'] = API_BASE_URL
+ if BACKEND_API_KEY:
+ env['JARVIS_API_KEY'] = BACKEND_API_KEY
+ if config_api_url and config_write_token:
+ env['S1_CONFIG_API_URL'] = config_api_url
+ env['S1_CONFIG_WRITE_TOKEN'] = config_write_token
+ else:
+ yield "INFO: ⚠️ Parser sync requested but destination is missing Config API URL or write token.\n"
if continuous:
# Batch mode for continuous
diff --git a/Frontend/templates/log_generator.html b/Frontend/templates/log_generator.html
index 5194526..5bfe58a 100644
--- a/Frontend/templates/log_generator.html
+++ b/Frontend/templates/log_generator.html
@@ -371,6 +371,14 @@
+
+
Start Log Generation
@@ -2209,6 +2217,7 @@ Select Parser Version
const eps = parseFloat(document.getElementById('eps').value);
const continuousMode = continuousModeCheckbox.checked;
const speedMode = speedModeCheckbox.checked;
+ const ensureParser = document.getElementById('ensure-parser')?.checked === true;
const selectedOpt = destSelect.options[destSelect.selectedIndex];
const destinationId = selectedOpt ? selectedOpt.value : '';
const destinationType = selectedOpt ? selectedOpt.dataset.type : '';
@@ -2270,7 +2279,8 @@ Select Parser Version
speed_mode: speedMode,
destination_id: destinationId,
hec_token: localToken, // Pass local token if available
- metadata: metadataFields // Pass metadata fields if provided
+ metadata: metadataFields, // Pass metadata fields if provided
+ ensure_parser: ensureParser
} : {
destination: 'syslog',
script: scriptPath,