diff --git a/Cargo.lock b/Cargo.lock index 78015cc2..59ced6e0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,8 +4,8 @@ version = 4 [[package]] name = "TinyUFO" -version = "0.8.0" -source = "git+https://github.com/gen0sec/pingora?rev=e82e0689ce62280f85c659dbfd862e49197133ad#e82e0689ce62280f85c659dbfd862e49197133ad" +version = "0.6.0" +source = "git+https://github.com/gen0sec/pingora?rev=c92146d621542303dd9b93a4cb5252e1eef46c81#c92146d621542303dd9b93a4cb5252e1eef46c81" dependencies = [ "ahash", "crossbeam-queue", @@ -55,9 +55,9 @@ dependencies = [ [[package]] name = "actix-http" -version = "3.12.0" +version = "3.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f860ee6746d0c5b682147b2f7f8ef036d4f92fe518251a3a35ffa3650eafdf0e" +checksum = "7926860314cbe2fb5d1f13731e387ab43bd32bca224e82e6e2db85de0a3dba49" dependencies = [ "actix-codec", "actix-rt", @@ -99,14 +99,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb" dependencies = [ "quote", - "syn 2.0.117", + "syn 2.0.116", ] [[package]] name = "actix-router" -version = "0.5.4" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14f8c75c51892f18d9c46150c5ac7beb81c95f78c8b83a634d49f4ca32551fe7" +checksum = "13d324164c51f63867b57e73ba5936ea151b8a41a1d23d1031eeb9f70d0236f8" dependencies = [ "bytestring", "cfg-if", @@ -166,9 +166,9 @@ dependencies = [ [[package]] name = "actix-web" -version = "4.13.0" +version = "4.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff87453bc3b56e9b2b23c1cc0b1be8797184accf51d2abe0f8a33ec275d316bf" +checksum = "1654a77ba142e37f049637a3e5685f864514af11fcbc51cb51eb6596afe5b8d6" dependencies = [ "actix-codec", "actix-http", @@ -216,7 +216,7 @@ dependencies = [ "actix-router", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.116", ] [[package]] @@ -294,22 +294,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a" dependencies = [ "anstyle", - "anstyle-parse 0.2.7", - "anstyle-query", - "anstyle-wincon", - "colorchoice", - "is_terminal_polyfill", - "utf8parse", -] - -[[package]] -name = "anstream" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" -dependencies = [ - "anstyle", - "anstyle-parse 1.0.0", + "anstyle-parse", "anstyle-query", "anstyle-wincon", "colorchoice", @@ -332,15 +317,6 @@ dependencies = [ "utf8parse", ] -[[package]] -name = "anstyle-parse" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" -dependencies = [ - "utf8parse", -] - [[package]] name = "anstyle-query" version = "1.1.5" @@ -363,9 +339,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.102" +version = "1.0.101" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +checksum = "5f0e0fee31ef5ed1ba1316088939cea399010ed7731dba877ed44aeb407a75ea" [[package]] name = "arc-swap" @@ -388,13 +364,29 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" +[[package]] +name = "asn1-rs" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5493c3bedbacf7fd7382c6346bbd66687d12bbaad3a89a2d2c303ee6cf20b048" +dependencies = [ + "asn1-rs-derive 0.5.1", + "asn1-rs-impl", + "displaydoc", + "nom", + "num-traits", + "rusticata-macros", + "thiserror 1.0.69", + "time", +] + [[package]] name = "asn1-rs" version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56624a96882bb8c26d61312ae18cb45868e5a9992ea73c58e45c3101e56a1e60" dependencies = [ - "asn1-rs-derive", + "asn1-rs-derive 0.6.0", "asn1-rs-impl", "displaydoc", "nom", @@ -404,6 +396,18 @@ dependencies = [ "time", ] +[[package]] +name = "asn1-rs-derive" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "965c2d33e53cb6b267e148a4cb0760bc01f4904c1cd4bb4002a085bb016d1490" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.116", + "synstructure", +] + [[package]] name = "asn1-rs-derive" version = "0.6.0" @@ -412,7 +416,7 @@ checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.116", "synstructure", ] @@ -424,7 +428,7 @@ checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.116", ] [[package]] @@ -435,7 +439,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.116", ] [[package]] @@ -444,6 +448,17 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi 0.1.19", + "libc", + "winapi", +] + [[package]] name = "autocfg" version = "1.5.0" @@ -452,9 +467,9 @@ checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" [[package]] name = "aws-lc-rs" -version = "1.16.1" +version = "1.15.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94bffc006df10ac2a68c83692d734a465f8ee6c5b384d8545a636f81d858f4bf" +checksum = "7b7b6141e96a8c160799cc2d5adecd5cbbe5054cb8c7c4af53da0f83bb7ad256" dependencies = [ "aws-lc-sys", "untrusted 0.7.1", @@ -463,9 +478,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.38.0" +version = "0.37.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4321e568ed89bb5a7d291a7f37997c2c0df89809d7b6d12062c81ddb54aa782e" +checksum = "b092fe214090261288111db7a2b2c2118e5a7f30dc2569f1732c4069a6840549" dependencies = [ "cc", "cmake", @@ -682,22 +697,11 @@ dependencies = [ "alloc-stdlib", ] -[[package]] -name = "bstr" -version = "1.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63044e1ae8e69f3b5a92c736ca6269b8d12fa7efe39bf34ddb06d102cf0e2cab" -dependencies = [ - "memchr", - "regex-automata", - "serde", -] - [[package]] name = "bumpalo" -version = "3.20.2" +version = "3.19.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" +checksum = "5dd9dc738b7a8311c7ade152424974d8115f2cdad61e8dab8dac9f2362298510" [[package]] name = "byteorder" @@ -823,9 +827,9 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.44" +version = "0.4.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" +checksum = "fac4744fb15ae8337dc853fee7fb3f4e48c0fbaa23d0afe49c447b4fab126118" dependencies = [ "iana-time-zone", "js-sys", @@ -844,6 +848,12 @@ dependencies = [ "serde", ] +[[package]] +name = "circular" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fc239e0f6cb375d2402d48afb92f76f5404fd1df208a41930ec81eda078bea" + [[package]] name = "clamav-tcp" version = "0.2.1" @@ -856,36 +866,75 @@ dependencies = [ [[package]] name = "clap" -version = "4.6.0" +version = "3.2.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b193af5b67834b676abd72466a96c1024e6a6ad978a1f484bd90b85c94041351" +checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123" +dependencies = [ + "atty", + "bitflags 1.3.2", + "clap_derive 3.2.25", + "clap_lex 0.2.4", + "indexmap 1.9.3", + "once_cell", + "strsim 0.10.0", + "termcolor", + "textwrap", +] + +[[package]] +name = "clap" +version = "4.5.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63be97961acde393029492ce0be7a1af7e323e6bae9511ebfac33751be5e6806" dependencies = [ "clap_builder", - "clap_derive", + "clap_derive 4.5.55", ] [[package]] name = "clap_builder" -version = "4.6.0" +version = "4.5.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +checksum = "7f13174bda5dfd69d7e947827e5af4b0f2f94a4a3ee92912fba07a66150f21e2" dependencies = [ - "anstream 1.0.0", + "anstream", "anstyle", - "clap_lex", - "strsim", + "clap_lex 1.0.0", + "strsim 0.11.1", ] [[package]] name = "clap_derive" -version = "4.6.0" +version = "3.2.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1110bd8a634a1ab8cb04345d8d878267d57c3cf1b38d91b71af6686408bbca6a" +checksum = "ae6371b8bdc8b7d3959e9cf7b22d4435ef3e79e138688421ec654acf8c81b008" dependencies = [ - "heck", + "heck 0.4.1", + "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.117", + "syn 1.0.109", +] + +[[package]] +name = "clap_derive" +version = "4.5.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a92793da1a46a5f2a02a6f4c46c6496b28c43638adea8306fcb0caa1634f24e5" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.116", +] + +[[package]] +name = "clap_lex" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5" +dependencies = [ + "os_str_bytes", ] [[package]] @@ -1022,6 +1071,25 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-channel" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + [[package]] name = "crossbeam-epoch" version = "0.9.18" @@ -1091,7 +1159,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e0b1fab2ae45819af2d0731d60f2afe17227ebb1a1538a236da84c93e9a60162" dependencies = [ "dispatch2", - "nix 0.31.2", + "nix 0.31.1", "windows-sys 0.61.2", ] @@ -1119,7 +1187,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.116", ] [[package]] @@ -1131,15 +1199,6 @@ dependencies = [ "libc", ] -[[package]] -name = "daggy" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70def8d72740e44d9f676d8dab2c933a236663d86dd24319b57a2bed4d694774" -dependencies = [ - "petgraph", -] - [[package]] name = "darling" version = "0.20.11" @@ -1160,8 +1219,8 @@ dependencies = [ "ident_case", "proc-macro2", "quote", - "strsim", - "syn 2.0.117", + "strsim 0.11.1", + "syn 2.0.116", ] [[package]] @@ -1172,7 +1231,7 @@ checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ "darling_core", "quote", - "syn 2.0.117", + "syn 2.0.116", ] [[package]] @@ -1206,13 +1265,27 @@ dependencies = [ "zeroize", ] +[[package]] +name = "der-parser" +version = "9.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cd0a5c643689626bec213c4d8bd4d96acc8ffdb4ad4bb6bc16abf27d5f4b553" +dependencies = [ + "asn1-rs 0.6.2", + "displaydoc", + "nom", + "num-bigint", + "num-traits", + "rusticata-macros", +] + [[package]] name = "der-parser" version = "10.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07da5016415d5a3c4dd39b11ed26f915f52fc4e0dc197d87908bc916e51bc1a6" dependencies = [ - "asn1-rs", + "asn1-rs 0.7.1", "displaydoc", "nom", "num-bigint", @@ -1222,9 +1295,9 @@ dependencies = [ [[package]] name = "deranged" -version = "0.5.8" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +checksum = "cc3dc5ad92c2e2d1c193bbbbdf2ea477cb81331de4f3103f267ca18368b988c4" dependencies = [ "powerfmt", ] @@ -1258,7 +1331,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.116", ] [[package]] @@ -1268,7 +1341,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" dependencies = [ "derive_builder_core", - "syn 2.0.117", + "syn 2.0.116", ] [[package]] @@ -1290,7 +1363,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version", - "syn 2.0.117", + "syn 2.0.116", "unicode-xid", ] @@ -1314,9 +1387,9 @@ dependencies = [ [[package]] name = "dispatch2" -version = "0.3.1" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" +checksum = "89a09f22a6c6069a18470eb92d2298acf25463f14256d24778e1230d789a2aec" dependencies = [ "bitflags 2.11.0", "block2", @@ -1332,7 +1405,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.116", ] [[package]] @@ -1427,10 +1500,10 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1e6a265c649f3f5979b601d26f1d05ada116434c87741c9493cb56218f76cbc" dependencies = [ - "heck", + "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.116", ] [[package]] @@ -1448,7 +1521,7 @@ version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b2daee4ea451f429a58296525ddf28b45a3b64f1acf6587e2067437bb11e218d" dependencies = [ - "anstream 0.6.21", + "anstream", "anstyle", "env_filter", "jiff", @@ -1471,6 +1544,15 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "etherparse" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b119b9796ff800751a220394b8b3613f26dd30c48f254f6837e64c464872d1c7" +dependencies = [ + "arrayvec", +] + [[package]] name = "fastrand" version = "2.3.0" @@ -1499,12 +1581,6 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" -[[package]] -name = "fixedbitset" -version = "0.5.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" - [[package]] name = "flate2" version = "1.1.9" @@ -1525,7 +1601,7 @@ dependencies = [ "ahash", "num_cpus", "parking_lot", - "seize", + "seize 0.3.3", ] [[package]] @@ -1651,7 +1727,7 @@ checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.116", ] [[package]] @@ -1737,20 +1813,20 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "r-efi 5.3.0", + "r-efi", "wasip2", "wasm-bindgen", ] [[package]] name = "getrandom" -version = "0.4.2" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +checksum = "139ef39800118c7683f2fd3c98c1b23c09ae076556b435f8e9064ae108aaeeec" dependencies = [ "cfg-if", "libc", - "r-efi 6.0.0", + "r-efi", "rand_core 0.10.0", "wasip2", "wasip3", @@ -1765,7 +1841,7 @@ dependencies = [ "proc-macro-error2", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.116", ] [[package]] @@ -1774,6 +1850,12 @@ version = "0.32.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" +[[package]] +name = "glob" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" + [[package]] name = "group" version = "0.13.0" @@ -1835,6 +1917,8 @@ version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ + "allocator-api2", + "equivalent", "foldhash 0.1.5", ] @@ -1849,12 +1933,27 @@ dependencies = [ "foldhash 0.2.0", ] +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + [[package]] name = "heck" version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + [[package]] name = "hermit-abi" version = "0.5.2" @@ -2247,9 +2346,9 @@ dependencies = [ [[package]] name = "instant-acme" -version = "0.8.5" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f05ad37c421b962354c358d347d4a6130151df9407978372d3ad7f0c8f71a64" +checksum = "e9e04488259349908dd13fadaf3a97523b61da296a468c490e112ecc73d28b47" dependencies = [ "async-trait", "aws-lc-rs", @@ -2285,9 +2384,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.12.0" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" +checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" [[package]] name = "ipnetwork" @@ -2329,9 +2428,9 @@ checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" [[package]] name = "jiff" -version = "0.2.23" +version = "0.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a3546dc96b6d42c5f24902af9e2538e82e39ad350b0c766eb3fbf2d8f3d8359" +checksum = "c867c356cc096b33f4981825ab281ecba3db0acefe60329f044c1789d94c6543" dependencies = [ "jiff-static", "log", @@ -2342,13 +2441,13 @@ dependencies = [ [[package]] name = "jiff-static" -version = "0.2.23" +version = "0.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a8c8b344124222efd714b73bb41f8b5120b27a7cc1c75593a6ff768d9d05aa4" +checksum = "f7946b4325269738f270bb55b3c19ab5c5040525f83fd625259422a9d25d9be5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.116", ] [[package]] @@ -2385,9 +2484,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.91" +version = "0.3.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b49715b7073f385ba4bc528e5747d02e66cb39c6146efb66b781f131f0fb399c" +checksum = "8c942ebf8e95485ca0d52d97da7c5a2c387d0e7f0ba4c35e93bfcaee045955b3" dependencies = [ "once_cell", "wasm-bindgen", @@ -2459,13 +2558,13 @@ checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" [[package]] name = "libbpf-cargo" -version = "0.26.1" +version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdd59674b82a31cb53e512cee97e3d5143737b40feda35f89b5b7f5c439e51e8" +checksum = "2df559c3961ae45e06aaff479036261375dbf0226541aa4155e60aa277fb37d6" dependencies = [ "anyhow", "cargo_metadata", - "clap", + "clap 4.5.58", "libbpf-rs", "libbpf-sys", "memmap2", @@ -2478,9 +2577,9 @@ dependencies = [ [[package]] name = "libbpf-rs" -version = "0.26.1" +version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6af29f679fd713ba4466b0296436268f573f6c5a8ef2f316d237eb3019c3c6f" +checksum = "87c7c1017b506e5cbc007f65abb78b21bd5957dff9358882427ddbf634ba415b" dependencies = [ "bitflags 2.11.0", "libbpf-sys", @@ -2495,15 +2594,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95f55336f7dbde4fbbaae7b5e271f9efc65bf1d0b69c6fcb1e5026a385e95f70" dependencies = [ "cc", - "nix 0.31.2", + "nix 0.31.1", "pkg-config", ] [[package]] name = "libc" -version = "0.2.182" +version = "0.2.180" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6800badb6cb2082ffd7b6a67e6125bb39f18782f793520caee8cb8846be06112" +checksum = "bcc35a38544a891a5f7c865aca548a982ccb3b8650a5b06d0fd33a10283c56fc" [[package]] name = "libm" @@ -2523,9 +2622,9 @@ dependencies = [ [[package]] name = "libz-ng-sys" -version = "1.1.24" +version = "1.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52ff028f347c6ed1b98e17cede6f36b5e2706afde598914c87d2da04f86029ba" +checksum = "7bf914b7dd154ca9193afec311d8e39345c1bd93b48b3faa77329f0db8f553c0" dependencies = [ "cmake", "libc", @@ -2539,9 +2638,9 @@ checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" [[package]] name = "linux-raw-sys" -version = "0.12.1" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" +checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" [[package]] name = "litemap" @@ -2623,7 +2722,7 @@ dependencies = [ "serde", "serde-value", "serde_json", - "serde_yaml", + "serde_yaml 0.9.34+deprecated", "thiserror 2.0.18", "thread-id", "typemap-ors", @@ -2633,11 +2732,11 @@ dependencies = [ [[package]] name = "lru" -version = "0.16.3" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1dc47f592c06f33f8e3aea9591776ec7c9f9e4124778ff8a3c3b87159f7e593" +checksum = "9f8cc7106155f10bdf99a6f379688f543ad6596a415375b36a59a054ceda1198" dependencies = [ - "hashbrown 0.16.1", + "hashbrown 0.15.5", ] [[package]] @@ -2655,6 +2754,15 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + [[package]] name = "matchit" version = "0.8.4" @@ -2663,9 +2771,9 @@ checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" [[package]] name = "maxminddb" -version = "0.27.3" +version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76371bd37ce742f8954daabd0fde7f1594ee43ac2200e20c003ba5c3d65e2192" +checksum = "d5e371ee70dfbe063e098d1f90f01eee1458db7b0d7c03cd01e95453aa0e04e6" dependencies = [ "ipnetwork", "log", @@ -2674,6 +2782,12 @@ dependencies = [ "thiserror 2.0.18", ] +[[package]] +name = "md5" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" + [[package]] name = "memchr" version = "2.8.0" @@ -2791,9 +2905,9 @@ dependencies = [ [[package]] name = "native-tls" -version = "0.2.18" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "465500e14ea162429d264d44189adc38b199b62b1c21eea9f69e4b73cb03bbf2" +checksum = "9d5d26952a508f321b4d3d2e80e78fc2603eaefcdf0c30783867f19586518bdc" dependencies = [ "libc", "log", @@ -2832,7 +2946,7 @@ dependencies = [ "proc-macro2", "quote", "serde", - "syn 2.0.117", + "syn 2.0.116", ] [[package]] @@ -2876,9 +2990,9 @@ dependencies = [ [[package]] name = "nix" -version = "0.31.2" +version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d6d0705320c1e6ba1d912b5e37cf18071b6c2e9b7fa8215a1e8a7651966f5d3" +checksum = "225e7cfe711e0ba79a68baeddb2982723e4235247aefce1482f2f16c27865b66" dependencies = [ "bitflags 2.11.0", "cfg-if", @@ -2949,7 +3063,17 @@ dependencies = [ [[package]] name = "nstealth" version = "0.1.0" -source = "git+https://github.com/gen0sec/nstealth?rev=f9a6323d41acf8b7d45e33ae7e4e863ec716ba05#f9a6323d41acf8b7d45e33ae7e4e863ec716ba05" +source = "git+https://github.com/gen0sec/nstealth?rev=31d371277cf55b24d66b52028c857d67d37b561a#31d371277cf55b24d66b52028c857d67d37b561a" +dependencies = [ + "serde", + "sha2", + "thiserror 2.0.18", +] + +[[package]] +name = "nstealth" +version = "0.1.0" +source = "git+https://github.com/gen0sec/nstealth?rev=3c87751b9d9537b055a119f155a730360a7d0078#3c87751b9d9537b055a119f155a730360a7d0078" dependencies = [ "serde", "sha2", @@ -3033,7 +3157,7 @@ version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" dependencies = [ - "hermit-abi", + "hermit-abi 0.5.2", "libc", ] @@ -3056,7 +3180,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.116", ] [[package]] @@ -3070,9 +3194,9 @@ dependencies = [ [[package]] name = "objc2" -version = "0.6.4" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a12a8ed07aefc768292f076dc3ac8c48f3781c8f2d5851dd3d98950e8c5a89f" +checksum = "b7c2599ce0ec54857b29ce62166b0ed9b4f6f1a70ccc9a71165b6154caca8c05" dependencies = [ "objc2-encode", ] @@ -3092,20 +3216,29 @@ dependencies = [ "memchr", ] +[[package]] +name = "oid-registry" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8d8034d9489cdaf79228eb9f6a3b8d7bb32ba00d6645ebd48eef4077ceb5bd9" +dependencies = [ + "asn1-rs 0.6.2", +] + [[package]] name = "oid-registry" version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "12f40cff3dde1b6087cc5d5f5d4d65712f34016a03ed60e9c08dcc392736b5b7" dependencies = [ - "asn1-rs", + "asn1-rs 0.7.1", ] [[package]] name = "once_cell" -version = "1.21.4" +version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" [[package]] name = "once_cell_polyfill" @@ -3136,7 +3269,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.116", ] [[package]] @@ -3182,6 +3315,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "os_str_bytes" +version = "6.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1" + [[package]] name = "p256" version = "0.13.2" @@ -3206,6 +3345,16 @@ dependencies = [ "sha2", ] +[[package]] +name = "papaya" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f92dd0b07c53a0a0c764db2ace8c541dc47320dad97c2200c2a637ab9dd2328f" +dependencies = [ + "equivalent", + "seize 0.5.1", +] + [[package]] name = "parking_lot" version = "0.12.5" @@ -3235,6 +3384,39 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" +[[package]] +name = "pcap-parser" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f8d57cc6bdf76d7abd6d3cc1113278047dab29c2ff6d97190e8d1c29d4efdac" +dependencies = [ + "circular", + "nom", + "rusticata-macros", +] + +[[package]] +name = "pcre2" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e970b0fcce0c7ee6ef662744ff711f21ccd6f11b7cf03cd187a80e89797fc67" +dependencies = [ + "libc", + "log", + "pcre2-sys", +] + +[[package]] +name = "pcre2-sys" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18b9073c1a2549bd409bf4a32c94d903bb1a09bf845bc306ae148897fa0760a4" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "pem" version = "3.0.6" @@ -3260,16 +3442,6 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" -[[package]] -name = "petgraph" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" -dependencies = [ - "fixedbitset", - "indexmap 2.13.0", -] - [[package]] name = "phf" version = "0.11.3" @@ -3310,29 +3482,29 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.11" +version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1749c7ed4bcaf4c3d0a3efc28538844fb29bcdd7d2b67b2be7e20ba861ff517" +checksum = "677f1add503faace112b9f1373e43e9e054bfdd22ff1a63c1bc485eaec6a6a8a" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.11" +version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b20ed30f105399776b9c883e68e536ef602a16ae6f596d2c473591d6ad64c6" +checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.116", ] [[package]] name = "pin-project-lite" -version = "0.2.17" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" [[package]] name = "pin-utils" @@ -3342,8 +3514,8 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pingora" -version = "0.8.0" -source = "git+https://github.com/gen0sec/pingora?rev=e82e0689ce62280f85c659dbfd862e49197133ad#e82e0689ce62280f85c659dbfd862e49197133ad" +version = "0.6.0" +source = "git+https://github.com/gen0sec/pingora?rev=c92146d621542303dd9b93a4cb5252e1eef46c81#c92146d621542303dd9b93a4cb5252e1eef46c81" dependencies = [ "pingora-cache", "pingora-core", @@ -3355,13 +3527,12 @@ dependencies = [ [[package]] name = "pingora-cache" -version = "0.8.0" -source = "git+https://github.com/gen0sec/pingora?rev=e82e0689ce62280f85c659dbfd862e49197133ad#e82e0689ce62280f85c659dbfd862e49197133ad" +version = "0.6.0" +source = "git+https://github.com/gen0sec/pingora?rev=c92146d621542303dd9b93a4cb5252e1eef46c81#c92146d621542303dd9b93a4cb5252e1eef46c81" dependencies = [ "ahash", "async-trait", "blake2", - "bstr", "bytes", "cf-rustracing", "cf-rustracing-jaeger", @@ -3391,18 +3562,16 @@ dependencies = [ [[package]] name = "pingora-core" -version = "0.8.0" -source = "git+https://github.com/gen0sec/pingora?rev=e82e0689ce62280f85c659dbfd862e49197133ad#e82e0689ce62280f85c659dbfd862e49197133ad" +version = "0.6.0" +source = "git+https://github.com/gen0sec/pingora?rev=c92146d621542303dd9b93a4cb5252e1eef46c81#c92146d621542303dd9b93a4cb5252e1eef46c81" dependencies = [ "ahash", "async-trait", "brotli 3.5.0", - "bstr", "bytes", "chrono", - "clap", + "clap 3.2.25", "daemonize", - "daggy", "derivative", "flate2", "futures", @@ -3428,13 +3597,12 @@ dependencies = [ "rand 0.8.5", "regex", "serde", - "serde_yaml", + "serde_yaml 0.8.26", "sfv", "socket2 0.6.2", "strum 0.26.3", "strum_macros 0.26.4", "tokio", - "tokio-stream", "tokio-test", "unicase", "windows-sys 0.59.0", @@ -3443,13 +3611,13 @@ dependencies = [ [[package]] name = "pingora-error" -version = "0.8.0" -source = "git+https://github.com/gen0sec/pingora?rev=e82e0689ce62280f85c659dbfd862e49197133ad#e82e0689ce62280f85c659dbfd862e49197133ad" +version = "0.6.0" +source = "git+https://github.com/gen0sec/pingora?rev=c92146d621542303dd9b93a4cb5252e1eef46c81#c92146d621542303dd9b93a4cb5252e1eef46c81" [[package]] name = "pingora-header-serde" -version = "0.8.0" -source = "git+https://github.com/gen0sec/pingora?rev=e82e0689ce62280f85c659dbfd862e49197133ad#e82e0689ce62280f85c659dbfd862e49197133ad" +version = "0.6.0" +source = "git+https://github.com/gen0sec/pingora?rev=c92146d621542303dd9b93a4cb5252e1eef46c81#c92146d621542303dd9b93a4cb5252e1eef46c81" dependencies = [ "bytes", "http 1.4.0", @@ -3463,8 +3631,8 @@ dependencies = [ [[package]] name = "pingora-http" -version = "0.8.0" -source = "git+https://github.com/gen0sec/pingora?rev=e82e0689ce62280f85c659dbfd862e49197133ad#e82e0689ce62280f85c659dbfd862e49197133ad" +version = "0.6.0" +source = "git+https://github.com/gen0sec/pingora?rev=c92146d621542303dd9b93a4cb5252e1eef46c81#c92146d621542303dd9b93a4cb5252e1eef46c81" dependencies = [ "bytes", "http 1.4.0", @@ -3473,24 +3641,24 @@ dependencies = [ [[package]] name = "pingora-ketama" -version = "0.8.0" -source = "git+https://github.com/gen0sec/pingora?rev=e82e0689ce62280f85c659dbfd862e49197133ad#e82e0689ce62280f85c659dbfd862e49197133ad" +version = "0.6.0" +source = "git+https://github.com/gen0sec/pingora?rev=c92146d621542303dd9b93a4cb5252e1eef46c81#c92146d621542303dd9b93a4cb5252e1eef46c81" dependencies = [ "crc32fast", ] [[package]] name = "pingora-limits" -version = "0.8.0" -source = "git+https://github.com/gen0sec/pingora?rev=e82e0689ce62280f85c659dbfd862e49197133ad#e82e0689ce62280f85c659dbfd862e49197133ad" +version = "0.6.0" +source = "git+https://github.com/gen0sec/pingora?rev=c92146d621542303dd9b93a4cb5252e1eef46c81#c92146d621542303dd9b93a4cb5252e1eef46c81" dependencies = [ "ahash", ] [[package]] name = "pingora-load-balancing" -version = "0.8.0" -source = "git+https://github.com/gen0sec/pingora?rev=e82e0689ce62280f85c659dbfd862e49197133ad#e82e0689ce62280f85c659dbfd862e49197133ad" +version = "0.6.0" +source = "git+https://github.com/gen0sec/pingora?rev=c92146d621542303dd9b93a4cb5252e1eef46c81#c92146d621542303dd9b93a4cb5252e1eef46c81" dependencies = [ "arc-swap", "async-trait", @@ -3510,8 +3678,8 @@ dependencies = [ [[package]] name = "pingora-lru" -version = "0.8.0" -source = "git+https://github.com/gen0sec/pingora?rev=e82e0689ce62280f85c659dbfd862e49197133ad#e82e0689ce62280f85c659dbfd862e49197133ad" +version = "0.6.0" +source = "git+https://github.com/gen0sec/pingora?rev=c92146d621542303dd9b93a4cb5252e1eef46c81#c92146d621542303dd9b93a4cb5252e1eef46c81" dependencies = [ "arrayvec", "hashbrown 0.16.1", @@ -3521,8 +3689,8 @@ dependencies = [ [[package]] name = "pingora-memory-cache" -version = "0.8.0" -source = "git+https://github.com/gen0sec/pingora?rev=e82e0689ce62280f85c659dbfd862e49197133ad#e82e0689ce62280f85c659dbfd862e49197133ad" +version = "0.6.0" +source = "git+https://github.com/gen0sec/pingora?rev=c92146d621542303dd9b93a4cb5252e1eef46c81#c92146d621542303dd9b93a4cb5252e1eef46c81" dependencies = [ "TinyUFO", "ahash", @@ -3536,8 +3704,8 @@ dependencies = [ [[package]] name = "pingora-openssl" -version = "0.8.0" -source = "git+https://github.com/gen0sec/pingora?rev=e82e0689ce62280f85c659dbfd862e49197133ad#e82e0689ce62280f85c659dbfd862e49197133ad" +version = "0.6.0" +source = "git+https://github.com/gen0sec/pingora?rev=c92146d621542303dd9b93a4cb5252e1eef46c81#c92146d621542303dd9b93a4cb5252e1eef46c81" dependencies = [ "foreign-types", "libc", @@ -3548,8 +3716,8 @@ dependencies = [ [[package]] name = "pingora-pool" -version = "0.8.0" -source = "git+https://github.com/gen0sec/pingora?rev=e82e0689ce62280f85c659dbfd862e49197133ad#e82e0689ce62280f85c659dbfd862e49197133ad" +version = "0.6.0" +source = "git+https://github.com/gen0sec/pingora?rev=c92146d621542303dd9b93a4cb5252e1eef46c81#c92146d621542303dd9b93a4cb5252e1eef46c81" dependencies = [ "crossbeam-queue", "log", @@ -3562,12 +3730,12 @@ dependencies = [ [[package]] name = "pingora-proxy" -version = "0.8.0" -source = "git+https://github.com/gen0sec/pingora?rev=e82e0689ce62280f85c659dbfd862e49197133ad#e82e0689ce62280f85c659dbfd862e49197133ad" +version = "0.6.0" +source = "git+https://github.com/gen0sec/pingora?rev=c92146d621542303dd9b93a4cb5252e1eef46c81#c92146d621542303dd9b93a4cb5252e1eef46c81" dependencies = [ "async-trait", "bytes", - "clap", + "clap 3.2.25", "futures", "h2 0.4.13", "http 1.4.0", @@ -3584,8 +3752,8 @@ dependencies = [ [[package]] name = "pingora-runtime" -version = "0.8.0" -source = "git+https://github.com/gen0sec/pingora?rev=e82e0689ce62280f85c659dbfd862e49197133ad#e82e0689ce62280f85c659dbfd862e49197133ad" +version = "0.6.0" +source = "git+https://github.com/gen0sec/pingora?rev=c92146d621542303dd9b93a4cb5252e1eef46c81#c92146d621542303dd9b93a4cb5252e1eef46c81" dependencies = [ "once_cell", "rand 0.8.5", @@ -3595,8 +3763,8 @@ dependencies = [ [[package]] name = "pingora-timeout" -version = "0.8.0" -source = "git+https://github.com/gen0sec/pingora?rev=e82e0689ce62280f85c659dbfd862e49197133ad#e82e0689ce62280f85c659dbfd862e49197133ad" +version = "0.6.0" +source = "git+https://github.com/gen0sec/pingora?rev=c92146d621542303dd9b93a4cb5252e1eef46c81#c92146d621542303dd9b93a4cb5252e1eef46c81" dependencies = [ "once_cell", "parking_lot", @@ -3690,7 +3858,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn 2.0.117", + "syn 2.0.116", ] [[package]] @@ -3714,13 +3882,37 @@ dependencies = [ [[package]] name = "proc-macro-crate" -version = "3.5.0" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" +checksum = "219cb19e96be00ab2e37d6e299658a0cfa83e52429179969b0f0121b4ac46983" dependencies = [ "toml_edit", ] +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn 1.0.109", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + [[package]] name = "proc-macro-error-attr2" version = "2.0.0" @@ -3740,7 +3932,7 @@ dependencies = [ "proc-macro-error-attr2", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.116", ] [[package]] @@ -3848,9 +4040,9 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.14" +version = "0.11.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +checksum = "f1906b49b0c3bc04b5fe5d86a77925ae6524a19b816ae38ce1e426255f1d8a31" dependencies = [ "aws-lc-rs", "bytes", @@ -3884,9 +4076,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.45" +version = "1.0.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +checksum = "21b2ebcf727b7760c461f091f9f0f539b77b8e87f2fd88131e7f1b433b3cece4" dependencies = [ "proc-macro2", ] @@ -3897,12 +4089,6 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" -[[package]] -name = "r-efi" -version = "6.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" - [[package]] name = "r2d2" version = "0.8.10" @@ -3942,7 +4128,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc266eb313df6c5c09c1c7b1fbe2510961e5bcd3add930c1e31f7ed9da0feff8" dependencies = [ "chacha20", - "getrandom 0.4.2", + "getrandom 0.4.1", "rand_core 0.10.0", ] @@ -3990,6 +4176,26 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c8d0fd677905edcbeedbf2edb6494d676f0e98d54d5cf9bda0b061cb8fb8aba" +[[package]] +name = "rayon" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "rcgen" version = "0.14.7" @@ -4000,15 +4206,15 @@ dependencies = [ "pem", "rustls-pki-types", "time", - "x509-parser", + "x509-parser 0.18.1", "yasna", ] [[package]] name = "redis" -version = "1.0.5" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b36964393906eb775b89b25b05b7b95685b8dd14062f1663a31ff93e75c452e5" +checksum = "e969d1d702793536d5fda739a82b88ad7cbe7d04f8386ee8cd16ad3eff4854a5" dependencies = [ "arc-swap", "arcstr", @@ -4060,7 +4266,7 @@ checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.116", ] [[package]] @@ -4094,9 +4300,9 @@ checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" [[package]] name = "regex-syntax" -version = "0.8.10" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" +checksum = "a96887878f22d7bad8a3b6dc5b7440e0ada9a245242924394987b21cf2210a4c" [[package]] name = "reqwest" @@ -4250,9 +4456,9 @@ dependencies = [ [[package]] name = "rustix" -version = "1.1.4" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +checksum = "146c9e247ccc180c1f61615433868c99f3de3ae256a30a43b49f67c2d9171f34" dependencies = [ "bitflags 2.11.0", "errno", @@ -4263,9 +4469,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.37" +version = "0.23.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4" +checksum = "c665f33d38cea657d9614f766881e4d510e0eda4239891eea56b4cadcf01801b" dependencies = [ "aws-lc-rs", "log", @@ -4417,7 +4623,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 2.0.117", + "syn 2.0.116", ] [[package]] @@ -4448,9 +4654,9 @@ dependencies = [ [[package]] name = "security-framework" -version = "3.7.0" +version = "3.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +checksum = "d17b898a6d6948c3a8ee4372c17cb384f90d2e6e912ef00895b14fd7ab54ec38" dependencies = [ "bitflags 2.11.0", "core-foundation 0.10.1", @@ -4461,9 +4667,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.17.0" +version = "2.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +checksum = "321c8673b092a9a42605034a9879d73cb79101ed5fd117bc9a597b89b4e9e61a" dependencies = [ "core-foundation-sys", "libc", @@ -4475,6 +4681,16 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "689224d06523904ebcc9b482c6a3f4f7fb396096645c4cd10c0d2ff7371a34d3" +[[package]] +name = "seize" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b55fb86dfd3a2f5f76ea78310a88f96c4ea21a3031f8d212443d56123fd0521" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + [[package]] name = "semver" version = "1.0.27" @@ -4528,7 +4744,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.116", ] [[package]] @@ -4539,7 +4755,7 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.116", ] [[package]] @@ -4588,6 +4804,18 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_yaml" +version = "0.8.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "578a7433b776b56a35785ed5ce9a7e777ac0598aac5a6dd1b4b18a307c7fc71b" +dependencies = [ + "indexmap 1.9.3", + "ryu", + "serde", + "yaml-rust", +] + [[package]] name = "serde_yaml" version = "0.9.34+deprecated" @@ -4603,9 +4831,9 @@ dependencies = [ [[package]] name = "serial_test" -version = "3.4.0" +version = "3.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "911bd979bf1070a3f3aa7b691a3b3e9968f339ceeec89e08c280a8a22207a32f" +checksum = "0d0b343e184fc3b7bb44dff0705fffcf4b3756ba6aff420dddd8b24ca145e555" dependencies = [ "futures-executor", "futures-util", @@ -4618,13 +4846,13 @@ dependencies = [ [[package]] name = "serial_test_derive" -version = "3.4.0" +version = "3.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a7d91949b85b0d2fb687445e448b40d322b6b3e4af6b44a29b21d9a5f33e6d9" +checksum = "6f50427f258fb77356e4cd4aa0e87e2bd2c66dbcee41dc405282cae2bfc26c83" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.116", ] [[package]] @@ -4764,10 +4992,10 @@ version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451" dependencies = [ - "heck", + "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.116", ] [[package]] @@ -4812,6 +5040,12 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + [[package]] name = "strsim" version = "0.11.1" @@ -4839,11 +5073,11 @@ version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" dependencies = [ - "heck", + "heck 0.5.0", "proc-macro2", "quote", "rustversion", - "syn 2.0.117", + "syn 2.0.116", ] [[package]] @@ -4852,10 +5086,10 @@ version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" dependencies = [ - "heck", + "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.116", ] [[package]] @@ -4877,9 +5111,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.117" +version = "2.0.116" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +checksum = "3df424c70518695237746f84cede799c9c58fcb37450d7b23716568cc8bc69cb" dependencies = [ "proc-macro2", "quote", @@ -4888,7 +5122,7 @@ dependencies = [ [[package]] name = "synapse" -version = "0.5.2" +version = "0.4.2" dependencies = [ "actix-files", "actix-web", @@ -4902,7 +5136,8 @@ dependencies = [ "bytes", "chrono", "clamav-tcp", - "clap", + "clap 4.5.58", + "crossbeam-channel", "ctrlc", "daemonize", "dashmap", @@ -4930,9 +5165,9 @@ dependencies = [ "multer", "native-tls", "nftables", - "nix 0.31.2", + "nix 0.31.1", "notify", - "nstealth", + "nstealth 0.1.0 (git+https://github.com/gen0sec/nstealth?rev=3c87751b9d9537b055a119f155a730360a7d0078)", "once_cell", "pingora", "pingora-core", @@ -4954,11 +5189,12 @@ dependencies = [ "serde", "serde_ignored", "serde_json", - "serde_yaml", + "serde_yaml 0.9.34+deprecated", "serial_test", "sha2", "syslog", "tempfile", + "thalamus", "tls-parser", "tokio", "tokio-rustls", @@ -4975,7 +5211,7 @@ dependencies = [ "vmlinux", "webpki-roots", "wirefilter-engine", - "x509-parser", + "x509-parser 0.18.1", ] [[package]] @@ -4995,7 +5231,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.116", ] [[package]] @@ -5033,17 +5269,72 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.27.0" +version = "3.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +checksum = "0136791f7c95b1f6dd99f9cc786b91bb81c3800b639b3478e561ddb7be95e5f1" dependencies = [ "fastrand", - "getrandom 0.4.2", + "getrandom 0.4.1", "once_cell", "rustix", "windows-sys 0.61.2", ] +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "textwrap" +version = "0.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c13547615a44dc9c452a8a534638acdf07120d4b6847c8178705da06306a3057" + +[[package]] +name = "thalamus" +version = "0.1.0" +source = "git+https://github.com/gen0sec/thalamus?branch=blocking#efafff03bc11a153daa3fd44bd4960a8bfc56b5e" +dependencies = [ + "ahash", + "aho-corasick", + "anyhow", + "base64 0.22.1", + "bytes", + "chrono", + "clap 4.5.58", + "crossbeam-channel", + "etherparse", + "glob", + "ipnetwork", + "libc", + "md5", + "nix 0.31.1", + "nstealth 0.1.0 (git+https://github.com/gen0sec/nstealth?rev=31d371277cf55b24d66b52028c857d67d37b561a)", + "papaya", + "parking_lot", + "pcap-parser", + "pcre2", + "rayon", + "regex", + "reqwest", + "serde", + "serde_json", + "serde_yaml 0.9.34+deprecated", + "sha1", + "sha2", + "tempfile", + "thiserror 2.0.18", + "tokio", + "tracing", + "tracing-subscriber", + "x509-parser 0.16.0", +] + [[package]] name = "thiserror" version = "1.0.69" @@ -5070,7 +5361,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.116", ] [[package]] @@ -5081,7 +5372,7 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.116", ] [[package]] @@ -5196,9 +5487,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.50.0" +version = "1.49.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27ad5e34374e03cfffefc301becb44e9dc3c17584f414349ebe29ed26661822d" +checksum = "72a2903cd7736441aac9df9d7688bd0ce48edccaadf181c3b90be801e81d3d86" dependencies = [ "bytes", "libc", @@ -5213,13 +5504,13 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.6.1" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c55a2eff8b69ce66c84f85e1da1c233edc36ceb85a2058d11b0d6a3c7e7569c" +checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.116", ] [[package]] @@ -5290,18 +5581,18 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "1.0.0+spec-1.1.0" +version = "0.7.5+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32c2555c699578a4f59f0cc68e5116c8d7cabbd45e1409b989d4be085b53f13e" +checksum = "92e1cfed4a3038bc5a127e35a2d360f145e1f4b971b551a2ba5fd7aedf7e1347" dependencies = [ "serde_core", ] [[package]] name = "toml_edit" -version = "0.25.4+spec-1.1.0" +version = "0.23.10+spec-1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7193cbd0ce53dc966037f54351dbbcf0d5a642c7f0038c382ef9e677ce8c13f2" +checksum = "84c8b9f757e028cee9fa244aea147aab2a9ec09d5325a9b01e0a49730c2b5269" dependencies = [ "indexmap 2.13.0", "toml_datetime", @@ -5311,18 +5602,18 @@ dependencies = [ [[package]] name = "toml_parser" -version = "1.0.9+spec-1.1.0" +version = "1.0.8+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "702d4415e08923e7e1ef96cd5727c0dfed80b4d2fa25db9647fe5eb6f7c5a4c4" +checksum = "0742ff5ff03ea7e67c8ae6c93cac239e0d9784833362da3f9a9c1da8dfefcbdc" dependencies = [ "winnow", ] [[package]] name = "tonic" -version = "0.14.5" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fec7c61a0695dc1887c1b53952990f3ad2e3a31453e1f49f10e75424943a93ec" +checksum = "7f32a6f80051a4111560201420c7885d0082ba9efe2ab61875c587bb6b18b9a0" dependencies = [ "async-trait", "axum", @@ -5426,7 +5717,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.116", ] [[package]] @@ -5450,19 +5741,36 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-serde" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704b1aeb7be0d0a84fc9828cae51dab5970fee5088f83d1dd7ee6f6246fc6ff1" +dependencies = [ + "serde", + "tracing-core", +] + [[package]] name = "tracing-subscriber" -version = "0.3.23" +version = "0.3.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +checksum = "2f30143827ddab0d256fd843b7a66d164e9f271cfa0dde49142c5ca0ca291f1e" dependencies = [ "chrono", + "matchers", "nu-ansi-term", + "once_cell", + "regex-automata", + "serde", + "serde_json", "sharded-slab", "smallvec", "thread_local", + "tracing", "tracing-core", "tracing-log", + "tracing-serde", ] [[package]] @@ -5565,9 +5873,9 @@ checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" [[package]] name = "unicode-ident" -version = "1.0.24" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +checksum = "537dd038a89878be9b64dd4bd1b260315c1bb94f4d784956b81e27a088d9a09e" [[package]] name = "unicode-normalization" @@ -5649,11 +5957,11 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.22.0" +version = "1.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a68d3c8f01c0cfa54a75291d83601161799e4a89a39e0929f4b0354d88757a37" +checksum = "b672338555252d43fd2240c714dc444b8c6fb0a5c5335e65a07bba7742735ddb" dependencies = [ - "getrandom 0.4.2", + "getrandom 0.4.1", "js-sys", "serde_core", "wasm-bindgen", @@ -5743,9 +6051,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.114" +version = "0.2.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6532f9a5c1ece3798cb1c2cfdba640b9b3ba884f5db45973a6f442510a87d38e" +checksum = "64024a30ec1e37399cf85a7ffefebdb72205ca1c972291c51512360d90bd8566" dependencies = [ "cfg-if", "once_cell", @@ -5756,9 +6064,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.64" +version = "0.4.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9c5522b3a28661442748e09d40924dfb9ca614b21c00d3fd135720e48b67db8" +checksum = "70a6e77fd0ae8029c9ea0063f87c46fde723e7d887703d74ad2616d792e51e6f" dependencies = [ "cfg-if", "futures-util", @@ -5770,9 +6078,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.114" +version = "0.2.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18a2d50fcf105fb33bb15f00e7a77b772945a2ee45dcf454961fd843e74c18e6" +checksum = "008b239d9c740232e71bd39e8ef6429d27097518b6b30bdf9086833bd5b6d608" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -5780,22 +6088,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.114" +version = "0.2.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03ce4caeaac547cdf713d280eda22a730824dd11e6b8c3ca9e42247b25c631e3" +checksum = "5256bae2d58f54820e6490f9839c49780dff84c65aeab9e772f15d5f0e913a55" dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.116", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.114" +version = "0.2.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75a326b8c223ee17883a4251907455a2431acc2791c98c26279376490c378c16" +checksum = "1f01b580c9ac74c8d8f0c0e4afb04eeef2acf145458e52c03845ee9cd23e3d12" dependencies = [ "unicode-ident", ] @@ -5836,9 +6144,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.91" +version = "0.3.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "854ba17bb104abfb26ba36da9729addc7ce7f06f5c0f90f3c391f8461cca21f9" +checksum = "312e32e551d92129218ea9a2452120f4aabc03529ef03e4d0d82fb2780608598" dependencies = [ "js-sys", "wasm-bindgen", @@ -5939,7 +6247,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.116", ] [[package]] @@ -5950,7 +6258,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.116", ] [[package]] @@ -6344,7 +6652,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" dependencies = [ "anyhow", - "heck", + "heck 0.5.0", "wit-parser", ] @@ -6355,10 +6663,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" dependencies = [ "anyhow", - "heck", + "heck 0.5.0", "indexmap 2.13.0", "prettyplease", - "syn 2.0.117", + "syn 2.0.116", "wasm-metadata", "wit-bindgen-core", "wit-component", @@ -6374,7 +6682,7 @@ dependencies = [ "prettyplease", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.116", "wit-bindgen-core", "wit-bindgen-rust", ] @@ -6422,19 +6730,36 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" +[[package]] +name = "x509-parser" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcbc162f30700d6f3f82a24bf7cc62ffe7caea42c0b2cba8bf7f3ae50cf51f69" +dependencies = [ + "asn1-rs 0.6.2", + "data-encoding", + "der-parser 9.0.0", + "lazy_static", + "nom", + "oid-registry 0.7.1", + "rusticata-macros", + "thiserror 1.0.69", + "time", +] + [[package]] name = "x509-parser" version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d43b0f71ce057da06bc0851b23ee24f3f86190b07203dd8f567d0b706a185202" dependencies = [ - "asn1-rs", + "asn1-rs 0.7.1", "aws-lc-rs", "data-encoding", - "der-parser", + "der-parser 10.0.0", "lazy_static", "nom", - "oid-registry", + "oid-registry 0.8.1", "rusticata-macros", "thiserror 2.0.18", "time", @@ -6446,6 +6771,15 @@ version = "0.8.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3" +[[package]] +name = "yaml-rust" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85" +dependencies = [ + "linked-hash-map", +] + [[package]] name = "yasna" version = "0.5.2" @@ -6474,28 +6808,28 @@ checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.116", "synstructure", ] [[package]] name = "zerocopy" -version = "0.8.40" +version = "0.8.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a789c6e490b576db9f7e6b6d661bcc9799f7c0ac8352f56ea20193b2681532e5" +checksum = "db6d35d663eadb6c932438e763b262fe1a70987f9ae936e60158176d710cae4a" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.40" +version = "0.8.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f65c489a7071a749c849713807783f70672b28094011623e200cb86dcb835953" +checksum = "4122cd3169e94605190e77839c9a40d40ed048d305bfdc146e7df40ab0f3e517" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.116", ] [[package]] @@ -6515,7 +6849,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.116", "synstructure", ] @@ -6555,7 +6889,7 @@ checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.116", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 4e4cc762..3ce19c75 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -46,7 +46,7 @@ serde = { version = "1", features = ["derive"] } serde_json = "1" serde_yaml = "0.9" serde_ignored = "0.1" -clap = { version = "4.6.0", features = ["derive"] } +clap = { version = "4.5.54", features = ["derive"] } nix = { version = "0.31.1", features = ["net", "fs", "resource"] } redis = { version = "1.0", features = [ "tokio-native-tls-comp", @@ -55,7 +55,7 @@ redis = { version = "1.0", features = [ ] } native-tls = "0.2" tokio-rustls = "0.26.4" -rustls = { version = "0.23.37", default-features = false, features = [ +rustls = { version = "0.23.36", default-features = false, features = [ "std", "ring", "logging", @@ -93,7 +93,7 @@ log4rs = { version = "1.3", features = [ ] } syslog = "7.0" jsonwebtoken = { version = "10.1", features = ["rust_crypto"] } -uuid = { version = "1.22", features = ["v4", "serde"] } +uuid = { version = "1.20", features = ["v4", "serde"] } url = "2.5" clamav-tcp = "0.2" multer = "3.0" @@ -110,27 +110,28 @@ daemonize = "0.5.0" # pingora-memory-cache = { path = "../pingora/pingora-memory-cache" } wirefilter-engine = { git = "https://github.com/gen0sec/wirefilter", rev = "ab901470a24aad789cb9c03dd214d6c7d4cab589" } -pingora = { git = "https://github.com/gen0sec/pingora", rev = "e82e0689ce62280f85c659dbfd862e49197133ad", features = [ +pingora = { git = "https://github.com/gen0sec/pingora", rev = "c92146d621542303dd9b93a4cb5252e1eef46c81", features = [ "lb", "openssl", "proxy", ] } -pingora-core = { git = "https://github.com/gen0sec/pingora", rev = "e82e0689ce62280f85c659dbfd862e49197133ad" } -pingora-proxy = { git = "https://github.com/gen0sec/pingora", rev = "e82e0689ce62280f85c659dbfd862e49197133ad" } -pingora-limits = { git = "https://github.com/gen0sec/pingora", rev = "e82e0689ce62280f85c659dbfd862e49197133ad" } -pingora-http = { git = "https://github.com/gen0sec/pingora", rev = "e82e0689ce62280f85c659dbfd862e49197133ad" } -pingora-memory-cache = { git = "https://github.com/gen0sec/pingora", rev = "e82e0689ce62280f85c659dbfd862e49197133ad" } +pingora-core = { git = "https://github.com/gen0sec/pingora", rev = "c92146d621542303dd9b93a4cb5252e1eef46c81" } +pingora-proxy = { git = "https://github.com/gen0sec/pingora", rev = "c92146d621542303dd9b93a4cb5252e1eef46c81" } +pingora-limits = { git = "https://github.com/gen0sec/pingora", rev = "c92146d621542303dd9b93a4cb5252e1eef46c81" } +pingora-http = { git = "https://github.com/gen0sec/pingora", rev = "c92146d621542303dd9b93a4cb5252e1eef46c81" } +pingora-memory-cache = { git = "https://github.com/gen0sec/pingora", rev = "c92146d621542303dd9b93a4cb5252e1eef46c81" } # JA4+ fingerprinting library #nstealth = { path = "../nstealth" } -nstealth = { git = "https://github.com/gen0sec/nstealth", rev = "f9a6323d41acf8b7d45e33ae7e4e863ec716ba05" } +nstealth = { git = "https://github.com/gen0sec/nstealth", rev = "3c87751b9d9537b055a119f155a730360a7d0078" } mimalloc = { version = "0.1.48", default-features = false } +crossbeam-channel = "0.5" dashmap = "7.0.0-rc2" ctrlc = "3.5.0" arc-swap = "1.8.0" prometheus = "0.14.0" -once_cell = "1.21.4" +once_cell = "1.21.3" maxminddb = "0.27" memmap2 = "0.9" axum-server = { version = "0.8.0", features = ["tls-openssl"] } @@ -155,12 +156,14 @@ libbpf-rs = { version = "0.26.0", optional = true } [target.'cfg(target_os = "linux")'.dependencies] nftables = "0.6" iptables = "0.6" +thalamus = { git = "https://github.com/gen0sec/thalamus", branch = "blocking", default-features = false, optional = true } [dev-dependencies] serial_test = "3.3" -tempfile = "3.27.0" +tempfile = "3.25.0" [features] default = ["bpf"] bpf = ["dep:libbpf-rs", "dep:libbpf-cargo", "dep:vmlinux"] disable-bpf = [] +thalamus-ids = ["dep:thalamus"] diff --git a/build.rs b/build.rs index cfa2eb21..0b1475b7 100644 --- a/build.rs +++ b/build.rs @@ -18,17 +18,7 @@ fn main() { println!("cargo:rerun-if-changed={}", JA4TS_SRC); println!("cargo:rerun-if-changed={}/filter.h", HEADER_DIR); println!("cargo:rerun-if-changed={}/xdp_maps.h", HEADER_DIR); - println!( - "cargo:rerun-if-changed={}/lib/tcp_fingerprinting.h", - HEADER_DIR - ); - println!("cargo:rerun-if-changed={}/lib/firewall.h", HEADER_DIR); - println!("cargo:rerun-if-changed={}/lib/helper.h", HEADER_DIR); - println!( - "cargo:rerun-if-changed={}/lib/latency_tracking.h", - HEADER_DIR - ); - println!("cargo:rerun-if-changed={}/include/common.h", HEADER_DIR); + println!("cargo:rerun-if-changed={}/lib/ids_export.h", HEADER_DIR); let bpf_disabled = env::var_os("CARGO_FEATURE_DISABLE_BPF").is_some(); let bpf_feature_set = env::var_os("CARGO_FEATURE_BPF").is_some(); diff --git a/src/bpf/lib/ratelimit.h b/src/bpf/lib/ratelimit.h new file mode 100644 index 00000000..bf18d28d --- /dev/null +++ b/src/bpf/lib/ratelimit.h @@ -0,0 +1,108 @@ +#pragma once + +#include "common.h" + +#include "../xdp_maps.h" +#include "firewall.h" +#include "vmlinux.h" + +struct ratelimiter_config_t { + __u64 TOKENS_PER_REQUEST; // tokens consumed per request + __u64 REFILL_RATE; // tokens added per sec + __u64 MAX_BUCKET_CAPACITY; // refill_rate * (max_bucket_capacity / + // refill_rate) = allow x request burst +}; + +// set this before loading the script +volatile struct ratelimiter_config_t ratelimiter_config = {1, 10, 100}; + +static __always_inline void refill_tokens(struct ratelimit_bucket_value *rl_val, + __u64 now) { + __u64 elapsed_ns = now - rl_val->last_topup; + + // Calculate tokens to add: (elapsed_seconds * REFILL_RATE) + __u64 tokens_to_add = + (elapsed_ns * ratelimiter_config.REFILL_RATE) / 1000000000ULL; + + if (tokens_to_add > 0) { + rl_val->num_of_tokens += tokens_to_add; + + if (rl_val->num_of_tokens > ratelimiter_config.MAX_BUCKET_CAPACITY) { + rl_val->num_of_tokens = ratelimiter_config.MAX_BUCKET_CAPACITY; + } + + rl_val->last_topup = now; + } +} + +static __noinline __u8 ipv4_syn_ratelimit(__be32 *addr, struct tcphdr *tcph) { + __u64 now = bpf_ktime_get_ns(); + + struct ratelimit_bucket_value *bucket = + bpf_map_lookup_elem(&ipv4_syn_bucket_store, addr); + + if (!bucket) { + struct ratelimit_bucket_value new_bucket = {}; + new_bucket.last_topup = now; + new_bucket.num_of_tokens = ratelimiter_config.MAX_BUCKET_CAPACITY; + bpf_map_update_elem(&ipv4_syn_bucket_store, &addr, &new_bucket, BPF_ANY); + bpf_printk("Bucket created for addr: %d", addr); + return XDP_PASS; + } + + refill_tokens(bucket, now); + + if (bucket->num_of_tokens >= ratelimiter_config.TOKENS_PER_REQUEST) { + bucket->num_of_tokens -= ratelimiter_config.TOKENS_PER_REQUEST; + bpf_printk("Packet passed for addr: %d", addr); + return XDP_PASS; + } + + bpf_printk("Packet dropped for addr: %d", addr); + return XDP_DROP; +} + +static __noinline __u8 ipv6_syn_ratelimit(ipv6_addr *addr, + struct tcphdr *tcph) { + __u64 now = bpf_ktime_get_ns(); + + struct ratelimit_bucket_value *bucket = + bpf_map_lookup_elem(&ipv6_syn_bucket_store, addr); + + if (!bucket) { + struct ratelimit_bucket_value new_bucket = {}; + new_bucket.last_topup = now; + new_bucket.num_of_tokens = ratelimiter_config.MAX_BUCKET_CAPACITY; + bpf_map_update_elem(&ipv6_syn_bucket_store, addr, &new_bucket, BPF_ANY); + + return XDP_PASS; + } + + refill_tokens(bucket, now); + + if (bucket->num_of_tokens >= ratelimiter_config.TOKENS_PER_REQUEST) { + bucket->num_of_tokens -= ratelimiter_config.TOKENS_PER_REQUEST; + return XDP_PASS; + } + + return XDP_DROP; +} + +int __noinline xdp_ratelimit(struct iphdr *iph, struct ipv6hdr *ip6h, + struct tcphdr *tcph) { + if (tcph) { + if (iph) { + if (ipv4_syn_ratelimit(&iph->saddr, tcph) == XDP_DROP) { + return XDP_DROP; + } + } else if (ip6h) { + ipv6_addr *ipv6_saddr_ptr = (ipv6_addr *)&ip6h->saddr; + + if (ipv6_syn_ratelimit(ipv6_saddr_ptr, tcph) == XDP_DROP) { + return XDP_DROP; + } + } + } + + return XDP_PASS; +} \ No newline at end of file diff --git a/src/cli.rs b/src/cli.rs index e73fa7c4..8fc8ab80 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -175,7 +175,7 @@ fn default_log_sending_enabled() -> bool { } fn default_include_response_body() -> bool { - false + true } fn default_max_body_size() -> usize { @@ -295,7 +295,7 @@ impl Config { base_url: "https://api.gen0sec.com/v1".to_string(), threat: GeoipDatabaseConfig::default(), log_sending_enabled: true, - include_response_body: false, + include_response_body: true, max_body_size: 1024 * 1024, // 1MB captcha: CaptchaConfig { site_key: None, @@ -550,7 +550,7 @@ impl Config { } } if let Ok(val) = env::var("AX_ARXIGNIS_INCLUDE_RESPONSE_BODY") { - self.arxignis.include_response_body = val.parse().unwrap_or(false); + self.arxignis.include_response_body = val.parse().unwrap_or(true); } if let Ok(val) = env::var("AX_ARXIGNIS_MAX_BODY_SIZE") { self.arxignis.max_body_size = val.parse().unwrap_or(1024 * 1024); @@ -682,8 +682,8 @@ pub struct Args { #[arg(long)] pub arxignis_log_sending_enabled: Option, - /// Include response body in access logs (may capture sensitive data) - #[arg(long, default_value_t = false)] + /// Include response body in access logs + #[arg(long, default_value_t = true)] pub arxignis_include_response_body: bool, /// Maximum size for request/response bodies in access logs (bytes) @@ -905,15 +905,6 @@ pub struct PingoraConfig { pub healthcheck_interval: u16, #[serde(default)] pub proxy_protocol: ProxyProtocolConfig, - /// Enable HTTP/2 cleartext (h2c) on the plaintext HTTP listener - #[serde(default)] - pub h2c: bool, - /// Allow proxying HTTP CONNECT requests (tunneling/WebSocket upgrades) - #[serde(default)] - pub allow_connect_method_proxying: bool, - /// Maximum requests per connection before closing (default: no limit) - #[serde(default)] - pub keepalive_request_limit: Option, } fn default_pingora_tls_grade() -> String { @@ -994,9 +985,6 @@ impl PingoraConfig { app_config.healthcheck_method = self.healthcheck_method.clone(); app_config.healthcheck_interval = self.healthcheck_interval; app_config.proxy_protocol_enabled = self.proxy_protocol.enabled; - app_config.h2c = self.h2c; - app_config.allow_connect_method_proxying = self.allow_connect_method_proxying; - app_config.keepalive_request_limit = self.keepalive_request_limit; // Parse config_address to local_server if let Some((ip, port_str)) = self.config_address.split_once(':') { diff --git a/src/core/cli.rs b/src/core/cli.rs index 7a96f293..92427b01 100644 --- a/src/core/cli.rs +++ b/src/core/cli.rs @@ -52,6 +52,8 @@ pub struct Config { pub network: NetworkConfig, #[serde(default)] pub firewall: FirewallConfig, + #[serde(default)] + pub ids: IdsConfig, #[serde(default, alias = "arxignis")] pub platform: Gen0SecConfig, #[serde(default)] @@ -148,6 +150,135 @@ pub struct NetworkConfig { pub ip_version: String, } +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RateLimiterConfig { + #[serde(default = "default_status")] + pub enabled: bool, + #[serde(default = "default_request_per_sec")] + pub request_per_sec: u64, + #[serde(default = "default_burst_factor")] + pub burst_factor: f32, + #[serde(default = "default_ratelimit_map_size")] + pub ipv4_map_size: u32, + #[serde(default = "default_ratelimit_map_size")] + pub ipv6_map_size: u32, +} + +fn default_status() -> bool { + return true; +} + +fn default_request_per_sec() -> u64 { + 1000 +} + +fn default_burst_factor() -> f32 { + 1.5 +} + +fn default_ratelimit_map_size() -> u32 { + 50000 +} + +impl Default for RateLimiterConfig { + fn default() -> Self { + Self { + enabled: true, + request_per_sec: default_request_per_sec(), + burst_factor: default_burst_factor(), + ipv4_map_size: default_ratelimit_map_size(), + ipv6_map_size: default_ratelimit_map_size(), + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum IdsCaptureMode { + Xdp, +} + +impl Default for IdsCaptureMode { + fn default() -> Self { + Self::Xdp + } +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct IdsConfig { + #[serde(default)] + pub enabled: bool, + #[serde(default)] + pub capture_mode: IdsCaptureMode, + #[serde(default)] + pub rule_paths: Vec, + #[serde(default)] + pub address_vars: std::collections::HashMap, + #[serde(default)] + pub port_vars: std::collections::HashMap, + #[serde(default = "default_ids_snaplen")] + pub snaplen: u32, + #[serde(default = "default_ids_poll_timeout_ms")] + pub poll_timeout_ms: u64, + #[serde(default = "default_ids_flow_timeout_secs")] + pub flow_timeout_secs: u64, + #[serde(default = "default_ids_max_flows")] + pub max_flows: usize, + #[serde(default = "default_ids_cleanup_interval_secs")] + pub cleanup_interval_secs: u64, + #[serde(default = "default_ids_stats_log_interval_secs")] + pub stats_log_interval_secs: u64, + #[serde(default)] + pub enforce_block: bool, +} + +fn default_ids_snaplen() -> u32 { + 512 +} + +fn default_ids_poll_timeout_ms() -> u64 { + 100 +} + +fn default_ids_flow_timeout_secs() -> u64 { + 120 +} + +fn default_ids_max_flows() -> usize { + 200_000 +} + +fn default_ids_cleanup_interval_secs() -> u64 { + 30 +} + +fn default_ids_stats_log_interval_secs() -> u64 { + 5 +} + +impl Default for IdsConfig { + fn default() -> Self { + Self { + enabled: false, + capture_mode: IdsCaptureMode::default(), + rule_paths: Vec::new(), + address_vars: std::collections::HashMap::new(), + port_vars: std::collections::HashMap::new(), + snaplen: default_ids_snaplen(), + poll_timeout_ms: default_ids_poll_timeout_ms(), + flow_timeout_secs: default_ids_flow_timeout_secs(), + max_flows: default_ids_max_flows(), + cleanup_interval_secs: default_ids_cleanup_interval_secs(), + stats_log_interval_secs: default_ids_stats_log_interval_secs(), + enforce_block: false, + } + } +} + +fn is_ids_config_default(cfg: &IdsConfig) -> bool { + cfg == &IdsConfig::default() +} + #[derive(Debug, Clone, Serialize, Deserialize, Default)] pub struct FirewallConfig { /// Firewall backend mode: auto, xdp, nftables, iptables, none @@ -155,6 +286,10 @@ pub struct FirewallConfig { pub mode: crate::firewall::FirewallMode, #[serde(default)] pub disable_xdp: bool, + #[serde(default)] + pub ratelimiter: RateLimiterConfig, + #[serde(default)] + pub ids: IdsConfig, } fn default_ip_version() -> String { @@ -187,7 +322,7 @@ fn default_log_sending_enabled() -> bool { } fn default_include_response_body() -> bool { - false + true } fn default_max_body_size() -> usize { @@ -390,8 +525,39 @@ pub struct CaptchaConfig { pub cache_ttl: u64, } -impl Default for Config { - fn default() -> Self { +impl Config { + pub fn load_from_file(path: &PathBuf) -> Result<(Self, ConfigDiagnostics)> { + let content = std::fs::read_to_string(path)?; + let mut unused = Vec::new(); + let deserializer = serde_yaml::Deserializer::from_str(&content); + let mut config: Config = + serde_ignored::deserialize(deserializer, |path| unused.push(path.to_string()))?; + let mut diagnostics = ConfigDiagnostics::default(); + if !is_ids_config_default(&config.firewall.ids) { + if is_ids_config_default(&config.ids) { + config.ids = config.firewall.ids.clone(); + diagnostics.warnings.push( + "Deprecated config path 'firewall.ids' detected; migrated to top-level 'ids'" + .to_string(), + ); + } else { + diagnostics.warnings.push( + "Both 'ids' and deprecated 'firewall.ids' are set; using top-level 'ids'" + .to_string(), + ); + } + } + if !unused.is_empty() { + diagnostics.warnings.push(format!( + "Unused config options in {}: {}", + path.display(), + unused.join(", ") + )); + } + Ok((config, diagnostics)) + } + + pub fn default() -> Self { Self { mode: "agent".to_string(), multi_thread: None, @@ -404,13 +570,16 @@ impl Default for Config { firewall: FirewallConfig { mode: crate::firewall::FirewallMode::default(), disable_xdp: false, + ratelimiter: RateLimiterConfig::default(), + ids: IdsConfig::default(), }, + ids: IdsConfig::default(), platform: Gen0SecConfig { api_key: "".to_string(), base_url: "https://api.gen0sec.com/v1".to_string(), threat: GeoipDatabaseConfig::default(), log_sending_enabled: true, - include_response_body: false, + include_response_body: true, max_body_size: 1024 * 1024, // 1MB }, logging: LoggingConfig { @@ -431,25 +600,6 @@ impl Default for Config { proxy: ProxyConfig::default(), } } -} - -impl Config { - pub fn load_from_file(path: &PathBuf) -> Result<(Self, ConfigDiagnostics)> { - let content = std::fs::read_to_string(path)?; - let mut unused = Vec::new(); - let deserializer = serde_yaml::Deserializer::from_str(&content); - let config: Config = - serde_ignored::deserialize(deserializer, |path| unused.push(path.to_string()))?; - let mut diagnostics = ConfigDiagnostics::default(); - if !unused.is_empty() { - diagnostics.warnings.push(format!( - "Unused config options in {}: {}", - path.display(), - unused.join(", ") - )); - } - Ok((config, diagnostics)) - } pub fn merge_with_args(&mut self, args: &Args) { // Override config values with command line arguments if provided @@ -829,17 +979,17 @@ impl Config { } // Multi-thread override - if let Some(val) = get_env("MULTI_THREAD") - && let Ok(parsed) = val.parse::() - { - self.multi_thread = Some(parsed); + if let Some(val) = get_env("MULTI_THREAD") { + if let Ok(parsed) = val.parse::() { + self.multi_thread = Some(parsed); + } } // Worker threads override - if let Some(val) = get_env("WORKER_THREADS") - && let Ok(parsed) = val.parse::() - { - self.worker_threads = Some(parsed); + if let Some(val) = get_env("WORKER_THREADS") { + if let Ok(parsed) = val.parse::() { + self.worker_threads = Some(parsed); + } } // Redis configuration overrides @@ -895,10 +1045,10 @@ impl Config { if let Some(val) = client_key_path { ssl.client_key_path = Some(val); } - if let Some(val) = insecure_val - && let Ok(insecure) = val.parse::() - { - ssl.insecure = insecure; + if let Some(val) = insecure_val { + if let Ok(insecure) = val.parse::() { + ssl.insecure = insecure; + } } } @@ -940,6 +1090,64 @@ impl Config { _ => crate::firewall::FirewallMode::Auto, }; } + if let Some(val) = get_env("IDS_ENABLED").or_else(|| get_env("FIREWALL_IDS_ENABLED")) { + self.ids.enabled = val.parse().unwrap_or(false); + } + if let Some(val) = get_env("IDS_RULE_PATHS").or_else(|| get_env("FIREWALL_IDS_RULE_PATHS")) + { + self.ids.rule_paths = val + .split(',') + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) + .collect(); + } + if let Some(val) = get_env("IDS_SNAPLEN").or_else(|| get_env("FIREWALL_IDS_SNAPLEN")) { + self.ids.snaplen = val.parse().unwrap_or(default_ids_snaplen()); + } + if let Some(val) = + get_env("IDS_CAPTURE_MODE").or_else(|| get_env("FIREWALL_IDS_CAPTURE_MODE")) + { + self.ids.capture_mode = match val.to_ascii_lowercase().as_str() { + "xdp" => IdsCaptureMode::Xdp, + _ => { + log::warn!( + "Invalid IDS_CAPTURE_MODE value '{}', only 'xdp' is supported", + val + ); + IdsCaptureMode::Xdp + } + }; + } + if let Some(val) = + get_env("IDS_POLL_TIMEOUT_MS").or_else(|| get_env("FIREWALL_IDS_POLL_TIMEOUT_MS")) + { + self.ids.poll_timeout_ms = val.parse().unwrap_or(default_ids_poll_timeout_ms()); + } + if let Some(val) = + get_env("IDS_FLOW_TIMEOUT_SECS").or_else(|| get_env("FIREWALL_IDS_FLOW_TIMEOUT_SECS")) + { + self.ids.flow_timeout_secs = val.parse().unwrap_or(default_ids_flow_timeout_secs()); + } + if let Some(val) = get_env("IDS_MAX_FLOWS").or_else(|| get_env("FIREWALL_IDS_MAX_FLOWS")) { + self.ids.max_flows = val.parse().unwrap_or(default_ids_max_flows()); + } + if let Some(val) = get_env("IDS_CLEANUP_INTERVAL_SECS") + .or_else(|| get_env("FIREWALL_IDS_CLEANUP_INTERVAL_SECS")) + { + self.ids.cleanup_interval_secs = + val.parse().unwrap_or(default_ids_cleanup_interval_secs()); + } + if let Some(val) = get_env("IDS_STATS_LOG_INTERVAL_SECS") + .or_else(|| get_env("FIREWALL_IDS_STATS_LOG_INTERVAL_SECS")) + { + self.ids.stats_log_interval_secs = + val.parse().unwrap_or(default_ids_stats_log_interval_secs()); + } + if let Some(val) = + get_env("IDS_ENFORCE_BLOCK").or_else(|| get_env("FIREWALL_IDS_ENFORCE_BLOCK")) + { + self.ids.enforce_block = val.parse().unwrap_or(false); + } // Gen0Sec configuration overrides // Supports: AX_API_KEY, API_KEY, AX_ARXIGNIS_API_KEY, ARXIGNIS_API_KEY (backward compat) @@ -949,13 +1157,13 @@ impl Config { if let Some(val) = get_env_arxignis("BASE_URL") { self.platform.base_url = val; } - if let Some(val) = get_env_arxignis("LOG_SENDING_ENABLED") - && let Ok(parsed) = val.parse::() - { - self.platform.log_sending_enabled = parsed; + if let Some(val) = get_env_arxignis("LOG_SENDING_ENABLED") { + if let Ok(parsed) = val.parse::() { + self.platform.log_sending_enabled = parsed; + } } if let Some(val) = get_env_arxignis("INCLUDE_RESPONSE_BODY") { - self.platform.include_response_body = val.parse().unwrap_or(false); + self.platform.include_response_body = val.parse().unwrap_or(true); } if let Some(val) = get_env_arxignis("MAX_BODY_SIZE") { self.platform.max_body_size = val.parse().unwrap_or(1024 * 1024); @@ -1065,83 +1273,83 @@ impl Config { if let Some(val) = get_env("THREAT_MMDB_PATH") { self.platform.threat.path = Some(PathBuf::from(val)); } - if let Some(val) = get_env("THREAT_MMDB_REFRESH_SECS") - && let Ok(refresh_secs) = val.parse::() - { - self.platform.threat.refresh_secs = Some(refresh_secs); + if let Some(val) = get_env("THREAT_MMDB_REFRESH_SECS") { + if let Ok(refresh_secs) = val.parse::() { + self.platform.threat.refresh_secs = Some(refresh_secs); + } } // BPF stats configuration overrides if let Some(val) = get_env("BPF_STATS_ENABLED") { self.logging.bpf_stats.enabled = val.parse().unwrap_or(true); } - if let Some(val) = get_env("BPF_STATS_LOG_INTERVAL") - && let Ok(interval) = val.parse::() - { - self.logging.bpf_stats.log_interval_secs = interval; + if let Some(val) = get_env("BPF_STATS_LOG_INTERVAL") { + if let Ok(interval) = val.parse::() { + self.logging.bpf_stats.log_interval_secs = interval; + } } if let Some(val) = get_env("BPF_STATS_ENABLE_DROPPED_IP_EVENTS") { self.logging.bpf_stats.enable_dropped_ip_events = val.parse().unwrap_or(true); } - if let Some(val) = get_env("BPF_STATS_DROPPED_IP_EVENTS_INTERVAL") - && let Ok(interval) = val.parse::() - { - self.logging.bpf_stats.dropped_ip_events_interval_secs = interval; + if let Some(val) = get_env("BPF_STATS_DROPPED_IP_EVENTS_INTERVAL") { + if let Ok(interval) = val.parse::() { + self.logging.bpf_stats.dropped_ip_events_interval_secs = interval; + } } // TCP fingerprint configuration overrides if let Some(val) = get_env("TCP_FINGERPRINT_ENABLED") { self.logging.tcp_fingerprint.enabled = val.parse().unwrap_or(true); } - if let Some(val) = get_env("TCP_FINGERPRINT_LOG_INTERVAL") - && let Ok(interval) = val.parse::() - { - self.logging.tcp_fingerprint.log_interval_secs = interval; + if let Some(val) = get_env("TCP_FINGERPRINT_LOG_INTERVAL") { + if let Ok(interval) = val.parse::() { + self.logging.tcp_fingerprint.log_interval_secs = interval; + } } if let Some(val) = get_env("TCP_FINGERPRINT_ENABLE_EVENTS") { self.logging.tcp_fingerprint.enable_fingerprint_events = val.parse().unwrap_or(true); } - if let Some(val) = get_env("TCP_FINGERPRINT_EVENTS_INTERVAL") - && let Ok(interval) = val.parse::() - { - self.logging - .tcp_fingerprint - .fingerprint_events_interval_secs = interval; + if let Some(val) = get_env("TCP_FINGERPRINT_EVENTS_INTERVAL") { + if let Ok(interval) = val.parse::() { + self.logging + .tcp_fingerprint + .fingerprint_events_interval_secs = interval; + } } - if let Some(val) = get_env("TCP_FINGERPRINT_MIN_PACKET_COUNT") - && let Ok(count) = val.parse::() - { - self.logging.tcp_fingerprint.min_packet_count = count; + if let Some(val) = get_env("TCP_FINGERPRINT_MIN_PACKET_COUNT") { + if let Ok(count) = val.parse::() { + self.logging.tcp_fingerprint.min_packet_count = count; + } } - if let Some(val) = get_env("TCP_FINGERPRINT_MIN_CONNECTION_DURATION") - && let Ok(duration) = val.parse::() - { - self.logging.tcp_fingerprint.min_connection_duration_secs = duration; + if let Some(val) = get_env("TCP_FINGERPRINT_MIN_CONNECTION_DURATION") { + if let Ok(duration) = val.parse::() { + self.logging.tcp_fingerprint.min_connection_duration_secs = duration; + } } // SSH fingerprint configuration overrides if let Some(val) = get_env("SSH_FINGERPRINT_ENABLED") { self.logging.ssh_fingerprint.enabled = val.parse().unwrap_or(true); } - if let Some(val) = get_env("SSH_FINGERPRINT_LOG_INTERVAL") - && let Ok(interval) = val.parse::() - { - self.logging.ssh_fingerprint.log_interval_secs = interval; + if let Some(val) = get_env("SSH_FINGERPRINT_LOG_INTERVAL") { + if let Ok(interval) = val.parse::() { + self.logging.ssh_fingerprint.log_interval_secs = interval; + } } if let Some(val) = get_env("SSH_FINGERPRINT_ENABLE_EVENTS") { self.logging.ssh_fingerprint.enable_fingerprint_events = val.parse().unwrap_or(true); } - if let Some(val) = get_env("SSH_FINGERPRINT_EVENTS_INTERVAL") - && let Ok(interval) = val.parse::() - { - self.logging - .ssh_fingerprint - .fingerprint_events_interval_secs = interval; + if let Some(val) = get_env("SSH_FINGERPRINT_EVENTS_INTERVAL") { + if let Ok(interval) = val.parse::() { + self.logging + .ssh_fingerprint + .fingerprint_events_interval_secs = interval; + } } - if let Some(val) = get_env("SSH_FINGERPRINT_SESSION_TIMEOUT") - && let Ok(timeout) = val.parse::() - { - self.logging.ssh_fingerprint.session_timeout_secs = timeout; + if let Some(val) = get_env("SSH_FINGERPRINT_SESSION_TIMEOUT") { + if let Ok(timeout) = val.parse::() { + self.logging.ssh_fingerprint.session_timeout_secs = timeout; + } } // Proxy address configuration overrides @@ -1168,10 +1376,10 @@ impl Config { if let Some(val) = get_env("UPSTREAM_HEALTHCHECK_METHOD") { self.proxy.upstream.healthcheck.method = val; } - if let Some(val) = get_env("UPSTREAM_HEALTHCHECK_INTERVAL") - && let Ok(interval) = val.parse::() - { - self.proxy.upstream.healthcheck.interval = interval.min(u16::MAX as u64) as u16; + if let Some(val) = get_env("UPSTREAM_HEALTHCHECK_INTERVAL") { + if let Ok(interval) = val.parse::() { + self.proxy.upstream.healthcheck.interval = interval.min(u16::MAX as u64) as u16; + } } // GeoIP configuration overrides @@ -1193,20 +1401,20 @@ impl Config { if let Some(val) = get_env("GEOIP_CITY_PATH") { self.proxy.geoip.city.path = Some(PathBuf::from(val)); } - if let Some(val) = get_env("GEOIP_REFRESH_SECS") - && let Ok(refresh_secs) = val.parse::() - { - self.proxy.geoip.refresh_secs = refresh_secs; + if let Some(val) = get_env("GEOIP_REFRESH_SECS") { + if let Ok(refresh_secs) = val.parse::() { + self.proxy.geoip.refresh_secs = refresh_secs; + } } // ACME configuration overrides if let Some(val) = get_env("ACME_ENABLED") { self.proxy.acme.enabled = val.parse().unwrap_or(false); } - if let Some(val) = get_env("ACME_PORT") - && let Ok(port) = val.parse::() - { - self.proxy.acme.port = port; + if let Some(val) = get_env("ACME_PORT") { + if let Ok(port) = val.parse::() { + self.proxy.acme.port = port; + } } if let Some(val) = get_env("ACME_EMAIL") { self.proxy.acme.email = Some(val); @@ -1269,8 +1477,8 @@ pub struct Args { #[arg(long)] pub arxignis_log_sending_enabled: Option, - /// Include response body in access logs (may capture sensitive data) - #[arg(long, default_value_t = false)] + /// Include response body in access logs + #[arg(long, default_value_t = true)] pub arxignis_include_response_body: bool, /// Maximum size for request/response bodies in access logs (bytes) @@ -1835,19 +2043,6 @@ pub struct ProxyConfig { pub upstream: UpstreamConfig, #[serde(default)] pub protocol: ProxyProtocolConfig, - /// Enable HTTP/2 cleartext (h2c) on the plaintext HTTP listener. - /// When enabled, clients can use HTTP/2 prior knowledge on the non-TLS port. - #[serde(default)] - pub h2c: bool, - /// Allow proxying HTTP CONNECT requests (used for tunneling/WebSocket upgrades). - /// When disabled, CONNECT requests are rejected with 405. - #[serde(default)] - pub allow_connect_method_proxying: bool, - /// Maximum number of requests per connection before closing. - /// Closing connections periodically frees per-connection memory allocations. - /// Default: no limit. - #[serde(default)] - pub keepalive_request_limit: Option, /// GeoIP configuration #[serde(default)] pub geoip: GeoipConfig, @@ -1926,9 +2121,6 @@ impl Default for ProxyConfig { port: 9180, bind_ip: "127.0.0.1".to_string(), }, - h2c: false, - allow_connect_method_proxying: false, - keepalive_request_limit: None, } } } @@ -1986,29 +2178,27 @@ fn default_acme_storage_path() -> String { impl ProxyConfig { /// Convert ProxyConfig to AppConfig for compatibility with old proxy system pub fn to_app_config(&self) -> crate::utils::structs::AppConfig { + let mut app_config = crate::utils::structs::AppConfig::default(); + app_config.proxy_address_http = self.address_http.clone(); + app_config.proxy_address_tls = self.address_tls.clone(); + app_config.proxy_certificates = self.certificates.clone(); + app_config.proxy_tls_grade = Some(self.tls_grade.clone()); + app_config.default_certificate = self.default_certificate.clone(); + app_config.upstreams_conf = self.upstream.conf.clone(); + app_config.healthcheck_method = self.upstream.healthcheck.method.clone(); + app_config.healthcheck_interval = self.upstream.healthcheck.interval; + app_config.proxy_protocol_enabled = self.protocol.enabled; + // Parse proxy_address_tls to proxy_port_tls - let proxy_port_tls = self.address_tls.as_ref().and_then(|tls_addr| { - tls_addr - .split_once(':') - .and_then(|(_, port_str)| port_str.parse::().ok()) - }); - - crate::utils::structs::AppConfig { - proxy_address_http: self.address_http.clone(), - proxy_address_tls: self.address_tls.clone(), - proxy_certificates: self.certificates.clone(), - proxy_tls_grade: Some(self.tls_grade.clone()), - default_certificate: self.default_certificate.clone(), - upstreams_conf: self.upstream.conf.clone(), - healthcheck_method: self.upstream.healthcheck.method.clone(), - healthcheck_interval: self.upstream.healthcheck.interval, - proxy_protocol_enabled: self.protocol.enabled, - proxy_port_tls, - h2c: self.h2c, - allow_connect_method_proxying: self.allow_connect_method_proxying, - keepalive_request_limit: self.keepalive_request_limit, - ..crate::utils::structs::AppConfig::default() + if let Some(ref tls_addr) = self.address_tls { + if let Some((_, port_str)) = tls_addr.split_once(':') { + if let Ok(port) = port_str.parse::() { + app_config.proxy_port_tls = Some(port); + } + } } + + app_config } } @@ -2089,6 +2279,53 @@ prefix: "test:prefix" assert!(config.ssl.is_none()); } + #[test] + fn test_ids_top_level_deserialize() { + let yaml = r#" +mode: "proxy" +ids: + enabled: true + rule_paths: + - "/tmp/rules/*.rules" + snaplen: 256 +"#; + let config: Config = serde_yaml::from_str(yaml).unwrap(); + assert!(config.ids.enabled); + assert_eq!( + config.ids.rule_paths, + vec!["/tmp/rules/*.rules".to_string()] + ); + assert_eq!(config.ids.snaplen, 256); + } + + #[test] + fn test_ids_legacy_firewall_migration() { + let file = tempfile::NamedTempFile::new().unwrap(); + std::fs::write( + file.path(), + r#" +mode: "proxy" +firewall: + ids: + enabled: true + rule_paths: + - "/tmp/legacy.rules" +"#, + ) + .unwrap(); + let (config, diagnostics) = Config::load_from_file(&file.path().to_path_buf()).unwrap(); + assert!(config.ids.enabled); + assert_eq!(config.ids.rule_paths, vec!["/tmp/legacy.rules".to_string()]); + assert!( + diagnostics + .warnings + .iter() + .any(|w| w.contains("Deprecated config path 'firewall.ids'")), + "expected legacy migration warning, got: {:?}", + diagnostics.warnings + ); + } + // Helper function to clean up SSL environment variables for test isolation fn cleanup_redis_ssl_env_vars() { unsafe { @@ -2226,6 +2463,20 @@ prefix: "test:prefix" assert!(!config.insecure); } + #[test] + #[serial] + fn test_apply_env_overrides_ids_enabled_new_prefix() { + let mut config = Config::default(); + unsafe { + env::set_var("IDS_ENABLED", "true"); + } + config.apply_env_overrides(); + assert!(config.ids.enabled); + unsafe { + env::remove_var("IDS_ENABLED"); + } + } + #[test] fn test_upstream_healthcheck_defaults_when_upstream_block_missing() { let yaml = r#" diff --git a/src/logger/access_log.rs b/src/logger/access_log.rs index 5d94e625..67508c86 100644 --- a/src/logger/access_log.rs +++ b/src/logger/access_log.rs @@ -1,6 +1,6 @@ use std::collections::HashMap; use std::net::SocketAddr; -use std::time::{SystemTime, UNIX_EPOCH}; +use std::time::{Instant, SystemTime, UNIX_EPOCH}; use chrono::{DateTime, Utc}; use http_body_util::{BodyExt, Full}; @@ -8,7 +8,8 @@ use hyper::{Response, header::HeaderValue}; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; -use crate::utils::fingerprint::ja4_plus::Ja4hFingerprint; +use crate::utils::fingerprint::ja4_plus::{Ja4hFingerprint, Ja4tFingerprint}; +use crate::utils::fingerprint::tcp_fingerprint::TcpFingerprintData; use crate::worker::log::{UnifiedEvent, get_log_sender_config, send_event}; // Re-export for compatibility @@ -51,6 +52,13 @@ pub struct ServerCertInfo { /// performance: PerformanceInfo { /// request_time_ms: Some(150), /// upstream_time_ms: Some(120), +/// pre_upstream_ms: Some(20), +/// upstream_header_time_ms: Some(120), +/// post_upstream_header_ms: Some(10), +/// response_header_to_log_ms: Some(10), +/// logging_prepare_ms: Some(1), +/// post_tls_ids_time_us: Some(40), +/// upstream_time_scope: Some("response_header_ttfb".to_string()), /// }, /// }; /// @@ -107,6 +115,24 @@ pub struct NetworkSummary { pub struct PerformanceInfo { pub request_time_ms: Option, pub upstream_time_ms: Option, + pub pre_upstream_ms: Option, + pub request_filter_total_ms: Option, + pub request_filter_keepalive_ms: Option, + pub request_filter_access_rules_ms: Option, + pub request_filter_tls_fingerprint_ms: Option, + pub request_filter_threat_intel_ms: Option, + pub request_filter_waf_ms: Option, + pub request_filter_routing_ms: Option, + pub upstream_peer_select_ms: Option, + pub gap_request_filter_to_upstream_peer_ms: Option, + pub gap_upstream_peer_to_upstream_start_ms: Option, + pub pre_upstream_unattributed_ms: Option, + pub upstream_header_time_ms: Option, + pub post_upstream_header_ms: Option, + pub response_header_to_log_ms: Option, + pub logging_prepare_ms: Option, + pub post_tls_ids_time_us: Option, + pub upstream_time_scope: Option, } impl AccessLogSummary { @@ -137,6 +163,63 @@ impl AccessLogSummary { if let Some(ms) = self.performance.request_time_ms { parts.push(format!(r#""request_time_ms":{}"#, ms)); } + if let Some(ms) = self.performance.pre_upstream_ms { + parts.push(format!(r#""pre_upstream_ms":{}"#, ms)); + } + if let Some(ms) = self.performance.request_filter_total_ms { + parts.push(format!(r#""request_filter_total_ms":{}"#, ms)); + } + if let Some(ms) = self.performance.request_filter_keepalive_ms { + parts.push(format!(r#""request_filter_keepalive_ms":{}"#, ms)); + } + if let Some(ms) = self.performance.request_filter_access_rules_ms { + parts.push(format!(r#""request_filter_access_rules_ms":{}"#, ms)); + } + if let Some(ms) = self.performance.request_filter_tls_fingerprint_ms { + parts.push(format!(r#""request_filter_tls_fingerprint_ms":{}"#, ms)); + } + if let Some(ms) = self.performance.request_filter_threat_intel_ms { + parts.push(format!(r#""request_filter_threat_intel_ms":{}"#, ms)); + } + if let Some(ms) = self.performance.request_filter_waf_ms { + parts.push(format!(r#""request_filter_waf_ms":{}"#, ms)); + } + if let Some(ms) = self.performance.request_filter_routing_ms { + parts.push(format!(r#""request_filter_routing_ms":{}"#, ms)); + } + if let Some(ms) = self.performance.upstream_peer_select_ms { + parts.push(format!(r#""upstream_peer_select_ms":{}"#, ms)); + } + if let Some(ms) = self.performance.gap_request_filter_to_upstream_peer_ms { + parts.push(format!( + r#""gap_request_filter_to_upstream_peer_ms":{}"#, + ms + )); + } + if let Some(ms) = self.performance.gap_upstream_peer_to_upstream_start_ms { + parts.push(format!( + r#""gap_upstream_peer_to_upstream_start_ms":{}"#, + ms + )); + } + if let Some(ms) = self.performance.pre_upstream_unattributed_ms { + parts.push(format!(r#""pre_upstream_unattributed_ms":{}"#, ms)); + } + if let Some(ms) = self.performance.upstream_header_time_ms { + parts.push(format!(r#""upstream_header_time_ms":{}"#, ms)); + } + if let Some(ms) = self.performance.post_upstream_header_ms { + parts.push(format!(r#""post_upstream_header_ms":{}"#, ms)); + } + if let Some(ms) = self.performance.response_header_to_log_ms { + parts.push(format!(r#""response_header_to_log_ms":{}"#, ms)); + } + if let Some(ms) = self.performance.logging_prepare_ms { + parts.push(format!(r#""logging_prepare_ms":{}"#, ms)); + } + if let Some(us) = self.performance.post_tls_ids_time_us { + parts.push(format!(r#""post_tls_ids_time_us":{}"#, us)); + } format!("{{{}}}", parts.join(",")) } @@ -175,24 +258,59 @@ impl AccessLogSummary { if let Ok(value) = HeaderValue::from_str(&threat.score.to_string()) { headers.insert("X-Threat-Score", value); } - if let Some(country) = &threat.country - && let Ok(value) = HeaderValue::from_str(country) - { - headers.insert("X-Client-Country", value); + if let Some(country) = &threat.country { + if let Ok(value) = HeaderValue::from_str(country) { + headers.insert("X-Client-Country", value); + } } } // Add performance metrics - if let Some(ms) = self.performance.request_time_ms - && let Ok(value) = HeaderValue::from_str(&ms.to_string()) - { - headers.insert("X-Request-Time-Ms", value); + if let Some(ms) = self.performance.request_time_ms { + if let Ok(value) = HeaderValue::from_str(&ms.to_string()) { + headers.insert("X-Request-Time-Ms", value); + } } - if let Some(ms) = self.performance.upstream_time_ms - && let Ok(value) = HeaderValue::from_str(&ms.to_string()) - { - headers.insert("X-Upstream-Time-Ms", value); + if let Some(ms) = self.performance.upstream_time_ms { + if let Ok(value) = HeaderValue::from_str(&ms.to_string()) { + headers.insert("X-Upstream-Time-Ms", value); + } + } + if let Some(ms) = self.performance.pre_upstream_ms { + if let Ok(value) = HeaderValue::from_str(&ms.to_string()) { + headers.insert("X-Pre-Upstream-Ms", value); + } + } + if let Some(ms) = self.performance.request_filter_total_ms { + if let Ok(value) = HeaderValue::from_str(&ms.to_string()) { + headers.insert("X-Request-Filter-Total-Ms", value); + } + } + if let Some(ms) = self.performance.upstream_peer_select_ms { + if let Ok(value) = HeaderValue::from_str(&ms.to_string()) { + headers.insert("X-Upstream-Peer-Select-Ms", value); + } + } + if let Some(ms) = self.performance.pre_upstream_unattributed_ms { + if let Ok(value) = HeaderValue::from_str(&ms.to_string()) { + headers.insert("X-Pre-Upstream-Unattributed-Ms", value); + } + } + if let Some(ms) = self.performance.upstream_header_time_ms { + if let Ok(value) = HeaderValue::from_str(&ms.to_string()) { + headers.insert("X-Upstream-Header-Time-Ms", value); + } + } + if let Some(ms) = self.performance.post_upstream_header_ms { + if let Ok(value) = HeaderValue::from_str(&ms.to_string()) { + headers.insert("X-Post-Upstream-Header-Ms", value); + } + } + if let Some(ms) = self.performance.response_header_to_log_ms { + if let Ok(value) = HeaderValue::from_str(&ms.to_string()) { + headers.insert("X-Response-Header-To-Log-Ms", value); + } } // Add compact JSON summary @@ -219,7 +337,6 @@ pub struct HttpAccessLog { pub geoip: Option, pub upstream: Option, pub performance: Option, - pub fingerprints: Option, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -279,6 +396,7 @@ pub struct TlsDetails { pub sni: Option, pub ja4: Option, pub ja4_unsorted: Option, + pub ja4t: Option, pub server_cert: Option, } @@ -335,31 +453,6 @@ pub struct GeoIpDetails { pub ip_version: u8, } -/// All JA4+ fingerprints collected during request processing -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct FingerprintDetails { - #[serde(skip_serializing_if = "Option::is_none")] - pub ja4: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub ja4_raw: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub ja4h: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub ja4t: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub ja4t_hash: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub ja4s: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub ja4ts: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub ja4ts_hash: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub ja4l: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub ja4x: Option, -} - impl HttpAccessLog { /// Create access log from request parts and response data pub async fn create_from_parts( @@ -370,6 +463,7 @@ impl HttpAccessLog { tls_present: bool, http_valid: bool, _tls_fingerprint: Option<&crate::utils::fingerprint::ja4_plus::Ja4hFingerprint>, + tcp_fingerprint_data: Option<&TcpFingerprintData>, server_cert_info: Option<&ServerCertInfo>, response_data: ResponseData, error_details: Option, @@ -382,8 +476,7 @@ impl HttpAccessLog { tls_cipher: Option, tls_ja4: Option, tls_ja4_unsorted: Option, - fingerprints: Option, - ) -> Result<(), Box> { + ) -> Result> { let timestamp = Utc::now(); let request_id = format!( "req_{}", @@ -393,7 +486,7 @@ impl HttpAccessLog { .as_nanos() ); - let http_details = if http_valid { + let (http_details, ja4t) = if http_valid { // Extract request details let uri = &req_parts.uri; let method = req_parts.method.to_string(); @@ -495,6 +588,18 @@ impl HttpAccessLog { ) }; + // Generate JA4T from TCP fingerprint data if available + let ja4t = tcp_fingerprint_data.map(|tcp_data| { + let ja4t_fp = Ja4tFingerprint::from_tcp_data( + tcp_data.window_size, + tcp_data.ttl, + tcp_data.mss, + tcp_data.window_scale, + &tcp_data.options, + ); + ja4t_fp.fingerprint + }); + let http_details = HttpDetails { method, scheme, @@ -517,9 +622,9 @@ impl HttpAccessLog { body_truncated, }; - Some(http_details) + (Some(http_details), ja4t) } else { - None + (None, None) }; // Process TLS details (no inference; only factual data) @@ -538,6 +643,7 @@ impl HttpAccessLog { sni: tls_sni.clone(), ja4: ja4_value, ja4_unsorted: ja4_unsorted_value, + ja4t: ja4t.clone(), server_cert, }) } else { @@ -594,18 +700,36 @@ impl HttpAccessLog { geoip: geoip_details, upstream: upstream_info, performance: performance_info, - fingerprints, }; // Log to stdout (existing behavior) + let emit_started = Instant::now(); if let Err(e) = access_log.log_to_stdout() { log::warn!("Failed to log access log to stdout: {}", e); } + let emit_elapsed = emit_started.elapsed(); + let emit_ms = emit_elapsed.as_millis() as u64; + let emit_us = emit_elapsed.as_micros().min(u128::from(u64::MAX)) as u64; + let path_for_perf = access_log + .http + .as_ref() + .map(|h| h.path.as_str()) + .unwrap_or(""); + log::info!( + target: "access_log_perf", + "{{\"event\":\"access_log_emit\",\"request_id\":\"{}\",\"src_ip\":\"{}\",\"path\":\"{}\",\"emit_ms\":{},\"emit_us\":{}}}", + access_log.request_id, + access_log.network.src_ip, + path_for_perf, + emit_ms, + emit_us + ); // Send to unified event queue + let request_id = access_log.request_id.clone(); send_event(UnifiedEvent::HttpAccessLog(access_log)); - Ok(()) + Ok(request_id) } /// Create remediation details from WAF result and threat intelligence data. @@ -636,12 +760,13 @@ impl HttpAccessLog { }) .unwrap_or(false); - // Create remediation section if any of: - // 1. WAF action is Block/Challenge/RateLimit (will populate WAF fields) + // Create remediation section if: + // 1. WAF action is Block/Challenge/RateLimit (will populate WAF fields), OR // 2. There's meaningful threat intelligence data (will populate threat fields) - // 3. There's any threat response at all (carries GeoIP data: country, ASN, org) - let has_geoip_data = threat_data.is_some(); - if !has_waf_remediation && !has_meaningful_threat_data && !has_geoip_data { + // Note: WAF fields (waf_action, waf_rule_id, waf_rule_name) are populated for Block/Challenge/RateLimit + // RateLimit is included because it blocks requests when the limit is exceeded + // Allow actions don't populate WAF fields, but remediation section can still exist if there's meaningful threat data + if !has_waf_remediation && !has_meaningful_threat_data { return None; } @@ -804,6 +929,24 @@ impl HttpAccessLog { performance: self.performance.clone().unwrap_or(PerformanceInfo { request_time_ms: None, upstream_time_ms: None, + pre_upstream_ms: None, + request_filter_total_ms: None, + request_filter_keepalive_ms: None, + request_filter_access_rules_ms: None, + request_filter_tls_fingerprint_ms: None, + request_filter_threat_intel_ms: None, + request_filter_waf_ms: None, + request_filter_routing_ms: None, + upstream_peer_select_ms: None, + gap_request_filter_to_upstream_peer_ms: None, + gap_upstream_peer_to_upstream_start_ms: None, + pre_upstream_unattributed_ms: None, + upstream_header_time_ms: None, + post_upstream_header_ms: None, + response_header_to_log_ms: None, + logging_prepare_ms: None, + post_tls_ids_time_us: None, + upstream_time_scope: None, }), } } @@ -1038,8 +1181,25 @@ mod tests { performance: Some(PerformanceInfo { request_time_ms: Some(50), upstream_time_ms: Some(45), + pre_upstream_ms: Some(5), + request_filter_total_ms: Some(3), + request_filter_keepalive_ms: Some(0), + request_filter_access_rules_ms: Some(1), + request_filter_tls_fingerprint_ms: Some(0), + request_filter_threat_intel_ms: Some(0), + request_filter_waf_ms: Some(1), + request_filter_routing_ms: Some(1), + upstream_peer_select_ms: Some(1), + gap_request_filter_to_upstream_peer_ms: Some(0), + gap_upstream_peer_to_upstream_start_ms: Some(1), + pre_upstream_unattributed_ms: Some(0), + upstream_header_time_ms: Some(45), + post_upstream_header_ms: Some(0), + response_header_to_log_ms: Some(0), + logging_prepare_ms: Some(0), + post_tls_ids_time_us: Some(10), + upstream_time_scope: Some("response_header_ttfb".to_string()), }), - fingerprints: None, }; let json = log.to_json().unwrap(); @@ -1387,7 +1547,6 @@ mod tests { geoip: None, upstream: None, performance: None, - fingerprints: None, }; // Serialize to JSON and verify threat intelligence fields are present diff --git a/src/logger/bpf_stats.rs b/src/logger/bpf_stats.rs index a9ff22b0..b856826e 100644 --- a/src/logger/bpf_stats.rs +++ b/src/logger/bpf_stats.rs @@ -8,6 +8,10 @@ use std::sync::{Arc, Mutex}; use crate::security::firewall::bpf::XdpSkel; +const IDS_EXPORT_STAT_EXPORTED: u32 = 0; +const IDS_EXPORT_STAT_RINGBUF_DROPPED: u32 = 1; +const IDS_EXPORT_STAT_INVALID_EVENT: u32 = 2; + /// BPF statistics collected from kernel-level access rule enforcement #[derive(Debug, Clone, Serialize, Deserialize)] pub struct BpfAccessStats { @@ -20,6 +24,9 @@ pub struct BpfAccessStats { pub ipv6_recently_banned_hits: u64, pub tcp_fingerprint_blocks_ipv4: u64, pub tcp_fingerprint_blocks_ipv6: u64, + pub ids_exported_packets: u64, + pub ids_ringbuf_dropped: u64, + pub ids_invalid_events: u64, pub drop_rate_percentage: f64, pub dropped_ip_addresses: DroppedIpAddresses, } @@ -86,6 +93,16 @@ impl BpfAccessStats { Self::read_bpf_counter(&skel.maps.tcp_fingerprint_blocks_ipv4)?; let tcp_fingerprint_blocks_ipv6 = Self::read_bpf_counter(&skel.maps.tcp_fingerprint_blocks_ipv6)?; + let ids_exported_packets = + Self::read_bpf_counter_at_key(&skel.maps.ids_export_stats, IDS_EXPORT_STAT_EXPORTED)?; + let ids_ringbuf_dropped = Self::read_bpf_counter_at_key( + &skel.maps.ids_export_stats, + IDS_EXPORT_STAT_RINGBUF_DROPPED, + )?; + let ids_invalid_events = Self::read_bpf_counter_at_key( + &skel.maps.ids_export_stats, + IDS_EXPORT_STAT_INVALID_EVENT, + )?; // Collect dropped IP addresses let dropped_ip_addresses = Self::collect_dropped_ip_addresses(skel)?; @@ -107,6 +124,9 @@ impl BpfAccessStats { ipv6_recently_banned_hits, tcp_fingerprint_blocks_ipv4, tcp_fingerprint_blocks_ipv6, + ids_exported_packets, + ids_ringbuf_dropped, + ids_invalid_events, drop_rate_percentage, dropped_ip_addresses, }) @@ -114,10 +134,27 @@ impl BpfAccessStats { /// Read a counter value from a BPF array map fn read_bpf_counter(map: &impl libbpf_rs::MapCore) -> Result> { - let key = 0u32.to_le_bytes(); - if let Some(value_bytes) = map.lookup(&key, libbpf_rs::MapFlags::ANY)? { - if value_bytes.len() >= 8 { - let value = u64::from_le_bytes([ + Self::read_bpf_counter_at_key(map, 0) + } + + /// Read a counter value from a BPF map by key. + /// Supports plain array/hash (single u64) and per-cpu array (N x u64) layouts. + fn read_bpf_counter_at_key( + map: &impl libbpf_rs::MapCore, + key: u32, + ) -> Result> { + let key = key.to_le_bytes(); + if map.map_type().is_percpu() { + let Some(percpu_values) = map.lookup_percpu(&key, libbpf_rs::MapFlags::ANY)? else { + return Ok(0); + }; + + let mut sum = 0u64; + for value_bytes in percpu_values { + if value_bytes.len() < 8 { + continue; + } + sum = sum.saturating_add(u64::from_le_bytes([ value_bytes[0], value_bytes[1], value_bytes[2], @@ -126,14 +163,28 @@ impl BpfAccessStats { value_bytes[5], value_bytes[6], value_bytes[7], - ]); - Ok(value) - } else { - Ok(0) + ])); } - } else { - Ok(0) + return Ok(sum); } + + let Some(value_bytes) = map.lookup(&key, libbpf_rs::MapFlags::ANY)? else { + return Ok(0); + }; + if value_bytes.len() < 8 { + return Ok(0); + } + + Ok(u64::from_le_bytes([ + value_bytes[0], + value_bytes[1], + value_bytes[2], + value_bytes[3], + value_bytes[4], + value_bytes[5], + value_bytes[6], + value_bytes[7], + ])) } /// Collect dropped IP addresses from BPF maps @@ -188,33 +239,35 @@ impl BpfAccessStats { if !batch_worked { log::debug!("Batch lookup empty, trying keys iterator for IPv4"); for key_bytes in skel.maps.dropped_ipv4_addresses.keys() { - if key_bytes.len() >= 4 - && let Ok(Some(value_bytes)) = skel + if key_bytes.len() >= 4 { + if let Ok(Some(value_bytes)) = skel .maps .dropped_ipv4_addresses .lookup(&key_bytes, libbpf_rs::MapFlags::ANY) - && value_bytes.len() >= 8 - { - let ip_bytes = [key_bytes[0], key_bytes[1], key_bytes[2], key_bytes[3]]; - let ip_addr = Ipv4Addr::from(ip_bytes); - let drop_count = u64::from_le_bytes([ - value_bytes[0], - value_bytes[1], - value_bytes[2], - value_bytes[3], - value_bytes[4], - value_bytes[5], - value_bytes[6], - value_bytes[7], - ]); - if drop_count > 0 { - log::debug!( - "Found dropped IPv4 (via keys): {} (dropped {} times)", - ip_addr, - drop_count - ); - ipv4_addresses.insert(ip_addr.to_string(), drop_count); - count += 1; + { + if value_bytes.len() >= 8 { + let ip_bytes = [key_bytes[0], key_bytes[1], key_bytes[2], key_bytes[3]]; + let ip_addr = Ipv4Addr::from(ip_bytes); + let drop_count = u64::from_le_bytes([ + value_bytes[0], + value_bytes[1], + value_bytes[2], + value_bytes[3], + value_bytes[4], + value_bytes[5], + value_bytes[6], + value_bytes[7], + ]); + if drop_count > 0 { + log::debug!( + "Found dropped IPv4 (via keys): {} (dropped {} times)", + ip_addr, + drop_count + ); + ipv4_addresses.insert(ip_addr.to_string(), drop_count); + count += 1; + } + } } } } @@ -264,34 +317,36 @@ impl BpfAccessStats { if !batch_worked { log::debug!("Batch lookup empty, trying keys iterator for IPv6"); for key_bytes in skel.maps.dropped_ipv6_addresses.keys() { - if key_bytes.len() >= 16 - && let Ok(Some(value_bytes)) = skel + if key_bytes.len() >= 16 { + if let Ok(Some(value_bytes)) = skel .maps .dropped_ipv6_addresses .lookup(&key_bytes, libbpf_rs::MapFlags::ANY) - && value_bytes.len() >= 8 - { - let mut ip_bytes = [0u8; 16]; - ip_bytes.copy_from_slice(&key_bytes[..16]); - let ip_addr = Ipv6Addr::from(ip_bytes); - let drop_count = u64::from_le_bytes([ - value_bytes[0], - value_bytes[1], - value_bytes[2], - value_bytes[3], - value_bytes[4], - value_bytes[5], - value_bytes[6], - value_bytes[7], - ]); - if drop_count > 0 { - log::debug!( - "Found dropped IPv6 (via keys): {} (dropped {} times)", - ip_addr, - drop_count - ); - ipv6_addresses.insert(ip_addr.to_string(), drop_count); - ipv6_count += 1; + { + if value_bytes.len() >= 8 { + let mut ip_bytes = [0u8; 16]; + ip_bytes.copy_from_slice(&key_bytes[..16]); + let ip_addr = Ipv6Addr::from(ip_bytes); + let drop_count = u64::from_le_bytes([ + value_bytes[0], + value_bytes[1], + value_bytes[2], + value_bytes[3], + value_bytes[4], + value_bytes[5], + value_bytes[6], + value_bytes[7], + ]); + if drop_count > 0 { + log::debug!( + "Found dropped IPv6 (via keys): {} (dropped {} times)", + ip_addr, + drop_count + ); + ipv6_addresses.insert(ip_addr.to_string(), drop_count); + ipv6_count += 1; + } + } } } } @@ -321,7 +376,7 @@ impl BpfAccessStats { /// Create a summary string for logging pub fn summary(&self) -> String { let mut summary = format!( - "BPF Stats: {} packets processed, {} dropped ({:.2}%), IPv4 banned: {}, IPv4 recent: {}, IPv6 banned: {}, IPv6 recent: {}, TCP FP blocks (IPv4/IPv6): {}/{}", + "BPF Stats: {} packets processed, {} dropped ({:.2}%), IPv4 banned: {}, IPv4 recent: {}, IPv6 banned: {}, IPv6 recent: {}, TCP FP blocks (IPv4/IPv6): {}/{}, IDS export (exported/ringbuf_dropped/invalid): {}/{}/{}", self.total_packets_processed, self.total_packets_dropped, self.drop_rate_percentage, @@ -330,7 +385,10 @@ impl BpfAccessStats { self.ipv6_banned_hits, self.ipv6_recently_banned_hits, self.tcp_fingerprint_blocks_ipv4, - self.tcp_fingerprint_blocks_ipv6 + self.tcp_fingerprint_blocks_ipv6, + self.ids_exported_packets, + self.ids_ringbuf_dropped, + self.ids_invalid_events ); // Add top dropped IP addresses if any @@ -412,12 +470,6 @@ impl DroppedIpEvent { } } -impl Default for DroppedIpEvents { - fn default() -> Self { - Self::new() - } -} - impl DroppedIpEvents { /// Create a new collection of dropped IP events pub fn new() -> Self { @@ -495,7 +547,7 @@ impl BpfStatsCollector { } }; - match BpfAccessStats::from_bpf_maps(&skel_guard) { + match BpfAccessStats::from_bpf_maps(&*skel_guard) { Ok(stat) => stats.push(stat), Err(e) => { log::warn!("Failed to collect BPF stats from skeleton: {}", e); @@ -527,6 +579,9 @@ impl BpfStatsCollector { ipv6_recently_banned_hits: 0, tcp_fingerprint_blocks_ipv4: 0, tcp_fingerprint_blocks_ipv6: 0, + ids_exported_packets: 0, + ids_ringbuf_dropped: 0, + ids_invalid_events: 0, drop_rate_percentage: 0.0, dropped_ip_addresses: DroppedIpAddresses { ipv4_addresses: HashMap::new(), @@ -544,6 +599,9 @@ impl BpfStatsCollector { aggregated.ipv6_recently_banned_hits += stat.ipv6_recently_banned_hits; aggregated.tcp_fingerprint_blocks_ipv4 += stat.tcp_fingerprint_blocks_ipv4; aggregated.tcp_fingerprint_blocks_ipv6 += stat.tcp_fingerprint_blocks_ipv6; + aggregated.ids_exported_packets += stat.ids_exported_packets; + aggregated.ids_ringbuf_dropped += stat.ids_ringbuf_dropped; + aggregated.ids_invalid_events += stat.ids_invalid_events; // Merge IP addresses for (ip, count) in stat.dropped_ip_addresses.ipv4_addresses { @@ -626,7 +684,7 @@ impl BpfStatsCollector { } }; - BpfAccessStats::collect_dropped_ip_addresses(&skel_guard)? + BpfAccessStats::collect_dropped_ip_addresses(&*skel_guard)? }; // Convert IPv4 addresses to events @@ -818,6 +876,9 @@ mod tests { ipv6_recently_banned_hits: 5, tcp_fingerprint_blocks_ipv4: 0, tcp_fingerprint_blocks_ipv6: 0, + ids_exported_packets: 900, + ids_ringbuf_dropped: 2, + ids_invalid_events: 1, drop_rate_percentage: 5.0, dropped_ip_addresses: DroppedIpAddresses { ipv4_addresses, @@ -830,6 +891,7 @@ mod tests { assert!(summary.contains("1000 packets processed")); assert!(summary.contains("50 dropped")); assert!(summary.contains("5.00%")); + assert!(summary.contains("IDS export")); assert!(summary.contains("2 unique IPs dropped")); assert!(summary.contains("192.168.1.1:10")); } @@ -846,6 +908,9 @@ mod tests { ipv6_recently_banned_hits: 1, tcp_fingerprint_blocks_ipv4: 0, tcp_fingerprint_blocks_ipv6: 0, + ids_exported_packets: 95, + ids_ringbuf_dropped: 0, + ids_invalid_events: 0, drop_rate_percentage: 10.0, dropped_ip_addresses: DroppedIpAddresses { ipv4_addresses: HashMap::new(), @@ -857,6 +922,7 @@ mod tests { let json = stats.to_json().unwrap(); assert!(json.contains("total_packets_processed")); assert!(json.contains("drop_rate_percentage")); + assert!(json.contains("ids_exported_packets")); assert!(json.contains("dropped_ip_addresses")); } } diff --git a/src/logger/mod.rs b/src/logger/mod.rs index ff401583..8f450057 100644 --- a/src/logger/mod.rs +++ b/src/logger/mod.rs @@ -158,44 +158,44 @@ pub fn init_file_logging( // Add syslog appenders if enabled let mut root_appenders = vec!["console", "error_file", "app_file"]; let mut access_appenders = vec!["access_file"]; - if let Some(syslog_cfg) = syslog_config - && syslog_cfg.enabled - { - // Error syslog appender - let error_syslog = SyslogAppender::new( - &syslog_cfg.facility, - &syslog_cfg.identifier, - &syslog_cfg.levels.error, - )?; - - config_builder = config_builder.appender( - Appender::builder() - .filter(Box::new(ThresholdFilter::new(LevelFilter::Error))) - .build("error_syslog", Box::new(error_syslog)), - ); - root_appenders.push("error_syslog"); - - // App syslog appender - let app_syslog = SyslogAppender::new( - &syslog_cfg.facility, - &syslog_cfg.identifier, - &syslog_cfg.levels.app, - )?; - - config_builder = - config_builder.appender(Appender::builder().build("app_syslog", Box::new(app_syslog))); - root_appenders.push("app_syslog"); - - // Access syslog appender - let access_syslog = SyslogAppender::new( - &syslog_cfg.facility, - &syslog_cfg.identifier, - &syslog_cfg.levels.access, - )?; - - config_builder = config_builder - .appender(Appender::builder().build("access_syslog", Box::new(access_syslog))); - access_appenders.push("access_syslog"); + if let Some(syslog_cfg) = syslog_config { + if syslog_cfg.enabled { + // Error syslog appender + let error_syslog = SyslogAppender::new( + &syslog_cfg.facility, + &syslog_cfg.identifier, + &syslog_cfg.levels.error, + )?; + + config_builder = config_builder.appender( + Appender::builder() + .filter(Box::new(ThresholdFilter::new(LevelFilter::Error))) + .build("error_syslog", Box::new(error_syslog)), + ); + root_appenders.push("error_syslog"); + + // App syslog appender + let app_syslog = SyslogAppender::new( + &syslog_cfg.facility, + &syslog_cfg.identifier, + &syslog_cfg.levels.app, + )?; + + config_builder = config_builder + .appender(Appender::builder().build("app_syslog", Box::new(app_syslog))); + root_appenders.push("app_syslog"); + + // Access syslog appender + let access_syslog = SyslogAppender::new( + &syslog_cfg.facility, + &syslog_cfg.identifier, + &syslog_cfg.levels.access, + )?; + + config_builder = config_builder + .appender(Appender::builder().build("access_syslog", Box::new(access_syslog))); + access_appenders.push("access_syslog"); + } } // Logger for access logs - routes to access appenders (not console) diff --git a/src/main.rs b/src/main.rs index e52fe391..0919296e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -11,11 +11,7 @@ use daemonize::Daemonize; #[cfg(all(feature = "bpf", not(feature = "disable-bpf")))] use libbpf_rs::skel::{OpenSkel, SkelBuilder}; #[cfg(all(feature = "bpf", not(feature = "disable-bpf")))] -use libbpf_rs::{TC_INGRESS, TcHookBuilder}; -#[cfg(all(feature = "bpf", not(feature = "disable-bpf")))] use nix::net::if_::if_nametoindex; -#[cfg(all(feature = "bpf", not(feature = "disable-bpf")))] -use std::os::fd::AsFd; #[global_allocator] static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; @@ -43,10 +39,6 @@ pub mod firewall; pub mod bpf { pub use crate::security::firewall::bpf::*; } -#[cfg(all(feature = "bpf", not(feature = "disable-bpf")))] -pub mod ja4ts_bpf { - pub use crate::security::firewall::ja4ts_bpf::*; -} #[cfg(any(not(feature = "bpf"), feature = "disable-bpf"))] #[path = "bpf_stub.rs"] pub mod bpf; @@ -63,7 +55,7 @@ use crate::security::waf::actions::content_scanning::{ }; use crate::security::waf::wirefilter::init_config; #[cfg(all(feature = "bpf", not(feature = "disable-bpf")))] -use crate::utils::bpf_utils::{bpf_attach_to_xdp, bpf_detach_from_xdp}; +use crate::utils::bpf_utils::{bpf_attach_to_xdp, bpf_detach_from_xdp, xdp_link_state_snapshot}; use crate::utils::fingerprint::latency_fingerprint::LatencyFingerprintCollector; use crate::utils::fingerprint::latency_fingerprint::LatencyFingerprintConfig; use crate::utils::fingerprint::ssh_fingerprint::SshFingerprintCollector; @@ -80,6 +72,13 @@ use crate::security::waf::actions::captcha::{ use crate::utils::http_client::init_global_client; use crate::worker::agent_status::AgentStatusWorker; use crate::worker::log::set_log_sender_config; +#[cfg(all( + feature = "thalamus-ids", + feature = "bpf", + not(feature = "disable-bpf"), + target_os = "linux" +))] +use crate::worker::thalamus_ids::{ThalamusIdsWorker, configure_ids_export_map_for_skel}; fn main() -> Result<()> { // Initialize rustls crypto provider early (must be done before any rustls operations) @@ -301,11 +300,10 @@ fn main() -> Result<()> { // Multi-threaded runtime let mut builder = tokio::runtime::Builder::new_multi_thread(); builder.enable_all(); - builder.thread_stack_size(2 * 1024 * 1024); // 2 MB (default is 8 MB) - if let Some(threads) = config.worker_threads - && threads > 0 - { - builder.worker_threads(threads); + if let Some(threads) = config.worker_threads { + if threads > 0 { + builder.worker_threads(threads); + } } builder.build()? } else { @@ -390,10 +388,6 @@ async fn async_main(args: Args, config: Config) -> Result<()> { let mut skels: Vec>>> = Vec::new(); #[allow(unused_mut)] let mut ifindices: Vec = Vec::new(); - #[cfg(all(feature = "bpf", not(feature = "disable-bpf")))] - let mut ja4ts_skels: Vec>>> = Vec::new(); - #[cfg(all(feature = "bpf", not(feature = "disable-bpf")))] - let mut tc_hooks: Vec = Vec::new(); let mut firewall_backend = FirewallBackend::None; let mut nftables_firewall: Option>> = None; let mut iptables_firewall: Option>> = None; @@ -405,29 +399,119 @@ async fn async_main(args: Args, config: Config) -> Result<()> { if config.firewall.disable_xdp { log::warn!("XDP disabled by config, will use nftables fallback"); + if config.ids.enabled { + log::warn!( + "ids.enabled=true but firewall.disable_xdp=true; XDP IDS export will be unavailable (proxy post-TLS IDS can still run in proxy mode)" + ); + } } else { #[cfg(all(feature = "bpf", not(feature = "disable-bpf")))] { - // Log libbpf output for diagnostics (verifier errors appear at debug level) - fn libbpf_print_cb(level: libbpf_rs::PrintLevel, msg: String) { - match level { - libbpf_rs::PrintLevel::Warn => log::warn!("libbpf: {}", msg.trim_end()), - libbpf_rs::PrintLevel::Info => log::info!("libbpf: {}", msg.trim_end()), - libbpf_rs::PrintLevel::Debug => log::debug!("libbpf: {}", msg.trim_end()), - } + let bpf_debug = std::env::var("SYNAPSE_BPF_DEBUG") + .map(|v| matches!(v.as_str(), "1" | "true" | "TRUE" | "on" | "ON")) + .unwrap_or(false); + if bpf_debug { + log::warn!( + "SYNAPSE_BPF_DEBUG is enabled; libbpf verifier/loader logs will be printed" + ); + } else { + // Suppress libbpf output in normal mode; we emit structured fallback logs. + libbpf_rs::set_print(None); } - libbpf_rs::set_print(Some((libbpf_rs::PrintLevel::Debug, libbpf_print_cb))); for iface in &iface_names { + log::info!( + "XDP init [{}]: mode={}, disable_xdp={}, ids_enabled={}, ids_snaplen={}, ids_rule_paths={}", + iface, + firewall_mode, + config.firewall.disable_xdp, + config.ids.enabled, + config.ids.snaplen, + config.ids.rule_paths.len() + ); + log::info!( + "XDP init [{}]: pre-attach link state: {}", + iface, + xdp_link_state_snapshot(iface) + ); let boxed_open: Box> = Box::new(MaybeUninit::uninit()); let open_object: &'static mut MaybeUninit = Box::leak(boxed_open); let skel_builder = bpf::XdpSkelBuilder::default(); match skel_builder.open(open_object) { - Ok(open_skel) => { + Ok(mut open_skel) => { + // Set map sizes from config before loading + if let Err(e) = crate::security::ratelimiter::set_bucket_map_size_ipv4( + &mut open_skel, + config.firewall.ratelimiter.ipv4_map_size, + ) { + log::warn!( + "Failed to set IPv4 bucket map size for '{}': {} -> falling back to default", + iface, + e + ); + } else { + log::debug!( + "Successfully set ratelimiter ipv4 map size: {}", + config.firewall.ratelimiter.ipv4_map_size + ); + } + + if let Err(e) = crate::security::ratelimiter::set_bucket_map_size_ipv6( + &mut open_skel, + config.firewall.ratelimiter.ipv6_map_size, + ) { + log::warn!( + "Failed to set IPv6 bucket map size for '{}': {} -> falling back to default", + iface, + e + ); + } else { + log::debug!( + "Successfully set ratelimiter ipv6 map size: {}", + config.firewall.ratelimiter.ipv6_map_size + ); + } + match open_skel.load() { Ok(mut skel) => { + { + // Apply rate limiter configuration + let mut ratelimiter = + crate::security::ratelimiter::XDPRateLimit::new(&mut skel); + ratelimiter.set_request_per_sec( + config.firewall.ratelimiter.request_per_sec, + config.firewall.ratelimiter.burst_factor, + ); + + ratelimiter.set_ratelimiter_status( + !!config.firewall.ratelimiter.enabled, + ); + } + #[cfg(all( + feature = "thalamus-ids", + feature = "bpf", + not(feature = "disable-bpf"), + target_os = "linux" + ))] + if let Err(e) = + configure_ids_export_map_for_skel(&mut skel, &config.ids) + { + log::warn!( + "Failed to configure IDS export map for '{}': {}", + iface, + e + ); + } else { + log::debug!( + "Configured IDS export map for '{}' (enabled={}, snaplen={})", + iface, + config.ids.enabled, + config.ids.snaplen + ); + } + let ifindex = match if_nametoindex(iface.as_str()) { Ok(index) => index as i32, Err(e) => { @@ -451,6 +535,12 @@ async fn async_main(args: Args, config: Config) -> Result<()> { xdp_modes.push((iface.as_str(), mode.as_str())); skels.push(Arc::new(Mutex::new(skel))); ifindices.push(ifindex); + log::debug!( + "XDP attach [{}] success (mode={}): {}", + iface, + mode.as_str(), + xdp_link_state_snapshot(iface) + ); } Err(e) => { // Check if error is EAFNOSUPPORT (error 97) - IPv6 might be disabled @@ -463,12 +553,22 @@ async fn async_main(args: Args, config: Config) -> Result<()> { iface, e ); + log::warn!( + "XDP attach [{}] failed state snapshot: {}", + iface, + xdp_link_state_snapshot(iface) + ); } else { log::error!( "Failed to attach XDP to '{}': {}", iface, e ); + log::warn!( + "XDP attach [{}] failed state snapshot: {}", + iface, + xdp_link_state_snapshot(iface) + ); } } } @@ -489,11 +589,21 @@ async fn async_main(args: Args, config: Config) -> Result<()> { } else { log::warn!("failed to load BPF skeleton for '{}': {e}", iface); } + log::warn!( + "XDP load failure [{}] link state: {}", + iface, + xdp_link_state_snapshot(iface) + ); } } } Err(e) => { log::warn!("failed to open BPF skeleton for '{}': {e}", iface); + log::warn!( + "XDP open failure [{}] link state: {}", + iface, + xdp_link_state_snapshot(iface) + ); } } } @@ -506,89 +616,6 @@ async fn async_main(args: Args, config: Config) -> Result<()> { } } - // Load and attach JA4TS TC BPF program (separate from XDP for verifier budget) - #[cfg(all(feature = "bpf", not(feature = "disable-bpf")))] - if !config.firewall.disable_xdp && !skels.is_empty() { - for iface in &iface_names { - let ifindex = match if_nametoindex(iface.as_str()) { - Ok(index) => index as i32, - Err(e) => { - log::warn!( - "JA4TS: failed to get interface index for '{}': {}", - iface, - e - ); - continue; - } - }; - - let boxed_open: Box> = - Box::new(MaybeUninit::uninit()); - let open_object: &'static mut MaybeUninit = - Box::leak(boxed_open); - let skel_builder = ja4ts_bpf::Ja4tsSkelBuilder::default(); - match skel_builder.open(open_object) { - Ok(open_skel) => match open_skel.load() { - Ok(skel) => { - let prog_fd = skel.progs.ja4ts_capture.as_fd(); - let mut builder = TcHookBuilder::new(prog_fd); - builder.ifindex(ifindex).replace(true).handle(1).priority(1); - let mut hook = builder.hook(TC_INGRESS); - - // Create clsact qdisc (idempotent) - if let Err(e) = hook.create() { - // EEXIST is fine — qdisc already exists - if !e.to_string().contains("File exists") - && !e.to_string().contains("EEXIST") - && !e.to_string().contains("error 17") - { - log::warn!( - "JA4TS: failed to create clsact qdisc on '{}': {}", - iface, - e - ); - } - } - - match hook.attach() { - Ok(_) => { - log::info!( - "JA4TS: TC program attached to '{}' (ifindex {})", - iface, - ifindex - ); - ja4ts_skels.push(Arc::new(Mutex::new(skel))); - tc_hooks.push(hook); - } - Err(e) => { - log::warn!( - "JA4TS: failed to attach TC program to '{}': {}", - iface, - e - ); - } - } - } - Err(e) => { - log::warn!("JA4TS: failed to load BPF skeleton for '{}': {}", iface, e); - } - }, - Err(e) => { - log::warn!("JA4TS: failed to open BPF skeleton for '{}': {}", iface, e); - } - } - } - - if ja4ts_skels.is_empty() { - log::warn!("JA4TS: no TC programs loaded; SYN-ACK fingerprinting disabled"); - } else { - log::info!( - "JA4TS: {} TC program(s) loaded for SYN-ACK fingerprinting", - ja4ts_skels.len() - ); - } - } - // Determine firewall backend based on config and availability log::info!("Firewall mode configured: {}", firewall_mode); @@ -701,16 +728,8 @@ async fn async_main(args: Args, config: Config) -> Result<()> { BpfStatsCollector::new(skels.clone(), config.logging.bpf_stats.enabled); // Create TCP fingerprinting collector - #[cfg(all(feature = "bpf", not(feature = "disable-bpf")))] let tcp_fingerprint_collector = TcpFingerprintCollector::new_with_config( skels.clone(), - ja4ts_skels.clone(), - TcpFingerprintConfig::from_cli_config(&config.logging.tcp_fingerprint), - ); - #[cfg(any(not(feature = "bpf"), feature = "disable-bpf"))] - let tcp_fingerprint_collector = TcpFingerprintCollector::new_with_config( - skels.clone(), - vec![], TcpFingerprintConfig::from_cli_config(&config.logging.tcp_fingerprint), ); @@ -742,16 +761,18 @@ async fn async_main(args: Args, config: Config) -> Result<()> { ); // Initialize access rules for nftables or iptables backend if active - if firewall_backend == FirewallBackend::Nftables - && let Some(ref nft_fw) = nftables_firewall - && let Err(e) = crate::security::access_rules::init_access_rules_nftables(nft_fw) - { - log::error!("Failed to initialize nftables access rules: {}", e); - } else if firewall_backend == FirewallBackend::Iptables - && let Some(ref ipt_fw) = iptables_firewall - && let Err(e) = crate::security::access_rules::init_access_rules_iptables(ipt_fw) - { - log::error!("Failed to initialize iptables access rules: {}", e); + if firewall_backend == FirewallBackend::Nftables { + if let Some(ref nft_fw) = nftables_firewall { + if let Err(e) = crate::security::access_rules::init_access_rules_nftables(nft_fw) { + log::error!("Failed to initialize nftables access rules: {}", e); + } + } + } else if firewall_backend == FirewallBackend::Iptables { + if let Some(ref ipt_fw) = iptables_firewall { + if let Err(e) = crate::security::access_rules::init_access_rules_iptables(ipt_fw) { + log::error!("Failed to initialize iptables access rules: {}", e); + } + } } let state = AppState { @@ -975,12 +996,12 @@ async fn async_main(args: Args, config: Config) -> Result<()> { } // Validate API key if provided - if !config.platform.base_url.is_empty() - && !config.platform.api_key.is_empty() - && let Err(e) = validate_api_key(&config.platform.base_url, &config.platform.api_key).await - { - log::error!("API key validation failed: {}", e); - return Err(anyhow::anyhow!("API key validation failed: {}", e)); + if !config.platform.base_url.is_empty() && !config.platform.api_key.is_empty() { + if let Err(e) = validate_api_key(&config.platform.base_url, &config.platform.api_key).await + { + log::error!("API key validation failed: {}", e); + return Err(anyhow::anyhow!("API key validation failed: {}", e)); + } } // Initialize content scanning from CLI config (skip in agent mode) @@ -1084,6 +1105,18 @@ async fn async_main(args: Args, config: Config) -> Result<()> { if !state.skels.is_empty() { capabilities.push("xdp".to_string()); } + if config.mode == "proxy" && config.ids.enabled { + capabilities.push("thalamus_ids_post_tls".to_string()); + } + #[cfg(all( + feature = "thalamus-ids", + feature = "bpf", + not(feature = "disable-bpf"), + target_os = "linux" + ))] + if config.ids.enabled && !state.skels.is_empty() { + capabilities.push("thalamus_ids".to_string()); + } let mut metadata = HashMap::new(); metadata.insert("os".to_string(), std::env::consts::OS.to_string()); @@ -1304,14 +1337,15 @@ async fn async_main(args: Args, config: Config) -> Result<()> { } // Update access rules in XDP if available - if !state.skels.is_empty() - && let Err(e) = + if !state.skels.is_empty() { + if let Err(e) = crate::security::access_rules::apply_rules_from_global_with_state( &state.skels, is_agent_mode, ) - { - log::warn!("Failed to apply access rules from local file to XDP: {}", e); + { + log::warn!("Failed to apply access rules from local file to XDP: {}", e); + } } } Err(e) => { @@ -1372,12 +1406,46 @@ async fn async_main(args: Args, config: Config) -> Result<()> { } } + #[cfg(all( + feature = "thalamus-ids", + feature = "bpf", + not(feature = "disable-bpf"), + target_os = "linux" + ))] + if config.ids.enabled { + log::info!( + "thalamus_ids worker uses Synapse-owned XDP skeletons; avoid running standalone Thalamus AF_XDP on the same interface concurrently" + ); + if state.skels.is_empty() { + log::warn!( + "ids.enabled=true and ids.capture_mode=xdp but no active XDP skeletons are attached; XDP IDS worker not started" + ); + } else { + let worker_config = worker::WorkerConfig { + name: "thalamus_ids".to_string(), + interval_secs: 1, + enabled: true, + }; + let ids_worker = ThalamusIdsWorker::new( + state.skels.clone(), + config.ids.clone(), + state.firewall_backend, + state.nftables_firewall.clone(), + state.iptables_firewall.clone(), + ); + if let Err(e) = worker_manager.register_worker(worker_config, ids_worker) { + log::error!("Failed to register thalamus IDS worker: {}", e); + } + } + } + // Start the old Pingora proxy system in a separate thread (non-blocking) // Only start if mode is "proxy" (disabled in agent mode) if config.mode == "proxy" { let logging_config = config.logging.clone(); let platform_config = config.platform.clone(); let network_config = config.network.clone(); + let ids_config = config.ids.clone(); let proxy_config = config.proxy.clone(); // Run Pingora in a separate OS thread // Note: Pingora's run_forever() creates its own runtime, which may be single-threaded @@ -1388,6 +1456,7 @@ async fn async_main(args: Args, config: Config) -> Result<()> { worker_threads: None, network: network_config, firewall: Default::default(), + ids: ids_config, platform: platform_config, logging: logging_config, daemon: Default::default(), @@ -1565,6 +1634,27 @@ async fn async_main(args: Args, config: Config) -> Result<()> { } else { serde_json::json!(1) }; + #[cfg(all( + feature = "thalamus-ids", + feature = "bpf", + not(feature = "disable-bpf"), + target_os = "linux" + ))] + let thalamus_ids_xdp_enabled = config.ids.enabled + && matches!( + config.ids.capture_mode, + crate::core::cli::IdsCaptureMode::Xdp + ) + && !state.skels.is_empty(); + #[cfg(not(all( + feature = "thalamus-ids", + feature = "bpf", + not(feature = "disable-bpf"), + target_os = "linux" + )))] + let thalamus_ids_xdp_enabled = false; + let thalamus_ids_post_tls_enabled = config.mode == "proxy" && config.ids.enabled; + let thalamus_ids_enabled = thalamus_ids_xdp_enabled || thalamus_ids_post_tls_enabled; let startup_info = serde_json::json!({ "event": "startup_complete", @@ -1581,6 +1671,12 @@ async fn async_main(args: Args, config: Config) -> Result<()> { "tcp_fingerprint": config.logging.tcp_fingerprint.enabled && !state.skels.is_empty(), "ssh_fingerprint": config.logging.ssh_fingerprint.enabled && !state.skels.is_empty(), "latency_fingerprint": config.logging.latency_fingerprint.enabled && !state.skels.is_empty(), + "thalamus_ids": thalamus_ids_enabled, + "thalamus_ids_capture_mode": if config.ids.enabled { Some(match config.ids.capture_mode { + crate::core::cli::IdsCaptureMode::Xdp => "xdp", + }) } else { None }, + "thalamus_ids_xdp": thalamus_ids_xdp_enabled, + "thalamus_ids_post_tls": thalamus_ids_post_tls_enabled, "waf": waf_enabled, "threat_intel": threat_client_enabled, "internal_services": config.proxy.internal_services.enabled && !is_agent_mode, @@ -1743,37 +1839,23 @@ async fn async_main(args: Args, config: Config) -> Result<()> { } } - // Detach JA4TS TC programs (TC hooks persist after process exit, unlike XDP) - #[cfg(all(feature = "bpf", not(feature = "disable-bpf")))] - if !tc_hooks.is_empty() { - log::info!( - "Detaching JA4TS TC programs from {} hooks...", - tc_hooks.len() - ); - for mut hook in tc_hooks { - if let Err(e) = hook.detach() { - log::error!("Failed to detach JA4TS TC hook: {}", e); - } - } - } - // Cleanup nftables rules if using nftables backend if let Some(ref nft_fw) = nftables_firewall { log::info!("Cleaning up nftables firewall rules..."); - if let Ok(fw) = nft_fw.lock() - && let Err(e) = fw.cleanup() - { - log::error!("Failed to cleanup nftables rules: {}", e); + if let Ok(fw) = nft_fw.lock() { + if let Err(e) = fw.cleanup() { + log::error!("Failed to cleanup nftables rules: {}", e); + } } } // Cleanup iptables rules if using iptables backend if let Some(ref ipt_fw) = iptables_firewall { log::info!("Cleaning up iptables firewall rules..."); - if let Ok(fw) = ipt_fw.lock() - && let Err(e) = fw.cleanup() - { - log::error!("Failed to cleanup iptables rules: {}", e); + if let Ok(fw) = ipt_fw.lock() { + if let Err(e) = fw.cleanup() { + log::error!("Failed to cleanup iptables rules: {}", e); + } } } diff --git a/src/proxy/acme/domain_reader.rs b/src/proxy/acme/domain_reader.rs index 23e6c24e..ea9ea9fd 100644 --- a/src/proxy/acme/domain_reader.rs +++ b/src/proxy/acme/domain_reader.rs @@ -32,12 +32,10 @@ pub trait DomainReader: Send + Sync { async fn read_domains(&self) -> Result>; } -type CachedDomains = Arc, String)>>>; - /// File-based domain reader with file watching and hash-based change detection pub struct FileDomainReader { file_path: PathBuf, - cached_domains: CachedDomains, // (domains, hash) + cached_domains: Arc, String)>>>, // (domains, hash) } impl FileDomainReader { @@ -90,7 +88,7 @@ impl FileDomainReader { /// Internal struct for file watching task struct FileDomainReaderWatching { file_path: PathBuf, - cached_domains: CachedDomains, + cached_domains: Arc, String)>>>, } impl FileDomainReaderWatching { @@ -118,10 +116,10 @@ impl FileDomainReaderWatching { }; // Watch the parent directory to catch file renames/moves - if let Some(parent) = self.file_path.parent() - && let Err(e) = watcher.watch(parent, RecursiveMode::NonRecursive) - { - tracing::warn!("Failed to watch directory {:?}: {}", parent, e); + if let Some(parent) = self.file_path.parent() { + if let Err(e) = watcher.watch(parent, RecursiveMode::NonRecursive) { + tracing::warn!("Failed to watch directory {:?}: {}", parent, e); + } } // Also watch the file directly @@ -173,10 +171,10 @@ impl FileDomainReaderWatching { // Check if hash changed { let cache = cached_domains_clone.read().await; - if let Some((_, old_hash)) = cache.as_ref() - && *old_hash == new_hash - { - return; // No change + if let Some((_, old_hash)) = cache.as_ref() { + if *old_hash == new_hash { + return; // No change + } } } @@ -232,10 +230,10 @@ impl FileDomainReaderWatching { // Check if hash changed { let cache = self.cached_domains.read().await; - if let Some((_, old_hash)) = cache.as_ref() - && *old_hash == new_hash - { - return Ok(false); // No change + if let Some((_, old_hash)) = cache.as_ref() { + if *old_hash == new_hash { + return Ok(false); // No change + } } } @@ -290,7 +288,7 @@ pub struct RedisDomainReader { redis_key: String, redis_url: String, redis_ssl: Option, - cached_domains: CachedDomains, // (domains, hash) + cached_domains: Arc, String)>>>, // (domains, hash) } impl RedisDomainReader { @@ -405,9 +403,7 @@ impl RedisDomainReader { if ssl_config.insecure { tls_builder.danger_accept_invalid_certs(true); tls_builder.danger_accept_invalid_hostnames(true); - tracing::error!( - "Redis SSL: Certificate verification DISABLED (insecure mode) — connections are vulnerable to MITM attacks" - ); + tracing::warn!("Redis SSL: Certificate verification disabled (insecure mode)"); } let _tls_connector = tls_builder @@ -420,7 +416,7 @@ impl RedisDomainReader { // we need to add them to the system trust store or use a workaround. let client = redis::Client::open(redis_url) - .with_context(|| "Failed to create Redis client with SSL config".to_string())?; + .with_context(|| format!("Failed to create Redis client with SSL config"))?; Ok(client) } @@ -443,7 +439,7 @@ impl RedisDomainReader { let hash = Self::calculate_hash(&content); let domains: Vec = serde_json::from_str(&content) - .with_context(|| "Failed to parse domains JSON from Redis".to_string())?; + .with_context(|| format!("Failed to parse domains JSON from Redis"))?; Ok((domains, hash)) } @@ -454,7 +450,7 @@ struct RedisDomainReaderPolling { redis_key: String, redis_url: String, redis_ssl: Option, - cached_domains: CachedDomains, + cached_domains: Arc, String)>>>, } impl RedisDomainReaderPolling { @@ -523,10 +519,10 @@ impl RedisDomainReaderPolling { // Check if hash changed { let cache = self.cached_domains.read().await; - if let Some((_, old_hash)) = cache.as_ref() - && *old_hash == new_hash - { - return Ok(false); // No change + if let Some((_, old_hash)) = cache.as_ref() { + if *old_hash == new_hash { + return Ok(false); // No change + } } } @@ -599,7 +595,7 @@ impl HttpDomainReader { .with_context(|| format!("Failed to read response from {}", self.url))?; let domains: Vec = serde_json::from_str(&content) - .with_context(|| "Failed to parse domains JSON from HTTP response".to_string())?; + .with_context(|| format!("Failed to parse domains JSON from HTTP response"))?; Ok(domains) } diff --git a/src/proxy/acme/embedded.rs b/src/proxy/acme/embedded.rs index 20d9f297..6e23a0b2 100644 --- a/src/proxy/acme/embedded.rs +++ b/src/proxy/acme/embedded.rs @@ -12,8 +12,6 @@ use std::sync::Arc; use tokio::sync::RwLock; use tracing::{info, warn}; -type SharedDomainReader = Arc>>>; - #[derive(Debug, Clone, Serialize)] pub struct EmbeddedAcmeConfig { /// Port for ACME server (e.g., 9180) @@ -40,7 +38,7 @@ pub struct EmbeddedAcmeConfig { pub struct EmbeddedAcmeServer { config: EmbeddedAcmeConfig, - domain_reader: SharedDomainReader, + domain_reader: Arc>>>, } impl EmbeddedAcmeServer { @@ -52,7 +50,7 @@ impl EmbeddedAcmeServer { } /// Get a clone of the domain reader - pub fn get_domain_reader(&self) -> SharedDomainReader { + pub fn get_domain_reader(&self) -> Arc>>> { self.domain_reader.clone() } @@ -239,7 +237,7 @@ impl EmbeddedAcmeServer { /// HTTP handler for checking expiration of all domains async fn check_all_certs_expiration_handler( config: web::Data, - domain_reader: web::Data, + domain_reader: web::Data>>>>, ) -> impl Responder { let reader = domain_reader.read().await; let reader_ref = match reader.as_ref() { @@ -295,7 +293,7 @@ async fn check_all_certs_expiration_handler( /// HTTP handler for checking expiration of a specific domain async fn check_cert_expiration_handler( config: web::Data, - domain_reader: web::Data, + domain_reader: web::Data>>>>, path: web::Path, ) -> impl Responder { let domain = path.into_inner(); @@ -357,7 +355,7 @@ async fn check_cert_expiration_handler( /// HTTP handler for renewing a certificate async fn renew_cert_handler( config: web::Data, - domain_reader: web::Data, + domain_reader: web::Data>>>>, path: web::Path, ) -> impl Responder { let domain = path.into_inner(); diff --git a/src/proxy/acme/lib.rs b/src/proxy/acme/lib.rs index 71808e44..ab1240b7 100644 --- a/src/proxy/acme/lib.rs +++ b/src/proxy/acme/lib.rs @@ -12,7 +12,6 @@ use anyhow::{Context, anyhow}; use once_cell::sync::OnceCell; use serde::Serialize; use std::io::BufReader; -use std::path::Path; use std::sync::Arc; use std::sync::RwLock as StdRwLock; use tracing::{debug, info, warn}; @@ -139,7 +138,7 @@ async fn save_cert_to_proxy_path( /// Create RUSTLS server config from certificates in storage pub fn get_https_config(config: &Config) -> AtomicServerResult { - use rustls::pki_types::PrivateKeyDer; + use rustls::pki_types::{CertificateDer, PrivateKeyDer}; use rustls_pemfile::{certs, pkcs8_private_keys}; // Create storage backend (file system by default) @@ -161,7 +160,7 @@ pub fn get_https_config(config: &Config) -> AtomicServerResult = pkcs8_private_keys(key_file) @@ -288,7 +287,7 @@ struct CertificateExpirationInfo { async fn get_cert_expiration_info( app_config: &AppConfig, domain: &str, - base_https_path: &Path, + base_https_path: &std::path::PathBuf, ) -> anyhow::Result { let domain_cfg = { let domain_config = DomainConfig { @@ -297,7 +296,7 @@ async fn get_cert_expiration_info( dns: false, wildcard: false, }; - app_config.create_domain_config(&domain_config, base_https_path.to_path_buf()) + app_config.create_domain_config(&domain_config, base_https_path.clone()) }; let storage = StorageFactory::create_default(&domain_cfg)?; @@ -380,30 +379,30 @@ async fn check_cert_expiration_handler( } }; - if let Ok(domains) = domain_reader.read_domains().await - && let Some(domain_config) = domains.iter().find(|d| d.domain == domain) - { - let app_config_clone = app_config.clone(); - let base_path_clone = base_path.clone(); - let domain_config_clone = domain_config.clone(); - - // Spawn renewal task in background - tokio::spawn(async move { - if let Err(e) = renew_cert_if_needed( - &app_config_clone, - &domain_config_clone, - &base_path_clone, - ) - .await - { - warn!( - "Error renewing certificate for {}: {}", - domain_config_clone.domain, e - ); - } - }); + if let Ok(domains) = domain_reader.read_domains().await { + if let Some(domain_config) = domains.iter().find(|d| d.domain == domain) { + let app_config_clone = app_config.clone(); + let base_path_clone = base_path.clone(); + let domain_config_clone = domain_config.clone(); + + // Spawn renewal task in background + tokio::spawn(async move { + if let Err(e) = renew_cert_if_needed( + &app_config_clone, + &domain_config_clone, + &base_path_clone, + ) + .await + { + warn!( + "Error renewing certificate for {}: {}", + domain_config_clone.domain, e + ); + } + }); - info.renewing = true; // Mark as renewing + info.renewing = true; // Mark as renewing + } } } HttpResponse::Ok().json(info) @@ -424,9 +423,9 @@ async fn check_cert_expiration_handler( async fn renew_cert_if_needed( app_config: &AppConfig, domain_config: &DomainConfig, - base_path: &Path, + base_path: &std::path::PathBuf, ) -> anyhow::Result<()> { - let domain_cfg = app_config.create_domain_config(domain_config, base_path.to_path_buf()); + let domain_cfg = app_config.create_domain_config(domain_config, base_path.clone()); if should_renew_certs_check(&domain_cfg).await? { info!( @@ -618,11 +617,13 @@ async fn check_dns_txt_record( expected_value, found_values ); } - } else if attempt % 6 == 0 { - info!( - "DNS record not found yet (attempt {}/{})...", - attempt, max_attempts - ); + } else { + if attempt % 6 == 0 { + info!( + "DNS record not found yet (attempt {}/{})...", + attempt, max_attempts + ); + } } } Err(e) => { @@ -706,7 +707,7 @@ async fn check_acme_challenge_endpoint(config: &Config) -> anyhow::Result<()> { ); tokio::time::sleep(retry_delay).await; // Exponential backoff: 10ms, 20ms, 40ms, 80ms, 160ms (user changed from 500ms) - retry_delay *= 2; + retry_delay = retry_delay * 2; continue; } // Last attempt or non-connection error - return error @@ -834,8 +835,8 @@ fn parse_retry_after(error_msg: &str) -> Option> { return Some(dt.with_timezone(&chrono::Utc)); } // Try with "UTC" as separate word (common format: "2025-11-14 21:13:29 UTC") - if let Some(without_tz) = after_text.strip_suffix(" UTC") { - let without_tz = &without_tz.trim(); + if after_text.ends_with(" UTC") { + let without_tz = &after_text[..after_text.len() - 4].trim(); if let Ok(dt) = chrono::NaiveDateTime::parse_from_str(without_tz, "%Y-%m-%d %H:%M:%S") { return Some(chrono::DateTime::from_naive_utc_and_offset(dt, chrono::Utc)); } @@ -856,16 +857,16 @@ fn parse_retry_after(error_msg: &str) -> Option> { // Try parsing as ISO 8601 duration (e.g., "PT86225.992004616S" or "PT24H") // This happens when the error message contains a duration instead of a timestamp - if let Some(after_pt) = after_text.strip_prefix("PT") { + if after_text.starts_with("PT") { // Parse ISO 8601 duration: PT[nH][nM][nS] or PT[n]S // Handle case where duration ends with 'S' (seconds) let duration_str = if after_text.ends_with('S') && !after_text.ends_with("MS") && !after_text.ends_with("HS") { - &after_pt[..after_pt.len() - 1] // Remove "S" suffix + &after_text[2..after_text.len() - 1] // Remove "PT" prefix and "S" suffix } else { - after_pt // Already stripped "PT" prefix + &after_text[2..] // Just remove "PT" prefix }; // Try to parse as seconds (e.g., "86225.992004616") @@ -902,15 +903,14 @@ fn parse_retry_after(error_msg: &str) -> Option> { } // Process last unit - if !current_unit.is_empty() - && !current_num.is_empty() - && let Ok(val) = current_num.parse::() - { - match current_unit.as_str() { - "H" => total_seconds += val * 3600.0, - "M" => total_seconds += val * 60.0, - "S" => total_seconds += val, - _ => {} + if !current_unit.is_empty() && !current_num.is_empty() { + if let Ok(val) = current_num.parse::() { + match current_unit.as_str() { + "H" => total_seconds += val * 3600.0, + "M" => total_seconds += val * 60.0, + "S" => total_seconds += val, + _ => {} + } } } @@ -961,7 +961,7 @@ async fn check_account_exists( /// Helper function to create a new Let's Encrypt account and save credentials /// Handles rate limits by waiting for the retry-after time async fn create_new_account( - storage: &dyn Storage, + storage: &Box, email: &str, lets_encrypt_url: &str, ) -> AtomicServerResult<(instant_acme::Account, instant_acme::AccountCredentials)> { @@ -1197,8 +1197,7 @@ async fn request_cert_internal(config: &Config) -> AtomicServerResult<()> { warn!( "Stored credentials invalid and account doesn't exist. Creating new account." ); - create_new_account(storage.as_ref(), &email, lets_encrypt_url) - .await? + create_new_account(&storage, &email, lets_encrypt_url).await? } Err(e) => { let error_msg = format!("{}", e); @@ -1209,23 +1208,15 @@ async fn request_cert_internal(config: &Config) -> AtomicServerResult<()> { warn!( "Rate limit hit while checking account. Will wait and retry in create_new_account." ); - create_new_account( - storage.as_ref(), - &email, - lets_encrypt_url, - ) - .await? + create_new_account(&storage, &email, lets_encrypt_url) + .await? } else { warn!( "Failed to check account existence: {}. Creating new account.", e ); - create_new_account( - storage.as_ref(), - &email, - lets_encrypt_url, - ) - .await? + create_new_account(&storage, &email, lets_encrypt_url) + .await? } } } @@ -1237,13 +1228,13 @@ async fn request_cert_internal(config: &Config) -> AtomicServerResult<()> { "Failed to parse stored credentials: {}. Creating new account.", e ); - create_new_account(storage.as_ref(), &email, lets_encrypt_url).await? + create_new_account(&storage, &email, lets_encrypt_url).await? } } } None => { // No stored credentials, create a new account - create_new_account(storage.as_ref(), &email, lets_encrypt_url).await? + create_new_account(&storage, &email, lets_encrypt_url).await? } }; @@ -1808,7 +1799,7 @@ async fn write_certs( // Combine cert and chain to create fullchain let mut fullchain = domain_cert_pem.clone(); if !chain_pem.is_empty() { - fullchain.push('\n'); + fullchain.push_str("\n"); fullchain.push_str(&chain_pem); } diff --git a/src/proxy/acme/storage/redis.rs b/src/proxy/acme/storage/redis.rs index 795409b6..b60bdf2e 100644 --- a/src/proxy/acme/storage/redis.rs +++ b/src/proxy/acme/storage/redis.rs @@ -120,9 +120,7 @@ impl RedisStorage { if ssl_config.insecure { tls_builder.danger_accept_invalid_certs(true); tls_builder.danger_accept_invalid_hostnames(true); - tracing::error!( - "Redis SSL: Certificate verification DISABLED (insecure mode) — connections are vulnerable to MITM attacks" - ); + tracing::warn!("Redis SSL: Certificate verification disabled (insecure mode)"); } let _tls_connector = tls_builder diff --git a/src/proxy/bgservice.rs b/src/proxy/bgservice.rs index 9e5b5630..b1527c79 100644 --- a/src/proxy/bgservice.rs +++ b/src/proxy/bgservice.rs @@ -85,7 +85,7 @@ impl BackgroundService for LB { let file_load = FromFileProvider { path: self.config.upstreams_conf.clone(), }; - drop(tokio::spawn(async move { file_load.start(tx).await })); + let _ = tokio::spawn(async move { file_load.start(tx).await }); } _ => { error!("Unknown discovery type: {}", config.typecfg); @@ -109,7 +109,8 @@ impl BackgroundService for LB { break; } val = rx.next() => { - if let Some(ss) = val { + match val { + Some(ss) => { // Update healthcheck settings from upstreams config if available if let Some(interval) = ss.healthcheck_interval { healthcheck_interval = interval; @@ -130,16 +131,15 @@ impl BackgroundService for LB { let im_clone = self.ump_byid.clone(); let method_clone = healthcheck_method.clone(); let interval_clone = healthcheck_interval; - drop(tokio::spawn(async move { + let _ = tokio::spawn(async move { healthcheck::hc2( uu_clone, ff_clone, im_clone, (method_clone.as_str(), u64::from(interval_clone)), - false, ) .await - })); + }); healthcheck_started = true; } @@ -157,7 +157,6 @@ impl BackgroundService for LB { let mut new = (*current).clone(); new.sticky_sessions = ss.extraparams.sticky_sessions; new.https_proxy_enabled = ss.extraparams.https_proxy_enabled; - new.forward_fingerprints = ss.extraparams.forward_fingerprints; new.authentication = ss.extraparams.authentication.clone(); self.extraparams.store(Arc::new(new)); @@ -202,10 +201,11 @@ impl BackgroundService for LB { } // Update upstreams certificate mappings - if let Some(certs_arc) = &self.certificates - && let Some(certs) = certs_arc.load().as_ref() { - certs.set_upstreams_cert_map(ss.certificates.clone()); - info!("Updated upstreams certificate mappings: {} entries", ss.certificates.len()); + if let Some(certs_arc) = &self.certificates { + if let Some(certs) = certs_arc.load().as_ref() { + certs.set_upstreams_cert_map(ss.certificates.clone()); + info!("Updated upstreams certificate mappings: {} entries", ss.certificates.len()); + } } // Check and request certificates for new/updated domains @@ -213,6 +213,8 @@ impl BackgroundService for LB { // info!("Upstreams list is changed, updating to:"); // print_upstreams(&self.ump_full); + } + None => {} } } } diff --git a/src/proxy/gethosts.rs b/src/proxy/gethosts.rs index 55019005..52d153a6 100644 --- a/src/proxy/gethosts.rs +++ b/src/proxy/gethosts.rs @@ -95,11 +95,11 @@ pub trait GetHost { #[async_trait] impl GetHost for LB { fn get_host(&self, peer: &str, path: &str, backend_id: Option<&str>) -> Option { - if let Some(b) = backend_id - && let Some(bb) = self.ump_byid.get(b) - { - // println!("BIB :===> {:?}", Some(bb.value())); - return Some(bb.value().clone()); + if let Some(b) = backend_id { + if let Some(bb) = self.ump_byid.get(b) { + // println!("BIB :===> {:?}", Some(bb.value())); + return Some(bb.value().clone()); + } } // Check internal_paths first - these paths work regardless of hostname @@ -139,7 +139,7 @@ impl GetHost for LB { } else if path.len() == pattern_prefix.len() { // Exact match (already handled above, but keep for completeness) true - } else if let Some(next_char) = path.chars().nth(pattern_prefix.len()) { + } else if let Some(next_char) = path.chars().skip(pattern_prefix.len()).next() { // Next character after prefix should be / for proper path segment match next_char == '/' } else { @@ -153,7 +153,7 @@ impl GetHost for LB { // Keep the longest (most specific) match based on the prefix length if best_match .as_ref() - .is_none_or(|(_, _, best_len)| prefix_len > *best_len) + .map_or(true, |(_, _, best_len)| prefix_len > *best_len) { best_match = Some((pattern.clone(), matched_server, prefix_len)); } @@ -205,12 +205,12 @@ impl GetHost for LB { break; } } - if best_match.is_none() - && let Some(entry) = host_entry.get("/") - { - let (servers, index) = entry.value(); - if let Some(selected) = select_weighted_server(servers, index) { - best_match = Some(selected); + if best_match.is_none() { + if let Some(entry) = host_entry.get("/") { + let (servers, index) = entry.value(); + if let Some(selected) = select_weighted_server(servers, index) { + best_match = Some(selected); + } } } // println!("Best Match :===> {:?}", best_match); @@ -225,11 +225,11 @@ impl GetHost for LB { let mut current_path = path.to_string(); let mut best_match: Option> = None; loop { - if let Some(entry) = host_entry.get(¤t_path) - && !entry.value().is_empty() - { - best_match = Some(entry.value().clone()); - break; + if let Some(entry) = host_entry.get(¤t_path) { + if !entry.value().is_empty() { + best_match = Some(entry.value().clone()); + break; + } } if let Some(pos) = current_path.rfind('/') { current_path.truncate(pos); @@ -237,11 +237,12 @@ impl GetHost for LB { break; } } - if best_match.is_none() - && let Some(entry) = host_entry.get("/") - && !entry.value().is_empty() - { - best_match = Some(entry.value().clone()); + if best_match.is_none() { + if let Some(entry) = host_entry.get("/") { + if !entry.value().is_empty() { + best_match = Some(entry.value().clone()); + } + } } best_match } @@ -250,11 +251,11 @@ impl GetHost for LB { let mut current_path = path.to_string(); let mut best_match: Option> = None; loop { - if let Some(entry) = host_entry.get(¤t_path) - && !entry.value().is_empty() - { - best_match = Some(entry.value().clone()); - break; + if let Some(entry) = host_entry.get(¤t_path) { + if !entry.value().is_empty() { + best_match = Some(entry.value().clone()); + break; + } } if let Some(pos) = current_path.rfind('/') { current_path.truncate(pos); @@ -262,11 +263,12 @@ impl GetHost for LB { break; } } - if best_match.is_none() - && let Some(entry) = host_entry.get("/") - && !entry.value().is_empty() - { - best_match = Some(entry.value().clone()); + if best_match.is_none() { + if let Some(entry) = host_entry.get("/") { + if !entry.value().is_empty() { + best_match = Some(entry.value().clone()); + } + } } best_match } diff --git a/src/proxy/mod.rs b/src/proxy/mod.rs index de633ab0..65e79e5e 100644 --- a/src/proxy/mod.rs +++ b/src/proxy/mod.rs @@ -4,3 +4,4 @@ pub mod gethosts; pub mod proxy_protocol; pub mod proxyhttp; pub mod start; +pub mod thalamus_post_tls; diff --git a/src/proxy/proxy_protocol.rs b/src/proxy/proxy_protocol.rs index 0e7ea8f3..49093b49 100644 --- a/src/proxy/proxy_protocol.rs +++ b/src/proxy/proxy_protocol.rs @@ -223,19 +223,19 @@ impl AsyncRead for ChainedReader { let this = &mut *self; // First drain the prefix - if let Some(ref prefix) = this.prefix - && this.prefix_pos < prefix.len() - { - let remaining = &prefix[this.prefix_pos..]; - let to_copy = remaining.len().min(buf.remaining()); - buf.put_slice(&remaining[..to_copy]); - this.prefix_pos += to_copy; - - if this.prefix_pos >= prefix.len() { - this.prefix = None; - } + if let Some(ref prefix) = this.prefix { + if this.prefix_pos < prefix.len() { + let remaining = &prefix[this.prefix_pos..]; + let to_copy = remaining.len().min(buf.remaining()); + buf.put_slice(&remaining[..to_copy]); + this.prefix_pos += to_copy; + + if this.prefix_pos >= prefix.len() { + this.prefix = None; + } - return std::task::Poll::Ready(Ok(())); + return std::task::Poll::Ready(Ok(())); + } } // Then read from inner diff --git a/src/proxy/proxyhttp.rs b/src/proxy/proxyhttp.rs index 8e1c75e5..9afd27ac 100644 --- a/src/proxy/proxyhttp.rs +++ b/src/proxy/proxyhttp.rs @@ -27,148 +27,10 @@ use pingora_http::{RequestHeader, ResponseHeader, StatusCode}; use pingora_proxy::{FailToProxy, ProxyHttp, Session}; use serde_json; use std::sync::Arc; -use std::sync::OnceLock; -use std::sync::atomic::{AtomicU64, AtomicUsize}; +use std::sync::atomic::AtomicUsize; use std::time::Duration; use tokio::time::Instant; -/// Cached BPF-derived fingerprints per connection (survives keep-alive/H2 reuse). -struct CachedBpfFingerprints { - ja4t: String, - ja4t_hash: String, - ja4l: String, - stored_at: std::time::Instant, -} - -static BPF_FINGERPRINT_CACHE: OnceLock> = OnceLock::new(); - -fn bpf_cache() -> &'static DashMap { - BPF_FINGERPRINT_CACHE.get_or_init(DashMap::new) -} - -static BPF_CACHE_CLEANUP_COUNTER: AtomicU64 = AtomicU64::new(0); - -/// Cached JA4T fingerprint per client IP (stable across connections — OS-level TCP stack property). -struct CachedJa4tFingerprint { - ja4t: String, - ja4t_hash: String, - stored_at: std::time::Instant, -} - -static JA4T_IP_CACHE: OnceLock> = OnceLock::new(); - -fn ja4t_ip_cache() -> &'static DashMap { - JA4T_IP_CACHE.get_or_init(DashMap::new) -} - -/// Cached JA4TS fingerprint per upstream server (stable across connections). -struct CachedJa4tsFingerprint { - ja4ts: String, - ja4ts_hash: String, - stored_at: std::time::Instant, -} - -static JA4TS_CACHE: OnceLock> = OnceLock::new(); - -fn ja4ts_cache() -> &'static DashMap { - JA4TS_CACHE.get_or_init(DashMap::new) -} - -/// Resolve an upstream address string to an IpAddr. -/// Handles: plain IPs, "ip:port" format, hostnames, "[ipv6]:port" format. -/// Fast path: parse as IP directly. Fallback: DNS resolution for hostnames. -fn resolve_upstream_ip(address: &str) -> Option { - // Fast path: try parsing as IP address directly - if let Ok(ip) = address.parse::() { - return Some(ip); - } - // Handle "ip:port" format — strip port suffix and try again - if let Some(ip_part) = address.rsplit_once(':').map(|(ip, _port)| ip) { - // Handle "[ipv6]:port" bracket format - let clean = ip_part.trim_start_matches('[').trim_end_matches(']'); - if let Ok(ip) = clean.parse::() { - return Some(ip); - } - } - // Fallback: DNS resolution (blocking, but upstream addresses rarely change) - use std::net::ToSocketAddrs; - format!("{}:0", address) - .to_socket_addrs() - .ok() - .and_then(|mut addrs| addrs.next()) - .map(|addr| addr.ip()) -} - -/// Map BoringSSL/OpenSSL cipher name to IANA cipher suite ID. -fn cipher_name_to_iana_id(name: &str) -> u16 { - match name { - // TLS 1.3 - "TLS_AES_128_GCM_SHA256" => 0x1301, - "TLS_AES_256_GCM_SHA384" => 0x1302, - "TLS_CHACHA20_POLY1305_SHA256" => 0x1303, - "TLS_AES_128_CCM_SHA256" => 0x1304, - // TLS 1.2 ECDHE - "ECDHE-RSA-AES128-GCM-SHA256" => 0xc02f, - "ECDHE-RSA-AES256-GCM-SHA384" => 0xc030, - "ECDHE-ECDSA-AES128-GCM-SHA256" => 0xc02b, - "ECDHE-ECDSA-AES256-GCM-SHA384" => 0xc02c, - "ECDHE-RSA-CHACHA20-POLY1305" => 0xcca8, - "ECDHE-ECDSA-CHACHA20-POLY1305" => 0xcca9, - // TLS 1.2 RSA - "AES128-GCM-SHA256" => 0x009c, - "AES256-GCM-SHA384" => 0x009d, - "AES128-SHA" => 0x002f, - "AES256-SHA" => 0x0035, - "AES128-SHA256" => 0x003c, - "AES256-SHA256" => 0x003d, - // TLS 1.2 DHE - "DHE-RSA-AES128-GCM-SHA256" => 0x009e, - "DHE-RSA-AES256-GCM-SHA384" => 0x009f, - "DHE-RSA-CHACHA20-POLY1305" => 0xccaa, - // TLS 1.2 ECDHE CBC - "ECDHE-RSA-AES128-SHA256" => 0xc027, - "ECDHE-RSA-AES256-SHA384" => 0xc028, - "ECDHE-ECDSA-AES128-SHA256" => 0xc023, - "ECDHE-ECDSA-AES256-SHA384" => 0xc024, - "ECDHE-RSA-AES128-SHA" => 0xc013, - "ECDHE-RSA-AES256-SHA" => 0xc014, - "ECDHE-ECDSA-AES128-SHA" => 0xc009, - "ECDHE-ECDSA-AES256-SHA" => 0xc00a, - _ => 0x0000, - } -} - -/// Reconstruct ServerHello extensions from negotiated TLS state. -/// -/// - TLS 1.3: RFC 8446 §4.1.3 mandates only `supported_versions` and `key_share`. -/// - TLS 1.2: BoringSSL includes `renegotiation_info`, `extended_master_secret`, -/// optionally `ec_point_formats` (ECDHE ciphers), and `alpn` (if negotiated). -fn build_server_hello_extensions( - tls_version: &nstealth::TlsVersion, - cipher_name: Option<&str>, - has_alpn: bool, -) -> Vec { - match tls_version { - nstealth::TlsVersion::Tls13 => { - vec![0x002b, 0x0033] // supported_versions, key_share - } - _ => { - let mut exts = Vec::new(); - exts.push(0xff01); // renegotiation_info (always in BoringSSL) - exts.push(0x0017); // extended_master_secret - if let Some(name) = cipher_name - && (name.contains("ECDHE") || name.contains("ECDSA")) - { - exts.push(0x000b); // ec_point_formats - } - if has_alpn { - exts.push(0x0010); // application_layer_protocol_negotiation - } - exts - } - } -} - #[derive(Clone)] pub struct LB { pub ump_upst: Arc, @@ -191,27 +53,32 @@ pub struct Context { backend_id: String, start_time: Instant, upstream_start_time: Option, + request_filter_end_time: Option, + upstream_peer_start_time: Option, + upstream_peer_end_time: Option, hostname: Option, upstream_peer: Option, extraparams: arc_swap::Guard>, tls_fingerprint: Option>, - /// Instant when TLS ClientHello was received (for JA4L TLS latency) - tls_hello_instant: Option, request_body: Vec, malware_detected: bool, block_body: Option, waf_result: Option, threat_data: Option, upstream_time: Option, + response_header_time: Option, + post_tls_ids_time_us: Option, + request_filter_total_ms: Option, + request_filter_keepalive_ms: Option, + request_filter_access_rules_ms: Option, + request_filter_tls_fingerprint_ms: Option, + request_filter_threat_intel_ms: Option, + request_filter_waf_ms: Option, + request_filter_routing_ms: Option, + upstream_peer_select_ms: Option, disable_access_log: bool, keepalive_override: Option>, error_details: Option, - /// Resolved upstream IP address, stored during upstream_peer() for JA4TS lookup - upstream_resolved_ip: Option, - /// Cached JA4T fingerprint (TCP SYN), looked up once and shared between forwarding and logging - ja4t_fingerprint: Option, - /// Cached JA4T hash - ja4t_hash: Option, } #[async_trait] @@ -222,31 +89,40 @@ impl ProxyHttp for LB { backend_id: String::new(), start_time: Instant::now(), upstream_start_time: None, + request_filter_end_time: None, + upstream_peer_start_time: None, + upstream_peer_end_time: None, hostname: None, upstream_peer: None, extraparams: self.extraparams.load(), tls_fingerprint: None, - tls_hello_instant: None, request_body: Vec::new(), malware_detected: false, block_body: None, waf_result: None, threat_data: None, upstream_time: None, + response_header_time: None, + post_tls_ids_time_us: None, + request_filter_total_ms: None, + request_filter_keepalive_ms: None, + request_filter_access_rules_ms: None, + request_filter_tls_fingerprint_ms: None, + request_filter_threat_intel_ms: None, + request_filter_waf_ms: None, + request_filter_routing_ms: None, + upstream_peer_select_ms: None, disable_access_log: false, keepalive_override: None, error_details: None, - upstream_resolved_ip: None, - ja4t_fingerprint: None, - ja4t_hash: None, } } async fn request_filter(&self, session: &mut Session, _ctx: &mut Self::CTX) -> Result { - // Only enable body buffering when content scanning is active - let scanner = crate::security::waf::actions::content_scanning::get_global_content_scanner(); - if scanner.is_some_and(|s| s.is_enabled()) { - session.enable_retry_buffering(); - } + let request_filter_started = Instant::now(); + let keepalive_started = Instant::now(); + + // Enable body buffering for content scanning + session.enable_retry_buffering(); let req_header = session.req_header(); let conn_header = req_header @@ -302,13 +178,15 @@ impl ProxyHttp for LB { .set_close_on_response_before_downstream_finish(true); } } + _ctx.request_filter_keepalive_ms = Some(keepalive_started.elapsed().as_millis() as u64); let ep = _ctx.extraparams.clone(); // Userland access rules check (fallback when eBPF/XDP is not available) // Check if IP is blocked by access rules + let access_rules_started = Instant::now(); if let Some(peer_addr) = session.client_addr().and_then(|addr| addr.as_inet()) { - let client_ip: std::net::IpAddr = peer_addr.ip(); + let client_ip: std::net::IpAddr = peer_addr.ip().into(); // Check if IP is blocked if crate::security::access_rules::is_ip_blocked_by_access_rules(client_ip) { @@ -322,56 +200,110 @@ impl ProxyHttp for LB { session .write_response_header(Box::new(header), true) .await?; + mark_request_filter_done(_ctx, request_filter_started); return Ok(true); } } + _ctx.request_filter_access_rules_ms = + Some(access_rules_started.elapsed().as_millis() as u64); // Try to get TLS fingerprint if available // Use fallback lookup to handle PROXY protocol address mismatches - if _ctx.tls_fingerprint.is_none() + let tls_fp_started = Instant::now(); + if _ctx.tls_fingerprint.is_none() { + if let Some(peer_addr) = session.client_addr().and_then(|addr| addr.as_inet()) { + let std_addr = std::net::SocketAddr::new(peer_addr.ip().into(), peer_addr.port()); + if let Some(fingerprint) = + crate::utils::tls_client_hello::get_fingerprint_with_fallback(&std_addr) + { + _ctx.tls_fingerprint = Some(fingerprint.clone()); + debug!( + "TLS Fingerprint retrieved for session - Peer: {}, JA4: {}, SNI: {:?}, ALPN: {:?}", + std_addr, fingerprint.ja4, fingerprint.sni, fingerprint.alpn + ); + } else { + debug!( + "No TLS fingerprint found in storage for peer: {} (PROXY protocol may cause this)", + std_addr + ); + } + } + } + _ctx.request_filter_tls_fingerprint_ms = Some(tls_fp_started.elapsed().as_millis() as u64); + + // Run Thalamus IDS on decrypted HTTP requests (post-TLS termination path). + // This keeps the logical pipeline as firewall -> IDS -> WAF/... for proxy traffic. + if crate::proxy::thalamus_post_tls::is_enabled() && let Some(peer_addr) = session.client_addr().and_then(|addr| addr.as_inet()) { - let std_addr = std::net::SocketAddr::new(peer_addr.ip(), peer_addr.port()); - if let Some(fingerprint) = - crate::utils::tls_client_hello::get_fingerprint_with_fallback(&std_addr) - { - _ctx.tls_fingerprint = Some(fingerprint.clone()); - // Also retrieve ClientHello instant for JA4L TLS latency measurement - _ctx.tls_hello_instant = - crate::utils::tls_client_hello::get_hello_instant(&std_addr); - debug!( - "TLS Fingerprint retrieved for session - Peer: {}, JA4: {}, SNI: {:?}, ALPN: {:?}", - std_addr, fingerprint.ja4, fingerprint.sni, fingerprint.alpn - ); - } else { - debug!( - "No TLS fingerprint found in storage for peer: {} (PROXY protocol may cause this)", - std_addr + let ids_started = Instant::now(); + let tls_terminated = session + .stream() + .map(|stream| stream.get_ssl().is_some()) + .unwrap_or(false); + let socket_addr = std::net::SocketAddr::new(peer_addr.ip().into(), peer_addr.port()); + let ids_alert = crate::proxy::thalamus_post_tls::inspect_http_request( + session.req_header(), + socket_addr, + &[], + ) + .await; + _ctx.post_tls_ids_time_us = + Some(ids_started.elapsed().as_micros().min(u128::from(u64::MAX)) as u64); + if let Some(ids_alert) = ids_alert { + info!( + "Thalamus proxy IDS matched: sid={}, action={}, msg=\"{}\", uri={}, tls_terminated={}", + ids_alert.signature_id, + ids_alert.action, + ids_alert.signature_name, + session.req_header().uri, + tls_terminated ); + if ids_alert.should_block { + let mut header = ResponseHeader::build(403, None).unwrap(); + header + .insert_header("X-IDS-Rule-ID", ids_alert.signature_id.to_string()) + .ok(); + header + .insert_header("X-IDS-Rule", ids_alert.signature_name) + .ok(); + header.insert_header("X-IDS-Action", ids_alert.action).ok(); + session.set_keepalive(None); + session + .write_response_header(Box::new(header), true) + .await?; + mark_request_filter_done(_ctx, request_filter_started); + return Ok(true); + } } } // Get threat intelligence data BEFORE WAF evaluation // This ensures threat intelligence is available in access logs even when WAF blocks/challenges early - if let Some(peer_addr) = session.client_addr().and_then(|addr| addr.as_inet()) - && _ctx.threat_data.is_none() - { - match crate::security::waf::threat::get_threat_intel(&peer_addr.ip().to_string()).await - { - Ok(Some(threat_response)) => { - _ctx.threat_data = Some(threat_response); - debug!("Threat intelligence retrieved for IP: {}", peer_addr.ip()); - } - Ok(None) => { - debug!("No threat intelligence data for IP: {}", peer_addr.ip()); - } - Err(e) => { - debug!("Threat intelligence error for IP {}: {}", peer_addr.ip(), e); + let threat_intel_started = Instant::now(); + if let Some(peer_addr) = session.client_addr().and_then(|addr| addr.as_inet()) { + if _ctx.threat_data.is_none() { + match crate::security::waf::threat::get_threat_intel(&peer_addr.ip().to_string()) + .await + { + Ok(Some(threat_response)) => { + _ctx.threat_data = Some(threat_response); + debug!("Threat intelligence retrieved for IP: {}", peer_addr.ip()); + } + Ok(None) => { + debug!("No threat intelligence data for IP: {}", peer_addr.ip()); + } + Err(e) => { + debug!("Threat intelligence error for IP {}: {}", peer_addr.ip(), e); + } } } } + _ctx.request_filter_threat_intel_ms = + Some(threat_intel_started.elapsed().as_millis() as u64); // Evaluate WAF rules (skip for internal captcha verification endpoint to prevent challenge loops) + let waf_started = Instant::now(); let request_path = session.req_header().uri.path(); let skip_waf = request_path == "/cgi-bin/captcha/verify"; if !skip_waf { @@ -416,6 +348,7 @@ impl ProxyHttp for LB { session .write_response_header(Box::new(header), true) .await?; + mark_request_filter_done(_ctx, request_filter_started); return Ok(true); } WafAction::Challenge => { @@ -430,26 +363,29 @@ impl ProxyHttp for LB { let mut captcha_token: Option = None; // Check cookies for captcha_token - if let Some(cookies) = session.req_header().headers.get("cookie") - && let Ok(cookie_str) = cookies.to_str() - { - for cookie in cookie_str.split(';') { - let trimmed = cookie.trim(); - if let Some(value) = trimmed.strip_prefix("captcha_token=") - { - captcha_token = Some(value.to_string()); - break; + if let Some(cookies) = session.req_header().headers.get("cookie") { + if let Ok(cookie_str) = cookies.to_str() { + for cookie in cookie_str.split(';') { + let trimmed = cookie.trim(); + if let Some(value) = + trimmed.strip_prefix("captcha_token=") + { + captcha_token = Some(value.to_string()); + break; + } } } } // Check X-Captcha-Token header if not found in cookies - if captcha_token.is_none() - && let Some(token_header) = + if captcha_token.is_none() { + if let Some(token_header) = session.req_header().headers.get("x-captcha-token") - && let Ok(token_str) = token_header.to_str() - { - captcha_token = Some(token_str.to_string()); + { + if let Ok(token_str) = token_header.to_str() { + captcha_token = Some(token_str.to_string()); + } + } } // Validate token if present @@ -532,6 +468,10 @@ impl ProxyHttp for LB { true, ) .await?; + mark_request_filter_done( + _ctx, + request_filter_started, + ); return Ok(true); } Err(e) => { @@ -562,6 +502,10 @@ impl ProxyHttp for LB { true, ) .await?; + mark_request_filter_done( + _ctx, + request_filter_started, + ); return Ok(true); } } @@ -603,6 +547,7 @@ impl ProxyHttp for LB { session .write_response_body(Some(Bytes::from(html)), true) .await?; + mark_request_filter_done(_ctx, request_filter_started); return Ok(true); } Err(e) => { @@ -620,6 +565,7 @@ impl ProxyHttp for LB { session .write_response_header(Box::new(header), true) .await?; + mark_request_filter_done(_ctx, request_filter_started); return Ok(true); } } @@ -686,6 +632,7 @@ impl ProxyHttp for LB { session .write_response_body(Some(Bytes::from(body)), true) .await?; + mark_request_filter_done(_ctx, request_filter_started); return Ok(true); } else { debug!( @@ -726,56 +673,70 @@ impl ProxyHttp for LB { debug!("WAF: No peer address available for request"); } } // end skip_waf + _ctx.request_filter_waf_ms = Some(waf_started.elapsed().as_millis() as u64); - let hostname = return_header_host(session); + let routing_started = Instant::now(); + let hostname = return_header_host(&session); _ctx.hostname = hostname; let mut backend_id = None; - if ep.sticky_sessions - && let Some(cookies) = session.req_header().headers.get("cookie") - && let Ok(cookie_str) = cookies.to_str() - { - for cookie in cookie_str.split(';') { - let trimmed = cookie.trim(); - if let Some(value) = trimmed.strip_prefix("backend_id=") { - backend_id = Some(value); - break; + if ep.sticky_sessions { + if let Some(cookies) = session.req_header().headers.get("cookie") { + if let Ok(cookie_str) = cookies.to_str() { + for cookie in cookie_str.split(';') { + let trimmed = cookie.trim(); + if let Some(value) = trimmed.strip_prefix("backend_id=") { + backend_id = Some(value); + break; + } + } } } } match _ctx.hostname.as_ref() { - None => return Ok(false), + None => { + _ctx.request_filter_routing_ms = Some(routing_started.elapsed().as_millis() as u64); + mark_request_filter_done(_ctx, request_filter_started); + return Ok(false); + } Some(host) => { // let optioninnermap = self.get_host(host.as_str(), host.as_str(), backend_id); let optioninnermap = self.get_host(host.as_str(), session.req_header().uri.path(), backend_id); match optioninnermap { - None => return Ok(false), + None => { + _ctx.request_filter_routing_ms = + Some(routing_started.elapsed().as_millis() as u64); + mark_request_filter_done(_ctx, request_filter_started); + return Ok(false); + } Some(ref innermap) => { // Check for HTTPS redirect before rate limiting - if (ep.https_proxy_enabled.unwrap_or(false) || innermap.https_proxy_enabled) - && let Some(stream) = session.stream() - && stream.get_ssl().is_none() - { - // HTTP request - redirect to HTTPS - let uri = session - .req_header() - .uri - .path_and_query() - .map_or("/", |pq| pq.as_str()); - let port = self.config.proxy_port_tls.unwrap_or(403); - let redirect_url = format!("https://{}:{}{}", host, port, uri); - let mut redirect_response = - ResponseHeader::build(StatusCode::MOVED_PERMANENTLY, None)?; - redirect_response.insert_header("Location", redirect_url)?; - redirect_response.insert_header("Content-Length", "0")?; - session.set_keepalive(None); - session - .write_response_header(Box::new(redirect_response), false) - .await?; - return Ok(true); + if ep.https_proxy_enabled.unwrap_or(false) || innermap.https_proxy_enabled { + if let Some(stream) = session.stream() { + if stream.get_ssl().is_none() { + // HTTP request - redirect to HTTPS + let uri = session + .req_header() + .uri + .path_and_query() + .map_or("/", |pq| pq.as_str()); + let port = self.config.proxy_port_tls.unwrap_or(403); + let redirect_url = format!("https://{}:{}{}", host, port, uri); + let mut redirect_response = + ResponseHeader::build(StatusCode::MOVED_PERMANENTLY, None)?; + redirect_response.insert_header("Location", redirect_url)?; + redirect_response.insert_header("Content-Length", "0")?; + session.set_keepalive(None); + session + .write_response_header(Box::new(redirect_response), false) + .await?; + mark_request_filter_done(_ctx, request_filter_started); + return Ok(true); + } + } } } } @@ -786,6 +747,8 @@ impl ProxyHttp for LB { } } } + _ctx.request_filter_routing_ms = Some(routing_started.elapsed().as_millis() as u64); + mark_request_filter_done(_ctx, request_filter_started); Ok(false) } async fn upstream_peer( @@ -793,6 +756,9 @@ impl ProxyHttp for LB { session: &mut Session, ctx: &mut Self::CTX, ) -> Result> { + let upstream_peer_started = Instant::now(); + ctx.upstream_peer_start_time = Some(upstream_peer_started); + // let host_name = return_header_host(&session); match ctx.hostname.as_ref() { Some(_hostname) => { @@ -800,7 +766,7 @@ impl ProxyHttp for LB { // Some((address, port, ssl, is_h2, https_proxy_enabled)) => { Some(innermap) => { let mut peer = Box::new(HttpPeer::new( - (innermap.address.clone(), innermap.port), + (innermap.address.clone(), innermap.port.clone()), innermap.ssl_enabled, String::new(), )); @@ -820,19 +786,19 @@ impl ProxyHttp for LB { // Defaults: connection_timeout: 30s, read_timeout: 120s, write_timeout: 30s, idle_timeout: 60s peer.options.connection_timeout = innermap .connection_timeout - .map(Duration::from_secs) + .map(|s| Duration::from_secs(s)) .or(Some(Duration::from_secs(30))); peer.options.read_timeout = innermap .read_timeout - .map(Duration::from_secs) + .map(|s| Duration::from_secs(s)) .or(Some(Duration::from_secs(120))); peer.options.write_timeout = innermap .write_timeout - .map(Duration::from_secs) + .map(|s| Duration::from_secs(s)) .or(Some(Duration::from_secs(30))); peer.options.idle_timeout = innermap .idle_timeout - .map(Duration::from_secs) + .map(|s| Duration::from_secs(s)) .or(Some(Duration::from_secs(60))); ctx.backend_id = format!( @@ -841,8 +807,7 @@ impl ProxyHttp for LB { innermap.port.clone(), innermap.ssl_enabled ); - // Pre-resolve upstream IP for JA4TS lookup (avoids DNS mismatch) - ctx.upstream_resolved_ip = resolve_upstream_ip(&innermap.address); + mark_upstream_peer_done(ctx, upstream_peer_started); Ok(peer) } None => { @@ -852,6 +817,7 @@ impl ProxyHttp for LB { { error!("Failed to send error response: {:?}", e); } + mark_upstream_peer_done(ctx, upstream_peer_started); Err(Box::new(Error { etype: HTTPStatus(502), esource: Upstream, @@ -870,6 +836,7 @@ impl ProxyHttp for LB { { error!("Failed to send error response: {:?}", e); } + mark_upstream_peer_done(ctx, upstream_peer_started); Err(Box::new(Error { etype: HTTPStatus(502), esource: Upstream, @@ -905,439 +872,16 @@ impl ProxyHttp for LB { } // Only set default Host if config doesn't override it - if !config_has_host && let Some(hostname) = ctx.hostname.as_ref() { - upstream_request.insert_header("Host", hostname)?; + if !config_has_host { + if let Some(hostname) = ctx.hostname.as_ref() { + upstream_request.insert_header("Host", hostname)?; + } } if let Some(peer) = ctx.upstream_peer.as_ref() { upstream_request.insert_header("X-Forwarded-For", peer.address.as_str())?; } - // Forward all JA4+ fingerprint headers to origin if enabled - log::debug!( - "Fingerprints: forward_fingerprints={}", - ctx.extraparams.forward_fingerprints - ); - if ctx.extraparams.forward_fingerprints { - // JA4: TLS fingerprint (from ClientHello, already in context) - if let Some(tls_fp) = ctx.tls_fingerprint.as_ref() { - log::debug!("JA4: {}", tls_fp.ja4); - upstream_request.insert_header("X-JA4", &tls_fp.ja4)?; - upstream_request.insert_header("X-JA4-Raw", &tls_fp.ja4_raw)?; - } else { - log::debug!("JA4: no TLS fingerprint in context"); - } - - // JA4T: TCP fingerprint (from BPF SYN capture, cached for keep-alive/H2) - let ja4t_collector = - crate::utils::fingerprint::tcp_fingerprint::get_global_tcp_fingerprint_collector(); - if ja4t_collector.is_none() { - log::debug!("JA4T: no global collector"); - } - if let Some(collector) = ja4t_collector { - let client_inet = _session.client_addr().and_then(|addr| addr.as_inet()); - if client_inet.is_none() { - log::debug!( - "JA4T: client_addr not available as inet (addr={:?})", - _session.client_addr() - ); - } - if let Some(peer_addr) = client_inet { - let cache_key = format!("{}:{}", peer_addr.ip(), peer_addr.port()); - - let mut ja4t_set = false; - if let Some(tcp_data) = - collector.lookup_fingerprint(peer_addr.ip(), peer_addr.port()) - { - let ja4t_inner = if !tcp_data.options.is_empty() && tcp_data.options[0] != 0 - { - nstealth::Ja4t::from_raw_options( - tcp_data.window_size, - &tcp_data.options, - ) - } else { - let mss = if tcp_data.mss > 0 { - Some(tcp_data.mss) - } else { - None - }; - let ws = if tcp_data.window_scale > 0 { - Some(tcp_data.window_scale) - } else { - None - }; - nstealth::Ja4t::new(tcp_data.window_size, &[], mss, ws) - }; - let fp = ja4t_inner.fingerprint(); - let hash = nstealth::hash12(&fp); - if fp != "0_00_00_00" { - // Fresh BPF data — cache per-connection and per-IP - let mut entry = - bpf_cache().entry(cache_key.clone()).or_insert_with(|| { - CachedBpfFingerprints { - ja4t: String::new(), - ja4t_hash: String::new(), - ja4l: String::new(), - stored_at: std::time::Instant::now(), - } - }); - entry.ja4t = fp.clone(); - entry.ja4t_hash = hash.clone(); - entry.stored_at = std::time::Instant::now(); - // Also cache by IP only (JA4T is OS-level, survives reconnects) - ja4t_ip_cache().insert( - peer_addr.ip().to_string(), - CachedJa4tFingerprint { - ja4t: fp.clone(), - ja4t_hash: hash.clone(), - stored_at: std::time::Instant::now(), - }, - ); - log::debug!("JA4T: {} (hash: {})", fp, hash); - upstream_request.insert_header("X-JA4T", &fp)?; - upstream_request.insert_header("X-JA4T-Hash", &hash)?; - // Store in context for logging reuse - ctx.ja4t_fingerprint = Some(fp); - ctx.ja4t_hash = Some(hash); - ja4t_set = true; - } - // If fp == "0_00_00_00", BPF returned stale data — fall through to cache - } - if !ja4t_set { - // Try per-connection cache first (keep-alive/H2 reuse) - if let Some(mut cached) = bpf_cache().get_mut(&cache_key) - && !cached.ja4t.is_empty() - { - upstream_request.insert_header("X-JA4T", &cached.ja4t)?; - upstream_request.insert_header("X-JA4T-Hash", &cached.ja4t_hash)?; - ctx.ja4t_fingerprint = Some(cached.ja4t.clone()); - ctx.ja4t_hash = Some(cached.ja4t_hash.clone()); - cached.stored_at = std::time::Instant::now(); - ja4t_set = true; - } - } - if !ja4t_set { - // Last resort: IP-keyed cache (survives keepalive rebuild / port change) - let ip_key = peer_addr.ip().to_string(); - if let Some(cached) = ja4t_ip_cache().get(&ip_key) - && cached.stored_at.elapsed() < std::time::Duration::from_secs(3600) - { - log::debug!("JA4T: {} (ip-cache)", cached.ja4t); - upstream_request.insert_header("X-JA4T", &cached.ja4t)?; - upstream_request.insert_header("X-JA4T-Hash", &cached.ja4t_hash)?; - ctx.ja4t_fingerprint = Some(cached.ja4t.clone()); - ctx.ja4t_hash = Some(cached.ja4t_hash.clone()); - // Promote to per-connection cache - let mut entry = - bpf_cache().entry(cache_key.clone()).or_insert_with(|| { - CachedBpfFingerprints { - ja4t: String::new(), - ja4t_hash: String::new(), - ja4l: String::new(), - stored_at: std::time::Instant::now(), - } - }); - entry.ja4t = cached.ja4t.clone(); - entry.ja4t_hash = cached.ja4t_hash.clone(); - entry.stored_at = std::time::Instant::now(); - } else { - log::debug!( - "JA4T: not found for {}:{} after all lookup tiers", - peer_addr.ip(), - peer_addr.port() - ); - } - } - } - } - - // JA4TS: TCP SYN-ACK server fingerprint (from BPF capture of upstream's SYN-ACK) - if let Some(peer) = ctx.upstream_peer.as_ref() { - log::debug!("JA4TS: upstream_peer={}:{}", peer.address, peer.port); - let ja4ts_cache_key = format!("{}:{}", peer.address, peer.port); - - // Check server-level cache first (1-hour TTL, server fingerprints are stable) - let mut ja4ts_set = false; - if let Some(cached) = ja4ts_cache().get(&ja4ts_cache_key) - && cached.stored_at.elapsed() < std::time::Duration::from_secs(3600) - { - log::debug!("JA4TS: {} (cached)", cached.ja4ts); - upstream_request.insert_header("X-JA4TS", &cached.ja4ts)?; - upstream_request.insert_header("X-JA4TS-Hash", &cached.ja4ts_hash)?; - ja4ts_set = true; - } - - if !ja4ts_set { - let ja4ts_collector = crate::utils::fingerprint::tcp_fingerprint::get_global_tcp_fingerprint_collector(); - if ja4ts_collector.is_none() { - log::debug!("JA4TS: no global collector"); - } - if let Some(collector) = ja4ts_collector { - // Use pre-resolved IP from upstream_peer() to avoid DNS mismatch - let server_ip_opt = ctx - .upstream_resolved_ip - .or_else(|| resolve_upstream_ip(&peer.address)); - log::debug!( - "JA4TS: resolved_ip={:?} (pre-resolved={:?})", - server_ip_opt, - ctx.upstream_resolved_ip - ); - if let Some(server_ip) = server_ip_opt { - match collector.lookup_synack_fingerprint(server_ip, peer.port) { - Some(synack_data) => { - log::debug!( - "JA4TS: BPF hit for {}:{} — win={} mss={} ws={} opts_len={}", - server_ip, - peer.port, - synack_data.window_size, - synack_data.mss, - synack_data.window_scale, - synack_data.options_len - ); - let ja4ts = nstealth::Ja4ts::from_raw_options( - synack_data.window_size, - &synack_data.options, - ); - let fp = ja4ts.fingerprint(); - let hash = nstealth::hash12(&fp); - if fp != "0_00_00_00" { - log::debug!("JA4TS: {} (hash: {})", fp, hash); - ja4ts_cache().insert( - ja4ts_cache_key.clone(), - CachedJa4tsFingerprint { - ja4ts: fp.clone(), - ja4ts_hash: hash.clone(), - stored_at: std::time::Instant::now(), - }, - ); - upstream_request.insert_header("X-JA4TS", &fp)?; - upstream_request.insert_header("X-JA4TS-Hash", &hash)?; - } else { - log::debug!( - "JA4TS: zero fingerprint for {}:{}", - server_ip, - peer.port - ); - } - } - None => { - log::debug!( - "JA4TS: no BPF data for {}:{}", - server_ip, - peer.port - ); - } - } - } - } - } - } else { - log::debug!("JA4TS: no upstream_peer in context"); - } - - // JA4H: HTTP header fingerprint (generated from client request headers) - let method = _session.req_header().method.as_str(); - let version = match _session.req_header().version { - http::Version::HTTP_10 => "HTTP/1.0", - http::Version::HTTP_11 => "HTTP/1.1", - http::Version::HTTP_2 => "HTTP/2.0", - http::Version::HTTP_3 => "HTTP/3.0", - _ => "HTTP/1.1", - }; - let ja4h = crate::utils::fingerprint::ja4_plus::Ja4hFingerprint::from_http_request( - method, - version, - &_session.req_header().headers, - ); - log::debug!("JA4H: {}", ja4h.fingerprint); - upstream_request.insert_header("X-JA4H", &ja4h.fingerprint)?; - - // JA4S: TLS ServerHello fingerprint (proxy's negotiated TLS parameters) - // Try stream().get_ssl() first (HTTP/1), fall back to digest() (HTTP/2) - let ja4s_generated = if let Some(stream) = _session.stream() { - if let Some(ssl) = stream.get_ssl() { - let tls_version = match ssl.version_str() { - "TLSv1.3" => nstealth::TlsVersion::Tls13, - "TLSv1.2" => nstealth::TlsVersion::Tls12, - "TLSv1.1" => nstealth::TlsVersion::Tls11, - "TLSv1" => nstealth::TlsVersion::Tls10, - _ => nstealth::TlsVersion::Unknown(0), - }; - let cipher_id = ssl - .current_cipher() - .map(|c| cipher_name_to_iana_id(c.name())) - .unwrap_or(0); - let alpn = ssl - .selected_alpn_protocol() - .and_then(|bytes| String::from_utf8(bytes.to_vec()).ok()); - // Reconstruct ServerHello extensions from negotiated TLS state - let extensions = build_server_hello_extensions( - &tls_version, - ssl.current_cipher().map(|c| c.name()), - alpn.is_some(), - ); - let ja4s = nstealth::Ja4s::new(false, tls_version, cipher_id, extensions, alpn); - let ja4s_fp = ja4s.fingerprint(); - log::debug!("JA4S: {} (stream)", ja4s_fp); - upstream_request.insert_header("X-JA4S", &ja4s_fp)?; - true - } else { - false - } - } else { - false - }; - // Fallback for HTTP/2: stream() returns None, use digest() for TLS info - if !ja4s_generated { - if let Some(digest) = _session.digest() { - if let Some(ssl_digest) = &digest.ssl_digest { - let tls_version = match ssl_digest.version.as_ref() { - "TLSv1.3" => nstealth::TlsVersion::Tls13, - "TLSv1.2" => nstealth::TlsVersion::Tls12, - "TLSv1.1" => nstealth::TlsVersion::Tls11, - "TLSv1" => nstealth::TlsVersion::Tls10, - _ => nstealth::TlsVersion::Unknown(0), - }; - let cipher_id = cipher_name_to_iana_id(&ssl_digest.cipher); - let alpn = Some("h2".to_string()); - let extensions = build_server_hello_extensions( - &tls_version, - Some(&ssl_digest.cipher), - true, // HTTP/2 always has ALPN - ); - let ja4s = - nstealth::Ja4s::new(false, tls_version, cipher_id, extensions, alpn); - let ja4s_fp = ja4s.fingerprint(); - log::debug!("JA4S: {} (digest)", ja4s_fp); - upstream_request.insert_header("X-JA4S", &ja4s_fp)?; - } else { - log::debug!("JA4S: no ssl_digest in digest"); - } - } else { - log::debug!("JA4S: no digest available"); - } - } - - // JA4L: Latency fingerprint (JA4L-S: server measuring client distance, cached for keep-alive/H2) - // TCP timing from BPF, TLS timing from ClientHello callback - // Format: {tcp_half_rtt}_{ttl} or {tcp_half_rtt}_{ttl}_{tls_half_rtt} - // TLS timing is a connection-level property computed once on the first request, - // then cached for all subsequent requests on the same connection. - if let Some(collector) = crate::utils::fingerprint::latency_fingerprint::get_global_latency_fingerprint_collector() { - if let Some(client_addr) = _session.client_addr().and_then(|addr| addr.as_inet()) { - let ja4l_cache_key = format!("{}:{}", client_addr.ip(), client_addr.port()); - - // Use cached JA4L if available (preserves TLS component across keep-alive) - let cached_ja4l = bpf_cache().get_mut(&ja4l_cache_key) - .and_then(|mut entry| if entry.ja4l.is_empty() { None } else { - // Refresh TTL so keep-alive/H2 entries survive cleanup - entry.stored_at = std::time::Instant::now(); - Some(entry.ja4l.clone()) - }); - - if let Some(ref cached_fp) = cached_ja4l { - upstream_request.insert_header("X-JA4L", cached_fp)?; - } else { - // First request on this connection — compute and cache - let lookup_result = collector.lookup_connection( - client_addr.ip(), - client_addr.port(), - ); - if let Some(latency_data) = &lookup_result { - log::debug!( - "JA4L: BPF latency data found for {}:{} - state={}, syn={}ns, synack={}ns, ack={}ns, ttl={}", - client_addr.ip(), client_addr.port(), - latency_data.state, latency_data.syn_time_ns, - latency_data.synack_time_ns, latency_data.ack_time_ns, - latency_data.client_ttl - ); - if latency_data.is_complete() { - let syn_us = latency_data.syn_time_ns / 1000; - let synack_us = latency_data.synack_time_ns / 1000; - let ack_us = latency_data.ack_time_ns / 1000; - - let mut ja4l = nstealth::Ja4l::new( - syn_us, - synack_us, - ack_us, - latency_data.client_ttl, - latency_data.server_ttl, - ); - - // Add TLS handshake timing if available - if let Some(hello_instant) = ctx.tls_hello_instant { - let request_start: std::time::Instant = ctx.start_time.into_std(); - if let Some(tls_elapsed) = request_start.checked_duration_since(hello_instant) { - let tls_elapsed_us = tls_elapsed.as_micros() as u64; - if tls_elapsed_us > 0 && tls_elapsed_us < 30_000_000 { - ja4l.set_app_handshake(0, 0, tls_elapsed_us); - } - } - } - - let fp = ja4l.client_fingerprint(); - log::info!("JA4L: Generated fingerprint: {} for {}:{}", fp, client_addr.ip(), client_addr.port()); - // Cache for all subsequent requests on this connection - if let Some(mut entry) = bpf_cache().get_mut(&ja4l_cache_key) { - entry.ja4l = fp.clone(); - entry.stored_at = std::time::Instant::now(); - } else { - bpf_cache().insert(ja4l_cache_key.clone(), CachedBpfFingerprints { - ja4t: String::new(), - ja4t_hash: String::new(), - ja4l: fp.clone(), - stored_at: std::time::Instant::now(), - }); - } - upstream_request.insert_header("X-JA4L", &fp)?; - } else { - log::debug!("JA4L: Handshake not complete (state={}) for {}:{}", latency_data.state, client_addr.ip(), client_addr.port()); - } - } else { - log::debug!( - "JA4L: No BPF latency data found for {}:{} (collector enabled={})", - client_addr.ip(), client_addr.port(), - collector.is_enabled(), - ); - } - } - } - } else { - log::debug!("JA4L: No global latency fingerprint collector available"); - } - - // JA4X: X.509 certificate fingerprint (proxy's server certificate) - if let Some(hostname) = ctx.hostname.as_ref() - && let Ok(store) = crate::worker::certificate::get_certificate_store().try_read() - && let Some(certs) = store.as_ref() - { - if let Some(cert_path) = certs.get_cert_path_for_hostname(hostname) { - if let Some(ja4x_fp) = crate::utils::tls::extract_ja4x_fingerprint(&cert_path) { - log::debug!("JA4X: {}", ja4x_fp); - upstream_request.insert_header("X-JA4X", &ja4x_fp)?; - } else { - log::debug!("JA4X: no fingerprint from cert"); - } - } else { - log::debug!("JA4X: no cert path for {}", hostname); - } - } - - // Periodic cleanup of stale BPF fingerprint cache entries (10 min TTL) - let count = - BPF_CACHE_CLEANUP_COUNTER.fetch_add(1, std::sync::atomic::Ordering::Relaxed); - if count.is_multiple_of(1000) || bpf_cache().len() > 50_000 { - let cutoff = std::time::Instant::now() - std::time::Duration::from_secs(600); - bpf_cache().retain(|_, v| v.stored_at > cutoff); - } - // Periodic cleanup of JA4TS server cache and JA4T IP cache (1 hour TTL, every 5000 requests) - if count.is_multiple_of(5000) { - let hour_cutoff = std::time::Instant::now() - std::time::Duration::from_secs(3600); - ja4ts_cache().retain(|_, v| v.stored_at > hour_cutoff); - ja4t_ip_cache().retain(|_, v| v.stored_at > hour_cutoff); - } - } - // Apply configured request headers from upstreams.yaml (will override default Host if present) if let Some(hostname) = ctx.hostname.as_ref() { let path = _session.req_header().uri.path(); @@ -1376,9 +920,10 @@ impl ProxyHttp for LB { { // Check if content scanning is active let scanner = crate::security::waf::actions::content_scanning::get_global_content_scanner(); - let scanning_active = scanner.is_some_and(|s| s.is_enabled()); + let scanning_active = scanner.map_or(false, |s| s.is_enabled()); + let ids_active = crate::proxy::thalamus_post_tls::is_enabled(); - if !scanning_active { + if !scanning_active && !ids_active { return Ok(()); } @@ -1389,15 +934,15 @@ impl ProxyHttp for LB { // could scan at end_of_stream. By forwarding normally, the upstream // waits for the full body before responding, so our scan completes // before response_filter runs. - if let Some(body_bytes) = body.as_ref() - && !body_bytes.is_empty() - { - debug!( - "Content scanner: accumulating {} bytes (total: {})", - body_bytes.len(), - ctx.request_body.len() + body_bytes.len() - ); - ctx.request_body.extend_from_slice(body_bytes); + if let Some(body_bytes) = body.as_ref() { + if !body_bytes.is_empty() { + debug!( + "Content scanner: accumulating {} bytes (total: {})", + body_bytes.len(), + ctx.request_body.len() + body_bytes.len() + ); + ctx.request_body.extend_from_slice(body_bytes); + } } if !end_of_stream { @@ -1409,6 +954,43 @@ impl ProxyHttp for LB { return Ok(()); } + if ids_active + && let Some(peer_addr) = _session.client_addr().and_then(|addr| addr.as_inet()) + { + let ids_started = Instant::now(); + let socket_addr = std::net::SocketAddr::new(peer_addr.ip().into(), peer_addr.port()); + let ids_alert = crate::proxy::thalamus_post_tls::inspect_http_request( + _session.req_header(), + socket_addr, + &ctx.request_body, + ) + .await; + let elapsed_us = ids_started.elapsed().as_micros().min(u128::from(u64::MAX)) as u64; + ctx.post_tls_ids_time_us = Some( + ctx.post_tls_ids_time_us + .unwrap_or(0) + .saturating_add(elapsed_us), + ); + + if let Some(ids_alert) = ids_alert { + info!( + "Thalamus proxy IDS (body) matched: sid={}, action={}, msg=\"{}\", uri={}", + ids_alert.signature_id, + ids_alert.action, + ids_alert.signature_name, + _session.req_header().uri + ); + if ids_alert.should_block { + ctx.malware_detected = true; + } + } + } + + if !scanning_active { + ctx.request_body.clear(); + return Ok(()); + } + let scanner = scanner.unwrap(); if ctx.request_body.len() > scanner.max_file_size() { @@ -1502,6 +1084,7 @@ impl ProxyHttp for LB { _upstream_response: &mut ResponseHeader, ctx: &mut Self::CTX, ) -> Result<()> { + ctx.response_header_time = Some(Instant::now()); // Calculate upstream response time if let Some(upstream_start) = ctx.upstream_start_time { ctx.upstream_time = Some(upstream_start.elapsed()); @@ -1554,41 +1137,45 @@ impl ProxyHttp for LB { } } // Apply configured response headers from upstreams.yaml - if let Some(host) = ctx.hostname.as_ref() { - let path = session.req_header().uri.path(); - let host_header = host; - let split_header = host_header.split_once(':'); - - match split_header { - Some(sh) => { - if let Some(response_headers) = self.get_response_headers(sh.0, path) { - for (key, value) in response_headers { - if let Err(e) = - _upstream_response.insert_header(key.clone(), value.clone()) - { - warn!( - "Failed to insert response header {}: {}, preserving original upstream status code", - key, e - ); + match ctx.hostname.as_ref() { + Some(host) => { + let path = session.req_header().uri.path(); + let host_header = host; + let split_header = host_header.split_once(':'); + + match split_header { + Some(sh) => { + if let Some(response_headers) = self.get_response_headers(sh.0, path) { + for (key, value) in response_headers { + if let Err(e) = + _upstream_response.insert_header(key.clone(), value.clone()) + { + warn!( + "Failed to insert response header {}: {}, preserving original upstream status code", + key, e + ); + } } } } - } - None => { - if let Some(response_headers) = self.get_response_headers(host_header, path) { - for (key, value) in response_headers { - if let Err(e) = - _upstream_response.insert_header(key.clone(), value.clone()) - { - warn!( - "Failed to insert response header {}: {}, preserving original upstream status code", - key, e - ); + None => { + if let Some(response_headers) = self.get_response_headers(host_header, path) + { + for (key, value) in response_headers { + if let Err(e) = + _upstream_response.insert_header(key.clone(), value.clone()) + { + warn!( + "Failed to insert response header {}: {}, preserving original upstream status code", + key, e + ); + } } } } } } + None => {} } // Apply global response headers (headers to add to responses) let global_response_headers = self.global_response_headers.load(); @@ -1622,28 +1209,6 @@ impl ProxyHttp for LB { pingora_core::ErrorSource::Unset => "Unset", }; let error_msg = e.to_string(); - - // Silently drop invalid/unsupported downstream connections. - // These are typically port scanners, bots sending garbage, or clients - // using unsupported protocols (e.g. h2c when disabled). No point sending - // an HTTP response to something that doesn't speak valid HTTP. - let is_downstream_invalid = matches!(e.esource(), pingora_core::ErrorSource::Downstream) - && matches!( - e.etype(), - ErrorType::InvalidHTTPHeader | ErrorType::H2Error | ErrorType::InvalidH2 - ); - - if is_downstream_invalid { - debug!( - "Dropping invalid downstream connection: {} {}", - error_type_str, error_msg - ); - return FailToProxy { - error_code: 0, - can_reuse_downstream: false, - }; - } - // Determine appropriate HTTP status code based on error type let code = match e.etype() { HTTPStatus(code) => *code, @@ -1656,8 +1221,8 @@ impl ProxyHttp for LB { 503 } ErrorType::InvalidHTTPHeader | ErrorType::H2Error | ErrorType::InvalidH2 => { - // Upstream sent invalid HTTP -> 502 Bad Gateway - 502 + // Invalid HTTP headers or HTTP/2 errors -> 400 Bad Request + 400 } _ => { // For other upstream errors, determine based on error source @@ -1689,10 +1254,10 @@ impl ProxyHttp for LB { }); // Send error response to downstream if code is valid - if code > 0 - && let Err(resp_err) = session.respond_error(code).await - { - let _ = resp_err; + if code > 0 { + if let Err(resp_err) = session.respond_error(code).await { + let _ = resp_err; + } } FailToProxy { @@ -1708,18 +1273,7 @@ impl ProxyHttp for LB { _e: Option<&pingora_core::Error>, ctx: &mut Self::CTX, ) { - // Skip logging for invalid/unsupported downstream connections (already dropped) - if let Some(err) = _e { - if matches!(err.esource(), pingora_core::ErrorSource::Downstream) - && matches!( - err.etype(), - ErrorType::InvalidHTTPHeader | ErrorType::H2Error | ErrorType::InvalidH2 - ) - { - return; - } - } - + let logging_started = Instant::now(); let response_code = if let Some(resp) = session.response_written() { resp.status.as_u16() } else if let Some(err) = _e { @@ -1729,7 +1283,7 @@ impl ProxyHttp for LB { 504 } ErrorType::ConnectRefused | ErrorType::ConnectNoRoute => 503, - ErrorType::InvalidHTTPHeader | ErrorType::H2Error | ErrorType::InvalidH2 => 502, + ErrorType::InvalidHTTPHeader | ErrorType::H2Error | ErrorType::InvalidH2 => 400, _ => match err.esource() { pingora_core::ErrorSource::Upstream => 502, pingora_core::ErrorSource::Downstream => 400, @@ -1816,7 +1370,8 @@ impl ProxyHttp for LB { { // Try to retrieve TLS fingerprint again if not in context // Use fallback lookup to handle PROXY protocol address mismatches - let std_addr = std::net::SocketAddr::new(peer_addr.ip(), peer_addr.port()); + let std_addr = + std::net::SocketAddr::new(peer_addr.ip().into(), peer_addr.port()); if let Some(fingerprint) = crate::utils::tls_client_hello::get_fingerprint_with_fallback(&std_addr) { @@ -1896,10 +1451,71 @@ impl ProxyHttp for LB { } }); + let request_time_ms = ctx.start_time.elapsed().as_millis() as u64; + let pre_upstream_ms = ctx + .upstream_start_time + .and_then(|upstream_start| upstream_start.checked_duration_since(ctx.start_time)) + .map(|d| d.as_millis() as u64); + let gap_request_filter_to_upstream_peer_ms = + match (ctx.request_filter_end_time, ctx.upstream_peer_start_time) { + (Some(filter_end), Some(peer_start)) => peer_start + .checked_duration_since(filter_end) + .map(|d| d.as_millis() as u64), + _ => None, + }; + let gap_upstream_peer_to_upstream_start_ms = + match (ctx.upstream_peer_end_time, ctx.upstream_start_time) { + (Some(peer_end), Some(upstream_start)) => upstream_start + .checked_duration_since(peer_end) + .map(|d| d.as_millis() as u64), + _ => None, + }; + let pre_upstream_accounted_ms = ctx + .request_filter_total_ms + .unwrap_or(0) + .saturating_add(gap_request_filter_to_upstream_peer_ms.unwrap_or(0)) + .saturating_add(ctx.upstream_peer_select_ms.unwrap_or(0)) + .saturating_add(gap_upstream_peer_to_upstream_start_ms.unwrap_or(0)); + let pre_upstream_unattributed_ms = + pre_upstream_ms.map(|pre| pre.saturating_sub(pre_upstream_accounted_ms)); + let upstream_header_time_ms = ctx.upstream_time.map(|d| d.as_millis() as u64); + let post_upstream_header_ms = match (pre_upstream_ms, upstream_header_time_ms) { + (Some(pre), Some(up_header)) => { + Some(request_time_ms.saturating_sub(pre.saturating_add(up_header))) + } + _ => None, + }; + let response_header_to_log_ms = ctx + .response_header_time + .map(|header_time| header_time.elapsed().as_millis() as u64); + let logging_prepare_elapsed = logging_started.elapsed(); + let logging_prepare_ms = logging_prepare_elapsed.as_millis() as u64; + let logging_prepare_us = logging_prepare_elapsed + .as_micros() + .min(u128::from(u64::MAX)) as u64; + // Build performance info let performance_info = crate::logger::access_log::PerformanceInfo { - request_time_ms: Some(ctx.start_time.elapsed().as_millis() as u64), - upstream_time_ms: ctx.upstream_time.map(|d| d.as_millis() as u64), + request_time_ms: Some(request_time_ms), + upstream_time_ms: upstream_header_time_ms, + pre_upstream_ms, + request_filter_total_ms: ctx.request_filter_total_ms, + request_filter_keepalive_ms: ctx.request_filter_keepalive_ms, + request_filter_access_rules_ms: ctx.request_filter_access_rules_ms, + request_filter_tls_fingerprint_ms: ctx.request_filter_tls_fingerprint_ms, + request_filter_threat_intel_ms: ctx.request_filter_threat_intel_ms, + request_filter_waf_ms: ctx.request_filter_waf_ms, + request_filter_routing_ms: ctx.request_filter_routing_ms, + upstream_peer_select_ms: ctx.upstream_peer_select_ms, + gap_request_filter_to_upstream_peer_ms, + gap_upstream_peer_to_upstream_start_ms, + pre_upstream_unattributed_ms, + upstream_header_time_ms, + post_upstream_header_ms, + response_header_to_log_ms, + logging_prepare_ms: Some(logging_prepare_ms), + post_tls_ids_time_us: ctx.post_tls_ids_time_us, + upstream_time_scope: Some("response_header_ttfb".to_string()), }; let error_details = ctx.error_details.clone().or_else(|| { @@ -1967,273 +1583,12 @@ impl ProxyHttp for LB { (None, None, None, None, None) }; - // Collect all JA4+ fingerprints for access log - let fingerprints = { - // JA4/JA4_raw: from TLS ClientHello fingerprint - let fp_ja4 = tls_fp_for_log.as_ref().map(|fp| fp.ja4.clone()); - let fp_ja4_raw = tls_fp_for_log.as_ref().map(|fp| fp.ja4_raw.clone()); - - // JA4H: HTTP header fingerprint - let method = session.req_header().method.as_str(); - let version = match session.req_header().version { - http::Version::HTTP_10 => "HTTP/1.0", - http::Version::HTTP_11 => "HTTP/1.1", - http::Version::HTTP_2 => "HTTP/2.0", - http::Version::HTTP_3 => "HTTP/3.0", - _ => "HTTP/1.1", - }; - let ja4h_fp = - crate::utils::fingerprint::ja4_plus::Ja4hFingerprint::from_http_request( - method, - version, - &session.req_header().headers, - ); - let fp_ja4h = Some(ja4h_fp.fingerprint); - - // JA4T/JA4T_hash: TCP SYN fingerprint - // Prefer context-cached value (set during upstream_request_filter), - // then fall back to BPF / connection cache / IP cache. - let (fp_ja4t, fp_ja4t_hash) = if ctx.ja4t_fingerprint.is_some() { - (ctx.ja4t_fingerprint.clone(), ctx.ja4t_hash.clone()) - } else { - let cache_key = format!("{}:{}", peer_addr.ip(), peer_addr.port()); - let ip_key = peer_addr.ip().to_string(); - let mut result = (None, None); - - // 1. Try fresh BPF data - if let Some(tcp_data) = &tcp_fingerprint_data { - let ja4t_inner = if !tcp_data.options.is_empty() && tcp_data.options[0] != 0 - { - nstealth::Ja4t::from_raw_options( - tcp_data.window_size, - &tcp_data.options, - ) - } else { - let mss = if tcp_data.mss > 0 { - Some(tcp_data.mss) - } else { - None - }; - let ws = if tcp_data.window_scale > 0 { - Some(tcp_data.window_scale) - } else { - None - }; - nstealth::Ja4t::new(tcp_data.window_size, &[], mss, ws) - }; - let fp = ja4t_inner.fingerprint(); - if fp != "0_00_00_00" { - let hash = nstealth::hash12(&fp); - // Populate IP cache with fresh data - ja4t_ip_cache().insert( - ip_key.clone(), - CachedJa4tFingerprint { - ja4t: fp.clone(), - ja4t_hash: hash.clone(), - stored_at: std::time::Instant::now(), - }, - ); - result = (Some(fp), Some(hash)); - } - } - - // 2. Fall back to per-connection cache - if result.0.is_none() - && let Some(cached) = bpf_cache().get(&cache_key) - && !cached.ja4t.is_empty() - { - result = (Some(cached.ja4t.clone()), Some(cached.ja4t_hash.clone())); - } - - // 3. Last resort: IP-keyed cache (survives keepalive rebuild / port change) - if result.0.is_none() - && let Some(cached) = ja4t_ip_cache().get(&ip_key) - && cached.stored_at.elapsed() < std::time::Duration::from_secs(3600) - { - result = (Some(cached.ja4t.clone()), Some(cached.ja4t_hash.clone())); - } - - result - }; - - // JA4S: TLS ServerHello fingerprint - let fp_ja4s = if tls_present { - if let Some(stream) = session.stream() { - if let Some(ssl) = stream.get_ssl() { - let tls_version = match ssl.version_str() { - "TLSv1.3" => nstealth::TlsVersion::Tls13, - "TLSv1.2" => nstealth::TlsVersion::Tls12, - "TLSv1.1" => nstealth::TlsVersion::Tls11, - "TLSv1" => nstealth::TlsVersion::Tls10, - _ => nstealth::TlsVersion::Unknown(0), - }; - let cipher_id = ssl - .current_cipher() - .map(|c| cipher_name_to_iana_id(c.name())) - .unwrap_or(0); - let alpn = ssl - .selected_alpn_protocol() - .and_then(|bytes| String::from_utf8(bytes.to_vec()).ok()); - let extensions = build_server_hello_extensions( - &tls_version, - ssl.current_cipher().map(|c| c.name()), - alpn.is_some(), - ); - let ja4s = nstealth::Ja4s::new( - false, - tls_version, - cipher_id, - extensions, - alpn, - ); - Some(ja4s.fingerprint()) - } else { - None - } - } else if let Some(digest) = session.digest() { - if let Some(ssl_digest) = &digest.ssl_digest { - let tls_version = match ssl_digest.version.as_ref() { - "TLSv1.3" => nstealth::TlsVersion::Tls13, - "TLSv1.2" => nstealth::TlsVersion::Tls12, - "TLSv1.1" => nstealth::TlsVersion::Tls11, - "TLSv1" => nstealth::TlsVersion::Tls10, - _ => nstealth::TlsVersion::Unknown(0), - }; - let cipher_id = cipher_name_to_iana_id(&ssl_digest.cipher); - let alpn = Some("h2".to_string()); - let extensions = build_server_hello_extensions( - &tls_version, - Some(&ssl_digest.cipher), - true, - ); - let ja4s = nstealth::Ja4s::new( - false, - tls_version, - cipher_id, - extensions, - alpn, - ); - Some(ja4s.fingerprint()) - } else { - None - } - } else { - None - } - } else { - None - }; - - // JA4TS: TCP SYN-ACK server fingerprint (from cache or BPF) - let (fp_ja4ts, fp_ja4ts_hash) = if let Some(peer) = ctx.upstream_peer.as_ref() { - let ja4ts_cache_key = format!("{}:{}", peer.address, peer.port); - // Try cache first (populated during upstream_request_filter) - if let Some(cached) = ja4ts_cache().get(&ja4ts_cache_key) { - if cached.stored_at.elapsed() < std::time::Duration::from_secs(3600) { - (Some(cached.ja4ts.clone()), Some(cached.ja4ts_hash.clone())) - } else { - (None, None) - } - } else { - // Try BPF lookup directly - let server_ip_opt = ctx - .upstream_resolved_ip - .or_else(|| resolve_upstream_ip(&peer.address)); - if let Some(server_ip) = server_ip_opt { - if let Some(collector) = crate::utils::fingerprint::tcp_fingerprint::get_global_tcp_fingerprint_collector() { - if let Some(synack_data) = collector.lookup_synack_fingerprint(server_ip, peer.port) { - let ja4ts = nstealth::Ja4ts::from_raw_options(synack_data.window_size, &synack_data.options); - let fp = ja4ts.fingerprint(); - if fp != "0_00_00_00" { - let hash = nstealth::hash12(&fp); - (Some(fp), Some(hash)) - } else { (None, None) } - } else { (None, None) } - } else { (None, None) } - } else { - (None, None) - } - } - } else { - (None, None) - }; - - // JA4L: Latency fingerprint (from BPF cache or collector) - let fp_ja4l = { - let ja4l_cache_key = format!("{}:{}", peer_addr.ip(), peer_addr.port()); - // Try cache first (populated during upstream_request_filter) - if let Some(entry) = bpf_cache().get(&ja4l_cache_key) { - if !entry.ja4l.is_empty() { - Some(entry.ja4l.clone()) - } else { - None - } - } else { - // Try BPF lookup directly - if let Some(collector) = crate::utils::fingerprint::latency_fingerprint::get_global_latency_fingerprint_collector() { - if let Some(latency_data) = collector.lookup_connection(peer_addr.ip(), peer_addr.port()) { - if latency_data.is_complete() { - let syn_us = latency_data.syn_time_ns / 1000; - let synack_us = latency_data.synack_time_ns / 1000; - let ack_us = latency_data.ack_time_ns / 1000; - let mut ja4l = nstealth::Ja4l::new( - syn_us, synack_us, ack_us, - latency_data.client_ttl, latency_data.server_ttl, - ); - if let Some(hello_instant) = ctx.tls_hello_instant { - let request_start: std::time::Instant = ctx.start_time.into_std(); - if let Some(tls_elapsed) = request_start.checked_duration_since(hello_instant) { - let tls_elapsed_us = tls_elapsed.as_micros() as u64; - if tls_elapsed_us > 0 && tls_elapsed_us < 30_000_000 { - ja4l.set_app_handshake(0, 0, tls_elapsed_us); - } - } - } - Some(ja4l.client_fingerprint()) - } else { None } - } else { None } - } else { None } - } - }; - - // JA4X: X.509 certificate fingerprint - let fp_ja4x = ctx.hostname.as_ref().and_then(|hostname| { - if let Ok(store) = - crate::worker::certificate::get_certificate_store().try_read() - { - if let Some(certs) = store.as_ref() { - if let Some(cert_path) = certs.get_cert_path_for_hostname(hostname) { - crate::utils::tls::extract_ja4x_fingerprint(&cert_path) - } else { - None - } - } else { - None - } - } else { - None - } - }); - - Some(crate::logger::access_log::FingerprintDetails { - ja4: fp_ja4, - ja4_raw: fp_ja4_raw, - ja4h: fp_ja4h, - ja4t: fp_ja4t, - ja4t_hash: fp_ja4t_hash, - ja4s: fp_ja4s, - ja4ts: fp_ja4ts, - ja4ts_hash: fp_ja4ts_hash, - ja4l: fp_ja4l, - ja4x: fp_ja4x, - }) - }; - // Create access log with upstream and performance info // Note: tls_fingerprint parameter is for Ja4hFingerprint (HTTP header fingerprint), // but we pass None since the function generates its own JA4H from headers. // TLS info is passed via the separate tls_sni, tls_alpn, tls_cipher, tls_ja4, tls_ja4_unsorted parameters. - if let Err(e) = crate::logger::access_log::HttpAccessLog::create_from_parts( + let access_log_call_started = Instant::now(); + match crate::logger::access_log::HttpAccessLog::create_from_parts( &req_parts, &req_body_bytes, peer_socket_addr, @@ -2244,6 +1599,7 @@ impl ProxyHttp for LB { Some(ErrorType::InvalidHTTPHeader | ErrorType::H2Error | ErrorType::InvalidH2) ), None, // Ja4hFingerprint is generated from HTTP headers inside the function + tcp_fingerprint_data.as_ref(), server_cert_info_opt.as_ref(), response_data, error_details, @@ -2256,11 +1612,51 @@ impl ProxyHttp for LB { tls_cipher, tls_ja4, tls_ja4_unsorted, - fingerprints, ) .await { - warn!("Failed to create access log: {}", e); + Ok(request_id) => { + let access_log_call_elapsed = access_log_call_started.elapsed(); + let access_log_call_ms = access_log_call_elapsed.as_millis() as u64; + let access_log_call_us = access_log_call_elapsed + .as_micros() + .min(u128::from(u64::MAX)) + as u64; + log::info!( + target: "access_log_perf", + "{}", + serde_json::json!({ + "event": "access_log_pipeline", + "request_id": request_id, + "src_ip": peer_socket_addr.ip().to_string(), + "path": session.req_header().uri.path(), + "request_time_ms": request_time_ms, + "pre_upstream_ms": pre_upstream_ms, + "request_filter_total_ms": ctx.request_filter_total_ms, + "request_filter_keepalive_ms": ctx.request_filter_keepalive_ms, + "request_filter_access_rules_ms": ctx.request_filter_access_rules_ms, + "request_filter_tls_fingerprint_ms": ctx.request_filter_tls_fingerprint_ms, + "request_filter_threat_intel_ms": ctx.request_filter_threat_intel_ms, + "request_filter_waf_ms": ctx.request_filter_waf_ms, + "request_filter_routing_ms": ctx.request_filter_routing_ms, + "upstream_peer_select_ms": ctx.upstream_peer_select_ms, + "gap_request_filter_to_upstream_peer_ms": gap_request_filter_to_upstream_peer_ms, + "gap_upstream_peer_to_upstream_start_ms": gap_upstream_peer_to_upstream_start_ms, + "pre_upstream_unattributed_ms": pre_upstream_unattributed_ms, + "upstream_header_time_ms": upstream_header_time_ms, + "post_upstream_header_ms": post_upstream_header_ms, + "response_header_to_log_ms": response_header_to_log_ms, + "logging_prepare_ms": logging_prepare_ms, + "logging_prepare_us": logging_prepare_us, + "post_tls_ids_time_us": ctx.post_tls_ids_time_us, + "access_log_call_ms": access_log_call_ms, + "access_log_call_us": access_log_call_us + }) + ); + } + Err(e) => { + warn!("Failed to create access log: {}", e); + } } } } @@ -2282,6 +1678,16 @@ impl ProxyHttp for LB { impl LB {} +fn mark_request_filter_done(ctx: &mut Context, started: Instant) { + ctx.request_filter_total_ms = Some(started.elapsed().as_millis() as u64); + ctx.request_filter_end_time = Some(Instant::now()); +} + +fn mark_upstream_peer_done(ctx: &mut Context, started: Instant) { + ctx.upstream_peer_select_ms = Some(started.elapsed().as_millis() as u64); + ctx.upstream_peer_end_time = Some(Instant::now()); +} + fn return_header_host(session: &Session) -> Option { if session.is_http2() { match session.req_header().uri.host() { diff --git a/src/proxy/start.rs b/src/proxy/start.rs index 3c4b8a01..f26db15a 100644 --- a/src/proxy/start.rs +++ b/src/proxy/start.rs @@ -8,7 +8,6 @@ use arc_swap::ArcSwap; use ctrlc; use dashmap::DashMap; use log::{debug, info, warn}; -use pingora_core::apps::HttpServerOptions; use pingora_core::listeners::tls::TlsSettings; use pingora_core::prelude::{Opt, background_service}; use pingora_core::server::Server; @@ -25,10 +24,21 @@ pub fn run() { pub fn run_with_config(config: Option) { // default_provider().install_default().expect("Failed to install rustls crypto provider"); let maincfg = if let Some(cfg) = config { + if let Err(e) = crate::proxy::thalamus_post_tls::configure(&cfg.ids) { + warn!("Failed to initialize Thalamus post-TLS IDS runtime: {}", e); + } cfg.proxy.to_app_config() } else { + if let Err(e) = + crate::proxy::thalamus_post_tls::configure(&crate::core::cli::IdsConfig::default()) + { + warn!( + "Failed to reset Thalamus post-TLS IDS runtime in legacy startup path: {}", + e + ); + } // Fallback to old parsing method for backward compatibility - let parameters = Opt::parse_args(); + let parameters = Some(Opt::parse_args()).unwrap(); let file = parameters.conf.clone().unwrap(); crate::utils::parceyaml::parce_main_config(file.as_str()) }; @@ -81,7 +91,6 @@ pub fn run_with_config(config: Option) { let ec_config = Arc::new(ArcSwap::from_pointee(Extraparams { sticky_sessions: false, https_proxy_enabled: None, - forward_fingerprints: false, authentication: DashMap::new(), })); @@ -113,179 +122,165 @@ pub fn run_with_config(config: Option) { info!("TLS grade set to: [ {} ]", grade); let bg_srvc = background_service("bgsrvc", lb.clone()); - let mut server_options = HttpServerOptions::default(); - if cfg.h2c { - info!("HTTP/2 cleartext (h2c) enabled on plaintext listener"); - server_options.h2c = true; - } - if cfg.allow_connect_method_proxying { - info!("HTTP CONNECT method proxying enabled"); - server_options.allow_connect_method_proxying = true; - } - if let Some(limit) = cfg.keepalive_request_limit { - info!("Keepalive request limit set to {}", limit); - server_options.keepalive_request_limit = Some(limit); - } - let mut proxy = pingora_proxy::ProxyServiceBuilder::new(&server.configuration, lb.clone()) - .server_options(server_options) - .build(); + let mut proxy = pingora_proxy::http_proxy_service(&server.configuration, lb.clone()); let bind_address_http = cfg.proxy_address_http.clone(); let bind_address_tls = cfg.proxy_address_tls.clone(); crate::utils::tools::check_priv(bind_address_http.as_str()); - if let Some(bind_address_tls) = bind_address_tls { - crate::utils::tools::check_priv(bind_address_tls.as_str()); - let (tx, rx): ( - Sender>, - Receiver>, - ) = channel(); - let certs_path = cfg.proxy_certificates.clone().unwrap(); - - // Check if directory exists before watching - let certs_path_exists = fs::metadata(&certs_path).is_ok(); - let certs_path_clone = certs_path.clone(); - - if certs_path_exists { - // Start watcher thread - it will send initial configs - thread::spawn(move || { - if let Err(e) = crate::utils::tools::watch_folder(certs_path_clone, tx) { - warn!("Failed to watch certificate directory: {:?}", e); - } - }); - } else { - warn!( - "Certificate directory does not exist: {}. TLS will be disabled until certificates are added.", - certs_path - ); - // Send empty configs so receiver doesn't block - if tx.send(vec![]).is_err() { - warn!("Failed to send initial certificate configs"); - } - } - - // Receive initial certificate configs - let certificate_configs = match rx.recv() { - Ok(configs) => configs, - Err(e) => { + match bind_address_tls { + Some(bind_address_tls) => { + crate::utils::tools::check_priv(bind_address_tls.as_str()); + let (tx, rx): ( + Sender>, + Receiver>, + ) = channel(); + let certs_path = cfg.proxy_certificates.clone().unwrap(); + + // Check if directory exists before watching + let certs_path_exists = fs::metadata(&certs_path).is_ok(); + let certs_path_clone = certs_path.clone(); + + if certs_path_exists { + // Start watcher thread - it will send initial configs + thread::spawn(move || { + if let Err(e) = crate::utils::tools::watch_folder(certs_path_clone, tx) { + warn!("Failed to watch certificate directory: {:?}", e); + } + }); + } else { warn!( - "Failed to receive certificate configs: {:?}. TLS will be disabled.", - e + "Certificate directory does not exist: {}. TLS will be disabled until certificates are added.", + certs_path ); - vec![] + // Send empty configs so receiver doesn't block + if tx.send(vec![]).is_err() { + warn!("Failed to send initial certificate configs"); + } } - }; - - if let Some(first_set) = tls::Certificates::new( - &certificate_configs, - grade.as_str(), - cfg.default_certificate.as_ref(), - ) { - let first_set_arc: Arc = Arc::new(first_set); - certificates_arc.store(Arc::new( - Some(first_set_arc.clone()) as Option> - )); - - // Set global certificates for SNI callback - tls::set_global_certificates(first_set_arc.clone()); - - let default_cert_path = first_set_arc.default_cert_path.clone(); - let default_key_path = first_set_arc.default_key_path.clone(); - - // Create TlsSettings with SNI callback for certificate selection - let tls_settings = match tls::create_tls_settings_with_sni( - &default_cert_path, - &default_key_path, - grade.as_str(), - Some(first_set_arc.clone()), - ) { - Ok(settings) => settings, + + // Receive initial certificate configs + let certificate_configs = match rx.recv() { + Ok(configs) => configs, Err(e) => { warn!( - "Failed to create TlsSettings with SNI callback: {}, falling back to default", + "Failed to receive certificate configs: {:?}. TLS will be disabled.", e ); - let mut settings = - TlsSettings::intermediate(&default_cert_path, &default_key_path) - .expect("unable to load or parse cert/key"); - tls::set_tsl_grade(&mut settings, grade.as_str()); - tls::set_alpn_prefer_h2(&mut settings); - settings + vec![] } }; - // Register ClientHello callback to generate fingerprints - // Note: When PROXY protocol is enabled, ClientHello extraction may fail if the connection - // is reset before TLS handshake completes. The "Failed to peek at socket" warnings are - // expected in this case and are non-fatal - the TLS handshake will still proceed. - #[cfg(unix)] - { - use log::info; - use pingora_core::listeners::set_client_hello_callback; - use pingora_core::protocols::l4::socket::SocketAddr; - use pingora_core::protocols::tls::client_hello::ClientHello; - - set_client_hello_callback(Some( - |hello: &ClientHello, peer_addr: Option| { - let peer_str = peer_addr - .as_ref() - .and_then(|a| a.as_inet()) - .map(|inet| format!("{}:{}", inet.ip(), inet.port())) - .unwrap_or_else(|| "unknown".to_string()); - debug!( - "ClientHello callback invoked for peer: {}, SNI: {:?}, ALPN: {:?}, raw_len={}", - peer_str, - hello.sni, - hello.alpn, - hello.raw.len() + if let Some(first_set) = tls::Certificates::new( + &certificate_configs, + grade.as_str(), + cfg.default_certificate.as_ref(), + ) { + let first_set_arc: Arc = Arc::new(first_set); + certificates_arc.store(Arc::new( + Some(first_set_arc.clone()) as Option> + )); + + // Set global certificates for SNI callback + tls::set_global_certificates(first_set_arc.clone()); + + let default_cert_path = first_set_arc.default_cert_path.clone(); + let default_key_path = first_set_arc.default_key_path.clone(); + + // Create TlsSettings with SNI callback for certificate selection + let tls_settings = match tls::create_tls_settings_with_sni( + &default_cert_path, + &default_key_path, + grade.as_str(), + Some(first_set_arc.clone()), + ) { + Ok(settings) => settings, + Err(e) => { + warn!( + "Failed to create TlsSettings with SNI callback: {}, falling back to default", + e ); - // Generate fingerprint from ClientHello - if let Some(_fp) = - crate::utils::tls_client_hello::generate_fingerprint_from_client_hello( - hello, peer_addr, - ) - { - debug!("Fingerprint generated successfully for peer: {}", peer_str); - } else { - // Log at debug level - failures are more common with PROXY protocol - // due to connection resets, but this is non-fatal + let mut settings = + TlsSettings::intermediate(&default_cert_path, &default_key_path) + .expect("unable to load or parse cert/key"); + tls::set_tsl_grade(&mut settings, grade.as_str()); + tls::set_alpn_prefer_h2(&mut settings); + settings + } + }; + + // Register ClientHello callback to generate fingerprints + // Note: When PROXY protocol is enabled, ClientHello extraction may fail if the connection + // is reset before TLS handshake completes. The "Failed to peek at socket" warnings are + // expected in this case and are non-fatal - the TLS handshake will still proceed. + #[cfg(unix)] + { + use log::info; + use pingora_core::listeners::set_client_hello_callback; + use pingora_core::protocols::l4::socket::SocketAddr; + use pingora_core::protocols::tls::client_hello::ClientHello; + + set_client_hello_callback(Some( + |hello: &ClientHello, peer_addr: Option| { + let peer_str = peer_addr + .as_ref() + .and_then(|a| a.as_inet()) + .map(|inet| format!("{}:{}", inet.ip(), inet.port())) + .unwrap_or_else(|| "unknown".to_string()); debug!( - "Failed to generate fingerprint for peer: {} (non-fatal, TLS handshake will continue)", - peer_str + "ClientHello callback invoked for peer: {}, SNI: {:?}, ALPN: {:?}, raw_len={}", + peer_str, + hello.sni, + hello.alpn, + hello.raw.len() ); - } - }, - )); - if proxy_protocol_enabled { - info!( - "TLS ClientHello callback registered for fingerprint generation (PROXY protocol enabled - some extraction failures are expected and non-fatal)" - ); - } else { - info!("TLS ClientHello callback registered for fingerprint generation"); + // Generate fingerprint from ClientHello + if let Some(_fp) = + crate::utils::tls_client_hello::generate_fingerprint_from_client_hello( + hello, + peer_addr, + ) + { + debug!("Fingerprint generated successfully for peer: {}", peer_str); + } else { + // Log at debug level - failures are more common with PROXY protocol + // due to connection resets, but this is non-fatal + debug!("Failed to generate fingerprint for peer: {} (non-fatal, TLS handshake will continue)", peer_str); + } + }, + )); + if proxy_protocol_enabled { + info!( + "TLS ClientHello callback registered for fingerprint generation (PROXY protocol enabled - some extraction failures are expected and non-fatal)" + ); + } else { + info!("TLS ClientHello callback registered for fingerprint generation"); + } } - } - proxy.add_tls_with_settings(&bind_address_tls, None, tls_settings); - } else { - info!( - "TLS listener disabled: no certificates found in directory. TLS will be enabled when certificates are added." - ); - } - - let certs_for_watcher = certificates_arc.clone(); - let default_cert_for_watcher = cfg.default_certificate.clone(); - thread::spawn(move || { - while let Ok(new_configs) = rx.recv() { - let new_certs = tls::Certificates::new( - &new_configs, - grade.as_str(), - default_cert_for_watcher.as_ref(), + proxy.add_tls_with_settings(&bind_address_tls, None, tls_settings); + } else { + info!( + "TLS listener disabled: no certificates found in directory. TLS will be enabled when certificates are added." ); - if let Some(new_certs) = new_certs { - certs_for_watcher.store(Arc::new(Some(Arc::new(new_certs)))); - } } - }); + + let certs_for_watcher = certificates_arc.clone(); + let default_cert_for_watcher = cfg.default_certificate.clone(); + thread::spawn(move || { + while let Ok(new_configs) = rx.recv() { + let new_certs = tls::Certificates::new( + &new_configs, + grade.as_str(), + default_cert_for_watcher.as_ref(), + ); + if let Some(new_certs) = new_certs { + certs_for_watcher.store(Arc::new(Some(Arc::new(new_certs)))); + } + } + }); + } + None => {} } proxy.add_tcp(bind_address_http.as_str()); diff --git a/src/proxy/thalamus_post_tls.rs b/src/proxy/thalamus_post_tls.rs new file mode 100644 index 00000000..90f2c315 --- /dev/null +++ b/src/proxy/thalamus_post_tls.rs @@ -0,0 +1,215 @@ +use crate::core::cli::IdsConfig; +use anyhow::Result; +use pingora_http::RequestHeader; +use std::net::SocketAddr; + +#[derive(Debug, Clone)] +pub struct ProxyIdsAlert { + pub signature_id: u32, + pub signature_name: String, + pub action: String, + pub should_block: bool, +} + +#[cfg(all(feature = "thalamus-ids", target_os = "linux"))] +mod enabled { + use super::*; + use once_cell::sync::Lazy; + use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; + use std::sync::{Arc, Mutex}; + use thalamus::engine::SignatureEngine; + use thalamus::rules::{RuleSet, RuleVars}; + + static PROXY_IDS_RUNTIME: Lazy>>> = + Lazy::new(|| Mutex::new(None)); + const CLAMAV_PROXY_IDS_SIGNATURE_ID: u32 = 9_990_001; + + struct ProxyIdsRuntime { + engine: Mutex, + enforce_block: bool, + } + + pub fn configure(ids: &IdsConfig) -> Result<()> { + if !ids.enabled || ids.rule_paths.is_empty() { + if let Ok(mut guard) = PROXY_IDS_RUNTIME.lock() { + *guard = None; + } + log::info!( + "Thalamus post-TLS IDS disabled (enabled={}, rule_paths={})", + ids.enabled, + ids.rule_paths.len() + ); + return Ok(()); + } + + let vars = RuleVars { + address_vars: ids.address_vars.clone(), + port_vars: ids.port_vars.clone(), + }; + let ruleset = RuleSet::load_from_paths_with_vars(&ids.rule_paths, &vars)?; + let rule_count = ruleset.rule_count(); + let engine = SignatureEngine::new(Arc::new(ruleset), ids.flow_timeout_secs, ids.max_flows); + let runtime = Arc::new(ProxyIdsRuntime { + engine: Mutex::new(engine), + enforce_block: ids.enforce_block, + }); + + if let Ok(mut guard) = PROXY_IDS_RUNTIME.lock() { + *guard = Some(runtime); + } + + log::info!( + "Thalamus post-TLS IDS enabled (rules={}, enforce_block={})", + rule_count, + ids.enforce_block + ); + Ok(()) + } + + pub fn is_enabled() -> bool { + PROXY_IDS_RUNTIME + .lock() + .ok() + .and_then(|g| g.as_ref().cloned()) + .is_some() + } + + pub async fn inspect_http_request( + req: &RequestHeader, + client_addr: SocketAddr, + body: &[u8], + ) -> Option { + let runtime = PROXY_IDS_RUNTIME + .lock() + .ok() + .and_then(|g| g.as_ref().cloned())?; + + let src_ip = client_addr.ip(); + let dst_ip = match src_ip { + IpAddr::V4(_) => IpAddr::V4(Ipv4Addr::LOCALHOST), + IpAddr::V6(_) => IpAddr::V6(Ipv6Addr::LOCALHOST), + }; + let src_port = client_addr.port(); + let dst_port = 80; + + let target = req + .uri + .path_and_query() + .map(|pq| pq.as_str()) + .unwrap_or("/"); + let mut headers_raw = Vec::with_capacity(512); + for (name, value) in req.headers.iter() { + headers_raw.extend_from_slice(name.as_str().as_bytes()); + headers_raw.extend_from_slice(b": "); + headers_raw.extend_from_slice(value.as_bytes()); + headers_raw.extend_from_slice(b"\r\n"); + } + + if let Some(alert) = runtime.engine.lock().ok()?.inspect_http( + src_ip, + dst_ip, + src_port, + dst_port, + req.method.as_str().as_bytes(), + target.as_bytes(), + &headers_raw, + body, + true, + ) { + let action_lower = alert.action.to_ascii_lowercase(); + let should_block = runtime.enforce_block + && matches!(action_lower.as_str(), "block" | "drop" | "reject"); + + return Some(ProxyIdsAlert { + signature_id: alert.signature_id, + signature_name: alert.signature_name, + action: alert.action, + should_block, + }); + } + + if body.is_empty() { + return None; + } + + let Some(scanner) = + crate::security::waf::actions::content_scanning::get_global_content_scanner() + else { + return None; + }; + if !scanner.is_enabled() { + return None; + } + + let content_type = req + .headers + .get("content-type") + .and_then(|v| v.to_str().ok()); + let scan_result = if let Some(ct) = content_type { + if let Some(boundary) = + crate::security::waf::actions::content_scanning::extract_multipart_boundary(ct) + { + scanner.scan_multipart_content(body, &boundary).await.ok()? + } else { + scanner.scan_content(body).await.ok()? + } + } else { + scanner.scan_content(body).await.ok()? + }; + + if !scan_result.malware_detected { + return None; + } + + let signature = scan_result + .signature + .unwrap_or_else(|| "unknown-signature".to_string()); + Some(ProxyIdsAlert { + signature_id: CLAMAV_PROXY_IDS_SIGNATURE_ID, + signature_name: format!("ClamAV: {}", signature), + action: if runtime.enforce_block { + "block".to_string() + } else { + "alert".to_string() + }, + should_block: runtime.enforce_block, + }) + } +} + +#[cfg(not(all(feature = "thalamus-ids", target_os = "linux")))] +mod enabled { + use super::*; + + pub fn configure(_ids: &IdsConfig) -> Result<()> { + Ok(()) + } + + pub fn is_enabled() -> bool { + false + } + + pub async fn inspect_http_request( + _req: &RequestHeader, + _client_addr: SocketAddr, + _body: &[u8], + ) -> Option { + None + } +} + +pub fn configure(ids: &IdsConfig) -> Result<()> { + enabled::configure(ids) +} + +pub fn is_enabled() -> bool { + enabled::is_enabled() +} + +pub async fn inspect_http_request( + req: &RequestHeader, + client_addr: SocketAddr, + body: &[u8], +) -> Option { + enabled::inspect_http_request(req, client_addr, body).await +} diff --git a/src/security/access_rules.rs b/src/security/access_rules.rs index b6d6ea5d..da8b84e1 100644 --- a/src/security/access_rules.rs +++ b/src/security/access_rules.rs @@ -4,18 +4,22 @@ use std::str::FromStr; use std::sync::{Arc, Mutex, OnceLock}; use crate::security::firewall::{Firewall, IptablesFirewall, NftablesFirewall, SYNAPSEFirewall}; +use crate::security::ratelimiter::XDPRateLimit; use crate::security::waf::wirefilter::update_http_filter_from_config_value; use crate::utils::http_utils::{is_ip_in_cidr, parse_ip_or_cidr}; -use crate::worker::config; use crate::worker::config::global_config; +use crate::worker::config::{self, XDPRateLimitConfig}; // Store previous rules state for comparison type PreviousRules = Arc>>; type PreviousRulesV6 = Arc>>; +type PreviousXDPRateLimiterConfig = Arc>; + // Global previous rules state for the access rules worker static PREVIOUS_RULES: OnceLock = OnceLock::new(); static PREVIOUS_RULES_V6: OnceLock = OnceLock::new(); +static PREVIOUS_RATE_LIMITER_CONFIG: OnceLock = OnceLock::new(); fn get_previous_rules() -> &'static PreviousRules { PREVIOUS_RULES.get_or_init(|| Arc::new(Mutex::new(HashSet::new()))) @@ -25,6 +29,16 @@ fn get_previous_rules_v6() -> &'static PreviousRulesV6 { PREVIOUS_RULES_V6.get_or_init(|| Arc::new(Mutex::new(HashSet::new()))) } +fn get_previous_xd_rate_limiter_config() -> &'static PreviousXDPRateLimiterConfig { + PREVIOUS_RATE_LIMITER_CONFIG.get_or_init(|| { + Arc::new(Mutex::new(XDPRateLimitConfig { + enabled: false, + requests: 1000, + burst_factor: 1.0, + })) + }) +} + /// Apply access rules once using the current global config snapshot (initial setup) pub fn init_access_rules_from_global( skels: &Vec>>>, @@ -38,12 +52,24 @@ pub fn init_access_rules_from_global( Arc::new(Mutex::new(std::collections::HashSet::new())); let previous_rules_v6: PreviousRulesV6 = Arc::new(Mutex::new(std::collections::HashSet::new())); + let previous_rate_limiter_config: PreviousXDPRateLimiterConfig = + Arc::new(Mutex::new(XDPRateLimitConfig { + enabled: false, + requests: 1000, + burst_factor: 1.0, + })); // Use Arc clone instead of full Config clone for efficiency let resp = config::ConfigApiResponse { success: true, config: (**cfg_arc).clone(), }; - apply_rules(skels, &resp, &previous_rules, &previous_rules_v6)?; + apply_rules( + skels, + &resp, + &previous_rules, + &previous_rules_v6, + &previous_rate_limiter_config, + )?; } } Ok(()) @@ -56,6 +82,7 @@ pub fn apply_rules_from_global( skels: &Vec>>>, previous_rules: &PreviousRules, previous_rules_v6: &PreviousRulesV6, + previous_rate_limiter_config: &PreviousXDPRateLimiterConfig, skip_waf_update: bool, ) -> Result<(), Box> { // Read from global config and apply if available @@ -79,6 +106,7 @@ pub fn apply_rules_from_global( }, previous_rules, previous_rules_v6, + previous_rate_limiter_config, )?; return Ok(()); } @@ -97,6 +125,7 @@ pub fn apply_rules_from_global_with_state( skels, get_previous_rules(), get_previous_rules_v6(), + get_previous_xd_rate_limiter_config(), skip_waf_update, ) } @@ -497,6 +526,7 @@ fn apply_rules( resp: &config::ConfigApiResponse, previous_rules: &PreviousRules, previous_rules_v6: &PreviousRulesV6, + previous_rules_rate_limiter: &PreviousXDPRateLimiterConfig, ) -> Result<(), Box> { fn parse_ipv4_ip_or_cidr(entry: &str) -> Option<(Ipv4Addr, u32)> { let s = entry.trim(); @@ -633,10 +663,12 @@ fn apply_rules( // Compare with previous rules to detect changes let mut previous_rules_guard = previous_rules.lock().unwrap(); let mut previous_rules_v6_guard = previous_rules_v6.lock().unwrap(); + let mut previous_rate_limiter_config = previous_rules_rate_limiter.lock().unwrap(); // Check if rules have changed let ipv4_changed = *previous_rules_guard != current_rules; let ipv6_changed = *previous_rules_v6_guard != current_rules_v6; + let rate_limiter_config_changed = *previous_rate_limiter_config != rule.config.rate_limit; // If neither family changed, skip quietly with a single log entry if ipv4_changed || ipv6_changed { @@ -710,6 +742,31 @@ fn apply_rules( log::debug!("No IPv4 or IPv6 access rule changes detected, skipping BPF map updates"); } + if rate_limiter_config_changed { + for s in skels.iter() { + let skel_ref_res = s.lock(); + let mut skel_ref = match skel_ref_res { + Ok(skel_ref) => skel_ref, + Err(e) => { + let err_msg = format!("Could not take skel mutex in thread: {}", e); + log::error!("{err_msg}"); + return Err(err_msg.into()); + } + }; + + let mut ratelimit = XDPRateLimit::new(&mut skel_ref); + ratelimit.setup_from_config(&rule.config.rate_limit); + + log::debug!("Successfully set XDP ratelimter config via access rules"); + } + + if rate_limiter_config_changed { + *previous_rate_limiter_config = rule.config.rate_limit.clone(); + } + } else { + log::debug!("No XDP rate limiter config cahnges detected, skipping update"); + } + Ok(()) } diff --git a/src/security/firewall/bpf/lib/ids_export.h b/src/security/firewall/bpf/lib/ids_export.h new file mode 100644 index 00000000..964d0d01 --- /dev/null +++ b/src/security/firewall/bpf/lib/ids_export.h @@ -0,0 +1,94 @@ +#pragma once + +#include "common.h" + +#include "../xdp_maps.h" + +static __always_inline void increment_ids_export_stat(__u32 idx) { + __u64 *value = bpf_map_lookup_elem(&ids_export_stats, &idx); + if (value) { + __sync_fetch_and_add(value, 1); + } +} + +static __always_inline void maybe_export_ids_packet(struct xdp_md *ctx, + void *data, + void *data_end, + const struct iphdr *iph, + const struct ipv6hdr *ip6h, + const struct tcphdr *tcph) { + __u32 key = 0; + struct ids_export_config *cfg = + bpf_map_lookup_elem(&ids_export_config_map, &key); + if (!cfg || cfg->enabled == 0) { + return; + } + + // Skip TCP packets with no payload (pure ACKs, keepalives, window updates). + // Always export SYN/FIN/RST — they carry state for flow tracking and rules. + if (tcph) { + __u32 ip_hdr_len = 0; + if (iph) + ip_hdr_len = sizeof(struct ethhdr) + ((__u32)iph->ihl * 4); + else if (ip6h) + ip_hdr_len = sizeof(struct ethhdr) + sizeof(struct ipv6hdr); + + __u32 tcp_hdr_len = (__u32)(tcph->doff) * 4; + __u32 headers_total = ip_hdr_len + tcp_hdr_len; + __u32 packet_total = (__u32)((__u64)data_end - (__u64)data); + + bool has_payload = packet_total > headers_total; + bool is_control = tcph->syn || tcph->fin || tcph->rst; + + if (!has_payload && !is_control) { + return; + } + } + + __u32 packet_len = (__u32)((__u64)data_end - (__u64)data); + __u32 snaplen = cfg->snaplen; + if (snaplen == 0 || snaplen > IDS_EXPORT_MAX_BYTES) { + snaplen = IDS_EXPORT_MAX_BYTES; + } + + __u32 captured_len = packet_len < snaplen ? packet_len : snaplen; + + struct ids_export_event *event = + bpf_ringbuf_reserve(&ids_export_events, sizeof(*event), 0); + if (!event) { + increment_ids_export_stat(IDS_EXPORT_STAT_RINGBUF_DROPPED); + return; + } + + event->ts_ns = bpf_ktime_get_ns(); + event->ifindex = ctx->ingress_ifindex; + event->packet_len = packet_len; + event->captured_len = captured_len; + event->_reserved = 0; + + if (captured_len == 0) { + goto submit_event; + } + + // Verifier-safe copy: first byte with constant len=1, then optional tail. + if (bpf_xdp_load_bytes(ctx, 0, event->packet, 1) < 0) { + bpf_ringbuf_discard(event, 0); + increment_ids_export_stat(IDS_EXPORT_STAT_INVALID_EVENT); + return; + } + __u32 tail_len = captured_len - 1; + tail_len &= (IDS_EXPORT_MAX_BYTES - 1); + if (tail_len == 0) { + goto submit_event; + } + + if (bpf_xdp_load_bytes(ctx, 1, event->packet + 1, tail_len) < 0) { + bpf_ringbuf_discard(event, 0); + increment_ids_export_stat(IDS_EXPORT_STAT_INVALID_EVENT); + return; + } + +submit_event: + bpf_ringbuf_submit(event, 0); + increment_ids_export_stat(IDS_EXPORT_STAT_EXPORTED); +} diff --git a/src/security/firewall/bpf/lib/ratelimit.h b/src/security/firewall/bpf/lib/ratelimit.h new file mode 100644 index 00000000..ceca571c --- /dev/null +++ b/src/security/firewall/bpf/lib/ratelimit.h @@ -0,0 +1,135 @@ +#pragma once + +#include "common.h" + +#include "../xdp_maps.h" +#include "vmlinux.h" + +struct ratelimiter_config_t { + __u64 TOKENS_PER_REQUEST; // tokens consumed per request + __u64 REFILL_RATE; // tokens added per sec + __u64 MAX_BUCKET_CAPACITY; // refill_rate * (max_bucket_capacity / + // refill_rate) = allow x request burst + __u8 ENABLED; +}; + +// set this before loading the script +volatile struct ratelimiter_config_t ratelimiter_config = {1, 1000, 3000, + false}; + +static __always_inline bool is_ipv4_whitelisted(__be32 *addr) { + return !!bpf_map_lookup_elem(&ipv4_ratelimit_whitelist, addr); +} + +static __always_inline bool is_ipv6_whitelisted(ipv6_addr *addr) { + return !!bpf_map_lookup_elem(&ipv6_ratelimit_whitelist, addr); +} + +static __always_inline void refill_tokens(struct ratelimit_bucket_value *rl_val, + __u64 now) { + + __u64 elapsed_ns = now - rl_val->last_topup; + + // Calculate tokens to add: (elapsed_seconds * REFILL_RATE) + __u64 tokens_to_add = + (elapsed_ns * ratelimiter_config.REFILL_RATE) / 1000000000ULL; + + if (tokens_to_add > 0) { + rl_val->num_of_tokens += tokens_to_add; + + if (rl_val->num_of_tokens > ratelimiter_config.MAX_BUCKET_CAPACITY) { + rl_val->num_of_tokens = ratelimiter_config.MAX_BUCKET_CAPACITY; + } + + rl_val->last_topup = now; + } +} + +static __noinline __u8 ipv4_syn_ratelimit(__be32 *addr, struct tcphdr *tcph) { + if (is_ipv4_whitelisted(addr)) { + return XDP_PASS; + } + + __u64 now = bpf_ktime_get_ns(); + + struct ratelimit_bucket_value *bucket = + bpf_map_lookup_elem(&ipv4_syn_bucket_store, addr); + + // bpf_printk("Bucket status -> tokens: %llu\n", bucket->num_of_tokens); + + if (!bucket) { + struct ratelimit_bucket_value new_bucket = {}; + new_bucket.last_topup = now; + new_bucket.num_of_tokens = ratelimiter_config.MAX_BUCKET_CAPACITY; + bpf_map_update_elem(&ipv4_syn_bucket_store, addr, &new_bucket, BPF_ANY); + // bpf_printk("Bucket created for addr: %u, topup: %ld\n", *addr, now); + return XDP_PASS; + } + + refill_tokens(bucket, now); + + if (bucket->num_of_tokens >= ratelimiter_config.TOKENS_PER_REQUEST) { + bucket->num_of_tokens -= ratelimiter_config.TOKENS_PER_REQUEST; + // bpf_printk("Packet passed for addr: %d, num of tokens: %llu\n", *addr, + // bucket->num_of_tokens); + return XDP_PASS; + } + + // bpf_printk("Packet dropped for addr: %u\n", *addr); + return XDP_DROP; +} + +static __noinline __u8 ipv6_syn_ratelimit(ipv6_addr *addr, + struct tcphdr *tcph) { + if (is_ipv6_whitelisted(addr)) { + return XDP_PASS; + } + + __u64 now = bpf_ktime_get_ns(); + struct ratelimit_bucket_value *bucket = + bpf_map_lookup_elem(&ipv6_syn_bucket_store, addr); + + if (!bucket) { + struct ratelimit_bucket_value new_bucket = {}; + new_bucket.last_topup = now; + new_bucket.num_of_tokens = ratelimiter_config.MAX_BUCKET_CAPACITY; + bpf_map_update_elem(&ipv6_syn_bucket_store, addr, &new_bucket, BPF_ANY); + + return XDP_PASS; + } + + refill_tokens(bucket, now); + + if (bucket->num_of_tokens >= ratelimiter_config.TOKENS_PER_REQUEST) { + bucket->num_of_tokens -= ratelimiter_config.TOKENS_PER_REQUEST; + return XDP_PASS; + } + + return XDP_DROP; +} + +int __noinline xdp_ratelimit(struct iphdr *iph, struct ipv6hdr *ip6h, + struct tcphdr *tcph) { + if (!ratelimiter_config.ENABLED) { + return XDP_PASS; + } + + if (tcph) { + if (!tcph->syn || tcph->ack) { + return XDP_PASS; + } + if (iph) { + if (ipv4_syn_ratelimit(&iph->saddr, tcph) == XDP_DROP) { + return XDP_DROP; + } + } else if (ip6h) { + ipv6_addr *ipv6_saddr_ptr = (ipv6_addr *)&ip6h->saddr; + + if (ipv6_syn_ratelimit(ipv6_saddr_ptr, tcph) == XDP_DROP) { + return XDP_DROP; + } + } + } + + return XDP_PASS; +} diff --git a/src/security/firewall/bpf/lib/tcp_fingerprinting.h b/src/security/firewall/bpf/lib/tcp_fingerprinting.h index 1ea0d18f..2bc90f9a 100644 --- a/src/security/firewall/bpf/lib/tcp_fingerprinting.h +++ b/src/security/firewall/bpf/lib/tcp_fingerprinting.h @@ -82,9 +82,9 @@ static __always_inline void increment_tcp_fingerprint_blocks_ipv6(void) { } } -static __noinline int parse_tcp_mss_wscale(struct tcphdr *tcp, - void *data_end, __u16 *mss_out, - __u8 *wscale_out) { +static __always_inline int parse_tcp_mss_wscale(struct tcphdr *tcp, + void *data_end, __u16 *mss_out, + __u8 *wscale_out) { if ((void *)tcp + sizeof(struct tcphdr) > data_end) { return 0; } @@ -109,10 +109,9 @@ static __noinline int parse_tcp_mss_wscale(struct tcphdr *tcp, return 0; } -// Parse options — limit to 10 iterations (enough for all standard TCP options: -// MSS(4B) + WScale(3B) + SACK_PERM(2B) + Timestamps(10B) + NOPs = ~8 iterations max) -// No #pragma unroll — use bounded loop (Linux 5.3+) to reduce verifier state explosion - for (int i = 0; i < 10; i++) { +// Parse options - limit to 20 iterations to handle NOPs +#pragma unroll + for (int i = 0; i < 20; i++) { if (ptr >= end || ptr >= (__u8 *)data_end) break; if (ptr + 1 > (__u8 *)data_end) @@ -150,42 +149,6 @@ static __noinline int parse_tcp_mss_wscale(struct tcphdr *tcp, return 0; } -/// Copy raw TCP options bytes into a buffer for JA4T option-kind extraction. -/// Sets *out_len to the number of bytes actually copied (up to -/// TCP_FP_MAX_OPTION_LEN). -static __noinline void copy_raw_tcp_options(struct tcphdr *tcp, - void *data_end, - __u8 *options_out, - __u8 *out_len) { - *out_len = 0; - - if ((void *)tcp + sizeof(struct tcphdr) > data_end) { - return; - } - - __u32 opt_total = (tcp->doff * 4) - sizeof(struct tcphdr); - if (opt_total > TCP_FP_MAX_OPTION_LEN) { - opt_total = TCP_FP_MAX_OPTION_LEN; - } - - __u8 *src = (__u8 *)tcp + sizeof(struct tcphdr); - if (src + opt_total > (__u8 *)data_end) { - opt_total = (__u8 *)data_end - src; - if (opt_total > TCP_FP_MAX_OPTION_LEN) { - opt_total = TCP_FP_MAX_OPTION_LEN; - } - } - - *out_len = (__u8)opt_total; - -// No #pragma unroll — use bounded loop (Linux 5.3+) to reduce verifier state explosion - for (int i = 0; i < 20; i++) { - if (i >= (int)opt_total || src + i + 1 > (__u8 *)data_end) - break; - options_out[i] = src[i]; - } -} - static __always_inline void generate_tcp_fingerprint(struct tcphdr *tcp, void *data_end, __u16 ttl, __u8 *fingerprint) { @@ -290,216 +253,107 @@ static __always_inline void record_tcp_fingerprint(__be32 src_ip, // Extract MSS and window scale from options parse_tcp_mss_wscale(tcp, data_end, &data.mss, &data.window_scale); - // Copy raw TCP options for JA4T option-kind fingerprinting - copy_raw_tcp_options(tcp, data_end, data.options, &data.options_len); - bpf_map_update_elem(&tcp_fingerprints, &key, &data, BPF_ANY); increment_unique_fingerprints(); - } - - // Also populate the simple (IP, port) map for O(1) userspace lookups. - struct tcp_fp_simple_key_v4 skey = {0}; - skey.src_ip = src_ip; - skey.src_port = src_port; - bpf_map_update_elem(&tcp_fingerprints_simple, &skey, &data, BPF_ANY); -} - - -/// Copy raw TCP options bytes (reduced to 20 iterations for verifier budget). -/// Covers all standard TCP options (MSS, SACK, timestamps, window scale, NOP). -/// The struct options field stays 40 bytes; unused bytes remain zeroed. -static __noinline void copy_raw_tcp_options_short(struct tcphdr *tcp, - void *data_end, - __u8 *options_out, - __u8 *out_len) { - *out_len = 0; - if ((void *)tcp + sizeof(struct tcphdr) > data_end) - return; - - __u32 opt_total = (tcp->doff * 4) - sizeof(struct tcphdr); - if (opt_total > 20) - opt_total = 20; - __u8 *src = (__u8 *)tcp + sizeof(struct tcphdr); - if (src + opt_total > (__u8 *)data_end) { - opt_total = (__u8 *)data_end - src; - if (opt_total > 20) - opt_total = 20; - } - - *out_len = (__u8)opt_total; - - for (int i = 0; i < 20; i++) { - if (i >= (int)opt_total || src + i + 1 > (__u8 *)data_end) - break; - options_out[i] = src[i]; + // Log new TCP fingerprint + // bpf_printk("TCP_FP: New fingerprint from %pI4:%d - TTL:%d MSS:%d WS:%d + // Window:%d", + // &src_ip, bpf_ntohs(src_port), ttl, data.mss, data.window_scale, + // data.window_size); } } -/// Refactored: computes everything ONCE to minimize verifier state explosion. -/// -/// Previous version inlined 4 loops (~100 iterations total): -/// generate_tcp_fingerprint -> parse_tcp_mss_wscale (20 iters) [1st] -/// record_tcp_fingerprint -> generate_tcp_fingerprint (20 iters) [2nd - DUPLICATE] -/// record_tcp_fingerprint -> parse_tcp_mss_wscale (20 iters) [3rd - TRIPLICATE] -/// record_tcp_fingerprint -> copy_raw_tcp_options (40 iters) -/// -/// Now: 1x parse_tcp_mss_wscale (20 iters) + 1x copy_raw_tcp_options_short (20 iters) -/// = 40 iterations total (~60% reduction in verifier states) static __noinline int xdp_tcp_fingerprinting(struct xdp_md *ctx, struct iphdr *iph, struct ipv6hdr *ip6h, struct tcphdr *tcph) { - if (!tcph || !tcph->syn || tcph->ack) - return XDP_PASS; - - void *data_end = (void *)(long)ctx->data_end; - if ((void *)tcph + sizeof(struct tcphdr) > data_end) - return XDP_PASS; - - increment_tcp_syn_stats(); - - // --- Parse MSS + window scale ONCE (single 20-iteration loop) --- - __u16 mss = 0; - __u8 wscale = 0; - parse_tcp_mss_wscale(tcph, data_end, &mss, &wscale); - - // --- Build fingerprint string ONCE (pure arithmetic, no loops) --- - __u16 ttl = iph ? iph->ttl : (ip6h ? ip6h->hop_limit : 0); - __u16 window = bpf_ntohs(tcph->window); - __u8 fingerprint[14] = {0}; - fingerprint[0] = '0' + (ttl / 100); - fingerprint[1] = '0' + ((ttl / 10) % 10); - fingerprint[2] = '0' + (ttl % 10); - fingerprint[3] = ':'; - fingerprint[4] = '0' + (mss / 1000); - fingerprint[5] = '0' + ((mss / 100) % 10); - fingerprint[6] = '0' + ((mss / 10) % 10); - fingerprint[7] = '0' + (mss % 10); - fingerprint[8] = ':'; - fingerprint[9] = '0' + (window / 10000); - fingerprint[10] = '0' + ((window / 1000) % 10); - fingerprint[11] = '0' + ((window / 100) % 10); - fingerprint[12] = '0' + ((window / 10) % 10); - fingerprint[13] = '0' + (window % 10); - - // --- IPv4 path --- - if (iph) { - if (is_tcp_fingerprint_blocked(fingerprint)) { - increment_tcp_fingerprint_blocks_ipv4(); - increment_total_packets_dropped(); - increment_dropped_ipv4_address(iph->saddr); - return XDP_DROP; - } - - // Skip localhost - if ((iph->saddr & bpf_htonl(0xff000000)) == bpf_htonl(0x7f000000)) - return XDP_PASS; - - struct tcp_fingerprint_key key = {0}; - key.src_ip = iph->saddr; - key.src_port = tcph->source; - __builtin_memcpy(key.fingerprint, fingerprint, 14); + if (tcph) { + if (iph) { + if (tcph->syn && !tcph->ack) { + increment_tcp_syn_stats(); + + __u8 fingerprint[14] = {0}; + generate_tcp_fingerprint(tcph, (void *)(long)ctx->data_end, iph->ttl, + fingerprint); + + if (is_tcp_fingerprint_blocked(fingerprint)) { + increment_tcp_fingerprint_blocks_ipv4(); + increment_total_packets_dropped(); + increment_dropped_ipv4_address(iph->saddr); + // bpf_printk("XDP: BLOCKED TCP fingerprint from IPv4 %pI4:%d - + // FP:%s", + // &iph->saddr, bpf_ntohs(tcph->source), fingerprint); + return XDP_DROP; + } + + record_tcp_fingerprint(iph->saddr, tcph->source, tcph, + (void *)(long)ctx->data_end, iph->ttl); + + // bpf_printk("TCP_FP: New IPv4 fingerprint from %pI4:%d - TTL:%d", + // &iph->saddr, bpf_ntohs(tcph->source), iph->ttl); + } + } else if (ip6h) { + __u8 fingerprint[14] = {0}; + generate_tcp_fingerprint(tcph, (void *)(long)ctx->data_end, + ip6h->hop_limit, fingerprint); + + if (is_tcp_fingerprint_blocked_v6(fingerprint)) { + increment_tcp_fingerprint_blocks_ipv6(); + increment_total_packets_dropped(); + increment_dropped_ipv6_address(ip6h->saddr); + // bpf_printk("XDP: BLOCKED TCP fingerprint from IPv6 %pI6:%d - + // FP:%s", + // &ip6h->saddr, bpf_ntohs(tcph->source), fingerprint); + return XDP_DROP; + } - __u64 now = bpf_ktime_get_ns(); - struct tcp_fingerprint_data *existing = - bpf_map_lookup_elem(&tcp_fingerprints, &key); - if (existing) { - struct tcp_fingerprint_data data = {0}; - data.first_seen = existing->first_seen; - data.last_seen = now; - data.packet_count = existing->packet_count + 1; - data.ttl = existing->ttl; - data.mss = existing->mss; - data.window_size = existing->window_size; - data.window_scale = existing->window_scale; - data.options_len = existing->options_len; - __builtin_memcpy(data.options, existing->options, TCP_FP_MAX_OPTION_LEN); - bpf_map_update_elem(&tcp_fingerprints, &key, &data, BPF_ANY); - } else { + struct tcp_fingerprint_key_v6 key = {0}; struct tcp_fingerprint_data data = {0}; - data.first_seen = now; - data.last_seen = now; - data.packet_count = 1; - data.ttl = ttl; - data.mss = mss; // Already parsed above - data.window_size = window; - data.window_scale = wscale; // Already parsed above - copy_raw_tcp_options_short(tcph, data_end, data.options, - &data.options_len); - bpf_map_update_elem(&tcp_fingerprints, &key, &data, BPF_ANY); - increment_unique_fingerprints(); - } - - // Populate simple (IP, port) map for O(1) userspace lookups - { - struct tcp_fp_simple_key_v4 skey = {0}; - skey.src_ip = iph->saddr; - skey.src_port = tcph->source; - struct tcp_fingerprint_data *latest = - bpf_map_lookup_elem(&tcp_fingerprints, &key); - if (latest) { - bpf_map_update_elem(&tcp_fingerprints_simple, &skey, latest, BPF_ANY); - } - } - } - // --- IPv6 path --- - else if (ip6h) { - if (is_tcp_fingerprint_blocked_v6(fingerprint)) { - increment_tcp_fingerprint_blocks_ipv6(); - increment_total_packets_dropped(); - increment_dropped_ipv6_address(ip6h->saddr); - return XDP_DROP; - } + __u64 timestamp = bpf_ktime_get_ns(); - struct tcp_fingerprint_key_v6 key = {0}; - copy_ipv6_addr_as_array(&key.src_ip, &ip6h->saddr); - key.src_port = tcph->source; - __builtin_memcpy(key.fingerprint, fingerprint, 14); + copy_ipv6_addr_as_array(&key.src_ip, &ip6h->saddr); + key.src_port = tcph->source; - __u64 now = bpf_ktime_get_ns(); - struct tcp_fingerprint_data *existing = - bpf_map_lookup_elem(&tcp_fingerprints_v6, &key); - if (existing) { - struct tcp_fingerprint_data data = {0}; - data.first_seen = existing->first_seen; - data.last_seen = now; - data.packet_count = existing->packet_count + 1; - data.ttl = existing->ttl; - data.mss = existing->mss; - data.window_size = existing->window_size; - data.window_scale = existing->window_scale; - data.options_len = existing->options_len; - __builtin_memcpy(data.options, existing->options, TCP_FP_MAX_OPTION_LEN); - bpf_map_update_elem(&tcp_fingerprints_v6, &key, &data, BPF_ANY); - } else { - struct tcp_fingerprint_data data = {0}; - data.first_seen = now; - data.last_seen = now; - data.packet_count = 1; - data.ttl = ip6h->hop_limit; - data.mss = mss; - data.window_size = window; - data.window_scale = wscale; - copy_raw_tcp_options_short(tcph, data_end, data.options, - &data.options_len); - bpf_map_update_elem(&tcp_fingerprints_v6, &key, &data, BPF_ANY); - increment_unique_fingerprints(); - } + __builtin_memcpy(key.fingerprint, fingerprint, sizeof(fingerprint)); - // Populate simple (IP, port) map for O(1) userspace lookups - { - struct tcp_fp_simple_key_v6 skey = {0}; - copy_ipv6_addr_as_array(&skey.src_ip, &ip6h->saddr); - skey.src_port = tcph->source; - struct tcp_fingerprint_data *latest = + struct tcp_fingerprint_data *existing = bpf_map_lookup_elem(&tcp_fingerprints_v6, &key); - if (latest) { - bpf_map_update_elem(&tcp_fingerprints_simple_v6, &skey, latest, - BPF_ANY); + if (existing) { + data.first_seen = existing->first_seen; + data.last_seen = timestamp; + data.packet_count = existing->packet_count + 1; + data.ttl = existing->ttl; + data.mss = existing->mss; + data.window_size = existing->window_size; + data.window_scale = existing->window_scale; + data.options_len = existing->options_len; + + __builtin_memcpy(data.options, existing->options, + TCP_FP_MAX_OPTION_LEN); + bpf_map_update_elem(&tcp_fingerprints_v6, &key, &data, BPF_ANY); + + } else { + data.first_seen = timestamp; + data.last_seen = timestamp; + data.packet_count = 1; + data.ttl = ip6h->hop_limit; + data.window_size = bpf_ntohs(tcph->window); + + parse_tcp_mss_wscale(tcph, (void *)(long)ctx->data_end, &data.mss, + &data.window_scale); + + bpf_map_update_elem(&tcp_fingerprints_v6, &key, &data, BPF_ANY); + increment_unique_fingerprints(); + + // Log new IPv6 TCP fingerprint + // bpf_printk("TCP_FP: New IPv6 fingerprint from %pI6:%d - TTL:%d MSS:%d + // " + // "WS:%d Window:%d", + // &ip6h->saddr, bpf_ntohs(tcph->source), data.ttl, data.mss, + // data.window_scale, data.window_size); } } } - return XDP_PASS; } diff --git a/src/security/firewall/bpf/xdp.bpf.c b/src/security/firewall/bpf/xdp.bpf.c index 06af6fa3..0a903d98 100644 --- a/src/security/firewall/bpf/xdp.bpf.c +++ b/src/security/firewall/bpf/xdp.bpf.c @@ -2,8 +2,9 @@ #include "lib/firewall.h" #include "lib/helper.h" +#include "lib/ids_export.h" #include "lib/tcp_fingerprinting.h" -#include "lib/latency_tracking.h" +#include "ratelimit.h" #include "vmlinux.h" #include "xdp_maps.h" @@ -64,15 +65,11 @@ int xdp_pipeline(struct xdp_md *ctx) { if (xdp_tcp_fingerprinting(ctx, iph, ip6h, tcph) == XDP_DROP) return XDP_DROP; - // JA4TS SYN-ACK capture moved to separate TC BPF program (ja4ts.bpf.c) + if (xdp_ratelimit(iph, ip6h, tcph) == XDP_DROP) + return XDP_DROP; - // Track TCP handshake latency for JA4L fingerprinting - if (tcph) { - if (iph) - track_latency_v4(iph, tcph, data_end); - else if (ip6h) - track_latency_v6(ip6h, tcph, data_end); - } + maybe_export_ids_packet(ctx, (void *)(long)ctx->data, data_end, iph, ip6h, + tcph); increment_total_packets_processed(); diff --git a/src/security/firewall/bpf/xdp_maps.h b/src/security/firewall/bpf/xdp_maps.h index bcc395cc..bf17b5ad 100644 --- a/src/security/firewall/bpf/xdp_maps.h +++ b/src/security/firewall/bpf/xdp_maps.h @@ -4,6 +4,51 @@ #include "vmlinux.h" #define TCP_FINGERPRINT_MAX_ENTRIES 10000 +#define IDS_EXPORT_MAX_BYTES 512 + +// IDS export event sent to userspace ring buffer. +// Contains a bounded packet slice to keep verifier/program size predictable. +struct ids_export_event { + __u64 ts_ns; + __u32 ifindex; + __u32 packet_len; + __u32 captured_len; + __u32 _reserved; + __u8 packet[IDS_EXPORT_MAX_BYTES]; +}; + +struct ids_export_config { + __u32 enabled; + __u32 snaplen; +}; + +enum ids_export_stat_idx { + IDS_EXPORT_STAT_EXPORTED = 0, + IDS_EXPORT_STAT_RINGBUF_DROPPED = 1, + IDS_EXPORT_STAT_INVALID_EVENT = 2, +}; + +// IDS packet export ring buffer consumed by userspace IDS worker. +struct { + __uint(type, BPF_MAP_TYPE_RINGBUF); + __uint(max_entries, 1 << 24); // 16MB +} ids_export_events SEC(".maps"); + +// Runtime IDS export toggle/snaplen. +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, __u32); + __type(value, struct ids_export_config); +} ids_export_config_map SEC(".maps"); + +// Export diagnostics counters. +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __uint(max_entries, 3); + __type(key, __u32); + __type(value, __u64); +} ids_export_stats SEC(".maps"); // IPv4 maps: permanently banned and recently banned struct { @@ -90,67 +135,6 @@ struct { __type(value, __u64); } total_packets_dropped SEC(".maps"); -// Connection latency tracking (JA4L) - tracks TCP handshake SYN/SYNACK/ACK timestamps -#define LATENCY_STATE_SYN_SEEN 0 -#define LATENCY_STATE_SYNACK_SEEN 1 -#define LATENCY_STATE_COMPLETE 2 -#define CONNECTION_LATENCY_MAX_ENTRIES 10000 - -// IPv4 key: client_ip(4) + client_port(2) + pad(2) = 8 bytes -// Client IP + ephemeral port is unique per connection, no server addr needed -struct __attribute__((packed)) connection_latency_key_v4 { - __be32 client_ip; - __be16 client_port; - __u16 _pad; -}; - -// IPv6 key: client_ip(16) + client_port(2) + pad(2) = 20 bytes -struct __attribute__((packed)) connection_latency_key_v6 { - __u8 client_ip[16]; - __be16 client_port; - __u16 _pad; -}; - -// Value: timestamps + TTLs + state = 32 bytes (with padding) -struct connection_latency_data { - __u64 syn_time_ns; - __u64 synack_time_ns; - __u64 ack_time_ns; - __u8 client_ttl; - __u8 server_ttl; - __u8 state; - __u8 _pad[5]; -}; - -struct { - __uint(type, BPF_MAP_TYPE_LRU_HASH); - __uint(max_entries, CONNECTION_LATENCY_MAX_ENTRIES); - __type(key, struct connection_latency_key_v4); - __type(value, struct connection_latency_data); -} connection_latency SEC(".maps"); - -struct { - __uint(type, BPF_MAP_TYPE_LRU_HASH); - __uint(max_entries, CONNECTION_LATENCY_MAX_ENTRIES); - __type(key, struct connection_latency_key_v6); - __type(value, struct connection_latency_data); -} connection_latency_v6 SEC(".maps"); - -// Latency statistics (total handshakes, complete, packets) -struct latency_stats_data { - __u64 total_handshakes; - __u64 complete_handshakes; - __u64 total_packets; - __u64 _reserved; -}; - -struct { - __uint(type, BPF_MAP_TYPE_ARRAY); - __uint(max_entries, 1); - __type(key, __u32); - __type(value, struct latency_stats_data); -} latency_stats SEC(".maps"); - // TCP fingerprinting maps struct { __uint(type, BPF_MAP_TYPE_LRU_HASH); @@ -166,39 +150,6 @@ struct { __type(value, struct tcp_fingerprint_data); } tcp_fingerprints_v6 SEC(".maps"); -// JA4T direct-lookup maps: keyed by (IP, port) only for O(1) userspace lookups. -// The compound-key maps above are kept for per-fingerprint accounting; these -// simple maps mirror the latest fingerprint data for each (IP, port) pair. -// IPv4 key: src_ip(4) + src_port(2) + pad(2) = 8 bytes -struct __attribute__((packed)) tcp_fp_simple_key_v4 { - __be32 src_ip; - __be16 src_port; - __u16 _pad; -}; - -// IPv6 key: src_ip(16) + src_port(2) + pad(2) = 20 bytes -struct __attribute__((packed)) tcp_fp_simple_key_v6 { - __u8 src_ip[16]; - __be16 src_port; - __u16 _pad; -}; - -struct { - __uint(type, BPF_MAP_TYPE_LRU_HASH); - __uint(max_entries, TCP_FINGERPRINT_MAX_ENTRIES); - __type(key, struct tcp_fp_simple_key_v4); - __type(value, struct tcp_fingerprint_data); -} tcp_fingerprints_simple SEC(".maps"); - -struct { - __uint(type, BPF_MAP_TYPE_LRU_HASH); - __uint(max_entries, TCP_FINGERPRINT_MAX_ENTRIES); - __type(key, struct tcp_fp_simple_key_v6); - __type(value, struct tcp_fingerprint_data); -} tcp_fingerprints_simple_v6 SEC(".maps"); - -// JA4TS SYN-ACK fingerprinting moved to separate TC BPF program (ja4ts.bpf.c) - struct { __uint(type, BPF_MAP_TYPE_ARRAY); __uint(max_entries, 1); @@ -279,3 +230,62 @@ struct { __type(value, __u8); } banned_outbound_ipv6_address_ports SEC(".maps"); +// RATE LIMITER MAPS + +struct ratelimiter_metrics { + __u64 requests_dropped; + __u64 total_ipv4_dropped; + __u64 total_ipv6_dropped; +}; + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(max_entries, 1); + __type(key, __u8); + __type(value, struct ratelimiter_metrics); +} ratelimit_metrics_map SEC(".maps"); + +#define DEFAULT_BUCKET_ENTRIES 50000 + +struct ratelimit_bucket_value { + __u64 last_topup; + __u64 num_of_tokens; +}; + +struct { + __uint(type, BPF_MAP_TYPE_LRU_PERCPU_HASH); + __uint(max_entries, + DEFAULT_BUCKET_ENTRIES); // to be modified in userspace before loading + // the script + __type(key, __be32); + __type(value, struct ratelimit_bucket_value); +} ipv4_syn_bucket_store SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_LRU_PERCPU_HASH); + __uint(max_entries, + DEFAULT_BUCKET_ENTRIES); // to be modified in userspace before loading + // the script + __type(key, ipv6_addr); + __type(value, struct ratelimit_bucket_value); +} ipv6_syn_bucket_store SEC(".maps"); + +#define DEFAULT_RATELIMITER_WHITELIST_ENTRIES_IPV4 100 +#define DEFAULT_RATELIMITER_WHITELIST_ENTRIES_IPV6 100 + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(max_entries, DEFAULT_RATELIMITER_WHITELIST_ENTRIES_IPV4); + __type(key, __be32); + __type(value, __u8); +} ipv4_ratelimit_whitelist SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(max_entries, DEFAULT_RATELIMITER_WHITELIST_ENTRIES_IPV6); + __type(key, ipv6_addr); + __type(value, __u8); +} ipv6_ratelimit_whitelist SEC(".maps"); + +// TODO: volumetric rate limiter, ratelimiter diagnostics, integration in the +// app. diff --git a/src/security/firewall/mod.rs b/src/security/firewall/mod.rs index 9ea5a338..5b515155 100644 --- a/src/security/firewall/mod.rs +++ b/src/security/firewall/mod.rs @@ -14,9 +14,6 @@ pub mod bpf { // Include the skeleton generated by build.rs into OUT_DIR at compile time include!(concat!(env!("OUT_DIR"), "/xdp.skel.rs")); } -pub mod ja4ts_bpf { - include!(concat!(env!("OUT_DIR"), "/ja4ts.skel.rs")); -} pub use iptables::IptablesFirewall; pub use nftables::NftablesFirewall; diff --git a/src/security/mod.rs b/src/security/mod.rs index 0596232e..fcc8d959 100644 --- a/src/security/mod.rs +++ b/src/security/mod.rs @@ -1,4 +1,6 @@ pub mod access_rules; #[cfg(all(feature = "bpf", not(feature = "disable-bpf")))] pub mod firewall; +#[cfg(all(feature = "bpf", not(feature = "disable-bpf")))] +pub mod ratelimiter; pub mod waf; diff --git a/src/security/ratelimiter/mod.rs b/src/security/ratelimiter/mod.rs new file mode 100644 index 00000000..3b71f5af --- /dev/null +++ b/src/security/ratelimiter/mod.rs @@ -0,0 +1,101 @@ +use crate::{ + security::firewall::bpf::{OpenXdpSkel, XdpSkel}, + worker::config::XDPRateLimitConfig, +}; +use anyhow::Result; +use libbpf_rs::{MapCore, MapFlags}; +use std::net::{Ipv4Addr, Ipv6Addr}; + +pub struct XDPRateLimit<'a, 'b> { + skel: &'a mut XdpSkel<'b>, +} + +impl<'a, 'b> XDPRateLimit<'a, 'b> { + pub fn new(skel: &'a mut XdpSkel<'b>) -> Self { + Self { skel } + } + + pub fn set_request_per_sec(&mut self, request_per_seq: u64, burst_multiplier: f32) { + if let Some(data) = self.skel.maps.data_data.as_mut() { + data.ratelimiter_config.TOKENS_PER_REQUEST = 1; + data.ratelimiter_config.REFILL_RATE = request_per_seq; + data.ratelimiter_config.MAX_BUCKET_CAPACITY = + (request_per_seq as f32 * burst_multiplier) as u64; + } + } + + pub fn set_ratelimiter_status(&mut self, enabled: bool) { + if let Some(data) = self.skel.maps.data_data.as_mut() { + data.ratelimiter_config.ENABLED = enabled as u8; + } + } + + pub fn setup_from_config(&mut self, config: &XDPRateLimitConfig) { + self.set_request_per_sec(config.requests, config.burst_factor); + self.set_ratelimiter_status(config.enabled); + } + + pub fn add_ipv4_to_whitelist(&mut self, ip: Ipv4Addr) -> Result<()> { + let key = u32::from(ip).to_be_bytes(); + let value: u8 = 1; + self.skel + .maps + .ipv4_ratelimit_whitelist + .update(&key, &[value], MapFlags::ANY)?; + Ok(()) + } + + pub fn add_ipv6_to_whitelist(&mut self, ip: Ipv6Addr) -> Result<()> { + let key = ip.octets(); + let value: u8 = 1; + self.skel + .maps + .ipv6_ratelimit_whitelist + .update(&key, &[value], MapFlags::ANY)?; + Ok(()) + } + + pub fn remove_ipv4_from_whitelist(&mut self, ip: Ipv4Addr) -> Result<()> { + let key = u32::from(ip).to_be_bytes(); + self.skel.maps.ipv4_ratelimit_whitelist.delete(&key)?; + Ok(()) + } + + pub fn remove_ipv6_from_whitelist(&mut self, ip: Ipv6Addr) -> Result<()> { + let key = ip.octets(); + self.skel.maps.ipv6_ratelimit_whitelist.delete(&key)?; + Ok(()) + } + + pub fn ipv4_bucket_max_entrie(&self) -> u32 { + self.skel.maps.ipv4_syn_bucket_store.max_entries() + } + + pub fn ipv6_bucket_max_entrie(&self) -> u32 { + self.skel.maps.ipv6_syn_bucket_store.max_entries() + } +} + +// setup before loading the ebpf program + +pub fn set_bucket_map_size_ipv4<'b>( + skel: &mut OpenXdpSkel<'b>, + bucket_map_size: u32, +) -> Result<()> { + skel.maps + .ipv4_syn_bucket_store + .set_max_entries(bucket_map_size)?; + + Ok(()) +} + +pub fn set_bucket_map_size_ipv6<'b>( + skel: &mut OpenXdpSkel<'b>, + bucket_map_size: u32, +) -> Result<()> { + skel.maps + .ipv6_syn_bucket_store + .set_max_entries(bucket_map_size)?; + + Ok(()) +} diff --git a/src/security/waf/actions/captcha.rs b/src/security/waf/actions/captcha.rs index bc306533..7921c0ea 100644 --- a/src/security/waf/actions/captcha.rs +++ b/src/security/waf/actions/captcha.rs @@ -14,9 +14,8 @@ use crate::storage::redis::RedisManager; use crate::utils::http_client::get_global_reqwest_client; /// Captcha provider types supported by Gen0Sec -#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, clap::ValueEnum)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, clap::ValueEnum)] pub enum CaptchaProvider { - #[default] #[serde(rename = "hcaptcha")] HCaptcha, #[serde(rename = "recaptcha")] @@ -27,6 +26,12 @@ pub enum CaptchaProvider { Prosopo, } +impl Default for CaptchaProvider { + fn default() -> Self { + CaptchaProvider::HCaptcha + } +} + impl std::str::FromStr for CaptchaProvider { type Err = anyhow::Error; @@ -1079,7 +1084,7 @@ pub async fn validate_captcha_response( let request = CaptchaValidationRequest { response_token, ip_address, - user_agent, + user_agent: user_agent, site_key: client.config.site_key.clone(), secret_key: client.config.secret_key.clone(), provider: client.config.provider.clone(), diff --git a/src/security/waf/geoip/mod.rs b/src/security/waf/geoip/mod.rs index bf720344..efd0c584 100644 --- a/src/security/waf/geoip/mod.rs +++ b/src/security/waf/geoip/mod.rs @@ -39,8 +39,6 @@ impl GeoIpManager { } /// Refresh all GeoIP databases - /// City DB is intentionally not loaded here — it is lazy-loaded on demand - /// as a fallback for country lookups, saving ~60 MB at startup. pub async fn refresh_all(&self) -> Result<()> { if let Err(e) = self.refresh_country().await { log::warn!("Failed to refresh GeoIP Country database: {}", e); @@ -48,6 +46,9 @@ impl GeoIpManager { if let Err(e) = self.refresh_asn().await { log::warn!("Failed to refresh GeoIP ASN database: {}", e); } + if let Err(e) = self.refresh_city().await { + log::warn!("Failed to refresh GeoIP City database: {}", e); + } Ok(()) } diff --git a/src/security/waf/threat/mod.rs b/src/security/waf/threat/mod.rs index 32190341..4d2cb6c8 100644 --- a/src/security/waf/threat/mod.rs +++ b/src/security/waf/threat/mod.rs @@ -68,135 +68,6 @@ pub struct ThreatContext { // GeoInfo is now in geoip module -/// Flat record layout as stored in the Threat MMDB. -/// Field names and types mirror the MMDB exactly so `maxminddb::Reader::lookup` can -/// decode them without error. -#[derive(Debug, Clone, Deserialize)] -#[allow(dead_code)] -struct MmdbThreatRecord { - #[serde(default)] - ip: String, - #[serde(default)] - score: u32, - #[serde(default)] - weight: u32, - #[serde(default)] - reason_code: String, - #[serde(default)] - rule_id: String, - #[serde(default)] - rule_name: String, - #[serde(default)] - description: String, - #[serde(default)] - action: String, - #[serde(default)] - num_sources: u32, - #[serde(default)] - source_tag_names: Vec, - #[serde(default)] - iso_code: String, - #[serde(default)] - country: String, - /// ASN string like "AS51396" - #[serde(default)] - asn: String, - #[serde(default)] - asn_org: String, - #[serde(default)] - asniso_code: String, - #[serde(default)] - expiration: u64, - #[serde(default)] - scored_at: String, - #[serde(default)] - created_at: String, - #[serde(default)] - updated_at: String, -} - -impl MmdbThreatRecord { - /// Convert the flat MMDB record into the canonical `ThreatResponse` shape used - /// everywhere else (access logs, WAF fields, cache). - fn into_threat_response(self, ip_addr: IpAddr) -> ThreatResponse { - let asn_num: u32 = self - .asn - .strip_prefix("AS") - .or_else(|| self.asn.strip_prefix("as")) - .and_then(|s| s.parse().ok()) - .unwrap_or(0); - - let confidence = if self.score > 0 { - (self.weight as f64 / 100.0).min(1.0) - } else { - 0.0 - }; - - let generated_at = parse_go_datetime(&self.scored_at).unwrap_or_else(Utc::now); - let first_seen = parse_go_datetime(&self.created_at); - let last_seen = parse_go_datetime(&self.updated_at); - - ThreatResponse { - schema_version: "1.0".to_string(), - tenant_id: "mmdb".to_string(), - ip: self.ip, - intel: ThreatIntel { - score: self.score, - confidence, - score_version: "mmdb".to_string(), - categories: self.source_tag_names, - tags: vec![], - first_seen, - last_seen, - source_count: self.num_sources, - reason_code: self.reason_code, - reason_summary: self.description, - rule_id: self.rule_id, - }, - context: ThreatContext { - asn: asn_num, - org: self.asn_org, - ip_version: if ip_addr.is_ipv4() { 4 } else { 6 }, - geo: GeoInfo { - country: self.country, - iso_code: self.iso_code.clone(), - asn_iso_code: if self.asniso_code.is_empty() { - self.iso_code - } else { - self.asniso_code - }, - }, - }, - advice: self.action, - ttl_s: if self.expiration > 0 { - self.expiration - } else { - 300 - }, - generated_at, - } - } -} - -/// Parse a Go-style datetime string like "2026-03-09 08:03:42.920798 +0000 UTC" -/// into a chrono DateTime. -fn parse_go_datetime(s: &str) -> Option> { - if s.is_empty() { - return None; - } - // Try Go format: "2006-01-02 15:04:05.999999 +0000 UTC" - let trimmed = s.trim_end_matches(" UTC"); - DateTime::parse_from_str(trimmed, "%Y-%m-%d %H:%M:%S%.f %z") - .map(|dt| dt.with_timezone(&Utc)) - .ok() - .or_else(|| { - // Fallback: try RFC3339 - DateTime::parse_from_rfc3339(s) - .map(|dt| dt.with_timezone(&Utc)) - .ok() - }) -} - /// WAF fields extracted from threat data #[derive(Debug, Clone)] pub struct WafFields { @@ -376,13 +247,11 @@ impl ThreatClient { } }; - // Perform blocking MMDB lookup in a separate thread. - // The MMDB stores a flat record (MmdbThreatRecord) which we decode first, - // then convert into the canonical ThreatResponse used everywhere else. + // Perform blocking MMDB lookup in a separate thread let result = tokio::task::spawn_blocking({ let reader = reader.clone(); let ip_addr_clone = ip_addr; - move || -> Result { + move || -> Result { let lookup_result = reader.lookup(ip_addr_clone)?; if !lookup_result.has_data() { return Err(maxminddb::MaxMindDbError::invalid_input( @@ -397,15 +266,9 @@ impl ThreatClient { .await; match result { - Ok(Ok(record)) => { - let response = record.into_threat_response(ip_addr); - log::debug!( - "🔍 [Threat MMDB] Found data for {}: score={}, reason={}", - ip, - response.intel.score, - response.intel.reason_code - ); - Ok(Some(response)) + Ok(Ok(threat_data)) => { + log::debug!("🔍 [Threat MMDB] Found data for {}: {:?}", ip, threat_data); + Ok(Some(threat_data)) } Ok(Err(e)) => { log::debug!("🔍 [Threat MMDB] IP {} not found or error: {}", ip, e); diff --git a/src/security/waf/wirefilter.rs b/src/security/waf/wirefilter.rs index 0808ac7c..644d6801 100644 --- a/src/security/waf/wirefilter.rs +++ b/src/security/waf/wirefilter.rs @@ -1,4 +1,4 @@ -use std::collections::HashMap; +use std::collections::HashSet; use std::net::SocketAddr; use std::sync::{Arc, OnceLock, RwLock}; @@ -16,18 +16,20 @@ use wirefilter::{ExecutionContext, Scheme, TypedArray, TypedMap}; /// Note: Interned strings are leaked to satisfy wirefilter's 'static lifetime requirement, /// but deduplication ensures each unique expression is only leaked once. struct ExpressionInterner { - interned: HashMap, + // Set of already interned expression hashes to avoid re-leaking + interned: HashSet, } impl ExpressionInterner { fn new() -> Self { Self { - interned: HashMap::new(), + interned: HashSet::new(), } } - /// Intern an expression string, returning a &'static str reference. - /// Returns the existing pointer on cache hit; only leaks on first insertion. + /// Intern an expression string, returning a &'static str reference + /// If the expression was already interned, recomputes the hash but doesn't re-leak + /// (The hash check prevents re-leaking the same expression) fn intern(&mut self, expr: &str) -> &'static str { use std::collections::hash_map::DefaultHasher; use std::hash::{Hash, Hasher}; @@ -36,13 +38,15 @@ impl ExpressionInterner { expr.hash(&mut hasher); let hash = hasher.finish(); - if let Some(existing) = self.interned.get(&hash) { - return *existing; + if self.interned.contains(&hash) { + // Already interned, but we still need to return a 'static reference + // We must leak again since we don't store the actual pointers + // This is a trade-off: we track hashes to log warnings, but still leak + log::trace!("Re-using previously interned expression (hash collision or same expr)"); } - let leaked: &'static str = Box::leak(expr.to_string().into_boxed_str()); - self.interned.insert(hash, leaked); - leaked + self.interned.insert(hash); + Box::leak(expr.to_string().into_boxed_str()) } /// Get the count of unique expressions interned @@ -841,8 +845,6 @@ pub async fn update_with_config(base_url: String, api_key: String) -> anyhow::Re /// Update the global HTTP filter using an already-fetched Config value pub fn update_http_filter_from_config_value(config: &Config) -> anyhow::Result<()> { - // Clean stale rate limiters — they'll be re-created on demand for active rules - crate::security::waf::actions::rate_limit::clear_all_rate_limiters(); if let Some(filter) = HTTP_FILTER.get() { filter.update_from_config(config)?; Ok(()) diff --git a/src/storage/redis.rs b/src/storage/redis.rs index 1188cdb9..2a0110b5 100644 --- a/src/storage/redis.rs +++ b/src/storage/redis.rs @@ -212,9 +212,7 @@ impl RedisManager { if ssl_config.insecure { tls_builder.danger_accept_invalid_certs(true); tls_builder.danger_accept_invalid_hostnames(true); - log::error!( - "Redis SSL: Certificate verification DISABLED (insecure mode) — connections are vulnerable to MITM attacks. Redis may contain certificates, tokens, and threat intel." - ); + log::warn!("Redis SSL: Certificate verification disabled (insecure mode)"); } // Build the TLS connector with our custom certificate configuration diff --git a/src/utils/bpf_utils.rs b/src/utils/bpf_utils.rs index fca21507..e4f88bcd 100644 --- a/src/utils/bpf_utils.rs +++ b/src/utils/bpf_utils.rs @@ -1,6 +1,7 @@ use std::fs; use std::net::{Ipv4Addr, Ipv6Addr}; use std::os::fd::AsFd; +use std::process::Command; use crate::security::firewall::bpf::{self, XdpSkel}; use libbpf_rs::{Xdp, XdpFlags}; @@ -32,29 +33,46 @@ impl XdpMode { } } -/// Validate that an interface name contains only safe characters (alphanumeric, '.', '-', '_'). -/// Prevents path traversal when the name is interpolated into /proc paths. -fn validate_iface_name(name: &str) -> Result<(), Box> { - if name.is_empty() { - return Err("Interface name is empty".into()); - } - if !name - .chars() - .all(|c| c.is_ascii_alphanumeric() || c == '.' || c == '-' || c == '_') - { - return Err(format!( - "Interface name '{}' contains invalid characters (only [a-zA-Z0-9._-] allowed)", - name - ) - .into()); +fn ip_binary_candidates() -> [&'static str; 3] { + ["/usr/sbin/ip", "/usr/bin/ip", "ip"] +} + +pub fn xdp_link_state_snapshot(iface_name: &str) -> String { + for ip in ip_binary_candidates() { + if ip.starts_with('/') && !std::path::Path::new(ip).exists() { + continue; + } + + match Command::new(ip) + .args(["-details", "link", "show", "dev", iface_name]) + .output() + { + Ok(out) if out.status.success() => { + let text = String::from_utf8_lossy(&out.stdout) + .replace('\n', " ") + .split_whitespace() + .collect::>() + .join(" "); + if !text.is_empty() { + return text; + } + } + Ok(out) => { + let stderr = String::from_utf8_lossy(&out.stderr).trim().to_string(); + return format!("ip query failed: {}", stderr); + } + Err(_) => { + continue; + } + } } - Ok(()) + + "ip command unavailable".to_string() } fn is_ipv6_disabled(iface: Option<&str>) -> bool { // Check if IPv6 is disabled for a specific interface or system-wide if let Some(iface_name) = iface - && validate_iface_name(iface_name).is_ok() && let Ok(content) = fs::read_to_string(format!( "/proc/sys/net/ipv6/conf/{}/disable_ipv6", iface_name @@ -70,9 +88,6 @@ fn is_ipv6_disabled(iface: Option<&str>) -> bool { } fn try_enable_ipv6_for_interface(iface: &str) -> Result<(), Box> { - // Validate interface name to prevent path traversal - validate_iface_name(iface)?; - // Try to enable IPv6 only for the specific interface (not system-wide) // This allows IPv4-only operation elsewhere while enabling XDP on this interface let disable_path = format!("/proc/sys/net/ipv6/conf/{}/disable_ipv6", iface); @@ -104,17 +119,39 @@ pub fn bpf_attach_to_xdp( iface_name: Option<&str>, ip_version: &str, ) -> Result> { + if let Some(iface) = iface_name { + log::debug!( + "XDP state before attach on {}: {}", + iface, + xdp_link_state_snapshot(iface) + ); + } + // Try hardware mode first, fall back to driver mode if not supported let xdp = Xdp::new(skel.progs.xdp_pipeline.as_fd().into()); // Try hardware offload mode first if let Ok(()) = xdp.attach(ifindex, XdpFlags::HW_MODE) { + if let Some(iface) = iface_name { + log::debug!( + "XDP state after hardware attach on {}: {}", + iface, + xdp_link_state_snapshot(iface) + ); + } return Ok(XdpMode::Hardware); } // Fall back to driver mode if hardware mode fails match xdp.attach(ifindex, XdpFlags::DRV_MODE) { Ok(()) => { + if let Some(iface) = iface_name { + log::debug!( + "XDP state after driver attach on {}: {}", + iface, + xdp_link_state_snapshot(iface) + ); + } return Ok(XdpMode::Driver); } Err(e) => { @@ -127,6 +164,13 @@ pub fn bpf_attach_to_xdp( // Try to replace existing XDP program match xdp.attach(ifindex, XdpFlags::DRV_MODE | XdpFlags::REPLACE) { Ok(()) => { + if let Some(iface) = iface_name { + log::debug!( + "XDP state after driver replace attach on {}: {}", + iface, + xdp_link_state_snapshot(iface) + ); + } return Ok(XdpMode::DriverReplace); } Err(e2) => { @@ -145,6 +189,13 @@ pub fn bpf_attach_to_xdp( // Try SKB mode (should work on all interfaces, including IPv4-only) match xdp.attach(ifindex, XdpFlags::SKB_MODE) { Ok(()) => { + if let Some(iface) = iface_name { + log::debug!( + "XDP state after SKB attach on {}: {}", + iface, + xdp_link_state_snapshot(iface) + ); + } return Ok(XdpMode::Skb); } Err(e) => { @@ -155,6 +206,13 @@ pub fn bpf_attach_to_xdp( // Try to replace existing XDP program in SKB mode match xdp.attach(ifindex, XdpFlags::SKB_MODE | XdpFlags::REPLACE) { Ok(()) => { + if let Some(iface) = iface_name { + log::debug!( + "XDP state after SKB replace attach on {}: {}", + iface, + xdp_link_state_snapshot(iface) + ); + } return Ok(XdpMode::SkbReplace); } Err(e2) => { @@ -191,6 +249,11 @@ pub fn bpf_attach_to_xdp( // Retry SKB mode after enabling IPv6 for the interface match xdp.attach(ifindex, XdpFlags::SKB_MODE) { Ok(()) => { + log::debug!( + "XDP state after SKB attach with IPv6-enable fallback on {}: {}", + iface, + xdp_link_state_snapshot(iface) + ); return Ok(XdpMode::SkbIpv6Enabled); } Err(e2) => { @@ -213,6 +276,13 @@ pub fn bpf_attach_to_xdp( // Try with UPDATE_IF_NOEXIST flag as last resort match xdp.attach(ifindex, XdpFlags::SKB_MODE | XdpFlags::UPDATE_IF_NOEXIST) { Ok(()) => { + if let Some(iface) = iface_name { + log::debug!( + "XDP state after SKB UPDATE_IF_NOEXIST attach on {}: {}", + iface, + xdp_link_state_snapshot(iface) + ); + } return Ok(XdpMode::SkbUpdateIfNoexist); } Err(e2) => { diff --git a/src/utils/bpf_utils_noop.rs b/src/utils/bpf_utils_noop.rs index ba4d8639..78abca46 100644 --- a/src/utils/bpf_utils_noop.rs +++ b/src/utils/bpf_utils_noop.rs @@ -14,3 +14,7 @@ pub fn bpf_attach_to_xdp( pub fn bpf_detach_from_xdp(_ifindex: i32) -> Result<(), Box> { Ok(()) } + +pub fn xdp_link_state_snapshot(_iface_name: &str) -> String { + "xdp state unavailable (bpf disabled)".to_string() +} diff --git a/src/utils/fingerprint/ja4_plus.rs b/src/utils/fingerprint/ja4_plus.rs index 70b9996b..7d0f3feb 100644 --- a/src/utils/fingerprint/ja4_plus.rs +++ b/src/utils/fingerprint/ja4_plus.rs @@ -1,13 +1,59 @@ -//! JA4H HTTP Header Fingerprint wrapper +//! JA4+ Fingerprinting types - using nstealth library //! -//! This module provides a convenience wrapper around nstealth::Ja4h -//! that accepts hyper::HeaderMap directly. +//! This module provides backward-compatible wrappers around nstealth types. use hyper::HeaderMap; -use nstealth::Ja4h; +use nstealth::{Ja4h, Ja4l, Ja4ssh, Ja4t, SshPacketStats}; + +// Re-export nstealth types that are API-compatible +pub use nstealth::{Ja4s as Ja4sFingerprint, Ja4x as Ja4xFingerprint}; + +/// JA4T: TCP Fingerprint from TCP options +/// Wrapper around nstealth::Ja4t with backward-compatible interface +#[derive(Debug, Clone)] +pub struct Ja4tFingerprint { + #[allow(dead_code)] + inner: Ja4t, + pub fingerprint: String, + pub window_size: u16, + pub ttl: u16, // Not in JA4T spec, kept for logging compatibility + pub mss: u16, + pub window_scale: u8, + pub options: Vec, +} + +impl Ja4tFingerprint { + /// Generate JA4T fingerprint from TCP parameters + pub fn from_tcp_data( + window_size: u16, + ttl: u16, + mss: u16, + window_scale: u8, + options: &[u8], + ) -> Self { + // Parse raw options to extract option kinds + let inner = Ja4t::from_raw_options(window_size, options); + let fingerprint = inner.fingerprint(); + + Self { + fingerprint, + window_size: inner.window_size, + ttl, // Store TTL for logging (not part of JA4T spec) + mss: inner.mss.unwrap_or(mss), + window_scale: inner.window_scale.unwrap_or(window_scale), + options: inner.tcp_options.clone(), + inner, + } + } + + /// Get the JA4T hash (first 12 characters of SHA-256) + pub fn hash(&self) -> String { + nstealth::hash12(&self.fingerprint) + } +} /// JA4H: HTTP Header Fingerprint -/// Wrapper around nstealth::Ja4h that extracts fields from hyper::HeaderMap +/// Wrapper around nstealth::Ja4h with backward-compatible interface #[derive(Debug, Clone)] pub struct Ja4hFingerprint { #[allow(dead_code)] @@ -89,10 +135,198 @@ impl Ja4hFingerprint { } } +/// JA4L: Latency Fingerprint +/// Wrapper around nstealth::Ja4l with backward-compatible builder interface +#[derive(Debug, Clone, Default)] +pub struct Ja4lMeasurement { + pub syn_time: Option, + pub synack_time: Option, + pub ack_time: Option, + pub ttl_client: Option, + pub ttl_server: Option, +} + +/// JA4SSH: SSH Session Fingerprint +/// Wrapper around nstealth::Ja4ssh with backward-compatible interface +#[derive(Debug, Clone)] +pub struct Ja4sshFingerprint { + #[allow(dead_code)] + inner: Ja4ssh, + pub fingerprint: String, + pub client_version: Option, + pub server_version: Option, + pub client_packets: u32, + pub server_packets: u32, + pub client_acks: u32, + pub server_acks: u32, + pub client_bytes: u64, + pub server_bytes: u64, +} + +impl Ja4sshFingerprint { + /// Create a new JA4SSH fingerprint from session statistics + pub fn from_session_stats( + client_version: Option, + server_version: Option, + client_packets: u32, + server_packets: u32, + client_acks: u32, + server_acks: u32, + client_bytes: u64, + server_bytes: u64, + ) -> Self { + let client_stats = SshPacketStats::new(client_packets, client_acks, client_bytes); + let server_stats = SshPacketStats::new(server_packets, server_acks, server_bytes); + + let inner = Ja4ssh::with_stats( + client_version.clone(), + server_version.clone(), + client_stats, + server_stats, + ); + let fingerprint = inner.fingerprint(); + + Self { + fingerprint, + client_version, + server_version, + client_packets, + server_packets, + client_acks, + server_acks, + client_bytes, + server_bytes, + inner, + } + } + + /// Create from raw BPF session data + pub fn from_bpf_session( + client_packets: u32, + server_packets: u32, + client_acks: u32, + server_acks: u32, + client_bytes: u64, + server_bytes: u64, + ) -> Self { + Self::from_session_stats( + None, + None, + client_packets, + server_packets, + client_acks, + server_acks, + client_bytes, + server_bytes, + ) + } + + /// Get the fingerprint string + pub fn to_string(&self) -> String { + self.fingerprint.clone() + } + + /// Get client software name (extracted from version string) + pub fn client_software(&self) -> Option<&str> { + self.inner.client_software() + } + + /// Get server software name (extracted from version string) + pub fn server_software(&self) -> Option<&str> { + self.inner.server_software() + } +} + +impl Default for Ja4sshFingerprint { + fn default() -> Self { + Self::from_bpf_session(0, 0, 0, 0, 0, 0) + } +} + +impl Ja4lMeasurement { + pub fn new() -> Self { + Self::default() + } + + /// Record SYN packet timestamp + pub fn set_syn(&mut self, timestamp_us: u64, ttl: u8) { + self.syn_time = Some(timestamp_us); + self.ttl_client = Some(ttl); + } + + /// Record SYNACK packet timestamp + pub fn set_synack(&mut self, timestamp_us: u64, ttl: u8) { + self.synack_time = Some(timestamp_us); + self.ttl_server = Some(ttl); + } + + /// Record ACK packet timestamp + pub fn set_ack(&mut self, timestamp_us: u64) { + self.ack_time = Some(timestamp_us); + } + + /// Generate JA4L client fingerprint using nstealth + pub fn fingerprint_client(&self) -> Option { + let ja4l = Ja4l::new( + self.syn_time?, + self.synack_time?, + self.ack_time?, + self.ttl_client?, + self.ttl_server?, + ); + Some(ja4l.client_fingerprint()) + } + + /// Generate JA4L server fingerprint using nstealth + pub fn fingerprint_server(&self) -> Option { + let ja4l = Ja4l::new( + self.syn_time?, + self.synack_time?, + self.ack_time?, + self.ttl_client?, + self.ttl_server?, + ); + Some(ja4l.server_fingerprint()) + } + + /// Legacy format for compatibility + pub fn fingerprint_combined(&self) -> Option { + let client = self.fingerprint_client()?; + let server = self.fingerprint_server()?; + Some(format!("c:{},s:{}", client, server)) + } +} + #[cfg(test)] mod tests { use super::*; + #[test] + fn test_ja4t_fingerprint() { + // MSS option: kind=2, len=4, value=1460 (0x05b4) + // Window Scale option: kind=3, len=3, value=7 + let options = vec![ + 2, 4, 0x05, 0xb4, // MSS + 3, 3, 7, // Window Scale + 0, // EOL + ]; + + let ja4t = Ja4tFingerprint::from_tcp_data( + 65535, // window_size + 64, // ttl + 1460, // mss (fallback) + 7, // window_scale (fallback) + &options, + ); + + assert_eq!(ja4t.window_size, 65535); + assert_eq!(ja4t.ttl, 64); + assert_eq!(ja4t.mss, 1460); + assert_eq!(ja4t.window_scale, 7); + assert!(!ja4t.hash().is_empty()); + assert_eq!(ja4t.hash().len(), 12); + } + #[test] fn test_ja4h_fingerprint() { let mut headers = HeaderMap::new(); @@ -114,4 +348,66 @@ mod tests { // Should have 4 parts separated by underscores assert_eq!(ja4h.fingerprint.matches('_').count(), 3); } + + #[test] + fn test_ja4l_measurement() { + let mut ja4l = Ja4lMeasurement::new(); + + ja4l.set_syn(1000000, 64); + ja4l.set_synack(1025000, 128); + ja4l.set_ack(1050000); + + let client_fp = ja4l.fingerprint_client().unwrap(); + let server_fp = ja4l.fingerprint_server().unwrap(); + + // Verify fingerprints are generated + assert!(!client_fp.is_empty()); + assert!(!server_fp.is_empty()); + assert!(client_fp.contains('_')); + assert!(server_fp.contains('_')); + } + + #[test] + fn test_ja4ssh_fingerprint() { + let ja4ssh = Ja4sshFingerprint::from_session_stats( + Some("SSH-2.0-OpenSSH_8.9".to_string()), + Some("SSH-2.0-dropbear".to_string()), + 14, // client_packets + 12, // server_packets + 10, // client_acks + 8, // server_acks + 1024, // client_bytes + 2048, // server_bytes + ); + + // Verify fingerprint format: c{client_pkts}s{server_pkts}_c{client_acks}s{server_acks}_c{client_bytes}s{server_bytes} + assert_eq!(ja4ssh.fingerprint, "c14s12_c10s8_c1024s2048"); + assert_eq!(ja4ssh.client_packets, 14); + assert_eq!(ja4ssh.server_packets, 12); + assert_eq!(ja4ssh.client_acks, 10); + assert_eq!(ja4ssh.server_acks, 8); + assert_eq!(ja4ssh.client_bytes, 1024); + assert_eq!(ja4ssh.server_bytes, 2048); + assert_eq!( + ja4ssh.client_version, + Some("SSH-2.0-OpenSSH_8.9".to_string()) + ); + assert_eq!(ja4ssh.client_software(), Some("OpenSSH_8.9")); + assert_eq!(ja4ssh.server_software(), Some("dropbear")); + } + + #[test] + fn test_ja4ssh_from_bpf_session() { + let ja4ssh = Ja4sshFingerprint::from_bpf_session(5, 4, 3, 2, 500, 400); + + assert_eq!(ja4ssh.fingerprint, "c5s4_c3s2_c500s400"); + assert!(ja4ssh.client_version.is_none()); + assert!(ja4ssh.server_version.is_none()); + } + + #[test] + fn test_ja4ssh_default() { + let ja4ssh = Ja4sshFingerprint::default(); + assert_eq!(ja4ssh.fingerprint, "c0s0_c0s0_c0s0"); + } } diff --git a/src/utils/fingerprint/latency_fingerprint.rs b/src/utils/fingerprint/latency_fingerprint.rs index 64fc8c42..1b5f8ddf 100644 --- a/src/utils/fingerprint/latency_fingerprint.rs +++ b/src/utils/fingerprint/latency_fingerprint.rs @@ -12,6 +12,7 @@ use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; use std::sync::{Arc, Mutex}; use crate::security::firewall::bpf::XdpSkel; +use crate::utils::fingerprint::ja4_plus::Ja4lMeasurement; /// Latency fingerprinting configuration #[derive(Debug, Clone, Serialize, Deserialize)] @@ -254,29 +255,47 @@ impl LatencyFingerprintCollector { self.enabled } - /// Lookup latency data for a specific connection by client IP + port - /// (client_ip + ephemeral port is unique per TCP connection) - pub fn lookup_connection(&self, client_ip: IpAddr, client_port: u16) -> Option { + /// Lookup latency data for a specific connection + pub fn lookup_connection( + &self, + client_ip: IpAddr, + client_port: u16, + server_ip: IpAddr, + server_port: u16, + ) -> Option { if !self.enabled || self.skels.is_empty() { return None; } - match client_ip { - IpAddr::V4(c_ip) => self.lookup_connection_v4(c_ip, client_port), - IpAddr::V6(c_ip) => self.lookup_connection_v6(c_ip, client_port), + match (client_ip, server_ip) { + (IpAddr::V4(c_ip), IpAddr::V4(s_ip)) => { + self.lookup_connection_v4(c_ip, client_port, s_ip, server_port) + } + (IpAddr::V6(c_ip), IpAddr::V6(s_ip)) => { + self.lookup_connection_v6(c_ip, client_port, s_ip, server_port) + } + _ => None, } } - fn lookup_connection_v4(&self, client_ip: Ipv4Addr, client_port: u16) -> Option { + fn lookup_connection_v4( + &self, + client_ip: Ipv4Addr, + client_port: u16, + server_ip: Ipv4Addr, + server_port: u16, + ) -> Option { for skel in &self.skels { let skel_guard = match skel.lock() { Ok(guard) => guard, Err(_) => continue, }; - // Build key: client_ip (4 bytes BE), client_port (2 bytes BE), padding (2 bytes) = 8 bytes - let mut key = [0u8; 8]; + // Build key: client_ip (4 bytes BE), client_port (2 bytes BE), server_ip (4 bytes BE), server_port (2 bytes BE), padding (2 bytes) + let mut key = [0u8; 14]; key[0..4].copy_from_slice(&client_ip.octets()); key[4..6].copy_from_slice(&client_port.to_be_bytes()); + key[6..10].copy_from_slice(&server_ip.octets()); + key[10..12].copy_from_slice(&server_port.to_be_bytes()); if let Ok(Some(value_bytes)) = skel_guard .maps @@ -289,16 +308,24 @@ impl LatencyFingerprintCollector { None } - fn lookup_connection_v6(&self, client_ip: Ipv6Addr, client_port: u16) -> Option { + fn lookup_connection_v6( + &self, + client_ip: Ipv6Addr, + client_port: u16, + server_ip: Ipv6Addr, + server_port: u16, + ) -> Option { for skel in &self.skels { let skel_guard = match skel.lock() { Ok(guard) => guard, Err(_) => continue, }; - // Build key: client_ip (16 bytes), client_port (2 bytes BE), padding (2 bytes) = 20 bytes - let mut key = [0u8; 20]; + // Build key: client_ip (16 bytes), client_port (2 bytes BE), server_ip (16 bytes), server_port (2 bytes BE), padding (2 bytes) + let mut key = [0u8; 38]; key[0..16].copy_from_slice(&client_ip.octets()); key[16..18].copy_from_slice(&client_port.to_be_bytes()); + key[18..34].copy_from_slice(&server_ip.octets()); + key[34..36].copy_from_slice(&server_port.to_be_bytes()); if let Ok(Some(value_bytes)) = skel_guard .maps @@ -434,14 +461,17 @@ impl LatencyFingerprintCollector { if let Ok(batch_iter) = batch_result { for (key_bytes, value_bytes) in batch_iter { - if key_bytes.len() >= 8 && value_bytes.len() >= 32 { - // Parse key: client_ip (4) + client_port (2) + pad (2) = 8 bytes + if key_bytes.len() >= 14 && value_bytes.len() >= 32 { + // Parse key let client_ip = Ipv4Addr::from([key_bytes[0], key_bytes[1], key_bytes[2], key_bytes[3]]); let client_port = u16::from_be_bytes([key_bytes[4], key_bytes[5]]); + let server_ip = + Ipv4Addr::from([key_bytes[6], key_bytes[7], key_bytes[8], key_bytes[9]]); + let server_port = u16::from_be_bytes([key_bytes[10], key_bytes[11]]); // Skip localhost - if client_ip.is_loopback() { + if client_ip.is_loopback() || server_ip.is_loopback() { continue; } @@ -451,23 +481,21 @@ impl LatencyFingerprintCollector { continue; } - // Generate JA4L fingerprints using nstealth directly - let ja4l = nstealth::Ja4l::new( - data.syn_time_ns / 1000, - data.synack_time_ns / 1000, - data.ack_time_ns / 1000, - data.client_ttl, - data.server_ttl, - ); - let ja4l_client = ja4l.client_fingerprint(); - let ja4l_server = ja4l.server_fingerprint(); + // Generate JA4L fingerprints using the ja4_plus module + let mut measurement = Ja4lMeasurement::new(); + measurement.set_syn(data.syn_time_ns / 1000, data.client_ttl); + measurement.set_synack(data.synack_time_ns / 1000, data.server_ttl); + measurement.set_ack(data.ack_time_ns / 1000); + + let ja4l_client = measurement.fingerprint_client().unwrap_or_default(); + let ja4l_server = measurement.fingerprint_server().unwrap_or_default(); connections.push(LatencyConnectionEntry { key: LatencyConnectionKey { client_ip: client_ip.to_string(), client_port, - server_ip: String::new(), - server_port: 0, + server_ip: server_ip.to_string(), + server_port, }, data, ja4l_client, @@ -494,15 +522,20 @@ impl LatencyFingerprintCollector { if let Ok(batch_iter) = batch_result { for (key_bytes, value_bytes) in batch_iter { - if key_bytes.len() >= 20 && value_bytes.len() >= 32 { - // Parse key: client_ip (16) + client_port (2) + pad (2) = 20 bytes + if key_bytes.len() >= 38 && value_bytes.len() >= 32 { + // Parse key let mut client_ip_bytes = [0u8; 16]; client_ip_bytes.copy_from_slice(&key_bytes[0..16]); let client_ip = Ipv6Addr::from(client_ip_bytes); let client_port = u16::from_be_bytes([key_bytes[16], key_bytes[17]]); + let mut server_ip_bytes = [0u8; 16]; + server_ip_bytes.copy_from_slice(&key_bytes[18..34]); + let server_ip = Ipv6Addr::from(server_ip_bytes); + let server_port = u16::from_be_bytes([key_bytes[34], key_bytes[35]]); + // Skip localhost - if client_ip.is_loopback() { + if client_ip.is_loopback() || server_ip.is_loopback() { continue; } @@ -512,23 +545,21 @@ impl LatencyFingerprintCollector { continue; } - // Generate JA4L fingerprints using nstealth directly - let ja4l = nstealth::Ja4l::new( - data.syn_time_ns / 1000, - data.synack_time_ns / 1000, - data.ack_time_ns / 1000, - data.client_ttl, - data.server_ttl, - ); - let ja4l_client = ja4l.client_fingerprint(); - let ja4l_server = ja4l.server_fingerprint(); + // Generate JA4L fingerprints + let mut measurement = Ja4lMeasurement::new(); + measurement.set_syn(data.syn_time_ns / 1000, data.client_ttl); + measurement.set_synack(data.synack_time_ns / 1000, data.server_ttl); + measurement.set_ack(data.ack_time_ns / 1000); + + let ja4l_client = measurement.fingerprint_client().unwrap_or_default(); + let ja4l_server = measurement.fingerprint_server().unwrap_or_default(); connections.push(LatencyConnectionEntry { key: LatencyConnectionKey { client_ip: client_ip.to_string(), client_port, - server_ip: String::new(), - server_port: 0, + server_ip: server_ip.to_string(), + server_port, }, data, ja4l_client, @@ -674,26 +705,18 @@ impl LatencyFingerprintCollector { log::debug!("Resetting latency counters"); - for (i, skel) in self.skels.iter().enumerate() { - let skel_guard = match skel.lock() { - Ok(guard) => guard, - Err(e) => { - log::warn!("Failed to lock XDP skeleton {} for latency reset: {}", i, e); - continue; - } - }; + for skel in &self.skels { // Reset latency stats let key = 0u32.to_le_bytes(); let zero_stats = vec![0u8; 32]; if let Err(e) = - skel_guard - .maps + skel.maps .latency_stats .update(&key, &zero_stats, libbpf_rs::MapFlags::ANY) { log::warn!("Failed to reset latency stats: {}", e); } else { - log::debug!("Reset latency stats for skeleton {}", i); + log::debug!("Reset latency stats"); } } diff --git a/src/utils/fingerprint/latency_fingerprint_noop.rs b/src/utils/fingerprint/latency_fingerprint_noop.rs index 5a5744d3..9dd89702 100644 --- a/src/utils/fingerprint/latency_fingerprint_noop.rs +++ b/src/utils/fingerprint/latency_fingerprint_noop.rs @@ -87,7 +87,13 @@ impl LatencyFingerprintCollector { self.config.enabled } - pub fn lookup_connection(&self, _client_ip: IpAddr, _client_port: u16) -> Option { + pub fn lookup_connection( + &self, + _client_ip: IpAddr, + _client_port: u16, + _server_ip: IpAddr, + _server_port: u16, + ) -> Option { None } diff --git a/src/utils/fingerprint/mod.rs b/src/utils/fingerprint/mod.rs index 54279c19..337bf223 100644 --- a/src/utils/fingerprint/mod.rs +++ b/src/utils/fingerprint/mod.rs @@ -10,9 +10,8 @@ pub mod tcp_fingerprint; #[path = "ssh_fingerprint_noop.rs"] pub mod ssh_fingerprint; -#[cfg(all(feature = "bpf", not(feature = "disable-bpf")))] -pub mod latency_fingerprint; -#[cfg(any(not(feature = "bpf"), feature = "disable-bpf"))] +// Latency fingerprinting requires BPF maps not yet implemented in XDP skeleton +// Using noop implementation until connection_latency maps are added #[path = "latency_fingerprint_noop.rs"] pub mod latency_fingerprint; diff --git a/src/utils/fingerprint/ssh_fingerprint.rs b/src/utils/fingerprint/ssh_fingerprint.rs index a4f2f613..c74a9804 100644 --- a/src/utils/fingerprint/ssh_fingerprint.rs +++ b/src/utils/fingerprint/ssh_fingerprint.rs @@ -12,6 +12,7 @@ use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; use std::sync::{Arc, Mutex}; use crate::security::firewall::bpf::XdpSkel; +use crate::utils::fingerprint::ja4_plus::Ja4sshFingerprint; /// SSH fingerprinting configuration #[derive(Debug, Clone, Serialize, Deserialize)] @@ -461,20 +462,15 @@ impl SshFingerprintCollector { } if let Some(data) = self.parse_session_data(&value_bytes) { - // Generate JA4SSH fingerprint using nstealth directly - let client_stats = nstealth::SshPacketStats::new( + // Generate JA4SSH fingerprint + let fingerprint = Ja4sshFingerprint::from_bpf_session( data.client_packets, - data.client_acks, - data.client_bytes, - ); - let server_stats = nstealth::SshPacketStats::new( data.server_packets, + data.client_acks, data.server_acks, + data.client_bytes, data.server_bytes, ); - let ja4ssh = nstealth::Ja4ssh::with_stats( - None, None, client_stats, server_stats, - ); sessions.push(SshSessionEntry { key: SshSessionKey { @@ -484,7 +480,7 @@ impl SshFingerprintCollector { server_port, }, data, - fingerprint: ja4ssh.fingerprint(), + fingerprint: fingerprint.fingerprint, }); } } @@ -525,20 +521,15 @@ impl SshFingerprintCollector { } if let Some(data) = self.parse_session_data(&value_bytes) { - // Generate JA4SSH fingerprint using nstealth directly - let client_stats = nstealth::SshPacketStats::new( + // Generate JA4SSH fingerprint + let fingerprint = Ja4sshFingerprint::from_bpf_session( data.client_packets, - data.client_acks, - data.client_bytes, - ); - let server_stats = nstealth::SshPacketStats::new( data.server_packets, + data.client_acks, data.server_acks, + data.client_bytes, data.server_bytes, ); - let ja4ssh = nstealth::Ja4ssh::with_stats( - None, None, client_stats, server_stats, - ); sessions.push(SshSessionEntry { key: SshSessionKey { @@ -548,7 +539,7 @@ impl SshFingerprintCollector { server_port, }, data, - fingerprint: ja4ssh.fingerprint(), + fingerprint: fingerprint.fingerprint, }); } } diff --git a/src/utils/fingerprint/tcp_fingerprint.rs b/src/utils/fingerprint/tcp_fingerprint.rs index a9d7ef81..b84fdc80 100644 --- a/src/utils/fingerprint/tcp_fingerprint.rs +++ b/src/utils/fingerprint/tcp_fingerprint.rs @@ -1,8 +1,7 @@ use crate::security::firewall::bpf::XdpSkel; -use crate::security::firewall::ja4ts_bpf::Ja4tsSkel; use crate::worker::log::{UnifiedEvent, send_event}; use chrono::{DateTime, Utc}; -use libbpf_rs::{MapCore, MapFlags}; +use libbpf_rs::MapCore; use nstealth::bpf::{BpfTcpFingerprint, BpfTcpFingerprintKeyV4, BpfTcpFingerprintKeyV6}; use serde::{Deserialize, Serialize}; use std::sync::{Arc, Mutex}; @@ -247,7 +246,6 @@ pub fn get_global_tcp_fingerprint_collector() -> Option>>>, - ja4ts_skels: Vec>>>, enabled: bool, config: TcpFingerprintConfig, } @@ -257,7 +255,6 @@ impl TcpFingerprintCollector { pub fn new(skels: Vec>>>, enabled: bool) -> Self { Self { skels, - ja4ts_skels: Vec::new(), enabled, config: TcpFingerprintConfig::default(), } @@ -266,12 +263,10 @@ impl TcpFingerprintCollector { /// Create a new TCP fingerprint collector with configuration pub fn new_with_config( skels: Vec>>>, - ja4ts_skels: Vec>>>, config: TcpFingerprintConfig, ) -> Self { Self { skels, - ja4ts_skels, enabled: config.enabled, config, } @@ -287,310 +282,86 @@ impl TcpFingerprintCollector { self.enabled } - /// Scan a BPF map for an IPv4 TCP fingerprint matching the given IP and port. - /// Tries batch lookup first, falls back to keys() iteration. - fn scan_map_v4( - map: &libbpf_rs::Map, - src_ip_host: u32, + /// Lookup TCP fingerprint for a specific source IP and port + pub fn lookup_fingerprint( + &self, + src_ip: std::net::IpAddr, src_port: u16, - map_name: &str, ) -> Option { - match map.lookup_batch(1000, libbpf_rs::MapFlags::ANY, libbpf_rs::MapFlags::ANY) { - Ok(iter) => { - for (key_bytes, value_bytes) in iter { - if let Some(key) = BpfTcpFingerprintKeyV4::from_bytes(&key_bytes) { - if key.src_ip == src_ip_host && key.src_port == src_port { - if let Some(data) = BpfTcpFingerprint::from_bytes(&value_bytes) { - return Some(Self::convert_bpf_data(&data)); - } + if !self.enabled || self.skels.is_empty() { + return None; + } + + match src_ip { + std::net::IpAddr::V4(ip) => { + let src_ip_be = u32::from_be_bytes(ip.octets()); + + for skel in &self.skels { + let skel_guard = match skel.lock() { + Ok(guard) => guard, + Err(e) => { + log::warn!( + "Failed to lock XDP skeleton for IPv4 fingerprint lookup: {}", + e + ); + continue; } - } - } - } - Err(e) => { - log::warn!( - "{}: batch lookup failed ({}), falling back to keys() scan", - map_name, - e - ); - for key_bytes in map.keys() { - if let Some(key) = BpfTcpFingerprintKeyV4::from_bytes(&key_bytes) { - if key.src_ip == src_ip_host && key.src_port == src_port { - if let Ok(Some(value_bytes)) = - map.lookup(&key_bytes, libbpf_rs::MapFlags::ANY) - { - if let Some(data) = BpfTcpFingerprint::from_bytes(&value_bytes) { - return Some(Self::convert_bpf_data(&data)); + }; + + if let Ok(iter) = skel_guard.maps.tcp_fingerprints.lookup_batch( + 1000, + libbpf_rs::MapFlags::ANY, + libbpf_rs::MapFlags::ANY, + ) { + for (key_bytes, value_bytes) in iter { + if let Some(key) = BpfTcpFingerprintKeyV4::from_bytes(&key_bytes) { + if key.src_ip == src_ip_be && key.src_port == src_port { + if let Some(data) = BpfTcpFingerprint::from_bytes(&value_bytes) + { + return Some(Self::convert_bpf_data(&data)); + } } } } } } + None } - } - None - } - - /// Scan a BPF map for an IPv6 TCP fingerprint matching the given IP and port. - /// Tries batch lookup first, falls back to keys() iteration. - fn scan_map_v6( - map: &libbpf_rs::Map, - src_ip_octets: [u8; 16], - src_port: u16, - map_name: &str, - ) -> Option { - match map.lookup_batch(1000, libbpf_rs::MapFlags::ANY, libbpf_rs::MapFlags::ANY) { - Ok(iter) => { - for (key_bytes, value_bytes) in iter { - if let Some(key) = BpfTcpFingerprintKeyV6::from_bytes(&key_bytes) { - if key.src_ip == src_ip_octets && key.src_port == src_port { - if let Some(data) = BpfTcpFingerprint::from_bytes(&value_bytes) { - return Some(Self::convert_bpf_data(&data)); - } + std::net::IpAddr::V6(ip) => { + let octets = ip.octets(); + + for skel in &self.skels { + let skel_guard = match skel.lock() { + Ok(guard) => guard, + Err(e) => { + log::warn!( + "Failed to lock XDP skeleton for IPv6 fingerprint lookup: {}", + e + ); + continue; } - } - } - } - Err(e) => { - log::warn!( - "{}: batch lookup failed ({}), falling back to keys() scan", - map_name, - e - ); - for key_bytes in map.keys() { - if let Some(key) = BpfTcpFingerprintKeyV6::from_bytes(&key_bytes) { - if key.src_ip == src_ip_octets && key.src_port == src_port { - if let Ok(Some(value_bytes)) = - map.lookup(&key_bytes, libbpf_rs::MapFlags::ANY) - { - if let Some(data) = BpfTcpFingerprint::from_bytes(&value_bytes) { - return Some(Self::convert_bpf_data(&data)); + }; + + if let Ok(iter) = skel_guard.maps.tcp_fingerprints_v6.lookup_batch( + 1000, + libbpf_rs::MapFlags::ANY, + libbpf_rs::MapFlags::ANY, + ) { + for (key_bytes, value_bytes) in iter { + if let Some(key) = BpfTcpFingerprintKeyV6::from_bytes(&key_bytes) { + if key.src_ip == octets && key.src_port == src_port { + if let Some(data) = BpfTcpFingerprint::from_bytes(&value_bytes) + { + return Some(Self::convert_bpf_data(&data)); + } } } } } } + None } } - None - } - - /// O(1) direct lookup from the simple (IP, port)-keyed BPF map. - /// Key layout matches `tcp_fp_simple_key_v4`: ip(4) + port(2) + pad(2) = 8 bytes - /// and `tcp_fp_simple_key_v6`: ip(16) + port(2) + pad(2) = 20 bytes. - fn lookup_fingerprint_direct( - &self, - src_ip: std::net::IpAddr, - src_port: u16, - ) -> Option { - for skel in &self.skels { - let skel_guard = match skel.lock() { - Ok(guard) => guard, - Err(e) => { - log::warn!("JA4T: failed to lock XDP skeleton: {}", e); - continue; - } - }; - - let result = match src_ip { - std::net::IpAddr::V4(ip) => { - let mut key = [0u8; 8]; - key[0..4].copy_from_slice(&ip.octets()); - key[4..6].copy_from_slice(&src_port.to_be_bytes()); - skel_guard - .maps - .tcp_fingerprints_simple - .lookup(&key, MapFlags::ANY) - } - std::net::IpAddr::V6(ip) => { - let mut key = [0u8; 20]; - key[0..16].copy_from_slice(&ip.octets()); - key[16..18].copy_from_slice(&src_port.to_be_bytes()); - skel_guard - .maps - .tcp_fingerprints_simple_v6 - .lookup(&key, MapFlags::ANY) - } - }; - - if let Ok(Some(bytes)) = result { - if let Some(data) = Self::parse_simple_fp_data(&bytes) { - return Some(data); - } - } - } - None - } - - /// Parse raw bytes from the simple fingerprint map using nstealth's - /// `BpfTcpFingerprint::from_bytes()` (same struct layout as the compound-key map value). - fn parse_simple_fp_data(bytes: &[u8]) -> Option { - BpfTcpFingerprint::from_bytes(bytes).map(|bpf| Self::convert_bpf_data(&bpf)) - } - - /// Lookup TCP fingerprint for a specific source IP and port. - /// - /// Tries O(1) direct lookup from the simple (IP, port) map first, - /// then falls back to the O(n) batch scan of the compound-key map. - pub fn lookup_fingerprint( - &self, - src_ip: std::net::IpAddr, - src_port: u16, - ) -> Option { - if !self.enabled || self.skels.is_empty() { - log::debug!( - "JA4T: collector disabled or no skels (enabled={}, skels={})", - self.enabled, - self.skels.len() - ); - return None; - } - - // 1. Try O(1) direct lookup from simple map - if let Some(data) = self.lookup_fingerprint_direct(src_ip, src_port) { - return Some(data); - } - - // 2. Fallback: O(n) batch scan of compound-key map - for skel in &self.skels { - let skel_guard = match skel.lock() { - Ok(guard) => guard, - Err(e) => { - log::warn!("JA4T: failed to lock XDP skeleton: {}", e); - continue; - } - }; - - let result = match src_ip { - std::net::IpAddr::V4(ip) => { - let src_ip_host = u32::from_be_bytes(ip.octets()); - Self::scan_map_v4( - &skel_guard.maps.tcp_fingerprints, - src_ip_host, - src_port, - "tcp_fingerprints", - ) - } - std::net::IpAddr::V6(ip) => Self::scan_map_v6( - &skel_guard.maps.tcp_fingerprints_v6, - ip.octets(), - src_port, - "tcp_fingerprints_v6", - ), - }; - - if result.is_some() { - return result; - } - } - - log::debug!("JA4T: no BPF data found for {}:{}", src_ip, src_port); - None - } - - /// Lookup TCP SYN-ACK fingerprint for a specific server IP and port (JA4TS) - /// - /// Uses the separate TC BPF program's map with a simple (ip, port) key - /// for O(1) direct lookup instead of O(n) map scan. - pub fn lookup_synack_fingerprint( - &self, - server_ip: std::net::IpAddr, - server_port: u16, - ) -> Option { - if !self.enabled || self.ja4ts_skels.is_empty() { - log::info!( - "JA4TS: lookup skipped (enabled={}, ja4ts_skels={})", - self.enabled, - self.ja4ts_skels.len() - ); - return None; - } - - for skel in &self.ja4ts_skels { - let guard = match skel.lock() { - Ok(g) => g, - Err(e) => { - log::warn!("JA4TS: failed to lock TC skeleton: {}", e); - continue; - } - }; - - let result = match server_ip { - std::net::IpAddr::V4(ip) => { - // Build 8-byte key: ip(4) + port(2) + pad(2) - let mut key = [0u8; 8]; - key[0..4].copy_from_slice(&ip.octets()); - key[4..6].copy_from_slice(&server_port.to_be_bytes()); - guard - .maps - .tcp_synack_fingerprints - .lookup(&key, MapFlags::ANY) - } - std::net::IpAddr::V6(ip) => { - // Build 20-byte key: ip(16) + port(2) + pad(2) - let mut key = [0u8; 20]; - key[0..16].copy_from_slice(&ip.octets()); - key[16..18].copy_from_slice(&server_port.to_be_bytes()); - guard - .maps - .tcp_synack_fingerprints_v6 - .lookup(&key, MapFlags::ANY) - } - }; - - if let Ok(Some(bytes)) = result { - if let Some(data) = Self::parse_ja4ts_data(&bytes) { - return Some(data); - } - } - } - - log::debug!( - "JA4TS: no SYN-ACK data in BPF map for {}:{} (checked {} skels)", - server_ip, - server_port, - self.ja4ts_skels.len() - ); - None - } - - /// Parse raw bytes from the JA4TS TC map into TcpFingerprintData. - /// Layout matches `struct ja4ts_data` in ja4ts.bpf.c: - /// first_seen(8) + last_seen(8) + packet_count(4) + ttl(2) + mss(2) + - /// window_size(2) + window_scale(1) + options_len(1) + options(40) = 68 bytes - fn parse_ja4ts_data(bytes: &[u8]) -> Option { - if bytes.len() < 28 { - return None; - } - - let first_seen_ns = u64::from_ne_bytes(bytes[0..8].try_into().ok()?); - let last_seen_ns = u64::from_ne_bytes(bytes[8..16].try_into().ok()?); - let packet_count = u32::from_ne_bytes(bytes[16..20].try_into().ok()?); - let ttl = u16::from_ne_bytes(bytes[20..22].try_into().ok()?); - let mss = u16::from_ne_bytes(bytes[22..24].try_into().ok()?); - let window_size = u16::from_ne_bytes(bytes[24..26].try_into().ok()?); - let window_scale = bytes[26]; - let options_len = bytes[27]; - - let options = if bytes.len() >= 28 + 40 { - bytes[28..68].to_vec() - } else if bytes.len() > 28 { - bytes[28..].to_vec() - } else { - vec![] - }; - - Some(TcpFingerprintData { - first_seen: DateTime::from_timestamp_nanos(first_seen_ns as i64), - last_seen: DateTime::from_timestamp_nanos(last_seen_ns as i64), - packet_count, - ttl, - mss, - window_size, - window_scale, - options_len, - options, - }) } /// Convert BpfTcpFingerprint to TcpFingerprintData diff --git a/src/utils/fingerprint/tcp_fingerprint_noop.rs b/src/utils/fingerprint/tcp_fingerprint_noop.rs index 8555cb61..4ce1dda6 100644 --- a/src/utils/fingerprint/tcp_fingerprint_noop.rs +++ b/src/utils/fingerprint/tcp_fingerprint_noop.rs @@ -55,9 +55,8 @@ impl TcpFingerprintCollector { Self { config } } - pub fn new_with_config( + pub fn new_with_config( _skels: Vec>>, - _ja4ts_skels: Vec, config: TcpFingerprintConfig, ) -> Self { Self { config } @@ -71,10 +70,6 @@ impl TcpFingerprintCollector { None } - pub fn lookup_synack_fingerprint(&self, _ip: IpAddr, _port: u16) -> Option { - None - } - pub fn log_stats(&self) -> Result<(), Box> { Ok(()) } diff --git a/src/utils/healthcheck.rs b/src/utils/healthcheck.rs index 815a5448..1587c9f5 100644 --- a/src/utils/healthcheck.rs +++ b/src/utils/healthcheck.rs @@ -14,17 +14,11 @@ pub async fn hc2( fullist: Arc, idlist: Arc, params: (&str, u64), - skip_tls_verify: bool, ) { let mut period = interval(Duration::from_secs(params.1)); - if skip_tls_verify { - warn!( - "Healthcheck TLS verification is disabled — upstream MITM attacks will not be detected" - ); - } let client = Client::builder() .timeout(Duration::from_secs(params.1)) - .danger_accept_invalid_certs(skip_tls_verify) + .danger_accept_invalid_certs(true) .build() .unwrap(); loop { @@ -53,7 +47,7 @@ pub async fn populate_upstreams( pub async fn initiate_upstreams(fullist: UpstreamsDashMap) -> UpstreamsDashMap { let client = Client::builder() .timeout(Duration::from_secs(2)) - .danger_accept_invalid_certs(false) + .danger_accept_invalid_certs(true) .build() .unwrap(); build_upstreams(&fullist, "HEAD", &client).await @@ -112,6 +106,16 @@ async fn build_upstreams( } else { innervec.push(scheme); } + + // let resp = http_request(&link, method, "", &client).await; + // if resp.0 { + // if resp.1 { + // scheme.is_http2 = is_h2; // could be adjusted further + // } + // innervec.push(scheme); + // } else { + // warn!("Dead Upstream : {}", link); + // } } inner.insert(path.clone(), (innervec, AtomicUsize::new(0))); } diff --git a/src/utils/maxmind.rs b/src/utils/maxmind.rs index a436690b..1f3acedf 100644 --- a/src/utils/maxmind.rs +++ b/src/utils/maxmind.rs @@ -35,19 +35,6 @@ impl MaxMindReader { .map(&file) .with_context(|| format!("Failed to memory-map MMDB from {:?}", path))? }; - // Validate MMDB file has correct magic bytes (metadata marker) - // The MMDB format contains a metadata marker "\xab\xcd\xefMaxMind.com" - const MMDB_METADATA_MARKER: &[u8] = b"\xab\xcd\xefMaxMind.com"; - if mmap.len() < MMDB_METADATA_MARKER.len() - || !mmap - .windows(MMDB_METADATA_MARKER.len()) - .any(|w| w == MMDB_METADATA_MARKER) - { - anyhow::bail!( - "MMDB file {:?} does not contain valid MaxMind metadata marker — file may be corrupted or tampered with", - path - ); - } Reader::from_source(mmap) .with_context(|| format!("Failed to parse MMDB from {:?}", path)) } diff --git a/src/utils/parceyaml.rs b/src/utils/parceyaml.rs index 76a02408..40da5504 100644 --- a/src/utils/parceyaml.rs +++ b/src/utils/parceyaml.rs @@ -7,7 +7,7 @@ use log::{error, info, warn}; use std::sync::OnceLock; use std::sync::atomic::AtomicUsize; // use std::sync::mpsc::{channel, Receiver, Sender}; -use std::fs; +use std::{env, fs}; // use tokio::sync::oneshot::{Receiver, Sender}; /// Global internal services address (bind_ip, port) set from main config. @@ -162,7 +162,6 @@ async fn populate_headers_and_auth(config: &mut Configuration, parsed: &Config) // Use values from config: section if present config.extraparams.sticky_sessions = global_config.sticky_sessions; config.extraparams.https_proxy_enabled = Some(global_config.https_proxy_enabled); - config.extraparams.forward_fingerprints = global_config.forward_fingerprints; // Parse global_request_headers (headers to add to upstream requests) if let Some(headers) = &global_config.global_request_headers { @@ -458,9 +457,7 @@ fn parce_tls_grades(what: Option) -> Option { Some("medium".to_string()) } "unsafe" => { - warn!( - "TLS grade set to UNSAFE/LEGACY — weak cipher suites are enabled, which may allow downgrade attacks. Only use for backward compatibility with legacy clients." - ); + // info!("TLS grade set to: [ UNSAFE ]"); Some("unsafe".to_string()) } _ => { @@ -481,21 +478,22 @@ fn log_builder(log_level: Option<&str>) { .map(|s| s.to_string()) .or_else(|| std::env::var("RUST_LOG").ok()) .unwrap_or_else(|| "info".to_string()); - let filter = match log_level.as_str() { - "error" => log::LevelFilter::Error, - "warn" => log::LevelFilter::Warn, - "info" => log::LevelFilter::Info, - "debug" => log::LevelFilter::Debug, - "trace" => log::LevelFilter::Trace, - "off" => log::LevelFilter::Off, - _ => { - println!("Error reading log level, defaulting to: INFO"); - log::LevelFilter::Info + unsafe { + match log_level.as_str() { + "info" => env::set_var("RUST_LOG", "info"), + "error" => env::set_var("RUST_LOG", "error"), + "warn" => env::set_var("RUST_LOG", "warn"), + "debug" => env::set_var("RUST_LOG", "debug"), + "trace" => env::set_var("RUST_LOG", "trace"), + "off" => env::set_var("RUST_LOG", "off"), + _ => { + println!("Error reading log level, defaulting to: INFO"); + env::set_var("RUST_LOG", "info") + } } - }; + } // Use try_init() to avoid panic if logger is already initialized (e.g., from main.rs) - // Configure filter directly instead of using env::set_var (unsafe in multi-threaded context) - let _ = env_logger::builder().filter_level(filter).try_init(); + let _ = env_logger::builder().try_init(); } pub fn build_headers( diff --git a/src/utils/structs.rs b/src/utils/structs.rs index 0c9c8d3d..6a6234d3 100644 --- a/src/utils/structs.rs +++ b/src/utils/structs.rs @@ -26,7 +26,6 @@ pub struct ServiceMapping { pub struct Extraparams { pub sticky_sessions: bool, pub https_proxy_enabled: Option, - pub forward_fingerprints: bool, pub authentication: DashMap>, } #[derive(Clone, Default, Debug, Serialize, Deserialize)] @@ -49,8 +48,6 @@ pub struct GlobalConfig { #[serde(default)] pub sticky_sessions: bool, #[serde(default)] - pub forward_fingerprints: bool, - #[serde(default)] #[serde(alias = "global_headers")] // Support old format for backward compatibility pub global_request_headers: Option>, #[serde(default)] @@ -191,12 +188,6 @@ pub struct AppConfig { pub runuser: Option, pub rungroup: Option, pub proxy_protocol_enabled: bool, - pub healthcheck_skip_tls_verify: bool, - #[serde(default)] - pub h2c: bool, - #[serde(default)] - pub allow_connect_method_proxying: bool, - pub keepalive_request_limit: Option, } #[derive(Debug, Clone, PartialEq, Eq, Hash)] diff --git a/src/utils/tls.rs b/src/utils/tls.rs index 9df9f6c1..e09bf513 100644 --- a/src/utils/tls.rs +++ b/src/utils/tls.rs @@ -85,7 +85,7 @@ impl TlsAccept for Certificates { name_str ); - match ssl.set_ssl_context(&ctx) { + match ssl.set_ssl_context(&*ctx) { Ok(_) => { log::debug!( "TlsAccept: Successfully set SSL context for hostname: {}", @@ -124,7 +124,7 @@ impl TlsAccept for Certificates { "TlsAccept: Using configured default certificate: {}", default_cert_name ); - if let Err(e) = ssl.set_ssl_context(ctx) { + if let Err(e) = ssl.set_ssl_context(&*ctx) { log::error!("TlsAccept: Failed to set default SSL context: {:?}", e); } else { log::debug!("TlsAccept: Successfully set default certificate"); @@ -137,7 +137,7 @@ impl TlsAccept for Certificates { ); if let Some(default_ctx) = self.cert_name_map.iter().next() { let ctx = default_ctx.value(); - if let Err(e) = ssl.set_ssl_context(ctx) { + if let Err(e) = ssl.set_ssl_context(&*ctx) { log::error!("TlsAccept: Failed to set fallback SSL context: {:?}", e); } else { log::debug!("TlsAccept: Using fallback certificate"); @@ -270,8 +270,8 @@ impl Certificates { log::debug!("Built cert_name_map with {} entries", cert_name_map.len()); Some(Self { - name_map, - cert_name_map, + name_map: name_map, + cert_name_map: cert_name_map, upstreams_cert_map: DashMap::new(), configs: cert_infos, default_cert_path: default_cert.cert_path.clone(), @@ -445,7 +445,7 @@ impl Certificates { "SNI callback: Setting SSL context for hostname: {}", name_str ); - ssl_ref.set_ssl_context(&ctx).map_err(|e| { + ssl_ref.set_ssl_context(&*ctx).map_err(|e| { log::error!( "SNI callback: Failed to set SSL context for hostname {}: {:?}", name_str, @@ -679,20 +679,13 @@ pub enum TlsGrade { LEGACY, } -impl std::str::FromStr for TlsGrade { - type Err = (); - - fn from_str(s: &str) -> Result { +impl TlsGrade { + pub fn from_str(s: &str) -> Option { match s.to_ascii_lowercase().as_str() { - "high" => Ok(TlsGrade::HIGH), - "medium" => Ok(TlsGrade::MEDIUM), - "unsafe" => { - log::warn!( - "TLS grade LEGACY/UNSAFE selected — weak cipher suites enabled, vulnerable to downgrade attacks" - ); - Ok(TlsGrade::LEGACY) - } - _ => Err(()), + "high" => Some(TlsGrade::HIGH), + "medium" => Some(TlsGrade::MEDIUM), + "unsafe" => Some(TlsGrade::LEGACY), + _ => None, } } } @@ -753,7 +746,7 @@ pub fn create_tls_settings_with_sni( } pub fn set_tsl_grade(tls_settings: &mut TlsSettings, grade: &str) { - let config_grade = grade.parse::().ok(); + let config_grade = TlsGrade::from_str(grade); match config_grade { Some(TlsGrade::HIGH) => { let _ = tls_settings.set_min_proto_version(Some(SslVersion::TLS1_2)); @@ -784,50 +777,6 @@ pub fn set_tsl_grade(tls_settings: &mut TlsSettings, grade: &str) { } } -/// Extract JA4X fingerprint from a certificate file. -/// -/// JA4X fingerprints X.509 certificates based on issuer RDN OIDs, -/// subject RDN OIDs, and extension OIDs. -pub fn extract_ja4x_fingerprint(cert_path: &str) -> Option { - let file = File::open(cert_path).ok()?; - let mut reader = BufReader::new(file); - - let item = read_one(&mut reader).ok()??; - let cert_der = match item { - Item::X509Certificate(der) => der, - _ => return None, - }; - - let (_, cert) = X509Certificate::from_der(&cert_der).ok()?; - - // Extract issuer RDN OIDs in dot notation - let issuer_oids: Vec = cert - .issuer() - .iter_attributes() - .map(|attr| attr.attr_type().to_id_string()) - .collect(); - let issuer_refs: Vec<&str> = issuer_oids.iter().map(|s| s.as_str()).collect(); - - // Extract subject RDN OIDs - let subject_oids: Vec = cert - .subject() - .iter_attributes() - .map(|attr| attr.attr_type().to_id_string()) - .collect(); - let subject_refs: Vec<&str> = subject_oids.iter().map(|s| s.as_str()).collect(); - - // Extract extension OIDs - let extension_oids: Vec = cert - .extensions() - .iter() - .map(|ext| ext.oid.to_id_string()) - .collect(); - let extension_refs: Vec<&str> = extension_oids.iter().map(|s| s.as_str()).collect(); - - let ja4x = nstealth::Ja4x::from_oid_strings(&issuer_refs, &subject_refs, &extension_refs); - Some(ja4x.fingerprint()) -} - /// Extract server certificate information for access logging pub fn extract_cert_info(cert_path: &str) -> Option { use sha2::{Digest, Sha256}; diff --git a/src/utils/tls_client_hello.rs b/src/utils/tls_client_hello.rs index 6a785468..f4fc1832 100644 --- a/src/utils/tls_client_hello.rs +++ b/src/utils/tls_client_hello.rs @@ -6,15 +6,13 @@ use std::net::SocketAddr; use std::sync::Arc; use std::sync::Mutex; use std::sync::OnceLock; -use std::time::{Instant, SystemTime, UNIX_EPOCH}; +use std::time::{SystemTime, UNIX_EPOCH}; /// TLS fingerprint entry with timestamp for fallback matching #[derive(Clone)] pub struct FingerprintEntry { pub fingerprint: Arc, pub stored_at: SystemTime, - /// Monotonic instant for TLS handshake latency measurement (JA4L) - pub created_instant: Instant, } /// Global storage for TLS fingerprints keyed by connection peer address @@ -75,19 +73,10 @@ pub fn generate_fingerprint_from_client_hello( let std_addr = SocketAddr::new(inet.ip().into(), inet.port()); let key = format!("{}", std_addr); if let Ok(mut map) = get_fingerprint_map().lock() { - // Cap at 50,000 entries to prevent unbounded growth - if map.len() >= 50_000 { - let cutoff = SystemTime::now() - .checked_sub(std::time::Duration::from_secs(300)) - .unwrap_or(UNIX_EPOCH); - map.retain(|_, entry| entry.stored_at > cutoff); - } let stored_at = SystemTime::now(); - let created_instant = Instant::now(); let entry = FingerprintEntry { fingerprint: fingerprint_arc.clone(), stored_at, - created_instant, }; map.insert(key, entry); debug!("Stored TLS fingerprint for {} at {:?}", std_addr, stored_at); @@ -159,33 +148,6 @@ pub fn get_fingerprint(peer_addr: &SocketAddr) -> Option> { } } -/// Get the ClientHello arrival Instant for a peer address (for JA4L TLS timing) -/// Falls back to IP-only match similar to get_fingerprint_with_fallback -pub fn get_hello_instant(peer_addr: &SocketAddr) -> Option { - let key = format!("{}", peer_addr); - if let Ok(map) = get_fingerprint_map().lock() { - // Exact match first - if let Some(entry) = map.get(&key) { - return Some(entry.created_instant); - } - // Fallback: match by IP, pick most recent - let peer_ip = peer_addr.ip(); - map.iter() - .filter_map(|(k, entry)| { - k.parse::().ok().and_then(|addr| { - if addr.ip() == peer_ip { - Some(entry.created_instant) - } else { - None - } - }) - }) - .max() // Pick the most recent Instant - } else { - None - } -} - /// Get stored TLS fingerprint with fallback strategies for PROXY protocol /// This tries multiple lookup strategies to handle cases where PROXY protocol /// might cause address mismatches between storage and retrieval diff --git a/src/utils/tools.rs b/src/utils/tools.rs index bada4015..9e4b74ad 100644 --- a/src/utils/tools.rs +++ b/src/utils/tools.rs @@ -161,13 +161,7 @@ pub fn clone_idmap_into(original: &UpstreamsDashMap, cloned: &UpstreamsIdMap) { pub fn listdir(dir: String) -> Vec { let mut certificate_configs: Vec = vec![]; - let paths = match fs::read_dir(&dir) { - Ok(paths) => paths, - Err(e) => { - log::error!("Failed to read certificate directory '{}': {}", dir, e); - return certificate_configs; - } - }; + let paths = fs::read_dir(dir).unwrap(); for path in paths { let path_str = path.unwrap().path().to_str().unwrap().to_owned(); if path_str.ends_with(".crt") { diff --git a/src/worker/README.md b/src/worker/README.md index 502ee39b..0a4c4dad 100644 --- a/src/worker/README.md +++ b/src/worker/README.md @@ -253,6 +253,7 @@ impl Worker for MmdbWorker { | `ConfigWorker` | Complex | `Worker` | Fetches config from API or watches local file | | `CertificateWorker` | Complex | `Worker` | Fetches certificates from Redis, multiple intervals | | `LogSenderWorker` | Event-driven | `Worker` | Batches and sends access logs to API | +| `ThalamusIdsWorker` | Event-driven | `Worker` | Consumes XDP ring-buffer exports, runs Thalamus rules, emits IDS alerts | ### When to Use Each Pattern diff --git a/src/worker/config.rs b/src/worker/config.rs index 2978313d..4c129fdb 100644 --- a/src/worker/config.rs +++ b/src/worker/config.rs @@ -30,7 +30,10 @@ pub struct Config { } #[derive(Debug, Clone, Deserialize, Serialize, Default)] -pub struct AccessRuleConfig {} +pub struct AccessRuleConfig { + #[serde(default, rename = "rateLimit")] + pub rate_limit: XDPRateLimitConfig, +} #[derive(Debug, Clone, Deserialize, Serialize)] pub struct AccessRule { @@ -90,6 +93,37 @@ impl RateLimitConfig { } } +/// These are the fields which can be set in runtime +/// We can expose these to the dashboard +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)] +pub struct XDPRateLimitConfig { + pub enabled: bool, + pub requests: u64, + pub burst_factor: f32, +} + +// TODO: ask arpad to rename the field from config +impl XDPRateLimitConfig { + pub fn from_json(value: &serde_json::Value) -> Result { + // Parse from nested structure: {"rateLimit": {"period": "25", ...}} + if let Some(rate_limit_obj) = value.get("rateLimit") { + serde_json::from_value(rate_limit_obj.clone()).map_err(|e| e.to_string()) + } else { + Err("rateLimit field not found".to_string()) + } + } +} + +impl Default for XDPRateLimitConfig { + fn default() -> Self { + Self { + enabled: true, + requests: 1000, + burst_factor: 1.5, + } + } +} + #[derive(Debug, Clone, Deserialize, Serialize)] pub struct RuleSet { pub asn: Vec>>, diff --git a/src/worker/log.rs b/src/worker/log.rs index a9e8acb6..a49b5391 100644 --- a/src/worker/log.rs +++ b/src/worker/log.rs @@ -46,7 +46,7 @@ impl LogSenderConfig { batch_timeout_secs: 10, // Default: 10 seconds include_request_body: false, // Default: disabled max_body_size: 1024 * 1024, // Default: 1MB - channel_capacity: Some(DEFAULT_CHANNEL_CAPACITY), // Default: bounded to prevent OOM + channel_capacity: None, // Default: unbounded (backward compatible) } } @@ -335,6 +335,42 @@ fn set_event_sender(sender: EventSender) { let _ = EVENT_SENDER.set(sender); } +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct IdsAlertEvent { + pub timestamp: DateTime, + pub signature_id: u32, + pub signature_name: String, + pub severity: u8, + pub category: String, + pub src_ip: String, + pub dst_ip: String, + pub src_port: u16, + pub dst_port: u16, + pub protocol: String, + pub action: String, + pub flow_id: Option, + pub ifindex: u32, + pub packet_len: u32, + pub captured_len: u32, + pub blocked: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub ja4t: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub ja4ts: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub ja4h: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub ja4d: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub ja4d6: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub ja4: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub ja4ssh: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub ja4l: Option, +} + /// Unified event types that can be sent to the /events endpoint #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(tag = "event_type")] @@ -351,6 +387,8 @@ pub enum UnifiedEvent { SshFingerprint(crate::utils::fingerprint::ssh_fingerprint::SshFingerprintEvent), #[serde(rename = "latency_fingerprint")] LatencyFingerprint(crate::utils::fingerprint::latency_fingerprint::LatencyFingerprintEvent), + #[serde(rename = "ids_alert")] + IdsAlert(IdsAlertEvent), } impl UnifiedEvent { @@ -363,6 +401,7 @@ impl UnifiedEvent { UnifiedEvent::AgentStatus(_) => "agent_status", UnifiedEvent::SshFingerprint(_) => "ssh_fingerprint", UnifiedEvent::LatencyFingerprint(_) => "latency_fingerprint", + UnifiedEvent::IdsAlert(_) => "ids_alert", } } @@ -375,6 +414,7 @@ impl UnifiedEvent { UnifiedEvent::AgentStatus(event) => event.timestamp, UnifiedEvent::SshFingerprint(event) => event.timestamp, UnifiedEvent::LatencyFingerprint(event) => event.timestamp, + UnifiedEvent::IdsAlert(event) => event.timestamp, } } @@ -501,6 +541,7 @@ fn estimate_event_size(event: &UnifiedEvent) -> usize { UnifiedEvent::AgentStatus(_) => base_size + 300, // Agent status events include system info UnifiedEvent::SshFingerprint(_) => base_size + 200, // SSH fingerprint events are medium size UnifiedEvent::LatencyFingerprint(_) => base_size + 150, // Latency fingerprint events are medium size + UnifiedEvent::IdsAlert(_) => base_size + 250, // IDS alerts include flow and signature metadata } } @@ -876,3 +917,60 @@ impl super::Worker for LogSenderWorker { }) } } + +#[cfg(test)] +mod tests { + use super::*; + use chrono::Utc; + + fn sample_ids_alert_event() -> IdsAlertEvent { + IdsAlertEvent { + timestamp: Utc::now(), + signature_id: 9_900_001, + signature_name: "test alert".to_string(), + severity: 3, + category: "test".to_string(), + src_ip: "203.0.113.10".to_string(), + dst_ip: "10.0.0.44".to_string(), + src_port: 54000, + dst_port: 443, + protocol: "tcp".to_string(), + action: "alert".to_string(), + flow_id: Some(1234), + ifindex: 2, + packet_len: 128, + captured_len: 96, + blocked: false, + ja4t: None, + ja4ts: None, + ja4h: None, + ja4d: None, + ja4d6: None, + ja4: None, + ja4ssh: None, + ja4l: None, + } + } + + #[test] + fn test_ids_alert_event_omits_empty_fingerprints() { + let event = UnifiedEvent::IdsAlert(sample_ids_alert_event()); + let json = event.to_json().expect("serialize ids alert"); + + assert!(json.contains("\"event_type\":\"ids_alert\"")); + assert!(!json.contains("\"ja4\"")); + assert!(!json.contains("\"ja4ssh\"")); + } + + #[test] + fn test_ids_alert_event_includes_fingerprints() { + let mut alert = sample_ids_alert_event(); + alert.ja4 = Some("t13d1516h2_8daaf6152771_b0da82dd1658".to_string()); + alert.ja4ssh = Some("c76s76_c0s0".to_string()); + let event = UnifiedEvent::IdsAlert(alert); + let json = event.to_json().expect("serialize ids alert"); + + assert!(json.contains("\"ja4\":\"t13d1516h2_8daaf6152771_b0da82dd1658\"")); + assert!(json.contains("\"ja4ssh\":\"c76s76_c0s0\"")); + } +} diff --git a/src/worker/mod.rs b/src/worker/mod.rs index 2f2ea61f..708eb61f 100644 --- a/src/worker/mod.rs +++ b/src/worker/mod.rs @@ -4,6 +4,13 @@ pub mod config; pub mod geoip_mmdb; pub mod log; pub mod manager; +#[cfg(all( + feature = "thalamus-ids", + feature = "bpf", + not(feature = "disable-bpf"), + target_os = "linux" +))] +pub mod thalamus_ids; pub mod threat_mmdb; pub use manager::{ diff --git a/src/worker/thalamus_ids.rs b/src/worker/thalamus_ids.rs new file mode 100644 index 00000000..35983da8 --- /dev/null +++ b/src/worker/thalamus_ids.rs @@ -0,0 +1,1366 @@ +#[cfg(all(feature = "bpf", not(feature = "disable-bpf"), target_os = "linux"))] +use crate::core::cli::IdsConfig; +#[cfg(all(feature = "bpf", not(feature = "disable-bpf"), target_os = "linux"))] +use crate::security::firewall::{ + Firewall, FirewallBackend, IptablesFirewall, NftablesFirewall, SYNAPSEFirewall, +}; +#[cfg(all(feature = "bpf", not(feature = "disable-bpf"), target_os = "linux"))] +use crate::worker::Worker; +#[cfg(all(feature = "bpf", not(feature = "disable-bpf"), target_os = "linux"))] +use crate::worker::log::{IdsAlertEvent, UnifiedEvent, send_event}; +#[cfg(all(feature = "bpf", not(feature = "disable-bpf"), target_os = "linux"))] +use libbpf_rs::{MapCore, MapFlags, RingBufferBuilder}; +#[cfg(all(feature = "bpf", not(feature = "disable-bpf"), target_os = "linux"))] +use nix::libc; +#[cfg(all(feature = "bpf", not(feature = "disable-bpf"), target_os = "linux"))] +use plain::Plain; +#[cfg(all(feature = "bpf", not(feature = "disable-bpf"), target_os = "linux"))] +use serde_json::json; +#[cfg(all(feature = "bpf", not(feature = "disable-bpf"), target_os = "linux"))] +use std::collections::{HashSet, hash_map::DefaultHasher}; +#[cfg(all(feature = "bpf", not(feature = "disable-bpf"), target_os = "linux"))] +use std::fs::{OpenOptions, create_dir_all}; +#[cfg(all(feature = "bpf", not(feature = "disable-bpf"), target_os = "linux"))] +use std::hash::{Hash, Hasher}; +#[cfg(all(feature = "bpf", not(feature = "disable-bpf"), target_os = "linux"))] +use std::io::Write; +#[cfg(all(feature = "bpf", not(feature = "disable-bpf"), target_os = "linux"))] +use std::net::IpAddr; +#[cfg(all(feature = "bpf", not(feature = "disable-bpf"), target_os = "linux"))] +use std::sync::atomic::{AtomicU64, Ordering}; +#[cfg(all(feature = "bpf", not(feature = "disable-bpf"), target_os = "linux"))] +use std::sync::{Arc, Mutex}; +#[cfg(all(feature = "bpf", not(feature = "disable-bpf"), target_os = "linux"))] +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; +#[cfg(all(feature = "bpf", not(feature = "disable-bpf"), target_os = "linux"))] +use thalamus::PacketVerdict; +#[cfg(all(feature = "bpf", not(feature = "disable-bpf"), target_os = "linux"))] +use thalamus::decoder::PacketDecoder; +#[cfg(all(feature = "bpf", not(feature = "disable-bpf"), target_os = "linux"))] +use thalamus::engine::{Alert, FlowKey as EngineFlowKey, SignatureEngine}; +#[cfg(all(feature = "bpf", not(feature = "disable-bpf"), target_os = "linux"))] +use thalamus::rules::{RuleSet, RuleVars}; +#[cfg(all(feature = "bpf", not(feature = "disable-bpf"), target_os = "linux"))] +use thalamus::trace; +#[cfg(all(feature = "bpf", not(feature = "disable-bpf"), target_os = "linux"))] +use tokio::sync::watch; +#[cfg(all(feature = "bpf", not(feature = "disable-bpf"), target_os = "linux"))] +use tokio::task::JoinHandle; + +#[cfg(all(feature = "bpf", not(feature = "disable-bpf"), target_os = "linux"))] +const IDS_EXPORT_MAX_BYTES: usize = 512; +#[cfg(all(feature = "bpf", not(feature = "disable-bpf"), target_os = "linux"))] +const IDS_LATENCY_HIST_BUCKETS: usize = 64; +#[cfg(all(feature = "bpf", not(feature = "disable-bpf"), target_os = "linux"))] +const RINGBUF_BATCH_LIMIT: u64 = 2048; + +#[cfg(all(feature = "bpf", not(feature = "disable-bpf"), target_os = "linux"))] +#[repr(C)] +#[derive(Copy, Clone)] +struct IdsExportEventRaw { + ts_ns: u64, + ifindex: u32, + packet_len: u32, + captured_len: u32, + _reserved: u32, + packet: [u8; IDS_EXPORT_MAX_BYTES], +} + +#[cfg(all(feature = "bpf", not(feature = "disable-bpf"), target_os = "linux"))] +impl Default for IdsExportEventRaw { + fn default() -> Self { + Self { + ts_ns: 0, + ifindex: 0, + packet_len: 0, + captured_len: 0, + _reserved: 0, + packet: [0u8; IDS_EXPORT_MAX_BYTES], + } + } +} + +#[cfg(all(feature = "bpf", not(feature = "disable-bpf"), target_os = "linux"))] +unsafe impl Plain for IdsExportEventRaw {} + +#[cfg(all(feature = "bpf", not(feature = "disable-bpf"), target_os = "linux"))] +struct IdsWorkerStats { + events_seen: AtomicU64, + packets_decoded: AtomicU64, + alerts: AtomicU64, + blocks_applied: AtomicU64, + decode_errors: AtomicU64, + malformed_events: AtomicU64, + latency_samples: AtomicU64, + latency_total_ns: AtomicU64, + latency_min_ns: AtomicU64, + latency_max_ns: AtomicU64, + latency_hist: [AtomicU64; IDS_LATENCY_HIST_BUCKETS], +} + +#[cfg(all(feature = "bpf", not(feature = "disable-bpf"), target_os = "linux"))] +impl Default for IdsWorkerStats { + fn default() -> Self { + Self { + events_seen: AtomicU64::new(0), + packets_decoded: AtomicU64::new(0), + alerts: AtomicU64::new(0), + blocks_applied: AtomicU64::new(0), + decode_errors: AtomicU64::new(0), + malformed_events: AtomicU64::new(0), + latency_samples: AtomicU64::new(0), + latency_total_ns: AtomicU64::new(0), + latency_min_ns: AtomicU64::new(u64::MAX), + latency_max_ns: AtomicU64::new(0), + latency_hist: std::array::from_fn(|_| AtomicU64::new(0)), + } + } +} + +#[cfg(all(feature = "bpf", not(feature = "disable-bpf"), target_os = "linux"))] +#[derive(Clone)] +struct IdsStatsSnapshot { + events_seen: u64, + packets_decoded: u64, + alerts: u64, + blocks_applied: u64, + decode_errors: u64, + malformed_events: u64, + latency_samples: u64, + latency_total_ns: u64, + latency_min_ns: u64, + latency_max_ns: u64, + latency_hist: [u64; IDS_LATENCY_HIST_BUCKETS], +} + +#[cfg(all(feature = "bpf", not(feature = "disable-bpf"), target_os = "linux"))] +impl Default for IdsStatsSnapshot { + fn default() -> Self { + Self { + events_seen: 0, + packets_decoded: 0, + alerts: 0, + blocks_applied: 0, + decode_errors: 0, + malformed_events: 0, + latency_samples: 0, + latency_total_ns: 0, + latency_min_ns: 0, + latency_max_ns: 0, + latency_hist: [0u64; IDS_LATENCY_HIST_BUCKETS], + } + } +} + +#[cfg(all(feature = "bpf", not(feature = "disable-bpf"), target_os = "linux"))] +#[derive(Clone, Copy, Default)] +struct IdsLatencySummary { + samples: u64, + avg_us: f64, + min_us: u64, + p90_us: u64, + p99_us: u64, + p999_us: u64, + max_us: u64, +} + +#[cfg(all(feature = "bpf", not(feature = "disable-bpf"), target_os = "linux"))] +impl IdsStatsSnapshot { + fn latency_summary(&self) -> IdsLatencySummary { + if self.latency_samples == 0 { + return IdsLatencySummary::default(); + } + + IdsLatencySummary { + samples: self.latency_samples, + avg_us: self.latency_total_ns as f64 / self.latency_samples as f64 / 1_000.0, + min_us: self.latency_min_ns / 1_000, + p90_us: percentile_from_histogram_us(&self.latency_hist, self.latency_samples, 0.90), + p99_us: percentile_from_histogram_us(&self.latency_hist, self.latency_samples, 0.99), + p999_us: percentile_from_histogram_us(&self.latency_hist, self.latency_samples, 0.999), + max_us: self.latency_max_ns / 1_000, + } + } +} + +#[cfg(all(feature = "bpf", not(feature = "disable-bpf"), target_os = "linux"))] +impl IdsWorkerStats { + fn snapshot(&self) -> IdsStatsSnapshot { + let mut latency_hist = [0u64; IDS_LATENCY_HIST_BUCKETS]; + for (idx, counter) in self.latency_hist.iter().enumerate() { + latency_hist[idx] = counter.load(Ordering::Relaxed); + } + + let latency_min_ns = self.latency_min_ns.load(Ordering::Relaxed); + IdsStatsSnapshot { + events_seen: self.events_seen.load(Ordering::Relaxed), + packets_decoded: self.packets_decoded.load(Ordering::Relaxed), + alerts: self.alerts.load(Ordering::Relaxed), + blocks_applied: self.blocks_applied.load(Ordering::Relaxed), + decode_errors: self.decode_errors.load(Ordering::Relaxed), + malformed_events: self.malformed_events.load(Ordering::Relaxed), + latency_samples: self.latency_samples.load(Ordering::Relaxed), + latency_total_ns: self.latency_total_ns.load(Ordering::Relaxed), + latency_min_ns: if latency_min_ns == u64::MAX { + 0 + } else { + latency_min_ns + }, + latency_max_ns: self.latency_max_ns.load(Ordering::Relaxed), + latency_hist, + } + } + + fn record_latency_ns(&self, latency_ns: u64) { + self.latency_samples.fetch_add(1, Ordering::Relaxed); + self.latency_total_ns + .fetch_add(latency_ns, Ordering::Relaxed); + self.latency_min_ns.fetch_min(latency_ns, Ordering::Relaxed); + self.latency_max_ns.fetch_max(latency_ns, Ordering::Relaxed); + + let bucket = latency_bucket_idx_ns(latency_ns); + self.latency_hist[bucket].fetch_add(1, Ordering::Relaxed); + } +} + +#[cfg(all(feature = "bpf", not(feature = "disable-bpf"), target_os = "linux"))] +fn monotonic_time_ns() -> Option { + let mut ts = libc::timespec { + tv_sec: 0, + tv_nsec: 0, + }; + let rc = unsafe { libc::clock_gettime(libc::CLOCK_MONOTONIC, &mut ts) }; + if rc != 0 || ts.tv_sec < 0 || ts.tv_nsec < 0 { + return None; + } + let sec = u64::try_from(ts.tv_sec).ok()?; + let nsec = u64::try_from(ts.tv_nsec).ok()?; + sec.checked_mul(1_000_000_000)?.checked_add(nsec) +} + +#[cfg(all(feature = "bpf", not(feature = "disable-bpf"), target_os = "linux"))] +fn latency_bucket_idx_ns(latency_ns: u64) -> usize { + let latency_us = latency_ns.div_ceil(1_000); + if latency_us == 0 { + return 0; + } + let idx = (u64::BITS - latency_us.saturating_sub(1).leading_zeros()) as usize; + idx.min(IDS_LATENCY_HIST_BUCKETS - 1) +} + +#[cfg(all(feature = "bpf", not(feature = "disable-bpf"), target_os = "linux"))] +fn latency_bucket_upper_us(bucket_idx: usize) -> u64 { + if bucket_idx == 0 { + return 1; + } + if bucket_idx >= 63 { + return u64::MAX; + } + 1u64 << bucket_idx +} + +#[cfg(all(feature = "bpf", not(feature = "disable-bpf"), target_os = "linux"))] +fn percentile_from_histogram_us( + hist: &[u64; IDS_LATENCY_HIST_BUCKETS], + total_samples: u64, + percentile: f64, +) -> u64 { + if total_samples == 0 { + return 0; + } + + let rank = ((total_samples as f64) * percentile).ceil() as u64; + let target = rank.max(1).min(total_samples); + + let mut seen = 0u64; + for (idx, count) in hist.iter().enumerate() { + seen = seen.saturating_add(*count); + if seen >= target { + return latency_bucket_upper_us(idx); + } + } + + latency_bucket_upper_us(IDS_LATENCY_HIST_BUCKETS - 1) +} + +#[cfg(all(feature = "bpf", not(feature = "disable-bpf"), target_os = "linux"))] +fn subtract_hist( + lhs: &[u64; IDS_LATENCY_HIST_BUCKETS], + rhs: &[u64; IDS_LATENCY_HIST_BUCKETS], +) -> [u64; IDS_LATENCY_HIST_BUCKETS] { + std::array::from_fn(|idx| lhs[idx].saturating_sub(rhs[idx])) +} + +#[cfg(all(feature = "bpf", not(feature = "disable-bpf"), target_os = "linux"))] +#[derive(Clone)] +struct BlockEnforcer { + firewall_backend: FirewallBackend, + skels: Vec>>>, + nftables_firewall: Option>>, + iptables_firewall: Option>>, + blocked_cache: Arc>>, +} + +#[cfg(all(feature = "bpf", not(feature = "disable-bpf"), target_os = "linux"))] +impl BlockEnforcer { + fn block_ip(&self, ip: IpAddr) -> bool { + if let Ok(cache) = self.blocked_cache.lock() + && cache.contains(&ip) + { + return true; + } + + let blocked = match self.firewall_backend { + FirewallBackend::Xdp => self.block_via_xdp(ip), + FirewallBackend::Nftables => self.block_via_nftables(ip), + FirewallBackend::Iptables => self.block_via_iptables(ip), + FirewallBackend::None => false, + }; + + if blocked && let Ok(mut cache) = self.blocked_cache.lock() { + cache.insert(ip); + } + + blocked + } + + fn block_via_xdp(&self, ip: IpAddr) -> bool { + if self.skels.is_empty() { + return false; + } + + let mut any_success = false; + let mut all_success = true; + + for skel in &self.skels { + let guard = match skel.lock() { + Ok(guard) => guard, + Err(e) => { + log::warn!("thalamus_ids: failed to lock XDP skeleton for block: {}", e); + all_success = false; + continue; + } + }; + + let mut fw = SYNAPSEFirewall::new(&guard); + let result = match ip { + IpAddr::V4(v4) => fw.ban_ip(v4, 32), + IpAddr::V6(v6) => fw.ban_ipv6(v6, 128), + }; + + match result { + Ok(()) => any_success = true, + Err(e) => { + log::warn!("thalamus_ids: failed to apply XDP block for {}: {}", ip, e); + all_success = false; + } + } + } + + any_success && all_success + } + + fn block_via_nftables(&self, ip: IpAddr) -> bool { + let Some(ref fw) = self.nftables_firewall else { + return false; + }; + + let mut guard = match fw.lock() { + Ok(guard) => guard, + Err(e) => { + log::warn!("thalamus_ids: failed to lock nftables firewall: {}", e); + return false; + } + }; + + let result = match ip { + IpAddr::V4(v4) => guard.ban_ip(v4, 32), + IpAddr::V6(v6) => guard.ban_ipv6(v6, 128), + }; + + if let Err(e) = result { + log::warn!( + "thalamus_ids: failed to apply nftables block for {}: {}", + ip, + e + ); + return false; + } + + true + } + + fn block_via_iptables(&self, ip: IpAddr) -> bool { + let Some(ref fw) = self.iptables_firewall else { + return false; + }; + + let mut guard = match fw.lock() { + Ok(guard) => guard, + Err(e) => { + log::warn!("thalamus_ids: failed to lock iptables firewall: {}", e); + return false; + } + }; + + let result = match ip { + IpAddr::V4(v4) => guard.ban_ip(v4, 32), + IpAddr::V6(v6) => guard.ban_ipv6(v6, 128), + }; + + if let Err(e) = result { + log::warn!( + "thalamus_ids: failed to apply iptables block for {}: {}", + ip, + e + ); + return false; + } + + true + } +} + +#[cfg(all(feature = "bpf", not(feature = "disable-bpf"), target_os = "linux"))] +#[derive(Clone, Copy)] +struct IdsPacketMeta { + ifindex: u32, + packet_len: u32, + captured_len: u32, +} + +#[cfg(all(feature = "bpf", not(feature = "disable-bpf"), target_os = "linux"))] +pub struct ThalamusIdsWorker { + skels: Vec>>>, + ids_config: IdsConfig, + firewall_backend: FirewallBackend, + nftables_firewall: Option>>, + iptables_firewall: Option>>, +} + +#[cfg(all(feature = "bpf", not(feature = "disable-bpf"), target_os = "linux"))] +impl ThalamusIdsWorker { + pub fn new( + skels: Vec>>>, + ids_config: IdsConfig, + firewall_backend: FirewallBackend, + nftables_firewall: Option>>, + iptables_firewall: Option>>, + ) -> Self { + Self { + skels, + ids_config, + firewall_backend, + nftables_firewall, + iptables_firewall, + } + } +} + +#[cfg(all(feature = "bpf", not(feature = "disable-bpf"), target_os = "linux"))] +impl Worker for ThalamusIdsWorker { + fn name(&self) -> &str { + "thalamus_ids" + } + + fn run(&self, shutdown: watch::Receiver) -> JoinHandle<()> { + let skels = self.skels.clone(); + let ids_config = self.ids_config.clone(); + let firewall_backend = self.firewall_backend; + let nftables_firewall = self.nftables_firewall.clone(); + let iptables_firewall = self.iptables_firewall.clone(); + let worker_name = self.name().to_string(); + + tokio::task::spawn_blocking(move || { + run_xdp_ids_worker( + worker_name, + shutdown, + skels, + ids_config, + firewall_backend, + nftables_firewall, + iptables_firewall, + ); + }) + } +} + +#[cfg(all(feature = "bpf", not(feature = "disable-bpf"), target_os = "linux"))] +fn write_trace_bootstrap_event(worker_name: &str, trace_snapshot: &serde_json::Value) { + let trace_dir = std::env::var("THALAMUS_TRACE_DIR").unwrap_or_else(|_| "./trace".to_string()); + let _ = create_dir_all(&trace_dir); + let path = std::path::Path::new(&trace_dir).join("trace_worker.ndjson"); + let mut file = match OpenOptions::new().create(true).append(true).open(path) { + Ok(file) => file, + Err(err) => { + log::warn!( + "[{}] Unable to open trace bootstrap file in '{}': {}", + worker_name, + trace_dir, + err + ); + return; + } + }; + let ts_ms = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_millis() as u64) + .unwrap_or(0); + let event = json!({ + "event": "worker_trace_bootstrap", + "worker": worker_name, + "timestamp_ms": ts_ms, + "trace_snapshot": trace_snapshot, + "raw_env": { + "THALAMUS_TRACE_ENABLED": std::env::var("THALAMUS_TRACE_ENABLED").ok(), + "THALAMUS_TRACE_DIR": std::env::var("THALAMUS_TRACE_DIR").ok(), + "THALAMUS_TRACE_PREVIEW_BYTES": std::env::var("THALAMUS_TRACE_PREVIEW_BYTES").ok(), + "THALAMUS_TRACE_SIDS": std::env::var("THALAMUS_TRACE_SIDS").ok(), + "THALAMUS_TRACE_FLOWS": std::env::var("THALAMUS_TRACE_FLOWS").ok(), + }, + }); + if let Ok(line) = serde_json::to_string(&event) { + let _ = writeln!(file, "{line}"); + } +} + +#[cfg(all(feature = "bpf", not(feature = "disable-bpf"), target_os = "linux"))] +fn run_xdp_ids_worker( + worker_name: String, + shutdown: watch::Receiver, + skels: Vec>>>, + ids_config: IdsConfig, + firewall_backend: FirewallBackend, + nftables_firewall: Option>>, + iptables_firewall: Option>>, +) { + if !ids_config.enabled { + log::info!("[{}] Disabled by configuration", worker_name); + return; + } + + if skels.is_empty() { + log::warn!( + "[{}] No active XDP skeletons available, IDS worker will not start", + worker_name + ); + return; + } + + let (engine, decoder, enforcer, stats, rules_count) = match init_ids_runtime( + &worker_name, + &ids_config, + firewall_backend, + skels.clone(), + nftables_firewall, + iptables_firewall, + ) { + Some(parts) => parts, + None => return, + }; + + // Channel capacity: large enough to absorb bursts while the inspect + // threads catch up. When full the ring buffer callback drops the event + // (counted in channel_drops) rather than blocking the consumer. + const INSPECT_CHANNEL_CAP: usize = 131_072; + const INSPECT_THREAD_COUNT: usize = 8; + + let channel_drops = Arc::new(AtomicU64::new(0)); + let mut shard_senders = Vec::with_capacity(INSPECT_THREAD_COUNT); + let mut inspect_handles = Vec::with_capacity(INSPECT_THREAD_COUNT); + + for tid in 0..INSPECT_THREAD_COUNT { + let (tx, rx) = crossbeam_channel::bounded::(INSPECT_CHANNEL_CAP); + shard_senders.push(tx); + let inspect_engine = engine.clone(); + let inspect_decoder = decoder.clone(); + let inspect_enforcer = enforcer.clone(); + let inspect_stats = stats.clone(); + let inspect_enforce_block = ids_config.enforce_block; + let inspect_shutdown = shutdown.clone(); + let inspect_worker_name = worker_name.clone(); + + let handle = std::thread::Builder::new() + .name(format!("{}_inspect_{}", worker_name, tid)) + .spawn(move || { + while !*inspect_shutdown.borrow() { + match rx.recv_timeout(Duration::from_millis(200)) { + Ok(raw) => { + let captured_len = raw.captured_len as usize; + if captured_len == 0 || captured_len > IDS_EXPORT_MAX_BYTES { + inspect_stats + .malformed_events + .fetch_add(1, Ordering::Relaxed); + continue; + } + let _ = inspect_packet( + &raw.packet[..captured_len], + IdsPacketMeta { + ifindex: raw.ifindex, + packet_len: raw.packet_len, + captured_len: raw.captured_len, + }, + &inspect_decoder, + &inspect_engine, + &inspect_enforcer, + inspect_enforce_block, + &inspect_stats, + false, + Some(tid), + ); + } + Err(crossbeam_channel::RecvTimeoutError::Timeout) => {} + Err(crossbeam_channel::RecvTimeoutError::Disconnected) => break, + } + } + log::info!( + "[{}_inspect_{}] Inspect thread exiting", + inspect_worker_name, + tid + ); + }) + .expect("failed to spawn IDS inspect thread"); + inspect_handles.push(handle); + } + + // --- Ring buffer setup: lightweight callback copies raw events to a flow shard --- + let mut ringbuf_builder = RingBufferBuilder::new(); + let mut skel_guards = Vec::with_capacity(skels.len()); + + for skel in &skels { + match skel.lock() { + Ok(guard) => skel_guards.push(guard), + Err(e) => { + log::error!( + "[{}] Failed to lock XDP skeleton for ring buffer setup: {}", + worker_name, + e + ); + return; + } + } + } + + let batch_counter = Arc::new(AtomicU64::new(0)); + + for guard in &skel_guards { + let shard_senders = shard_senders.clone(); + let stats = stats.clone(); + let chan_drops = channel_drops.clone(); + let batch_ctr = batch_counter.clone(); + + if let Err(e) = ringbuf_builder.add(&guard.maps.ids_export_events, move |data| { + if batch_ctr.fetch_add(1, Ordering::Relaxed) >= RINGBUF_BATCH_LIMIT { + return -1; + } + + let mut raw = IdsExportEventRaw::default(); + if plain::copy_from_bytes(&mut raw, data).is_err() { + stats.malformed_events.fetch_add(1, Ordering::Relaxed); + return 0; + } + + if let Some(now_ns) = monotonic_time_ns() { + if let Some(latency_ns) = now_ns.checked_sub(raw.ts_ns) { + stats.record_latency_ns(latency_ns); + } + } + + let captured_len = raw.captured_len as usize; + let shard = select_inspect_shard( + &raw.packet[..captured_len.min(IDS_EXPORT_MAX_BYTES)], + shard_senders.len(), + ); + + match shard_senders[shard].try_send(raw) { + Ok(()) => {} + Err(crossbeam_channel::TrySendError::Full(_)) => { + chan_drops.fetch_add(1, Ordering::Relaxed); + } + Err(crossbeam_channel::TrySendError::Disconnected(_)) => { + return -1; + } + } + 0 + }) { + log::error!( + "[{}] Failed to attach IDS ring buffer callback: {}", + worker_name, + e + ); + return; + } + } + + let ringbuf = match ringbuf_builder.build() { + Ok(ringbuf) => ringbuf, + Err(e) => { + log::error!("[{}] Failed to build IDS ring buffer: {}", worker_name, e); + return; + } + }; + let poll_timeout_ms = ids_config.poll_timeout_ms.max(1).min(5_000); + let poll_timeout = Duration::from_millis(poll_timeout_ms); + let cleanup_interval = Duration::from_secs(ids_config.cleanup_interval_secs.max(1)); + let stats_log_interval = Duration::from_secs(ids_config.stats_log_interval_secs.max(1)); + let mut last_cleanup = Instant::now(); + let mut last_stats_log = Instant::now(); + let mut last_stats_snapshot = stats.snapshot(); + let mut last_stats_ts = Instant::now(); + + log::info!( + "[{}] Started (source=xdp, rules={}, snaplen={}, enforce_block={}, skels={}, poll_timeout_ms={}, flow_timeout_secs={}, max_flows={}, stats_log_interval_secs={}, dispatch=flow_sharded, inspect_shards={}, inspect_channel_cap={})", + worker_name, + rules_count, + ids_config.snaplen.min(IDS_EXPORT_MAX_BYTES as u32), + ids_config.enforce_block, + skels.len(), + poll_timeout_ms, + ids_config.flow_timeout_secs, + ids_config.max_flows, + stats_log_interval.as_secs(), + INSPECT_THREAD_COUNT, + INSPECT_CHANNEL_CAP + ); + let trace_snapshot = trace::debug_snapshot(); + log::info!( + "[{}] Trace config (enabled={}, sid_filters={}, flow_filters={})", + worker_name, + trace::enabled(), + trace::has_sid_filters(), + trace::has_flow_filters() + ); + log::info!("[{}] Trace snapshot: {}", worker_name, trace_snapshot); + write_trace_bootstrap_event(&worker_name, &trace_snapshot); + log::debug!("[{}] rule_paths={:?}", worker_name, ids_config.rule_paths); + + let key = 0u32.to_ne_bytes(); + for (idx, guard) in skel_guards.iter().enumerate() { + match guard.maps.ids_export_config_map.lookup(&key, MapFlags::ANY) { + Ok(Some(value)) if value.len() >= 8 => { + let enabled = u32::from_ne_bytes([value[0], value[1], value[2], value[3]]); + let snaplen = u32::from_ne_bytes([value[4], value[5], value[6], value[7]]); + log::debug!( + "[{}] skel[{}] ids_export_config_map => enabled={}, snaplen={}", + worker_name, + idx, + enabled, + snaplen + ); + } + Ok(Some(value)) => { + log::warn!( + "[{}] skel[{}] ids_export_config_map has unexpected value length: {}", + worker_name, + idx, + value.len() + ); + } + Ok(None) => { + log::warn!( + "[{}] skel[{}] ids_export_config_map missing key=0 entry", + worker_name, + idx + ); + } + Err(e) => { + log::warn!( + "[{}] skel[{}] failed reading ids_export_config_map: {}", + worker_name, + idx, + e + ); + } + } + } + drop(skel_guards); + + // --- Main polling loop: fast ring buffer drain + stats/cleanup --- + loop { + if *shutdown.borrow() { + break; + } + + batch_counter.store(0, Ordering::Relaxed); + match ringbuf.poll(poll_timeout) { + Ok(()) => {} + Err(e) => { + if batch_counter.load(Ordering::Relaxed) >= RINGBUF_BATCH_LIMIT { + // Batch limit reached — yield to the main loop for stats/cleanup. + } else { + log::warn!("[{}] Ring buffer poll error: {}", worker_name, e); + } + } + } + + if last_cleanup.elapsed() >= cleanup_interval { + engine.cleanup_flows(); + last_cleanup = Instant::now(); + } + + if last_stats_log.elapsed() >= stats_log_interval { + let chan_drop_count = channel_drops.swap(0, Ordering::Relaxed); + if chan_drop_count > 0 { + log::warn!( + "[{}] Inspect channel dropped {} events (full)", + worker_name, + chan_drop_count + ); + } + log_ids_stats( + &worker_name, + &stats, + &mut last_stats_snapshot, + &mut last_stats_ts, + &mut last_stats_log, + ); + } + } + + // Drop the sender to signal the inspect threads to exit, then wait. + drop(shard_senders); + for handle in inspect_handles { + let _ = handle.join(); + } + log_ids_shutdown_stats(&worker_name, &stats); +} + +#[cfg(all(feature = "bpf", not(feature = "disable-bpf"), target_os = "linux"))] +fn init_ids_runtime( + worker_name: &str, + ids_config: &IdsConfig, + firewall_backend: FirewallBackend, + skels: Vec>>>, + nftables_firewall: Option>>, + iptables_firewall: Option>>, +) -> Option<( + Arc, + Arc, + BlockEnforcer, + Arc, + usize, +)> { + if ids_config.rule_paths.is_empty() { + log::warn!( + "[{}] No IDS rule paths configured, IDS worker will not start", + worker_name + ); + return None; + } + + let vars = RuleVars { + address_vars: ids_config.address_vars.clone(), + port_vars: ids_config.port_vars.clone(), + }; + + let ruleset = match RuleSet::load_from_paths_with_vars(&ids_config.rule_paths, &vars) { + Ok(ruleset) => ruleset, + Err(e) => { + log::error!("[{}] Failed to load IDS rules: {}", worker_name, e); + return None; + } + }; + + let rules_count = ruleset.rule_count(); + let ruleset = Arc::new(ruleset); + let engine = Arc::new(SignatureEngine::new( + ruleset, + ids_config.flow_timeout_secs, + ids_config.max_flows, + )); + let decoder = Arc::new(PacketDecoder::new()); + let enforcer = BlockEnforcer { + firewall_backend, + skels, + nftables_firewall, + iptables_firewall, + blocked_cache: Arc::new(Mutex::new(HashSet::new())), + }; + let stats = Arc::new(IdsWorkerStats::default()); + + Some((engine, decoder, enforcer, stats, rules_count)) +} + +#[cfg(all(feature = "bpf", not(feature = "disable-bpf"), target_os = "linux"))] +fn select_inspect_shard(packet: &[u8], shard_count: usize) -> usize { + if shard_count <= 1 { + return 0; + } + + let mut hasher = DefaultHasher::new(); + if let Some(flow_key) = packet_flow_shard_key(packet) { + flow_key.hash(&mut hasher); + } else { + packet[..packet.len().min(64)].hash(&mut hasher); + } + (hasher.finish() as usize) % shard_count +} + +#[cfg(all(feature = "bpf", not(feature = "disable-bpf"), target_os = "linux"))] +fn packet_flow_shard_key(packet: &[u8]) -> Option { + let (ethertype, l3_offset) = parse_ethertype(packet)?; + match ethertype { + 0x0800 => parse_ipv4_flow_shard_key(&packet[l3_offset..]), + 0x86DD => parse_ipv6_flow_shard_key(&packet[l3_offset..]), + _ => None, + } +} + +#[cfg(all(feature = "bpf", not(feature = "disable-bpf"), target_os = "linux"))] +fn parse_ethertype(packet: &[u8]) -> Option<(u16, usize)> { + if packet.len() < 14 { + return None; + } + + let mut ethertype = u16::from_be_bytes([packet[12], packet[13]]); + let mut offset = 14; + + while matches!(ethertype, 0x8100 | 0x88A8 | 0x9100) { + if packet.len() < offset + 4 { + return None; + } + ethertype = u16::from_be_bytes([packet[offset + 2], packet[offset + 3]]); + offset += 4; + } + + Some((ethertype, offset)) +} + +#[cfg(all(feature = "bpf", not(feature = "disable-bpf"), target_os = "linux"))] +fn parse_ipv4_flow_shard_key(data: &[u8]) -> Option { + if data.len() < 20 || (data[0] >> 4) != 4 { + return None; + } + + let header_len = ((data[0] & 0x0f) as usize) * 4; + if header_len < 20 || data.len() < header_len { + return None; + } + + let protocol = data[9]; + let src_ip = IpAddr::V4(std::net::Ipv4Addr::new( + data[12], data[13], data[14], data[15], + )); + let dst_ip = IpAddr::V4(std::net::Ipv4Addr::new( + data[16], data[17], data[18], data[19], + )); + let (src_port, dst_port) = transport_ports(protocol, &data[header_len..]); + + Some(EngineFlowKey::new( + src_ip, dst_ip, src_port, dst_port, protocol, + )) +} + +#[cfg(all(feature = "bpf", not(feature = "disable-bpf"), target_os = "linux"))] +fn parse_ipv6_flow_shard_key(data: &[u8]) -> Option { + if data.len() < 40 || (data[0] >> 4) != 6 { + return None; + } + + let protocol = data[6]; + let src_ip = IpAddr::V6(std::net::Ipv6Addr::from( + <[u8; 16]>::try_from(&data[8..24]).ok()?, + )); + let dst_ip = IpAddr::V6(std::net::Ipv6Addr::from( + <[u8; 16]>::try_from(&data[24..40]).ok()?, + )); + let (src_port, dst_port) = transport_ports(protocol, &data[40..]); + + Some(EngineFlowKey::new( + src_ip, dst_ip, src_port, dst_port, protocol, + )) +} + +#[cfg(all(feature = "bpf", not(feature = "disable-bpf"), target_os = "linux"))] +fn transport_ports(protocol: u8, data: &[u8]) -> (u16, u16) { + match protocol { + 6 | 17 => { + if data.len() < 4 { + return (0, 0); + } + ( + u16::from_be_bytes([data[0], data[1]]), + u16::from_be_bytes([data[2], data[3]]), + ) + } + _ => (0, 0), + } +} + +#[cfg(all(feature = "bpf", not(feature = "disable-bpf"), target_os = "linux"))] +fn log_ids_stats( + worker_name: &str, + stats: &IdsWorkerStats, + last_stats_snapshot: &mut IdsStatsSnapshot, + last_stats_ts: &mut Instant, + last_stats_log: &mut Instant, +) { + let now = Instant::now(); + let elapsed = now.duration_since(*last_stats_ts).as_secs_f64().max(1e-6); + let snapshot = stats.snapshot(); + + let delta_events = snapshot + .events_seen + .saturating_sub(last_stats_snapshot.events_seen); + let delta_decoded = snapshot + .packets_decoded + .saturating_sub(last_stats_snapshot.packets_decoded); + let delta_alerts = snapshot.alerts.saturating_sub(last_stats_snapshot.alerts); + let delta_blocks = snapshot + .blocks_applied + .saturating_sub(last_stats_snapshot.blocks_applied); + let delta_decode_errors = snapshot + .decode_errors + .saturating_sub(last_stats_snapshot.decode_errors); + let delta_malformed = snapshot + .malformed_events + .saturating_sub(last_stats_snapshot.malformed_events); + let delta_latency_samples = snapshot + .latency_samples + .saturating_sub(last_stats_snapshot.latency_samples); + let delta_latency_total_ns = snapshot + .latency_total_ns + .saturating_sub(last_stats_snapshot.latency_total_ns); + let delta_latency_hist = + subtract_hist(&snapshot.latency_hist, &last_stats_snapshot.latency_hist); + let latency = snapshot.latency_summary(); + let delta_latency_avg_us = if delta_latency_samples > 0 { + delta_latency_total_ns as f64 / delta_latency_samples as f64 / 1_000.0 + } else { + 0.0 + }; + let delta_latency_p90_us = + percentile_from_histogram_us(&delta_latency_hist, delta_latency_samples, 0.90); + let delta_latency_p99_us = + percentile_from_histogram_us(&delta_latency_hist, delta_latency_samples, 0.99); + let delta_latency_p999_us = + percentile_from_histogram_us(&delta_latency_hist, delta_latency_samples, 0.999); + log::info!( + "[{}] Stats (events={}, decoded={}, alerts={}, blocked={}, decode_errors={}, malformed={}, eps={:.0}, dps={:.0}, aps={:.0}, latency_avg_us={:.1}, latency_p90_us={}, latency_p99_us={}, latency_p999_us={}, latency_max_us={}, latency_window_avg_us={:.1}, latency_window_p90_us={}, latency_window_p99_us={}, latency_window_p999_us={})", + worker_name, + snapshot.events_seen, + snapshot.packets_decoded, + snapshot.alerts, + snapshot.blocks_applied, + snapshot.decode_errors, + snapshot.malformed_events, + delta_events as f64 / elapsed, + delta_decoded as f64 / elapsed, + delta_alerts as f64 / elapsed, + latency.avg_us, + latency.p90_us, + latency.p99_us, + latency.p999_us, + latency.max_us, + delta_latency_avg_us, + delta_latency_p90_us, + delta_latency_p99_us, + delta_latency_p999_us + ); + log::debug!( + "[{}] Stats delta (events={}, decoded={}, alerts={}, blocked={}, decode_errors={}, malformed={}, latency_samples={}, latency_avg_us={:.1}, latency_p90_us={}, latency_p99_us={})", + worker_name, + delta_events, + delta_decoded, + delta_alerts, + delta_blocks, + delta_decode_errors, + delta_malformed, + delta_latency_samples, + delta_latency_avg_us, + delta_latency_p90_us, + delta_latency_p99_us + ); + + *last_stats_log = now; + *last_stats_ts = now; + *last_stats_snapshot = snapshot; +} + +#[cfg(all(feature = "bpf", not(feature = "disable-bpf"), target_os = "linux"))] +fn log_ids_shutdown_stats(worker_name: &str, stats: &IdsWorkerStats) { + let final_snapshot = stats.snapshot(); + let final_latency = final_snapshot.latency_summary(); + log::info!( + "[{}] Stopped (events={}, decoded={}, alerts={}, blocked={}, decode_errors={}, malformed={}, latency_samples={}, latency_avg_us={:.1}, latency_min_us={}, latency_p90_us={}, latency_p99_us={}, latency_p999_us={}, latency_max_us={})", + worker_name, + final_snapshot.events_seen, + final_snapshot.packets_decoded, + final_snapshot.alerts, + final_snapshot.blocks_applied, + final_snapshot.decode_errors, + final_snapshot.malformed_events, + final_latency.samples, + final_latency.avg_us, + final_latency.min_us, + final_latency.p90_us, + final_latency.p99_us, + final_latency.p999_us, + final_latency.max_us + ); +} + +#[cfg(all(feature = "bpf", not(feature = "disable-bpf"), target_os = "linux"))] +pub fn configure_ids_export_map_for_skel( + skel: &mut crate::bpf::XdpSkel<'static>, + ids_config: &IdsConfig, +) -> Result<(), Box> { + let key = 0u32.to_ne_bytes(); + let enabled = if ids_config.enabled { 1u32 } else { 0u32 }; + let snaplen = ids_config.snaplen.clamp(1, IDS_EXPORT_MAX_BYTES as u32); + + let mut value = [0u8; 8]; + value[..4].copy_from_slice(&enabled.to_ne_bytes()); + value[4..].copy_from_slice(&snaplen.to_ne_bytes()); + + skel.maps + .ids_export_config_map + .update(&key, &value, MapFlags::ANY)?; + + Ok(()) +} + +#[cfg(all(feature = "bpf", not(feature = "disable-bpf"), target_os = "linux"))] +#[allow(dead_code)] +fn handle_ringbuf_event( + data: &[u8], + decoder: &PacketDecoder, + engine: &SignatureEngine, + enforcer: &BlockEnforcer, + enforce_block: bool, + stats: &IdsWorkerStats, +) -> i32 { + let mut raw = IdsExportEventRaw::default(); + if plain::copy_from_bytes(&mut raw, data).is_err() { + stats.malformed_events.fetch_add(1, Ordering::Relaxed); + return 0; + } + + let captured_len = raw.captured_len as usize; + if captured_len == 0 || captured_len > IDS_EXPORT_MAX_BYTES { + stats.malformed_events.fetch_add(1, Ordering::Relaxed); + return 0; + } + + if let Some(now_ns) = monotonic_time_ns() + && let Some(latency_ns) = now_ns.checked_sub(raw.ts_ns) + { + stats.record_latency_ns(latency_ns); + } + + let _ = inspect_packet( + &raw.packet[..captured_len], + IdsPacketMeta { + ifindex: raw.ifindex, + packet_len: raw.packet_len, + captured_len: raw.captured_len, + }, + decoder, + engine, + enforcer, + enforce_block, + stats, + false, + None, + ); + + 0 +} + +#[cfg(all(feature = "bpf", not(feature = "disable-bpf"), target_os = "linux"))] +fn inspect_packet( + packet: &[u8], + meta: IdsPacketMeta, + decoder: &PacketDecoder, + engine: &SignatureEngine, + enforcer: &BlockEnforcer, + enforce_block: bool, + stats: &IdsWorkerStats, + inline_verdict: bool, + shard_id: Option, +) -> PacketVerdict { + stats.events_seen.fetch_add(1, Ordering::Relaxed); + + let decoded = match decoder.decode(packet) { + Ok(decoded) => decoded, + Err(_) => { + stats.decode_errors.fetch_add(1, Ordering::Relaxed); + return PacketVerdict::Accept; + } + }; + stats.packets_decoded.fetch_add(1, Ordering::Relaxed); + + let trace_flow = if let Some(ip) = decoded.ip.as_ref() { + trace::should_trace_flow( + ip.src, + decoded.src_port().unwrap_or(0), + ip.dst, + decoded.dst_port().unwrap_or(0), + ) + } else { + false + }; + + let alerts = engine.inspect_all(&decoded); + let mut collector_emit_count = 0usize; + let mut blocked = false; + let mut verdict = PacketVerdict::Accept; + + if let Some(verdict_alert) = alerts.first() { + stats + .alerts + .fetch_add(alerts.len() as u64, Ordering::Relaxed); + + let should_block = enforce_block && should_block_action(&verdict_alert.action); + let immediate_drop = inline_verdict && should_block; + let persistent_block = if should_block && is_block_candidate(verdict_alert.block_ip()) { + enforcer.block_ip(verdict_alert.block_ip()) + } else { + false + }; + blocked = immediate_drop || persistent_block; + + if blocked { + stats.blocks_applied.fetch_add(1, Ordering::Relaxed); + } + + for alert in &alerts { + collector_emit_count += 1; + send_event(UnifiedEvent::IdsAlert(alert_to_event( + alert, + meta.ifindex, + meta.packet_len, + meta.captured_len, + blocked, + ))); + } + + if immediate_drop { + verdict = PacketVerdict::Drop; + } + } + + let trace_sid = trace::has_sid_filters() + && alerts + .iter() + .any(|alert| trace::should_trace_sid(alert.signature_id)); + if trace_flow || trace_sid { + let mut hasher = DefaultHasher::new(); + packet[..packet.len().min(64)].hash(&mut hasher); + let packet_hash = hasher.finish(); + let alert_sids: Vec = alerts.iter().map(|alert| alert.signature_id).collect(); + trace::write_ndjson( + "trace_worker.ndjson", + &json!({ + "shard_id": shard_id, + "packet_hash": packet_hash, + "decode_ok": true, + "alerts_count": alerts.len(), + "alert_sids": alert_sids, + "verdict_sid": alerts.first().map(|a| a.signature_id), + "blocked": blocked, + "collector_emit_count": collector_emit_count, + "verdict": match verdict { + PacketVerdict::Drop => "drop", + _ => "accept", + }, + }), + ); + } + + verdict +} + +#[cfg(all(feature = "bpf", not(feature = "disable-bpf"), target_os = "linux"))] +fn should_block_action(action: &str) -> bool { + matches!( + action.to_ascii_lowercase().as_str(), + "block" | "drop" | "reject" + ) +} + +#[cfg(all(feature = "bpf", not(feature = "disable-bpf"), target_os = "linux"))] +fn is_block_candidate(ip: IpAddr) -> bool { + match ip { + IpAddr::V4(v4) => !v4.is_unspecified(), + IpAddr::V6(v6) => !v6.is_unspecified(), + } +} + +#[cfg(all(feature = "bpf", not(feature = "disable-bpf"), target_os = "linux"))] +fn alert_to_event( + alert: &Alert, + ifindex: u32, + packet_len: u32, + captured_len: u32, + blocked: bool, +) -> IdsAlertEvent { + IdsAlertEvent { + timestamp: alert.timestamp, + signature_id: alert.signature_id, + signature_name: alert.signature_name.clone(), + severity: alert.severity, + category: alert.category.clone(), + src_ip: alert.src_ip.to_string(), + dst_ip: alert.dst_ip.to_string(), + src_port: alert.src_port, + dst_port: alert.dst_port, + protocol: alert.protocol.clone(), + action: alert.action.clone(), + flow_id: alert.flow_id, + ifindex, + packet_len, + captured_len, + blocked, + ja4t: alert.fingerprints.ja4t.clone(), + ja4ts: alert.fingerprints.ja4ts.clone(), + ja4h: alert.fingerprints.ja4h.clone(), + ja4d: alert.fingerprints.ja4d.clone(), + ja4d6: alert.fingerprints.ja4d6.clone(), + ja4: alert.fingerprints.ja4.clone(), + ja4ssh: alert.fingerprints.ja4ssh.clone(), + ja4l: alert.fingerprints.ja4l.clone(), + } +} + +#[cfg(all( + test, + feature = "bpf", + not(feature = "disable-bpf"), + target_os = "linux" +))] +mod tests { + use super::select_inspect_shard; + + fn build_ipv4_tcp_packet(src: [u8; 4], dst: [u8; 4], src_port: u16, dst_port: u16) -> Vec { + let mut packet = vec![0u8; 14 + 20 + 20]; + packet[12] = 0x08; + packet[13] = 0x00; + + let ip = &mut packet[14..34]; + ip[0] = 0x45; + ip[2..4].copy_from_slice(&(40u16).to_be_bytes()); + ip[8] = 64; + ip[9] = 6; + ip[12..16].copy_from_slice(&src); + ip[16..20].copy_from_slice(&dst); + + let tcp = &mut packet[34..54]; + tcp[0..2].copy_from_slice(&src_port.to_be_bytes()); + tcp[2..4].copy_from_slice(&dst_port.to_be_bytes()); + tcp[12] = 0x50; + + packet + } + + #[test] + fn same_flow_maps_to_same_shard_bidirectionally() { + let forward = build_ipv4_tcp_packet([10, 0, 0, 10], [203, 0, 113, 7], 42424, 6667); + let reverse = build_ipv4_tcp_packet([203, 0, 113, 7], [10, 0, 0, 10], 6667, 42424); + + let forward_shard = select_inspect_shard(&forward, 8); + let reverse_shard = select_inspect_shard(&reverse, 8); + + assert_eq!(forward_shard, reverse_shard); + } +} diff --git a/tests/e2e/bpf_test.rs b/tests/e2e/bpf_test.rs index 6933f5d3..d9faea78 100644 --- a/tests/e2e/bpf_test.rs +++ b/tests/e2e/bpf_test.rs @@ -1,7 +1,8 @@ //! eBPF/XDP E2E tests //! //! These tests verify BPF availability and basic XDP functionality. -//! Tests requiring root privileges will skip gracefully when run as non-root. +//! They require root privileges and kernel BPF support. +//! Run with: sudo cargo test -- --ignored use serial_test::serial; use std::path::Path; @@ -52,6 +53,7 @@ fn test_bpftool_available() { /// Check kernel BPF config (requires root) #[test] +#[ignore] fn test_kernel_bpf_config() { if !is_root() { eprintln!("Skipping: requires root"); @@ -120,6 +122,7 @@ fn test_kernel_bpf_config() { /// List loaded XDP programs (requires root) #[test] +#[ignore] #[serial] fn test_list_xdp_programs() { if !is_root() { @@ -172,6 +175,7 @@ fn test_list_xdp_programs() { /// Test XDP attachment to loopback (requires root) #[test] +#[ignore] #[serial] fn test_xdp_attach_loopback() { if !is_root() { @@ -199,6 +203,7 @@ fn test_xdp_attach_loopback() { /// List BPF maps (requires root) #[test] +#[ignore] #[serial] fn test_list_bpf_maps() { if !is_root() { @@ -234,6 +239,7 @@ fn test_list_bpf_maps() { /// Test BPF syscall availability (requires root) #[test] +#[ignore] fn test_bpf_syscall() { if !is_root() { eprintln!("Skipping: requires root"); @@ -271,6 +277,7 @@ fn test_bpf_syscall() { /// Check network interfaces for XDP support (requires root) #[test] +#[ignore] #[serial] fn test_interface_xdp_support() { if !is_root() { @@ -338,166 +345,6 @@ fn test_btf_availability() { } } -/// Regression test: BPF verifier must accept XDP program within 1M instruction limit. -/// -/// Previously, `__always_inline` on loop-heavy TCP fingerprinting functions caused -/// verifier state explosion exceeding the 1,000,000 instruction limit. The fix uses -/// `__noinline` subprograms and bounded loop iterations. -/// -/// This test compiles xdp.bpf.c with the same flags as build.rs and verifies the -/// kernel BPF verifier accepts the program via `bpftool prog load`. -#[test] -#[serial] -fn test_xdp_verifier_instruction_limit() { - if !is_root() { - eprintln!("Skipping: requires root"); - return; - } - - let manifest_dir = env!("CARGO_MANIFEST_DIR"); - let bpf_dir = format!("{}/src/security/firewall/bpf", manifest_dir); - let xdp_src = format!("{}/xdp.bpf.c", bpf_dir); - let include_dir = format!("{}/include", bpf_dir); - let lib_dir = format!("{}/lib", bpf_dir); - - // Find vmlinux.h — use the same crate path as build.rs - let vmlinux_dir = find_vmlinux_include(); - if vmlinux_dir.is_none() { - eprintln!("Skipping: vmlinux.h not found"); - return; - } - let vmlinux_dir = vmlinux_dir.unwrap(); - - // Compile BPF object with same flags as build.rs - let tmp_dir = std::env::temp_dir(); - let bpf_obj = tmp_dir.join("test_xdp_verifier.bpf.o"); - - let compile = Command::new("clang") - .args([ - "-target", - "bpf", - "-g", - "-O3", - "-fno-unroll-loops", - "-Wall", - "-Wextra", - "-DBPF_NO_PRESERVE_ACCESS_INDEX", - "-Ubpf", - &format!("-I{}", vmlinux_dir), - &format!("-I{}", bpf_dir), - &format!("-I{}", include_dir), - &format!("-I{}", lib_dir), - "-c", - &xdp_src, - "-o", - bpf_obj.to_str().unwrap(), - ]) - .output(); - - match compile { - Ok(output) if output.status.success() => { - println!("BPF object compiled successfully"); - } - Ok(output) => { - let stderr = String::from_utf8_lossy(&output.stderr); - panic!("Failed to compile BPF object:\n{}", stderr); - } - Err(e) => { - eprintln!("Skipping: clang not available: {}", e); - return; - } - } - - // Use bpftool to load the program and verify it passes the kernel verifier. - // Pin to /sys/fs/bpf/test_xdp_verifier so we can clean up after. - let pin_path = "/sys/fs/bpf/test_xdp_verifier"; - - // Clean up any stale pin from a previous failed run - let _ = std::fs::remove_file(pin_path); - - let load = Command::new("bpftool") - .args([ - "prog", - "load", - bpf_obj.to_str().unwrap(), - pin_path, - "type", - "xdp", - ]) - .output(); - - // Clean up compiled object - let _ = std::fs::remove_file(&bpf_obj); - - match load { - Ok(output) => { - // Always clean up the pinned program - let _ = std::fs::remove_file(pin_path); - - let stderr = String::from_utf8_lossy(&output.stderr); - if !output.status.success() { - if stderr.contains("too large") || stderr.contains("1000001") { - panic!( - "REGRESSION: BPF verifier rejected program — exceeded 1M instruction limit!\n\ - This means tcp_fingerprinting.h functions may need __noinline or loop bounds reduced.\n\ - Verifier output:\n{}", - stderr - ); - } - panic!( - "bpftool prog load failed (not instruction limit):\n{}", - stderr - ); - } - - println!("XDP program passed kernel BPF verifier (within 1M instruction limit)"); - } - Err(e) => { - let _ = std::fs::remove_file(pin_path); - eprintln!("Skipping: bpftool not available: {}", e); - } - } -} - -/// Find vmlinux.h include directory for BPF compilation -fn find_vmlinux_include() -> Option { - // Check the cargo git checkout (same location as the vmlinux crate used in build.rs) - let home = std::env::var("HOME").unwrap_or_else(|_| "/root".to_string()); - let cargo_git = format!("{}/.cargo/git/checkouts", home); - - if let Ok(entries) = std::fs::read_dir(&cargo_git) { - for entry in entries.flatten() { - let name = entry.file_name().to_string_lossy().to_string(); - if name.starts_with("vmlinux") { - // Look for the x86_64 directory inside - if let Ok(refs) = std::fs::read_dir(entry.path()) { - for ref_entry in refs.flatten() { - let candidate = ref_entry.path().join("include").join("x86_64"); - if candidate.join("vmlinux.h").exists() { - return Some(candidate.to_string_lossy().to_string()); - } - // Also try x86 - let candidate = ref_entry.path().join("include").join("x86"); - if candidate.join("vmlinux.h").exists() { - return Some(candidate.to_string_lossy().to_string()); - } - } - } - } - } - } - - // Fallback: check common system locations - let fallbacks = ["/usr/include/vmlinux", "/usr/local/include/vmlinux"]; - for path in &fallbacks { - if Path::new(path).join("vmlinux.h").exists() { - return Some(path.to_string()); - } - } - - None -} - /// Check BPF filesystem mount #[test] fn test_bpf_filesystem() { diff --git a/tests/e2e/firewall_test.rs b/tests/e2e/firewall_test.rs index daa7d0f6..15985207 100644 --- a/tests/e2e/firewall_test.rs +++ b/tests/e2e/firewall_test.rs @@ -1,7 +1,8 @@ //! Firewall E2E tests (nftables/iptables) //! //! These tests verify firewall functionality through system commands. -//! Tests requiring root privileges will skip gracefully when run as non-root. +//! They require root privileges and are marked with #[ignore]. +//! Run with: sudo cargo test -- --ignored use serial_test::serial; use std::process::Command; @@ -64,6 +65,7 @@ fn test_ip6tables_available() { /// Test nftables table creation (requires root) #[test] +#[ignore] #[serial] fn test_nftables_create_table() { if !is_root() { @@ -123,6 +125,7 @@ fn test_nftables_create_table() { /// Test nftables set creation (requires root) #[test] +#[ignore] #[serial] fn test_nftables_create_set() { if !is_root() { @@ -210,6 +213,7 @@ fn test_nftables_create_set() { /// Test iptables chain creation (requires root) #[test] +#[ignore] #[serial] fn test_iptables_create_chain() { if !is_root() { @@ -268,6 +272,7 @@ fn test_iptables_create_chain() { /// Test iptables rule addition (requires root) #[test] +#[ignore] #[serial] fn test_iptables_add_rule() { if !is_root() { @@ -330,6 +335,7 @@ fn test_iptables_add_rule() { /// Test nftables CIDR support (requires root) #[test] +#[ignore] #[serial] fn test_nftables_cidr() { if !is_root() { @@ -413,6 +419,7 @@ fn test_nftables_cidr() { /// Test nftables IPv6 support (requires root) #[test] +#[ignore] #[serial] fn test_nftables_ipv6() { if !is_root() { @@ -496,6 +503,7 @@ fn test_nftables_ipv6() { /// Test firewall cleanup (requires root) #[test] +#[ignore] #[serial] fn test_firewall_cleanup() { if !is_root() {