From df424faf08c31a14cbc78b5895470aea8a8794dd Mon Sep 17 00:00:00 2001 From: David Young Date: Sun, 30 Mar 2025 15:09:13 +0100 Subject: [PATCH 1/6] Create axum test server --- Cargo.lock | 385 ++++++++++++++++++++++++++++++++++++++++++++- Cargo.toml | 4 + src/main.rs | 7 +- src/test_server.rs | 89 +++++++++++ 4 files changed, 482 insertions(+), 3 deletions(-) create mode 100644 src/test_server.rs diff --git a/Cargo.lock b/Cargo.lock index 9d5a595..9bce174 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,15 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "addr2line" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" +dependencies = [ + "gimli", +] + [[package]] name = "adler2" version = "2.0.0" @@ -105,6 +114,75 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" +[[package]] +name = "axum" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de45108900e1f9b9242f7f2e254aa3e2c029c921c258fe9e6b4217eeebd54288" +dependencies = [ + "axum-core", + "bytes", + "form_urlencoded", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68464cd0412f486726fb3373129ef5d2993f90c34bc2bc1c1e9943b2f4fc7ca6" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "rustversion", + "sync_wrapper", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "backtrace" +version = "0.3.74" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a" +dependencies = [ + "addr2line", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", + "windows-targets", +] + [[package]] name = "bit-set" version = "0.5.3" @@ -137,6 +215,12 @@ dependencies = [ "serde", ] +[[package]] +name = "bytes" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" + [[package]] name = "cc" version = "1.2.16" @@ -391,14 +475,17 @@ name = "dirble" version = "1.4.2" dependencies = [ "atty", + "axum", "chardet", "clap", "colored", "ctrlc", "curl", "encoding", + "http", "log", "percent-encoding", + "phf 0.11.3", "pretty_assertions", "rand 0.9.0", "select", @@ -410,6 +497,7 @@ dependencies = [ "simplelog", "tempfile", "time", + "tokio", "url", "vergen-gix", ] @@ -573,6 +661,39 @@ dependencies = [ "new_debug_unreachable", ] +[[package]] +name = "futures-channel" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" + +[[package]] +name = "futures-task" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" + +[[package]] +name = "futures-util" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "pin-utils", +] + [[package]] name = "getrandom" version = "0.2.15" @@ -596,6 +717,12 @@ dependencies = [ "windows-targets", ] +[[package]] +name = "gimli" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" + [[package]] name = "gix" version = "0.69.1" @@ -1216,6 +1343,87 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "http" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4a85d31aea989eead29a3aaf9e1115a180df8282431156e533de47660892565" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc2b571658e38e0c01b1fdca3bbbe93c00d3d71693ff2770043f8c29bc7d6f80" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", +] + +[[package]] +name = "hyper-util" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df2dcfbe0677734ab2f3ffa7fa7bfd4706bfdc1ef393f2ee30184aed67e631b4" +dependencies = [ + "bytes", + "futures-util", + "http", + "http-body", + "hyper", + "pin-project-lite", + "tokio", + "tower-service", +] + [[package]] name = "icu_collections" version = "1.5.0" @@ -1478,7 +1686,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a2629bb1404f3d34c2e921f21fd34ba00b206124c81f65c50b43b6aaefeb016" dependencies = [ "log", - "phf", + "phf 0.10.1", "phf_codegen", "string_cache", "string_cache_codegen", @@ -1497,6 +1705,12 @@ dependencies = [ "xml5ever", ] +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + [[package]] name = "maybe-async" version = "0.2.10" @@ -1523,6 +1737,12 @@ dependencies = [ "libc", ] +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + [[package]] name = "miniz_oxide" version = "0.8.5" @@ -1532,6 +1752,17 @@ dependencies = [ "adler2", ] +[[package]] +name = "mio" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd" +dependencies = [ + "libc", + "wasi 0.11.0+wasi-snapshot-preview1", + "windows-sys 0.52.0", +] + [[package]] name = "new_debug_unreachable" version = "1.0.6" @@ -1574,6 +1805,15 @@ dependencies = [ "libc", ] +[[package]] +name = "object" +version = "0.36.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" +dependencies = [ + "memchr", +] + [[package]] name = "once_cell" version = "1.21.1" @@ -1636,6 +1876,16 @@ dependencies = [ "phf_shared 0.10.0", ] +[[package]] +name = "phf" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" +dependencies = [ + "phf_macros", + "phf_shared 0.11.3", +] + [[package]] name = "phf_codegen" version = "0.10.0" @@ -1666,6 +1916,19 @@ dependencies = [ "rand 0.8.5", ] +[[package]] +name = "phf_macros" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216" +dependencies = [ + "phf_generator 0.11.3", + "phf_shared 0.11.3", + "proc-macro2 1.0.94", + "quote 1.0.40", + "syn 2.0.100", +] + [[package]] name = "phf_shared" version = "0.10.0" @@ -1684,6 +1947,18 @@ dependencies = [ "siphasher 1.0.1", ] +[[package]] +name = "pin-project-lite" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + [[package]] name = "pkg-config" version = "0.3.32" @@ -1877,6 +2152,12 @@ version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" +[[package]] +name = "rustc-demangle" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" + [[package]] name = "rustix" version = "0.38.44" @@ -1982,6 +2263,16 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_path_to_error" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59fab13f937fa393d08645bf3a84bdfe86e296747b506ada67bb15f10f218b2a" +dependencies = [ + "itoa", + "serde", +] + [[package]] name = "serde_test" version = "1.0.177" @@ -1991,6 +2282,18 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + [[package]] name = "sha1_smol" version = "1.0.1" @@ -2153,6 +2456,12 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" + [[package]] name = "synstructure" version = "0.13.1" @@ -2289,6 +2598,80 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" +[[package]] +name = "tokio" +version = "1.44.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f382da615b842244d4b8738c82ed1275e6c5dd90c459a30941cd07080b06c91a" +dependencies = [ + "backtrace", + "libc", + "mio", + "pin-project-lite", + "socket2", + "tokio-macros", + "windows-sys 0.52.0", +] + +[[package]] +name = "tokio-macros" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" +dependencies = [ + "proc-macro2 1.0.94", + "quote 1.0.40", + "syn 2.0.100", +] + +[[package]] +name = "tower" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" +dependencies = [ + "log", + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c" +dependencies = [ + "once_cell", +] + [[package]] name = "unicode-bom" version = "2.0.3" diff --git a/Cargo.toml b/Cargo.toml index 17ee143..0e56c68 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,5 +35,9 @@ vergen-gix = { version = "1.0.6", features = ["build", "si"] } release_version_string = [] [dev-dependencies] +axum = "0.8.3" +http = "1.3.1" +phf = { version = "0.11.3", features = ["macros"] } pretty_assertions = "1.4.1" tempfile = "3.19.1" +tokio = { version = "1.44.1", features = ["net", "macros", "rt"] } diff --git a/src/main.rs b/src/main.rs index 2573979..0097c26 100644 --- a/src/main.rs +++ b/src/main.rs @@ -15,16 +15,16 @@ // You should have received a copy of the GNU General Public License // along with Dirble. If not, see . -use log::{LevelFilter, debug, error, info, warn}; +use log::{debug, error, info, warn, LevelFilter}; use simplelog::{ColorChoice, TermLogger, TerminalMode}; use std::{ collections::VecDeque, env::current_exe, path::Path, sync::{ - Arc, atomic::{AtomicBool, Ordering}, mpsc::{self, Receiver, Sender}, + Arc, }, thread, time::Duration, @@ -41,6 +41,9 @@ mod request_thread; mod validator_thread; mod wordlist; +#[cfg(test)] +mod test_server; + #[allow(clippy::cognitive_complexity)] fn main() { // Read the arguments in using the arg_parse module diff --git a/src/test_server.rs b/src/test_server.rs new file mode 100644 index 0000000..6d85554 --- /dev/null +++ b/src/test_server.rs @@ -0,0 +1,89 @@ +use axum::{extract::Path, Router}; +use http::StatusCode; +use tokio::net::TcpListener; + +pub const PATHS: phf::Map<&str, TestPath> = phf::phf_map! { + "ok" => TestPath { + code: StatusCode::OK, + length: 10, + }, + "201" => TestPath { + code: StatusCode::CREATED, + length: 11, + }, +}; + +pub struct TestPath { + pub code: StatusCode, + pub length: usize, +} + +pub fn launch() -> u16 { + let listener = std::net::TcpListener::bind("[::1]:0").unwrap(); + listener.set_nonblocking(true).unwrap(); + + let port = listener.local_addr().unwrap().port(); + let app = make_router(); + + std::thread::spawn(move || { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("Failed to create tokio runtime") + .block_on(async { + let listener = TcpListener::from_std(listener).unwrap(); + + axum::serve(listener, app).await.unwrap(); + }) + }); + + port +} + +fn make_router() -> Router { + use axum::routing::get; + + Router::new() + .route("/", get(|| async { "OK" })) + .route("/{*path}", get(get_test_path)) +} + +async fn get_test_path(Path(path): Path) -> (StatusCode, String) { + dbg!(&path); + let Some(params) = PATHS.get(&path) else { + return (StatusCode::NOT_FOUND, "Not found".into()); + }; + (params.code, "A".repeat(params.length)) +} + +mod test { + use super::*; + use curl::easy::Easy; + + #[test] + fn server_startup() { + let port = launch(); + + let mut easy = Easy::new(); + easy.url(&format!("http://localhost:{port}")).unwrap(); + easy.perform().unwrap(); + assert_eq!(easy.response_code().unwrap(), 200); + assert_eq!(easy.content_length_download().unwrap(), 2.0); + + easy.url(&format!("http://localhost:{port}/ok")).unwrap(); + easy.perform().unwrap(); + assert_eq!(easy.response_code().unwrap(), 200); + assert_eq!(easy.content_length_download().unwrap(), 10.0); + + easy.url(&format!("http://localhost:{port}/notfound")) + .unwrap(); + easy.perform().unwrap(); + assert_eq!(easy.response_code().unwrap(), 404); + assert_eq!(easy.content_length_download().unwrap(), 9.0); + + easy.url(&format!("http://localhost:{port}/201")).unwrap(); + easy.perform().unwrap(); + assert_eq!(easy.response_code().unwrap(), 201); + assert_eq!(easy.content_length_download().unwrap(), 11.0); + } +} From f1562e69b77c18f5859a112ee63af7d128e77b7d Mon Sep 17 00:00:00 2001 From: David Young Date: Sun, 30 Mar 2025 15:12:42 +0100 Subject: [PATCH 2/6] Refactor into main+lib --- src/lib.rs | 413 ++++++++++++++++++++++++++++++++++++++++++++++++++++ src/main.rs | 393 +------------------------------------------------ 2 files changed, 415 insertions(+), 391 deletions(-) create mode 100644 src/lib.rs diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..df24d82 --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,413 @@ +// This file is part of Dirble - https://www.github.com/nccgroup/dirble +// Copyright (C) 2019 Izzy Whistlecroft +// Released as open source by NCC Group Plc - https://www.nccgroup.com/ +// +// Dirble is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Dirble is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with Dirble. If not, see . + +use crate::arg_parse::GlobalOpts; +use log::{debug, error, info, warn, LevelFilter}; +use simplelog::{ColorChoice, TermLogger, TerminalMode}; +use std::{ + collections::VecDeque, + env::current_exe, + path::Path, + sync::{ + atomic::{AtomicBool, Ordering}, + mpsc::{self, Receiver, Sender}, + Arc, + }, + thread, + time::Duration, +}; +use url::Url; + +#[macro_use] +pub mod arg_parse; +mod content_parse; +mod output; +mod output_format; +mod output_thread; +mod request; +mod request_thread; +mod validator_thread; +mod wordlist; + +#[cfg(test)] +mod test_server; + +#[allow(clippy::cognitive_complexity)] +pub fn dirble_main(args: GlobalOpts) { + let global_opts = Arc::new(args); + + // Prepare the logging handler. Default to a pretty TermLogger, + // but if the TermLogger initialisation fails (e.g. if we are not + // connected to a TTY) then set up a SimpleLogger instead. + let log_config = simplelog::ConfigBuilder::new() + .set_time_level(LevelFilter::Debug) + .set_time_format_custom(time::macros::format_description!( + "[hour]:[minute]:[second]" + )) + .build(); + + // TermLogger::init() fails only if another Logger was initialised + TermLogger::init( + global_opts.log_level, + log_config, + TerminalMode::Mixed, + ColorChoice::Auto, + ) + .expect("Failed to init TermLogger"); + + // Get the wordlist file from the arguments. If it has not been set + // then try the default wordlist locations. + let mut wordlist: Vec = Vec::new(); + let wordlist_string: String; + if let Some(wordlist_files) = global_opts.wordlist_files.clone() { + // A wordlist has been set in the global opts + for wordlist_file in wordlist_files { + wordlist.append(&mut wordlist::lines_from_file(&wordlist_file)); + } + wordlist_string = "".into(); + } else { + // Otherwise try the directory containing the exe, then + // /usr/share/dirble, then /usr/share/wordlists, then finally + // /usr/share/wordlists/dirble before giving up. + let mut exe_path = current_exe().unwrap_or_else(|error| { + println!("Getting directory of exe failed: {}", error); + std::process::exit(2); + }); + exe_path.set_file_name("dirble_wordlist.txt"); + let usr_share_dirble = + Path::new("/usr/share/dirble/dirble_wordlist.txt"); + let usr_share_wordlists = + Path::new("/usr/share/wordlists/dirble_wordlist.txt"); + let usr_share_wordlists_dirble = + Path::new("/usr/share/wordlists/dirble/dirble_wordlist.txt"); + + debug!( + "Checking for wordlist in:\n - {}\n - {}\n - {}\n - {}", + exe_path.to_str().unwrap(), + usr_share_dirble.to_str().unwrap(), + usr_share_wordlists.to_str().unwrap(), + usr_share_wordlists_dirble.to_str().unwrap(), + ); + let wordlist_file = if exe_path.exists() { + // Prioritise the wordlist in the same directory as the exe + String::from(exe_path.to_str().unwrap()) + } else if usr_share_dirble.exists() { + String::from(usr_share_dirble.to_str().unwrap()) + } else if usr_share_wordlists.exists() { + String::from(usr_share_wordlists.to_str().unwrap()) + } else { + error!("Unable to find default wordlist"); + std::process::exit(1); + }; + wordlist.append(&mut wordlist::lines_from_file(&wordlist_file)); + wordlist_string = wordlist_file; + } + + if let Some(text) = + output::startup_text(global_opts.clone(), &wordlist_string) + { + println!("{}", text); + } + + // Remove leading and trailing slashes from words + for word in &mut wordlist { + if word.starts_with('/') { + word.remove(0); + } + + if word.ends_with('/') { + word.pop(); + } + } + + wordlist.sort(); + wordlist.dedup(); + + let wordlist = Arc::new(wordlist); + + // Create a channel for threads to communicate with the parent on + // This is used to send information about ending threads and + // information on responses + let (output_tx, output_rx): ( + Sender, + Receiver, + ) = mpsc::channel(); + let (to_validate_tx, to_validate_rx): ( + Sender, + Receiver, + ) = mpsc::channel(); + let (to_scan_tx, to_scan_rx): ( + Sender>, + Receiver>, + ) = mpsc::channel(); + + let validator_global_opts = global_opts.clone(); + let validator_thread = thread::spawn(|| { + validator_thread::validator_thread( + to_validate_rx, + to_scan_tx, + validator_global_opts, + ) + }); + + for (host_index, hostname) in global_opts.hostnames.iter().enumerate() { + let mut request = + request::fabricate_request_response(hostname.clone(), true, false); + let depth = hostname.path_segments().unwrap().count() as u32; + request.parent_index = host_index; + request.parent_depth = depth; + to_validate_tx.send(request).unwrap(); + } + + // Create a queue for URIs that need to be scanned + let mut scan_queue: VecDeque = VecDeque::new(); + + // Push the host URI to the scan queue + for _i in 0..global_opts.hostnames.len() { + let response = to_scan_rx.recv().unwrap(); + + match response { + None => continue, + Some(dir_info) => { + match &dir_info.validator { + Some(validator) => { + if validator.scan_folder(&global_opts.scan_opts) { + add_dir_to_scan_queue( + &mut scan_queue, + &global_opts, + &dir_info, + &wordlist, + true, + ); + } else { + info!( + "Skipping {}{}", + dir_info.url, + &validator.print_alert() + ) + } + } + // If there is no validator, then scan the folder + None => { + add_dir_to_scan_queue( + &mut scan_queue, + &global_opts, + &dir_info, + &wordlist, + true, + ); + } + } + } + } + } + // Define the max number of threads and the number of threads + // currently in use + let mut threads_in_use = 0; + + let file_handles = output::create_files(global_opts.clone()); + let output_global_opts = global_opts.clone(); + + let output_thread = thread::spawn(|| { + output_thread::output_thread( + output_rx, + output_global_opts, + file_handles, + ) + }); + + let caught_ctrl_c = Arc::new(AtomicBool::new(false)); + let caught_ctrl_c_clone_for_handler = caught_ctrl_c.clone(); + ctrlc::set_handler(move || { + warn!("Caught interrupt signal, cleaning up..."); + caught_ctrl_c_clone_for_handler.store(true, Ordering::SeqCst); + }) + .expect("Unable to attach interrupt signal handler"); + + // Loop of checking for messages from the threads, + // spawning new threads on items in the scan queue + // and checking if the program is done + while !caught_ctrl_c.load(Ordering::SeqCst) { + // Check for messages from the threads + let to_scan = to_scan_rx.try_recv(); + + // Ignore any errors - this happens if the message queue is + // empty, that's okay + if let Ok(Some(dir_info)) = to_scan { + // If a thread has sent end, then we can reduce the + // threads in use count + if dir_info.url.as_str() == "data:END" { + threads_in_use -= 1; + } + // Check the validator to see if the directory should + // be scanned + else { + match &dir_info.validator { + Some(validator) => { + if validator.scan_folder(&global_opts.scan_opts) { + add_dir_to_scan_queue( + &mut scan_queue, + &global_opts, + &dir_info, + &wordlist, + false, + ); + } else { + info!( + "Skipping {}{}", + dir_info.url, + &validator.print_alert() + ) + } + } + // If there is no validator, then scan the folder + None => { + add_dir_to_scan_queue( + &mut scan_queue, + &global_opts, + &dir_info, + &wordlist, + false, + ); + } + } + } + }; + + // If there are items in the scan queue and available threads + // Spawn a new thread to scan an item + if threads_in_use < global_opts.max_threads && !scan_queue.is_empty() { + // Clone a new sender to the channel and a new wordlist + // reference, then pop the scan target from the queue + let to_validate_tx_clone = mpsc::Sender::clone(&to_validate_tx); + let output_tx_clone = mpsc::Sender::clone(&output_tx); + let list_gen = scan_queue.pop_front().unwrap(); + let arg_clone = global_opts.clone(); + + // Spawn a thread with the arguments and increment the in + // use counter + thread::spawn(|| { + request_thread::thread_spawn( + to_validate_tx_clone, + output_tx_clone, + list_gen, + arg_clone, + ) + }); + threads_in_use += 1; + } + + // If there are no threads in use and the queue is empty then + // stop + if threads_in_use == 0 && scan_queue.is_empty() { + break; + } + + // Sleep to reduce CPU cycles used by main + thread::sleep(Duration::from_millis(1)); + } + + // loop to check that report printing has ended + output_tx.send(generate_end()).unwrap(); + to_validate_tx.send(generate_end()).unwrap(); + output_thread.join().unwrap(); + validator_thread.join().unwrap(); +} + +#[inline] +fn add_dir_to_scan_queue( + scan_queue: &mut VecDeque, + global_opts: &Arc, + dir_info: &validator_thread::DirectoryInfo, + wordlist: &Arc>, + first_run: bool, +) { + // first_run is true when the initial scans are being initialised + // on the base paths. We override the default wordlist_split to + // improve performance of the initial discovery phase. + let num_hosts = global_opts.hostnames.len() as u32; + let wordlist_split; + if first_run + && global_opts.max_threads >= 3 + && (global_opts.wordlist_split * num_hosts) + < (global_opts.max_threads - 2) + { + // If there's enough headroom to boost the split then do so + wordlist_split = (global_opts.max_threads - 2) / num_hosts; + info!( + "Increasing wordlist-split for initial scan of {} to {}", + dir_info.url, wordlist_split + ); + } else { + wordlist_split = global_opts.wordlist_split; + } + + for prefix in &global_opts.prefixes { + for extension in &global_opts.extensions { + for start_index in 0..wordlist_split { + scan_queue.push_back(wordlist::UriGenerator::new( + dir_info.url.clone(), + prefix.clone(), + extension.clone(), + wordlist.clone(), + start_index, + wordlist_split, + dir_info.parent_index, + dir_info.parent_depth, + dir_info.validator.clone(), + global_opts.extension_substitution, + )); + } + } + } +} + +fn generate_end() -> request::RequestResponse { + request::RequestResponse { + url: Url::parse("data:MAIN ENDING").unwrap(), + code: 0, + content_len: 0, + is_directory: false, + is_listable: false, + redirect_url: String::from(""), + found_from_listable: false, + parent_index: 0, + parent_depth: 0, + } +} + +#[cfg(test)] +mod test { + use crate::request::RequestResponse; + use url::Url; + + impl Default for RequestResponse { + fn default() -> Self { + RequestResponse { + url: Url::parse("http://example.com/").unwrap(), + code: 200, + content_len: 200, + is_directory: false, + is_listable: false, + redirect_url: "".into(), + found_from_listable: false, + parent_index: 0, + parent_depth: 0, + } + } + } +} diff --git a/src/main.rs b/src/main.rs index 0097c26..94da295 100644 --- a/src/main.rs +++ b/src/main.rs @@ -15,398 +15,9 @@ // You should have received a copy of the GNU General Public License // along with Dirble. If not, see . -use log::{debug, error, info, warn, LevelFilter}; -use simplelog::{ColorChoice, TermLogger, TerminalMode}; -use std::{ - collections::VecDeque, - env::current_exe, - path::Path, - sync::{ - atomic::{AtomicBool, Ordering}, - mpsc::{self, Receiver, Sender}, - Arc, - }, - thread, - time::Duration, -}; -use url::Url; -#[macro_use] -mod arg_parse; -mod content_parse; -mod output; -mod output_format; -mod output_thread; -mod request; -mod request_thread; -mod validator_thread; -mod wordlist; - -#[cfg(test)] -mod test_server; - #[allow(clippy::cognitive_complexity)] fn main() { // Read the arguments in using the arg_parse module - let global_opts = Arc::new(arg_parse::get_args(std::env::args_os())); - - // Prepare the logging handler. Default to a pretty TermLogger, - // but if the TermLogger initialisation fails (e.g. if we are not - // connected to a TTY) then set up a SimpleLogger instead. - let log_config = simplelog::ConfigBuilder::new() - .set_time_level(LevelFilter::Debug) - .set_time_format_custom(time::macros::format_description!( - "[hour]:[minute]:[second]" - )) - .build(); - - // TermLogger::init() fails only if another Logger was initialised - TermLogger::init( - global_opts.log_level, - log_config, - TerminalMode::Mixed, - ColorChoice::Auto, - ) - .expect("Failed to init TermLogger"); - - // Get the wordlist file from the arguments. If it has not been set - // then try the default wordlist locations. - let mut wordlist: Vec = Vec::new(); - let wordlist_string: String; - if let Some(wordlist_files) = global_opts.wordlist_files.clone() { - // A wordlist has been set in the global opts - for wordlist_file in wordlist_files { - wordlist.append(&mut wordlist::lines_from_file(&wordlist_file)); - } - wordlist_string = "".into(); - } else { - // Otherwise try the directory containing the exe, then - // /usr/share/dirble, then /usr/share/wordlists, then finally - // /usr/share/wordlists/dirble before giving up. - let mut exe_path = current_exe().unwrap_or_else(|error| { - println!("Getting directory of exe failed: {}", error); - std::process::exit(2); - }); - exe_path.set_file_name("dirble_wordlist.txt"); - let usr_share_dirble = - Path::new("/usr/share/dirble/dirble_wordlist.txt"); - let usr_share_wordlists = - Path::new("/usr/share/wordlists/dirble_wordlist.txt"); - let usr_share_wordlists_dirble = - Path::new("/usr/share/wordlists/dirble/dirble_wordlist.txt"); - - debug!( - "Checking for wordlist in:\n - {}\n - {}\n - {}\n - {}", - exe_path.to_str().unwrap(), - usr_share_dirble.to_str().unwrap(), - usr_share_wordlists.to_str().unwrap(), - usr_share_wordlists_dirble.to_str().unwrap(), - ); - let wordlist_file = if exe_path.exists() { - // Prioritise the wordlist in the same directory as the exe - String::from(exe_path.to_str().unwrap()) - } else if usr_share_dirble.exists() { - String::from(usr_share_dirble.to_str().unwrap()) - } else if usr_share_wordlists.exists() { - String::from(usr_share_wordlists.to_str().unwrap()) - } else { - error!("Unable to find default wordlist"); - std::process::exit(1); - }; - wordlist.append(&mut wordlist::lines_from_file(&wordlist_file)); - wordlist_string = wordlist_file; - } - - if let Some(text) = - output::startup_text(global_opts.clone(), &wordlist_string) - { - println!("{}", text); - } - - // Remove leading and trailing slashes from words - for word in &mut wordlist { - if word.starts_with('/') { - word.remove(0); - } - - if word.ends_with('/') { - word.pop(); - } - } - - wordlist.sort(); - wordlist.dedup(); - - let wordlist = Arc::new(wordlist); - - // Create a channel for threads to communicate with the parent on - // This is used to send information about ending threads and - // information on responses - let (output_tx, output_rx): ( - Sender, - Receiver, - ) = mpsc::channel(); - let (to_validate_tx, to_validate_rx): ( - Sender, - Receiver, - ) = mpsc::channel(); - let (to_scan_tx, to_scan_rx): ( - Sender>, - Receiver>, - ) = mpsc::channel(); - - let validator_global_opts = global_opts.clone(); - let validator_thread = thread::spawn(|| { - validator_thread::validator_thread( - to_validate_rx, - to_scan_tx, - validator_global_opts, - ) - }); - - for (host_index, hostname) in global_opts.hostnames.iter().enumerate() { - let mut request = - request::fabricate_request_response(hostname.clone(), true, false); - let depth = hostname.path_segments().unwrap().count() as u32; - request.parent_index = host_index; - request.parent_depth = depth; - to_validate_tx.send(request).unwrap(); - } - - // Create a queue for URIs that need to be scanned - let mut scan_queue: VecDeque = VecDeque::new(); - - // Push the host URI to the scan queue - for _i in 0..global_opts.hostnames.len() { - let response = to_scan_rx.recv().unwrap(); - - match response { - None => continue, - Some(dir_info) => { - match &dir_info.validator { - Some(validator) => { - if validator.scan_folder(&global_opts.scan_opts) { - add_dir_to_scan_queue( - &mut scan_queue, - &global_opts, - &dir_info, - &wordlist, - true, - ); - } else { - info!( - "Skipping {}{}", - dir_info.url, - &validator.print_alert() - ) - } - } - // If there is no validator, then scan the folder - None => { - add_dir_to_scan_queue( - &mut scan_queue, - &global_opts, - &dir_info, - &wordlist, - true, - ); - } - } - } - } - } - // Define the max number of threads and the number of threads - // currently in use - let mut threads_in_use = 0; - - let file_handles = output::create_files(global_opts.clone()); - let output_global_opts = global_opts.clone(); - - let output_thread = thread::spawn(|| { - output_thread::output_thread( - output_rx, - output_global_opts, - file_handles, - ) - }); - - let caught_ctrl_c = Arc::new(AtomicBool::new(false)); - let caught_ctrl_c_clone_for_handler = caught_ctrl_c.clone(); - ctrlc::set_handler(move || { - warn!("Caught interrupt signal, cleaning up..."); - caught_ctrl_c_clone_for_handler.store(true, Ordering::SeqCst); - }) - .expect("Unable to attach interrupt signal handler"); - - // Loop of checking for messages from the threads, - // spawning new threads on items in the scan queue - // and checking if the program is done - while !caught_ctrl_c.load(Ordering::SeqCst) { - // Check for messages from the threads - let to_scan = to_scan_rx.try_recv(); - - // Ignore any errors - this happens if the message queue is - // empty, that's okay - if let Ok(Some(dir_info)) = to_scan { - // If a thread has sent end, then we can reduce the - // threads in use count - if dir_info.url.as_str() == "data:END" { - threads_in_use -= 1; - } - // Check the validator to see if the directory should - // be scanned - else { - match &dir_info.validator { - Some(validator) => { - if validator.scan_folder(&global_opts.scan_opts) { - add_dir_to_scan_queue( - &mut scan_queue, - &global_opts, - &dir_info, - &wordlist, - false, - ); - } else { - info!( - "Skipping {}{}", - dir_info.url, - &validator.print_alert() - ) - } - } - // If there is no validator, then scan the folder - None => { - add_dir_to_scan_queue( - &mut scan_queue, - &global_opts, - &dir_info, - &wordlist, - false, - ); - } - } - } - }; - - // If there are items in the scan queue and available threads - // Spawn a new thread to scan an item - if threads_in_use < global_opts.max_threads && !scan_queue.is_empty() { - // Clone a new sender to the channel and a new wordlist - // reference, then pop the scan target from the queue - let to_validate_tx_clone = mpsc::Sender::clone(&to_validate_tx); - let output_tx_clone = mpsc::Sender::clone(&output_tx); - let list_gen = scan_queue.pop_front().unwrap(); - let arg_clone = global_opts.clone(); - - // Spawn a thread with the arguments and increment the in - // use counter - thread::spawn(|| { - request_thread::thread_spawn( - to_validate_tx_clone, - output_tx_clone, - list_gen, - arg_clone, - ) - }); - threads_in_use += 1; - } - - // If there are no threads in use and the queue is empty then - // stop - if threads_in_use == 0 && scan_queue.is_empty() { - break; - } - - // Sleep to reduce CPU cycles used by main - thread::sleep(Duration::from_millis(1)); - } - - // loop to check that report printing has ended - output_tx.send(generate_end()).unwrap(); - to_validate_tx.send(generate_end()).unwrap(); - output_thread.join().unwrap(); - validator_thread.join().unwrap(); -} - -#[inline] -fn add_dir_to_scan_queue( - scan_queue: &mut VecDeque, - global_opts: &Arc, - dir_info: &validator_thread::DirectoryInfo, - wordlist: &Arc>, - first_run: bool, -) { - // first_run is true when the initial scans are being initialised - // on the base paths. We override the default wordlist_split to - // improve performance of the initial discovery phase. - let num_hosts = global_opts.hostnames.len() as u32; - let wordlist_split; - if first_run - && global_opts.max_threads >= 3 - && (global_opts.wordlist_split * num_hosts) - < (global_opts.max_threads - 2) - { - // If there's enough headroom to boost the split then do so - wordlist_split = (global_opts.max_threads - 2) / num_hosts; - info!( - "Increasing wordlist-split for initial scan of {} to {}", - dir_info.url, wordlist_split - ); - } else { - wordlist_split = global_opts.wordlist_split; - } - - for prefix in &global_opts.prefixes { - for extension in &global_opts.extensions { - for start_index in 0..wordlist_split { - scan_queue.push_back(wordlist::UriGenerator::new( - dir_info.url.clone(), - prefix.clone(), - extension.clone(), - wordlist.clone(), - start_index, - wordlist_split, - dir_info.parent_index, - dir_info.parent_depth, - dir_info.validator.clone(), - global_opts.extension_substitution, - )); - } - } - } -} - -fn generate_end() -> request::RequestResponse { - request::RequestResponse { - url: Url::parse("data:MAIN ENDING").unwrap(), - code: 0, - content_len: 0, - is_directory: false, - is_listable: false, - redirect_url: String::from(""), - found_from_listable: false, - parent_index: 0, - parent_depth: 0, - } -} - -#[cfg(test)] -mod test { - use crate::request::RequestResponse; - use url::Url; - - impl Default for RequestResponse { - fn default() -> Self { - RequestResponse { - url: Url::parse("http://example.com/").unwrap(), - code: 200, - content_len: 200, - is_directory: false, - is_listable: false, - redirect_url: "".into(), - found_from_listable: false, - parent_index: 0, - parent_depth: 0, - } - } - } + let global_opts = dirble::arg_parse::get_args(std::env::args_os()); + dirble::dirble_main(global_opts) } From bf3402ea32152466a4be08976babd3d83a96a774 Mon Sep 17 00:00:00 2001 From: David Young Date: Sun, 30 Mar 2025 15:27:47 +0100 Subject: [PATCH 3/6] Basic integration test with snapshot test of output format --- Cargo.lock | 97 +++++++++++++++++++ Cargo.toml | 1 + src/integration_testing.rs | 56 +++++++++++ src/lib.rs | 2 + ...rble__integration_testing__basic_test.snap | 7 ++ 5 files changed, 163 insertions(+) create mode 100644 src/integration_testing.rs create mode 100644 src/snapshots/dirble__integration_testing__basic_test.snap diff --git a/Cargo.lock b/Cargo.lock index 9bce174..45762c9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -29,6 +29,15 @@ dependencies = [ "zerocopy 0.7.35", ] +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + [[package]] name = "allocator-api2" version = "0.2.21" @@ -309,6 +318,18 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "console" +version = "0.15.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "054ccb5b10f9f2cbf51eb355ca1d05c2d279ce1804688d0db74b4733a5aeafd8" +dependencies = [ + "encode_unicode", + "libc", + "once_cell", + "windows-sys 0.59.0", +] + [[package]] name = "core-foundation-sys" version = "0.8.7" @@ -483,6 +504,7 @@ dependencies = [ "curl", "encoding", "http", + "insta", "log", "percent-encoding", "phf 0.11.3", @@ -525,6 +547,12 @@ version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" +[[package]] +name = "encode_unicode" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" + [[package]] name = "encoding" version = "0.2.33" @@ -1569,6 +1597,20 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "insta" +version = "1.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50259abbaa67d11d2bcafc7ba1d094ed7a0c70e3ce893f0d0997f73558cb3084" +dependencies = [ + "console", + "linked-hash-map", + "once_cell", + "pin-project", + "regex", + "similar", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.1" @@ -1639,6 +1681,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "linked-hash-map" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" + [[package]] name = "linux-raw-sys" version = "0.4.15" @@ -1947,6 +1995,26 @@ dependencies = [ "siphasher 1.0.1", ] +[[package]] +name = "pin-project" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677f1add503faace112b9f1373e43e9e054bfdd22ff1a63c1bc485eaec6a6a8a" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" +dependencies = [ + "proc-macro2 1.0.94", + "quote 1.0.40", + "syn 2.0.100", +] + [[package]] name = "pin-project-lite" version = "0.2.16" @@ -2146,11 +2214,34 @@ dependencies = [ "bitflags", ] +[[package]] +name = "regex" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + [[package]] name = "regex-automata" version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "rustc-demangle" @@ -2331,6 +2422,12 @@ dependencies = [ "libc", ] +[[package]] +name = "similar" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" + [[package]] name = "simple_xml_serialize" version = "0.3.0" diff --git a/Cargo.toml b/Cargo.toml index 0e56c68..6d53e67 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,6 +37,7 @@ release_version_string = [] [dev-dependencies] axum = "0.8.3" http = "1.3.1" +insta = { version = "1.42.2", features = ["filters"] } phf = { version = "0.11.3", features = ["macros"] } pretty_assertions = "1.4.1" tempfile = "3.19.1" diff --git a/src/integration_testing.rs b/src/integration_testing.rs new file mode 100644 index 0000000..523de92 --- /dev/null +++ b/src/integration_testing.rs @@ -0,0 +1,56 @@ +// This file is part of Dirble - https://www.github.com/nccgroup/dirble +// Copyright (C) 2019 Izzy Whistlecroft +// Released as open source by NCC Group Plc - https://www.nccgroup.com/ +// +// Dirble is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Dirble is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with Dirble. If not, see . + +use crate::arg_parse::GlobalOpts; +use std::io::{Read, Write}; +use tempfile::NamedTempFile; + +#[test] +fn basic_test() { + let port = crate::test_server::launch(); + let wordlist = crate::test_server::PATHS + .keys() + .copied() + .chain(std::iter::once("notfound")) + .collect::>(); + + let mut wordlist_file = NamedTempFile::new().unwrap(); + for word in wordlist { + writeln!(&mut wordlist_file, "{word}").unwrap(); + } + + let mut output_file = NamedTempFile::new().unwrap(); + + let args = GlobalOpts { + hostnames: vec![format!("http://localhost:{port}").parse().unwrap()], + wordlist_files: Some(vec![wordlist_file.path().display().to_string()]), + output_file: Some(output_file.path().display().to_string()), + ..GlobalOpts::default() + }; + + crate::dirble_main(args); + + let mut output = String::new(); + output_file.read_to_string(&mut output).unwrap(); + + insta::with_settings!({ + filters => vec![ + ("localhost:[0-9]{1,5}","localhost"), + ]}, { + insta::assert_snapshot!(output); + }); +} diff --git a/src/lib.rs b/src/lib.rs index df24d82..967fac1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -43,6 +43,8 @@ mod request_thread; mod validator_thread; mod wordlist; +#[cfg(test)] +mod integration_testing; #[cfg(test)] mod test_server; diff --git a/src/snapshots/dirble__integration_testing__basic_test.snap b/src/snapshots/dirble__integration_testing__basic_test.snap new file mode 100644 index 0000000..4854407 --- /dev/null +++ b/src/snapshots/dirble__integration_testing__basic_test.snap @@ -0,0 +1,7 @@ +--- +source: src/integration_testing.rs +expression: output +--- +Dirble Scan Report for http://localhost/: ++ http://localhost/201 (CODE:201|SIZE:11) ++ http://localhost/ok (CODE:200|SIZE:10) From a2bb2a203709ca9383ac1abf946c0e35a24d1b23 Mon Sep 17 00:00:00 2001 From: David Young Date: Sun, 30 Mar 2025 15:30:14 +0100 Subject: [PATCH 4/6] Delete broken python test --- Makefile | 5 +---- test/aio-chunk-server.py | 39 ----------------------------------- test/check-output-file.sh | 43 --------------------------------------- test/flask-server.py | 32 ----------------------------- test/flask_wordlist.txt | 5 ----- test/requirements.txt | 2 -- 6 files changed, 1 insertion(+), 125 deletions(-) delete mode 100755 test/aio-chunk-server.py delete mode 100755 test/check-output-file.sh delete mode 100755 test/flask-server.py delete mode 100644 test/flask_wordlist.txt delete mode 100644 test/requirements.txt diff --git a/Makefile b/Makefile index 35996f8..09209d5 100644 --- a/Makefile +++ b/Makefile @@ -4,10 +4,7 @@ targets = x86_64-unknown-linux-gnu \ i686-unknown-linux-gnu \ x86_64-pc-windows-gnu \ - i686-pc-windows-gnu \ -# wasm32-unknown-emscripten -# ^ Potential bug in cross, openssl does not compile for wasm for some -# reason. + i686-pc-windows-gnu cargo_flags = --release \ --features release_version_string diff --git a/test/aio-chunk-server.py b/test/aio-chunk-server.py deleted file mode 100755 index 53253bb..0000000 --- a/test/aio-chunk-server.py +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env python3 - -import asyncio -from aiohttp import web -import subprocess - -async def chunk_handler(request): - res = web.StreamResponse( - status=200, - reason="OK", - headers={"Content-Type": "text/plain"} - ) - await res.prepare(request) - for i in range(10): - await res.write(b"This is a line: %d\n"%(i)) - - return res - -async def build_server(loop, address, port): - app = web.Application() - app.router.add_route('GET', "/chunked.html", chunk_handler) - runner = web.AppRunner(app) - await runner.setup() - - site = web.TCPSite(runner, address, port) - await site.start() - -if __name__ == '__main__': - listen_addr = "::1" - port = 5001 - loop = asyncio.get_event_loop() - loop.run_until_complete(build_server(loop, listen_addr, port)) - print("Server listening on %s, port %d"%(listen_addr, port)) - - try: - loop.run_forever() - except KeyboardInterrupt: - print("Shutting Down!") - loop.close() \ No newline at end of file diff --git a/test/check-output-file.sh b/test/check-output-file.sh deleted file mode 100755 index 0ed63af..0000000 --- a/test/check-output-file.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash - -# Check the output file against a couple of parameters - -# Dirble Scan Report for http://localhost:5000/: -# + http://localhost:5000/302.html (CODE:302|SIZE:209|DEST:http://localhost:5000/) -# + http://localhost:5000/401.html (CODE:401|SIZE:338) -# + http://localhost:5000/403.html (CODE:403|SIZE:234) -# + http://localhost:5000/429.html (CODE:429|SIZE:194) -# + http://localhost:5000/console (CODE:200|SIZE:1985) -# -# Dirble Scan Report for http://localhost:5001/: -# + http://localhost:5001/chunked.html (CODE:200|SIZE:180) - -FILE=$1 - -if [[ "$FILE" == "" ]] ; then - echo "Usage: ./$0 output-file.txt" - exit 1 -fi - -function mismatch() { - echo "Test failed, see earlier output". - exit 1 -} - -# The chunked server should be processed correctly -grep "http://localhost:5001/chunked.html (CODE:200|SIZE:180)" "$FILE" || mismatch - -# The tested response codes should be rendered correctly -grep \ - "http://localhost:5000/302.html (CODE:302|SIZE:209|DEST:http://localhost:5000/)" \ - "$FILE" \ - || mismatch -grep "ttp://localhost:5000/401.html (CODE:401|SIZE:338)" "$FILE" || mismatch -grep "http://localhost:5000/403.html (CODE:403|SIZE:234)" "$FILE" || mismatch -grep "http://localhost:5000/429.html (CODE:429|SIZE:194)" "$FILE" || mismatch - -# We don't care about the werkzeug console; that doesn't count as a test case - -# If it got this far then all the output should be correct. Print a nice -# message and exit normally -echo "All tests passed! :)" \ No newline at end of file diff --git a/test/flask-server.py b/test/flask-server.py deleted file mode 100755 index d5ffdb7..0000000 --- a/test/flask-server.py +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env python3 - -from flask import Flask, url_for, redirect, abort, Response - -# This needs to be global so that the decorators work -app = Flask(__name__) - -@app.route("/") -def index(): - return "Index page" - -@app.route("/302.html") -def code302(): - return redirect(url_for("index"), 302) - -@app.route("/401.html") -def code401(): - abort(401) - -@app.route("/403.html") -def code403(): - abort(403) - -@app.route("/429.html") -def code429(): - abort(429) - -def main(): - app.run(debug=True, host="::1", port=5000, threaded=True) - -if __name__ == "__main__": - main() \ No newline at end of file diff --git a/test/flask_wordlist.txt b/test/flask_wordlist.txt deleted file mode 100644 index e494fbe..0000000 --- a/test/flask_wordlist.txt +++ /dev/null @@ -1,5 +0,0 @@ -302.html -401.html -403.html -429.html -chunked.html \ No newline at end of file diff --git a/test/requirements.txt b/test/requirements.txt deleted file mode 100644 index 671e0fe..0000000 --- a/test/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -flask ~= 1.1 -aiohttp ~= 3.6 \ No newline at end of file From f984f3951be18754682474ff5f98b15f2d75412d Mon Sep 17 00:00:00 2001 From: David Young Date: Sun, 30 Mar 2025 15:32:22 +0100 Subject: [PATCH 5/6] fmt --- src/lib.rs | 4 ++-- src/test_server.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 967fac1..ad74da2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -16,16 +16,16 @@ // along with Dirble. If not, see . use crate::arg_parse::GlobalOpts; -use log::{debug, error, info, warn, LevelFilter}; +use log::{LevelFilter, debug, error, info, warn}; use simplelog::{ColorChoice, TermLogger, TerminalMode}; use std::{ collections::VecDeque, env::current_exe, path::Path, sync::{ + Arc, atomic::{AtomicBool, Ordering}, mpsc::{self, Receiver, Sender}, - Arc, }, thread, time::Duration, diff --git a/src/test_server.rs b/src/test_server.rs index 6d85554..7ae035f 100644 --- a/src/test_server.rs +++ b/src/test_server.rs @@ -1,4 +1,4 @@ -use axum::{extract::Path, Router}; +use axum::{Router, extract::Path}; use http::StatusCode; use tokio::net::TcpListener; From d86267cc1ff8e6824c8f31be338f7c898d1ba2f1 Mon Sep 17 00:00:00 2001 From: David Young Date: Sun, 30 Mar 2025 15:34:52 +0100 Subject: [PATCH 6/6] Remove python from ci --- .github/workflows/build.yml | 36 ------------------------------------ 1 file changed, 36 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 39ed26b..47b940f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -91,39 +91,3 @@ jobs: with: command: clippy args: -- -D warnings - - # flask: - # name: Test against flask server - # runs-on: ubuntu-latest - # steps: - # - name: Checkout sources - # uses: actions/checkout@v1 - - # - name: Install stable toolchain - # uses: actions-rs/toolchain@v1 - # with: - # toolchain: stable - # override: true - - # - name: Install python dependencies - # run: | - # python3 -m pip install setuptools - # python3 -m pip install -r test/requirements.txt - - # - name: Start server - # run: | - # python3 test/flask-server.py & - # python3 test/aio-chunk-server.py & - - # - name: Run Dirble - # run: | - # cargo run -- \ - # -u http://localhost:5000 \ - # -u http://localhost:5001 \ - # -w dirble_wordlist.txt \ - # -w test/flask_wordlist.txt \ - # -o test-output.txt - - # - name: Check that the output is correct - # run: | - # bash test/check-output-file.sh test-output.txt