From a81c1aafc4216d06ad049843f1c8cc3119027cb9 Mon Sep 17 00:00:00 2001 From: Matej Hrica Date: Thu, 13 Nov 2025 14:17:32 +0100 Subject: [PATCH 1/9] `make test`: Add the ability to pick which test to ran via var This allows you to run only a specific test e.g. `make test TEST=tsi-udp` If TEST is not specified all test will be run. Signed-off-by: Matej Hrica --- Makefile | 4 +++- tests/run.sh | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 69aa8660c..269edcb34 100644 --- a/Makefile +++ b/Makefile @@ -181,5 +181,7 @@ test-prefix/lib64/libkrun.pc: $(LIBRARY_RELEASE_$(OS)) test-prefix: test-prefix/lib64/libkrun.pc +TEST ?= all + test: test-prefix - cd tests; LD_LIBRARY_PATH="$$(realpath ../test-prefix/lib64/)" PKG_CONFIG_PATH="$$(realpath ../test-prefix/lib64/pkgconfig/)" ./run.sh + cd tests; LD_LIBRARY_PATH="$$(realpath ../test-prefix/lib64/)" PKG_CONFIG_PATH="$$(realpath ../test-prefix/lib64/pkgconfig/)" ./run.sh test --test-case "$(TEST)" diff --git a/tests/run.sh b/tests/run.sh index b977b658b..7e2b4f9d4 100755 --- a/tests/run.sh +++ b/tests/run.sh @@ -17,10 +17,10 @@ cargo build -p runner export KRUN_TEST_GUEST_AGENT_PATH="target/$GUEST_TARGET_ARCH/debug/guest-agent" if [ -z "${KRUN_NO_UNSHARE}" ] && which unshare 2>&1 >/dev/null; then - unshare --user --map-root-user --net -- /bin/sh -c "ifconfig lo 127.0.0.1 && exec target/debug/runner $@" + unshare --user --map-root-user --net -- /bin/sh -c "ifconfig lo 127.0.0.1 && exec target/debug/runner $*" else echo "WARNING: Running tests without a network namespace." echo "Tests may fail if the required network ports are already in use." echo - target/debug/runner $@ + target/debug/runner "$@" fi From 68cd5f70808a2faed7c2876509bcb3e71db7a917 Mon Sep 17 00:00:00 2001 From: Matej Hrica Date: Thu, 27 Nov 2025 16:01:59 +0100 Subject: [PATCH 2/9] tests: Crate all test dirs under a single temp dir Instead of creating a tmp dir with random suffix for each test, create a tmp dir with the suffix for the whole run of the test. This allows the user to more easily view results of a run under a single directory. This also adds a --base-dir flag for overriding the test directory, and --keep-all which doesn't delete any of the files generated by the test. Signed-off-by: Matej Hrica --- Makefile | 3 ++- tests/run.sh | 14 +++++++--- tests/runner/src/main.rs | 55 +++++++++++++++++++++++++++++++--------- 3 files changed, 56 insertions(+), 16 deletions(-) diff --git a/Makefile b/Makefile index 269edcb34..83dc540d0 100644 --- a/Makefile +++ b/Makefile @@ -182,6 +182,7 @@ test-prefix/lib64/libkrun.pc: $(LIBRARY_RELEASE_$(OS)) test-prefix: test-prefix/lib64/libkrun.pc TEST ?= all +TEST_FLAGS ?= test: test-prefix - cd tests; LD_LIBRARY_PATH="$$(realpath ../test-prefix/lib64/)" PKG_CONFIG_PATH="$$(realpath ../test-prefix/lib64/pkgconfig/)" ./run.sh test --test-case "$(TEST)" + cd tests; RUST_LOG=trace LD_LIBRARY_PATH="$$(realpath ../test-prefix/lib64/)" PKG_CONFIG_PATH="$$(realpath ../test-prefix/lib64/pkgconfig/)" ./run.sh test --test-case "$(TEST)" $(TEST_FLAGS) diff --git a/tests/run.sh b/tests/run.sh index 7e2b4f9d4..128b3e546 100755 --- a/tests/run.sh +++ b/tests/run.sh @@ -1,4 +1,4 @@ -#/bin/sh +#!/bin/sh # This script has to be run with the working directory being "test" # This runs the tests on the libkrun instance found by pkg-config. @@ -16,11 +16,19 @@ cargo build -p runner export KRUN_TEST_GUEST_AGENT_PATH="target/$GUEST_TARGET_ARCH/debug/guest-agent" +# Build runner args: pass through all arguments +RUNNER_ARGS="$*" + +# Add --base-dir if KRUN_TEST_BASE_DIR is set +if [ -n "${KRUN_TEST_BASE_DIR}" ]; then + RUNNER_ARGS="${RUNNER_ARGS} --base-dir ${KRUN_TEST_BASE_DIR}" +fi + if [ -z "${KRUN_NO_UNSHARE}" ] && which unshare 2>&1 >/dev/null; then - unshare --user --map-root-user --net -- /bin/sh -c "ifconfig lo 127.0.0.1 && exec target/debug/runner $*" + unshare --user --map-root-user --net -- /bin/sh -c "ifconfig lo 127.0.0.1 && exec target/debug/runner ${RUNNER_ARGS}" else echo "WARNING: Running tests without a network namespace." echo "Tests may fail if the required network ports are already in use." echo - target/debug/runner "$@" + target/debug/runner ${RUNNER_ARGS} fi diff --git a/tests/runner/src/main.rs b/tests/runner/src/main.rs index 449b5d50c..0956406dc 100644 --- a/tests/runner/src/main.rs +++ b/tests/runner/src/main.rs @@ -1,10 +1,11 @@ use anyhow::Context; use clap::Parser; use nix::sys::resource::{getrlimit, setrlimit, Resource}; +use std::env; +use std::fs; use std::panic::catch_unwind; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; -use std::{env, mem}; use tempdir::TempDir; use test_cases::{test_cases, Test, TestCase, TestSetup}; @@ -30,17 +31,17 @@ fn start_vm(test_setup: TestSetup) -> anyhow::Result<()> { Ok(()) } -fn run_single_test(test_case: &str) -> anyhow::Result { +fn run_single_test(test_case: &str, base_dir: &Path, keep_all: bool) -> anyhow::Result { let executable = env::current_exe().context("Failed to detect current executable")?; - let tmp_dir = - TempDir::new(&format!("krun-test-{test_case}")).context("Failed to create tmp dir")?; + let test_dir = base_dir.join(test_case); + fs::create_dir(&test_dir).context("Failed to create test directory")?; let child = Command::new(&executable) .arg("start-vm") .arg("--test-case") .arg(test_case) .arg("--tmp-dir") - .arg(tmp_dir.path()) + .arg(&test_dir) .stdin(Stdio::piped()) .stdout(Stdio::piped()) .stderr(Stdio::piped()) @@ -56,17 +57,30 @@ fn run_single_test(test_case: &str) -> anyhow::Result { match result { Ok(()) => { println!("[{test_case}]: OK"); + if !keep_all { + let _ = fs::remove_dir_all(&test_dir); + } Ok(true) } Err(_e) => { - println!("[{test_case}]: FAIL (dir {:?} kept)", tmp_dir.path()); - mem::forget(tmp_dir); + println!("[{test_case}]: FAIL (dir {:?} kept)", test_dir); Ok(false) } } } -fn run_tests(test_case: &str) -> anyhow::Result<()> { +fn run_tests(test_case: &str, base_dir: Option, keep_all: bool) -> anyhow::Result<()> { + // Create the base directory - either use provided path or create a temp one + let base_dir = match base_dir { + Some(path) => { + fs::create_dir_all(&path).context("Failed to create base directory")?; + path + } + None => TempDir::new("libkrun-tests") + .context("Failed to create temp base directory")? + .into_path(), + }; + let mut num_tests = 1; let mut num_ok: usize = 0; @@ -75,17 +89,22 @@ fn run_tests(test_case: &str) -> anyhow::Result<()> { num_tests = test_cases.len(); for TestCase { name, test: _ } in test_cases { - num_ok += run_single_test(name).context(name)? as usize; + num_ok += run_single_test(name, &base_dir, keep_all).context(name)? as usize; } } else { - num_ok += run_single_test(test_case).context(test_case.to_string())? as usize; + num_ok += run_single_test(test_case, &base_dir, keep_all).context(test_case.to_string())? + as usize; } let num_failures = num_tests - num_ok; if num_failures > 0 { + eprintln!("(See test artifacts at: {})", base_dir.display()); println!("\nFAIL (PASSED {num_ok}/{num_tests})"); anyhow::bail!("") } else { + if keep_all { + eprintln!("(See test artifacts at: {})", base_dir.display()); + } println!("\nOK (PASSED {num_ok}/{num_tests})"); } @@ -98,6 +117,12 @@ enum CliCommand { /// Specify which test to run or "all" #[arg(long, default_value = "all")] test_case: String, + /// Base directory for test artifacts + #[arg(long)] + base_dir: Option, + /// Keep all test artifacts even on success + #[arg(long)] + keep_all: bool, }, StartVm { #[arg(long)] @@ -111,6 +136,8 @@ impl Default for CliCommand { fn default() -> Self { Self::Test { test_case: "all".to_string(), + base_dir: None, + keep_all: false, } } } @@ -127,6 +154,10 @@ fn main() -> anyhow::Result<()> { match command { CliCommand::StartVm { test_case, tmp_dir } => start_vm(TestSetup { test_case, tmp_dir }), - CliCommand::Test { test_case } => run_tests(&test_case), + CliCommand::Test { + test_case, + base_dir, + keep_all, + } => run_tests(&test_case, base_dir, keep_all), } } From f7531390e46232c98a150dd21cff91902cd8c75a Mon Sep 17 00:00:00 2001 From: Matej Hrica Date: Fri, 28 Nov 2025 13:48:49 +0100 Subject: [PATCH 3/9] tests: Save the log of each test into a file instead of printing it Signed-off-by: Matej Hrica --- tests/runner/src/main.rs | 7 +++++-- tests/test_cases/src/lib.rs | 5 ----- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/tests/runner/src/main.rs b/tests/runner/src/main.rs index 0956406dc..51eb8dc78 100644 --- a/tests/runner/src/main.rs +++ b/tests/runner/src/main.rs @@ -2,7 +2,7 @@ use anyhow::Context; use clap::Parser; use nix::sys::resource::{getrlimit, setrlimit, Resource}; use std::env; -use std::fs; +use std::fs::{self, File}; use std::panic::catch_unwind; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; @@ -36,6 +36,9 @@ fn run_single_test(test_case: &str, base_dir: &Path, keep_all: bool) -> anyhow:: let test_dir = base_dir.join(test_case); fs::create_dir(&test_dir).context("Failed to create test directory")?; + let log_path = test_dir.join("log.txt"); + let log_file = File::create(&log_path).context("Failed to create log file")?; + let child = Command::new(&executable) .arg("start-vm") .arg("--test-case") @@ -44,7 +47,7 @@ fn run_single_test(test_case: &str, base_dir: &Path, keep_all: bool) -> anyhow:: .arg(&test_dir) .stdin(Stdio::piped()) .stdout(Stdio::piped()) - .stderr(Stdio::piped()) + .stderr(log_file) .spawn() .context("Failed to start subprocess for test")?; diff --git a/tests/test_cases/src/lib.rs b/tests/test_cases/src/lib.rs index 551a89b2d..dfe5211a0 100644 --- a/tests/test_cases/src/lib.rs +++ b/tests/test_cases/src/lib.rs @@ -78,11 +78,6 @@ pub trait Test { /// Checks the output of the (host) process which started the VM fn check(self: Box, child: Child) { let output = child.wait_with_output().unwrap(); - let err = String::from_utf8(output.stderr).unwrap(); - if !err.is_empty() { - eprintln!("{}", err); - } - assert_eq!(String::from_utf8(output.stdout).unwrap(), "OK\n"); } } From e58722030060f8928c5ec42deb84d4152ac3d9db Mon Sep 17 00:00:00 2001 From: Matej Hrica Date: Thu, 27 Nov 2025 16:10:58 +0100 Subject: [PATCH 4/9] tests: Make runner output pretty, tabulated, visible when test hangs Signed-off-by: Matej Hrica --- tests/runner/src/main.rs | 34 ++++++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/tests/runner/src/main.rs b/tests/runner/src/main.rs index 51eb8dc78..f756d473d 100644 --- a/tests/runner/src/main.rs +++ b/tests/runner/src/main.rs @@ -31,7 +31,12 @@ fn start_vm(test_setup: TestSetup) -> anyhow::Result<()> { Ok(()) } -fn run_single_test(test_case: &str, base_dir: &Path, keep_all: bool) -> anyhow::Result { +fn run_single_test( + test_case: &str, + base_dir: &Path, + keep_all: bool, + max_name_len: usize, +) -> anyhow::Result { let executable = env::current_exe().context("Failed to detect current executable")?; let test_dir = base_dir.join(test_case); fs::create_dir(&test_dir).context("Failed to create test directory")?; @@ -39,6 +44,12 @@ fn run_single_test(test_case: &str, base_dir: &Path, keep_all: bool) -> anyhow:: let log_path = test_dir.join("log.txt"); let log_file = File::create(&log_path).context("Failed to create log file")?; + eprint!( + "[{test_case}] {:. anyhow:: match result { Ok(()) => { - println!("[{test_case}]: OK"); + eprintln!("OK"); if !keep_all { let _ = fs::remove_dir_all(&test_dir); } Ok(true) } Err(_e) => { - println!("[{test_case}]: FAIL (dir {:?} kept)", test_dir); + eprintln!("FAIL"); Ok(false) } } @@ -88,15 +99,18 @@ fn run_tests(test_case: &str, base_dir: Option, keep_all: bool) -> anyh let mut num_ok: usize = 0; if test_case == "all" { - let test_cases = test_cases(); - num_tests = test_cases.len(); + let all_tests = test_cases(); + num_tests = all_tests.len(); + let max_name_len = all_tests.iter().map(|t| t.name.len()).max().unwrap_or(0); - for TestCase { name, test: _ } in test_cases { - num_ok += run_single_test(name, &base_dir, keep_all).context(name)? as usize; + for TestCase { name, test: _ } in all_tests { + num_ok += + run_single_test(name, &base_dir, keep_all, max_name_len).context(name)? as usize; } } else { - num_ok += run_single_test(test_case, &base_dir, keep_all).context(test_case.to_string())? - as usize; + let max_name_len = test_case.len(); + num_ok += run_single_test(test_case, &base_dir, keep_all, max_name_len) + .context(test_case.to_string())? as usize; } let num_failures = num_tests - num_ok; @@ -108,7 +122,7 @@ fn run_tests(test_case: &str, base_dir: Option, keep_all: bool) -> anyh if keep_all { eprintln!("(See test artifacts at: {})", base_dir.display()); } - println!("\nOK (PASSED {num_ok}/{num_tests})"); + eprintln!("\nOK ({num_ok}/{num_tests} passed)"); } Ok(()) From ad4b5099588b53eaf1275eca1981e59859e6bb1d Mon Sep 17 00:00:00 2001 From: Matej Hrica Date: Fri, 28 Nov 2025 14:03:43 +0100 Subject: [PATCH 5/9] tests: Add --github-summary to generate formatted markdown file Signed-off-by: Matej Hrica --- tests/runner/src/main.rs | 115 ++++++++++++++++++++++++++++++++------- 1 file changed, 94 insertions(+), 21 deletions(-) diff --git a/tests/runner/src/main.rs b/tests/runner/src/main.rs index f756d473d..d3d3a702a 100644 --- a/tests/runner/src/main.rs +++ b/tests/runner/src/main.rs @@ -3,12 +3,19 @@ use clap::Parser; use nix::sys::resource::{getrlimit, setrlimit, Resource}; use std::env; use std::fs::{self, File}; +use std::io::Write; use std::panic::catch_unwind; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; use tempdir::TempDir; use test_cases::{test_cases, Test, TestCase, TestSetup}; +struct TestResult { + name: String, + passed: bool, + log_path: PathBuf, +} + fn get_test(name: &str) -> anyhow::Result> { let tests = test_cases(); tests @@ -36,7 +43,7 @@ fn run_single_test( base_dir: &Path, keep_all: bool, max_name_len: usize, -) -> anyhow::Result { +) -> anyhow::Result { let executable = env::current_exe().context("Failed to detect current executable")?; let test_dir = base_dir.join(test_case); fs::create_dir(&test_dir).context("Failed to create test directory")?; @@ -68,22 +75,76 @@ fn run_single_test( test.check(child); }); - match result { - Ok(()) => { - eprintln!("OK"); - if !keep_all { - let _ = fs::remove_dir_all(&test_dir); - } - Ok(true) - } - Err(_e) => { - eprintln!("FAIL"); - Ok(false) + let passed = result.is_ok(); + if passed { + eprintln!("OK"); + if !keep_all { + let _ = fs::remove_dir_all(&test_dir); } + } else { + eprintln!("FAIL"); + } + + Ok(TestResult { + name: test_case.to_string(), + passed, + log_path, + }) +} + +fn write_github_summary( + results: &[TestResult], + num_ok: usize, + num_tests: usize, +) -> anyhow::Result<()> { + let summary_path = env::var("GITHUB_STEP_SUMMARY") + .context("GITHUB_STEP_SUMMARY environment variable not set")?; + + let mut file = fs::OpenOptions::new() + .create(true) + .append(true) + .open(&summary_path) + .context("Failed to open GITHUB_STEP_SUMMARY")?; + + let all_passed = num_ok == num_tests; + let status = if all_passed { "✅" } else { "❌" }; + + writeln!( + file, + "## {status} Integration Tests ({num_ok}/{num_tests} passed)\n" + )?; + + for result in results { + let icon = if result.passed { "✅" } else { "❌" }; + let log_content = fs::read_to_string(&result.log_path).unwrap_or_default(); + + writeln!(file, "
")?; + writeln!(file, "{icon} {}\n", result.name)?; + writeln!(file, "```")?; + // Limit log size to avoid huge summaries (2 MiB limit) + const MAX_LOG_SIZE: usize = 2 * 1024 * 1024; + let truncated = if log_content.len() > MAX_LOG_SIZE { + format!( + "... (truncated, showing last 1 MiB) ...\n{}", + &log_content[log_content.len() - MAX_LOG_SIZE..] + ) + } else { + log_content + }; + writeln!(file, "{truncated}")?; + writeln!(file, "```")?; + writeln!(file, "
\n")?; } + + Ok(()) } -fn run_tests(test_case: &str, base_dir: Option, keep_all: bool) -> anyhow::Result<()> { +fn run_tests( + test_case: &str, + base_dir: Option, + keep_all: bool, + github_summary: bool, +) -> anyhow::Result<()> { // Create the base directory - either use provided path or create a temp one let base_dir = match base_dir { Some(path) => { @@ -95,22 +156,29 @@ fn run_tests(test_case: &str, base_dir: Option, keep_all: bool) -> anyh .into_path(), }; - let mut num_tests = 1; - let mut num_ok: usize = 0; + let mut results: Vec = Vec::new(); if test_case == "all" { let all_tests = test_cases(); - num_tests = all_tests.len(); let max_name_len = all_tests.iter().map(|t| t.name.len()).max().unwrap_or(0); for TestCase { name, test: _ } in all_tests { - num_ok += - run_single_test(name, &base_dir, keep_all, max_name_len).context(name)? as usize; + results.push(run_single_test(name, &base_dir, keep_all, max_name_len).context(name)?); } } else { let max_name_len = test_case.len(); - num_ok += run_single_test(test_case, &base_dir, keep_all, max_name_len) - .context(test_case.to_string())? as usize; + results.push( + run_single_test(test_case, &base_dir, keep_all, max_name_len) + .context(test_case.to_string())?, + ); + } + + let num_tests = results.len(); + let num_ok = results.iter().filter(|r| r.passed).count(); + + // Write GitHub Actions summary if requested + if github_summary { + write_github_summary(&results, num_ok, num_tests)?; } let num_failures = num_tests - num_ok; @@ -140,6 +208,9 @@ enum CliCommand { /// Keep all test artifacts even on success #[arg(long)] keep_all: bool, + /// Write test results to GitHub Actions job summary ($GITHUB_STEP_SUMMARY) + #[arg(long)] + github_summary: bool, }, StartVm { #[arg(long)] @@ -155,6 +226,7 @@ impl Default for CliCommand { test_case: "all".to_string(), base_dir: None, keep_all: false, + github_summary: false, } } } @@ -175,6 +247,7 @@ fn main() -> anyhow::Result<()> { test_case, base_dir, keep_all, - } => run_tests(&test_case, base_dir, keep_all), + github_summary, + } => run_tests(&test_case, base_dir, keep_all, github_summary), } } From 6da81e04ee275fe278e23f91a14ca45b593abfc9 Mon Sep 17 00:00:00 2001 From: Matej Hrica Date: Fri, 28 Nov 2025 14:05:48 +0100 Subject: [PATCH 6/9] CI: Generate test summary Markdown and upload test logs Signed-off-by: Matej Hrica --- .github/workflows/integration_tests.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index d45bc691b..0d659cf64 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -52,4 +52,14 @@ jobs: run: curl -L -o /tmp/libkrunfw-5.0.0-x86_64.tgz https://github.com/containers/libkrunfw/releases/download/v5.0.0/libkrunfw-5.0.0-x86_64.tgz && mkdir tmp && tar xf /tmp/libkrunfw-5.0.0-x86_64.tgz -C tmp && sudo mv tmp/lib64/* /lib/x86_64-linux-gnu - name: Integration tests - run: RUST_LOG=trace KRUN_ENOMEM_WORKAROUND=1 KRUN_NO_UNSHARE=1 make test + run: KRUN_ENOMEM_WORKAROUND=1 KRUN_NO_UNSHARE=1 KRUN_TEST_BASE_DIR=/tmp/libkrun-tests make test TEST_FLAGS="--keep-all --github-summary" + + - name: Upload test logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: test-logs + path: | + /tmp/libkrun-tests/ + !/tmp/libkrun-tests/**/guest-agent + if-no-files-found: ignore From d1e569d928be2dcbdc2368af5ea7c8689ff35e97 Mon Sep 17 00:00:00 2001 From: Matej Hrica Date: Wed, 26 Nov 2025 13:57:48 +0100 Subject: [PATCH 7/9] tests: Add support for running tests with namespace isolation Signed-off-by: Matej Hrica --- tests/guest-agent/src/main.rs | 2 +- tests/runner/Cargo.toml | 2 +- tests/runner/src/main.rs | 71 +++++++++++++++++++++++-- tests/test_cases/Cargo.toml | 2 +- tests/test_cases/src/common.rs | 94 ++++++++++++++++++++++++++++++++++ tests/test_cases/src/lib.rs | 7 ++- 6 files changed, 170 insertions(+), 8 deletions(-) diff --git a/tests/guest-agent/src/main.rs b/tests/guest-agent/src/main.rs index 1f9b7965c..14015e7c4 100644 --- a/tests/guest-agent/src/main.rs +++ b/tests/guest-agent/src/main.rs @@ -8,7 +8,7 @@ fn run_guest_agent(test_name: &str) -> anyhow::Result<()> { .into_iter() .find(|t| t.name() == test_name) .context("No such test!")?; - let TestCase { test, name: _ } = test_case; + let TestCase { test, name: _, requires_namespace: _ } = test_case; test.in_guest(); Ok(()) } diff --git a/tests/runner/Cargo.toml b/tests/runner/Cargo.toml index b74e9ad7c..e5d10bbf9 100644 --- a/tests/runner/Cargo.toml +++ b/tests/runner/Cargo.toml @@ -5,7 +5,7 @@ edition = "2021" [dependencies] test_cases = { path = "../test_cases", features = ["host"] } anyhow = "1.0.95" -nix = { version = "0.29.0", features = ["resource", "fs"] } +nix = { version = "0.29.0", features = ["resource", "fs", "sched", "user", "process"] } macros = { path = "../macros" } clap = { version = "4.5.27", features = ["derive"] } tempdir = "0.3.7" diff --git a/tests/runner/src/main.rs b/tests/runner/src/main.rs index d3d3a702a..caec723bd 100644 --- a/tests/runner/src/main.rs +++ b/tests/runner/src/main.rs @@ -32,12 +32,75 @@ fn start_vm(test_setup: TestSetup) -> anyhow::Result<()> { setrlimit(Resource::RLIMIT_NOFILE, hard_limit, hard_limit) .context("setrlimit RLIMIT_NOFILE")?; - let test = get_test(&test_setup.test_case)?; - test.start_vm(test_setup.clone()) - .with_context(|| format!("testcase: {test_setup:?}"))?; + // Check if this test requires a namespace + let test_cases = test_cases(); + let requires_namespace = test_cases + .into_iter() + .find(|t| t.name == test_setup.test_case) + .map(|t| t.requires_namespace) + .unwrap_or(false); + + if requires_namespace { + setup_namespace_and_run(test_setup)?; + } else { + let test = get_test(&test_setup.test_case)?; + test.start_vm(test_setup.clone()) + .with_context(|| format!("testcase: {test_setup:?}"))?; + } Ok(()) } +fn setup_namespace_and_run(test_setup: TestSetup) -> anyhow::Result<()> { + use nix::sched::{unshare, CloneFlags}; + use nix::unistd::{fork, Gid, Uid, ForkResult}; + use std::fs; + + // Get our current uid/gid before entering the namespace + let uid = Uid::current(); + let gid = Gid::current(); + + // Create a new user namespace, mount namespace, and PID namespace (rootless) + unshare(CloneFlags::CLONE_NEWUSER | CloneFlags::CLONE_NEWNS | CloneFlags::CLONE_NEWPID) + .context("Failed to unshare user+mount+pid namespace")?; + + // Set up uid_map to map our uid to root (0) in the namespace + let uid_map = format!("0 {} 1", uid); + fs::write("/proc/self/uid_map", uid_map) + .context("Failed to write uid_map")?; + + // Disable setgroups (required before writing gid_map as non-root) + fs::write("/proc/self/setgroups", "deny") + .context("Failed to write setgroups")?; + + // Set up gid_map to map our gid to root (0) in the namespace + let gid_map = format!("0 {} 1", gid); + fs::write("/proc/self/gid_map", gid_map) + .context("Failed to write gid_map")?; + + // Fork so the child becomes PID 1 in the new PID namespace + // This is necessary to be able to mount procfs + match unsafe { fork() }.context("Failed to fork")? { + ForkResult::Parent { child } => { + // Parent waits for child and exits + use nix::sys::wait::waitpid; + let status = waitpid(child, None).context("Failed to wait for child")?; + // Exit with the child's exit code + use nix::sys::wait::WaitStatus; + match status { + WaitStatus::Exited(_, code) => std::process::exit(code), + _ => std::process::exit(1), + } + } + ForkResult::Child => { + // Child continues - we are now PID 1 in the PID namespace + let test = get_test(&test_setup.test_case)?; + test.start_vm(test_setup.clone()) + .with_context(|| format!("testcase: {test_setup:?}"))?; + Ok(()) + } + } +} + fn run_single_test( test_case: &str, base_dir: &Path, @@ -162,7 +225,7 @@ fn run_tests( let all_tests = test_cases(); let max_name_len = all_tests.iter().map(|t| t.name.len()).max().unwrap_or(0); - for TestCase { name, test: _ } in all_tests { + for TestCase { name, test: _, requires_namespace: _ } in all_tests { results.push(run_single_test(name, &base_dir, keep_all, max_name_len).context(name)?); } } else { diff --git a/tests/test_cases/Cargo.toml b/tests/test_cases/Cargo.toml index 34d646797..0e6da173c 100644 --- a/tests/test_cases/Cargo.toml +++ b/tests/test_cases/Cargo.toml @@ -12,6 +12,6 @@ name = "test_cases" [dependencies] krun-sys = { path = "../../krun-sys", optional = true } macros = { path = "../macros" } -nix = { version = "0.29.0", features = ["socket"] } +nix = { version = "0.29.0", features = ["socket", "sched", "user", "mount"] } anyhow = "1.0.95" tempdir = "0.3.7" \ No newline at end of file diff --git a/tests/test_cases/src/common.rs b/tests/test_cases/src/common.rs index 6a3ee2483..e78dcc261 100644 --- a/tests/test_cases/src/common.rs +++ b/tests/test_cases/src/common.rs @@ -11,6 +11,9 @@ use std::ptr::null; use crate::{krun_call, TestSetup}; use krun_sys::*; +use nix::unistd::{chroot, chdir}; +use std::path::PathBuf; + fn copy_guest_agent(dir: &Path) -> anyhow::Result<()> { let path = std::env::var_os("KRUN_TEST_GUEST_AGENT_PATH") .context("KRUN_TEST_GUEST_AGENT_PATH env variable not set")?; @@ -50,3 +53,94 @@ pub fn setup_fs_and_enter(ctx: u32, test_setup: TestSetup) -> anyhow::Result<()> } unreachable!() } + +/// Like setup_fs_and_enter, but changes the host process's root to the guest's root +/// before entering the VM. This is needed for Unix domain socket TSI tests where the +/// host process needs to access socket paths in the guest filesystem. +/// +/// This function: +/// 1. Creates a new user namespace and mount namespace (unshare CLONE_NEWUSER | CLONE_NEWNS) +/// 2. Sets up uid/gid mappings to become root in the namespace +/// 3. Changes root to the guest's root directory (chroot) +/// 4. Then calls krun_start_enter +/// +/// The before_enter callback is called after chroot but before krun_start_enter, allowing +/// setup of host-side resources (like Unix domain socket servers) that need to be accessible +/// at the same paths as the guest will use. +/// +/// Note: This uses rootless namespaces (user namespaces) so it doesn't require root. +pub fn setup_fs_and_enter_with_namespace( + ctx: u32, + test_setup: TestSetup, + before_enter: F, +) -> anyhow::Result<()> +where + F: FnOnce() -> anyhow::Result<()>, +{ + let root_dir = test_setup.tmp_dir.join("root"); + create_dir(&root_dir).context("Failed to create root directory")?; + + // Create necessary directories in the guest root + create_dir(root_dir.join("tmp")).context("Failed to create tmp directory")?; + create_dir(root_dir.join("dev")).context("Failed to create dev directory")?; + create_dir(root_dir.join("proc")).context("Failed to create proc directory")?; + create_dir(root_dir.join("sys")).context("Failed to create sys directory")?; + + copy_guest_agent(&root_dir)?; + + // The runner has already set up the namespace for us (user+mount+pid) + // We are now root in the user namespace and PID 1 in the PID namespace + // Make our mounts private so they don't affect the parent namespace + use nix::mount::{mount, MsFlags}; + mount( + None::<&str>, + "/", + None::<&str>, + MsFlags::MS_REC | MsFlags::MS_PRIVATE, + None::<&str>, + ).context("Failed to make / private")?; + + // Bind mount /dev into the guest root so /dev/kvm is accessible + // (we're root in the namespace now) + mount( + Some("/dev"), + root_dir.join("dev").as_path(), + None::<&str>, + MsFlags::MS_BIND | MsFlags::MS_REC, + None::<&str>, + ).context("Failed to bind mount /dev")?; + + // Now we can chroot + let root_path = PathBuf::from(&root_dir); + chroot(&root_path).context("Failed to chroot to guest root")?; + chdir("/").context("Failed to chdir to /")?; + + // Mount procfs after chroot with standard proc mount flags + mount( + Some("proc"), + "/proc", + Some("proc"), + MsFlags::MS_NOSUID | MsFlags::MS_NODEV | MsFlags::MS_NOEXEC, + None::<&str>, + ).context("Failed to mount procfs")?; + + // Call the before_enter callback to set up host-side resources + before_enter().context("before_enter callback failed")?; + + let path_str = CString::new("/").context("CString::new")?; + unsafe { + krun_call!(krun_set_root(ctx, path_str.as_ptr()))?; + krun_call!(krun_set_workdir(ctx, c"/".as_ptr()))?; + let test_case_cstr = CString::new(test_setup.test_case).context("CString::new")?; + let argv = [test_case_cstr.as_ptr(), null()]; + let envp = [null()]; + krun_call!(krun_set_exec( + ctx, + c"/guest-agent".as_ptr(), + argv.as_ptr(), + envp.as_ptr(), + ))?; + krun_call!(krun_start_enter(ctx))?; + } + unreachable!() +} diff --git a/tests/test_cases/src/lib.rs b/tests/test_cases/src/lib.rs index dfe5211a0..9eff3505d 100644 --- a/tests/test_cases/src/lib.rs +++ b/tests/test_cases/src/lib.rs @@ -91,13 +91,18 @@ pub trait Test { pub struct TestCase { pub name: &'static str, pub test: Box, + pub requires_namespace: bool, } impl TestCase { // Your test can be parametrized, so you can add the same test multiple times constructed with // different parameters with and specify a different name here. pub fn new(name: &'static str, test: Box) -> Self { - Self { name, test } + Self { name, test, requires_namespace: false } + } + + pub fn new_with_namespace(name: &'static str, test: Box) -> Self { + Self { name, test, requires_namespace: true } } #[allow(dead_code)] From 22b68fb47f3eb161c8d157958655ba1cc254265e Mon Sep 17 00:00:00 2001 From: Matej Hrica Date: Wed, 26 Nov 2025 14:44:50 +0100 Subject: [PATCH 8/9] WIP tests: Rewrite the TSI test and also test dgram and unix sockets Signed-off-by: Matej Hrica --- tests/TSI_TESTING_GUIDE.md | 171 ++++++++++++++++ tests/guest-agent/src/main.rs | 6 +- tests/runner/src/main.rs | 61 +++++- tests/test_cases/Cargo.toml | 2 +- tests/test_cases/src/common.rs | 149 ++++---------- tests/test_cases/src/datagram_tester.rs | 120 +++++++++++ tests/test_cases/src/lib.rs | 192 ++++++++++++++++-- tests/test_cases/src/stream_tester.rs | 111 ++++++++++ tests/test_cases/src/tcp_tester.rs | 79 ------- tests/test_cases/src/test_tsi.rs | 147 ++++++++++++++ .../src/test_tsi_tcp_guest_connect.rs | 54 ----- .../src/test_tsi_tcp_guest_listen.rs | 65 ------ .../src/test_tsi_unix_dgram_setsockopt.rs | 81 ++++++++ 13 files changed, 908 insertions(+), 330 deletions(-) create mode 100644 tests/TSI_TESTING_GUIDE.md create mode 100644 tests/test_cases/src/datagram_tester.rs create mode 100644 tests/test_cases/src/stream_tester.rs delete mode 100644 tests/test_cases/src/tcp_tester.rs create mode 100644 tests/test_cases/src/test_tsi.rs delete mode 100644 tests/test_cases/src/test_tsi_tcp_guest_connect.rs delete mode 100644 tests/test_cases/src/test_tsi_tcp_guest_listen.rs create mode 100644 tests/test_cases/src/test_tsi_unix_dgram_setsockopt.rs diff --git a/tests/TSI_TESTING_GUIDE.md b/tests/TSI_TESTING_GUIDE.md new file mode 100644 index 000000000..f21013607 --- /dev/null +++ b/tests/TSI_TESTING_GUIDE.md @@ -0,0 +1,171 @@ +# TSI Testing Guide for AI Agents + +This guide explains how to run and debug TSI (Transparent Socket Impersonation) tests for libkrun. + +## Directory Structure + +- **libkrun**: `~/Dev2/libkrun/t/test-fix-tsi` - contains test code and libkrun source +- **libkrunfw**: `~/Dev2/libkrunfw/t/tsi-dgram-fix` - contains kernel patches +- **test-prefix**: `~/Dev2/libkrun/t/test-fix-tsi/test-prefix` - where libraries are installed for testing + +## Running Tests + +**IMPORTANT: Always use `make test` from the libkrun directory. Do NOT use `./run.sh` directly - the Makefile handles library paths and environment setup correctly.** + +### Run all tests +```bash +cd ~/Dev2/libkrun/t/test-fix-tsi +make test +``` + +### Run a specific test +```bash +cd ~/Dev2/libkrun/t/test-fix-tsi +make test TEST=tsi-unix-dgram-setsockopt +``` + +### Run with timeout (useful for tests that may hang on kernel panic) +```bash +cd ~/Dev2/libkrun/t/test-fix-tsi +timeout 30 make test TEST=tsi-unix-dgram-setsockopt +``` + +### Specify custom output directory (recommended for debugging) + +Use `--base-dir` to specify where test artifacts are saved. Use a timestamped path to avoid conflicts: + +```bash +cd ~/Dev2/libkrun/t/test-fix-tsi +make test TEST=tsi-unix-dgram-setsockopt TEST_FLAGS="--base-dir /tmp/tsi-test-$(date +%Y%m%d-%H%M%S) --keep-all" +``` + +This creates a predictable output location like `/tmp/tsi-test-20251201-143628/` containing: +- `/log.txt` - full test log with kernel output +- `/root/` - guest filesystem artifacts + +### Keep test artifacts for debugging +```bash +make test TEST=tsi-unix-dgram-setsockopt TEST_FLAGS="--keep-all" +``` + +Note: Without `--keep-all`, artifacts are only kept for failed tests. + +## Switching Kernels + +Tests use whatever libkrunfw is installed in `test-prefix/lib64/`. + +### Install patched kernel (tsi-dgram-fix branch) +```bash +cd ~/Dev2/libkrunfw/t/tsi-dgram-fix +PREFIX="$(realpath ~/Dev2/libkrun/t/test-fix-tsi/test-prefix)" make install +``` + +### Install unpatched kernel (main branch) to verify bugs exist +```bash +cd ~/Dev2/libkrunfw/t/main +PREFIX="$(realpath ~/Dev2/libkrun/t/test-fix-tsi/test-prefix)" make install +``` + +### Safely remove libkrunfw before switching +```bash +rm -f ~/Dev2/libkrun/t/test-fix-tsi/test-prefix/lib64/libkrunfw* +``` + +## Debugging Kernel Panics + +### Viewing kernel output + +Filter for kernel messages in the log file: +```bash +grep "init_or_kernel]" /tmp/tsi-test-XXXXXXXX/tsi-unix-dgram-setsockopt/log.txt +``` + +This shows kernel boot messages and panic traces. + +### Enabling verbose kernel output + +By default, kernel output is suppressed. To see kernel panics and debug messages, edit: +``` +src/vmm/src/vmm_config/kernel_cmdline.rs +``` + +**Default (quiet, no panic output):** +```rust +pub const DEFAULT_KERNEL_CMDLINE: &str = "reboot=k panic=-1 panic_print=0 nomodule console=hvc0 \ + rootfstype=virtiofs rw quiet no-kvmapf"; +``` + +**For debugging (verbose, shows panics):** +```rust +pub const DEFAULT_KERNEL_CMDLINE: &str = "reboot=k panic=-1 nomodule console=hvc0 \ + rootfstype=virtiofs rw no-kvmapf"; +``` + +Changes: +- Remove `quiet` - shows all kernel boot messages +- Remove `panic_print=0` - shows full panic information + +**Remember to restore the original settings after debugging!** + +### Example: Finding a kernel panic + +1. Enable verbose kernel output (edit kernel_cmdline.rs as above) +2. Run the test with timeout and custom output dir: + ```bash + cd ~/Dev2/libkrun/t/test-fix-tsi + timeout 30 make test TEST=tsi-unix-dgram-setsockopt TEST_FLAGS="--base-dir /tmp/tsi-test-$(date +%Y%m%d-%H%M%S) --keep-all" + ``` +3. Filter for kernel messages: + ```bash + grep "init_or_kernel]" /tmp/tsi-test-XXXXXXXX/tsi-unix-dgram-setsockopt/log.txt + ``` +4. Look for `BUG:`, `Oops:`, `NULL pointer dereference`, `Call Trace:` etc. + +### Example kernel panic output +``` +[ 0.060929] BUG: kernel NULL pointer dereference, address: 0000000000000000 +[ 0.061037] #PF: supervisor instruction fetch in kernel mode +[ 0.061126] #PF: error_code(0x0010) - not-present page +[ 0.061239] Oops: Oops: 0010 [#1] PREEMPT SMP NOPTI +[ 0.061311] CPU: 0 UID: 0 PID: 318 Comm: guest-agent Not tainted 6.12.44 #1 +[ 0.061398] RIP: 0010:0x0 +[ 0.062476] Call Trace: +[ 0.062527] +[ 0.062561] ? tsi_dgram_setsockopt+0x6a/0x90 +[ 0.062644] ? do_sock_setsockopt+0xaa/0x190 +[ 0.062736] ? __sys_setsockopt+0x5d/0xb0 +``` + +This shows a NULL pointer dereference in `tsi_dgram_setsockopt`. + +## Test Files + +- Test cases: `tests/test_cases/src/` +- Test runner: `tests/runner/src/main.rs` + +### Adding a new test + +1. Create `tests/test_cases/src/test_.rs` +2. Register in `tests/test_cases/src/lib.rs`: + ```rust + mod test_; + use test_::Test; + + // In test_cases() function: + TestCase::new("test-name", Box::new(Test)), + ``` + +## Common Issues + +### Test hangs indefinitely +- Usually indicates a kernel panic or deadlock +- Use `timeout` to prevent waiting forever +- Check logs for kernel panic messages + +### "cannot open shared object file: libkrun.so.1" +- Always use `make test` which sets up LD_LIBRARY_PATH correctly +- Do NOT run `./run.sh` directly + +### Cargo build lock +- If you see "Blocking waiting for file lock on build directory" +- Another cargo process is running, wait for it or kill it diff --git a/tests/guest-agent/src/main.rs b/tests/guest-agent/src/main.rs index 14015e7c4..3c38e7bee 100644 --- a/tests/guest-agent/src/main.rs +++ b/tests/guest-agent/src/main.rs @@ -8,7 +8,11 @@ fn run_guest_agent(test_name: &str) -> anyhow::Result<()> { .into_iter() .find(|t| t.name() == test_name) .context("No such test!")?; - let TestCase { test, name: _, requires_namespace: _ } = test_case; + let TestCase { + test, + name: _, + requires_namespace: _, + } = test_case; test.in_guest(); Ok(()) } diff --git a/tests/runner/src/main.rs b/tests/runner/src/main.rs index caec723bd..8f490e649 100644 --- a/tests/runner/src/main.rs +++ b/tests/runner/src/main.rs @@ -25,7 +25,7 @@ fn get_test(name: &str) -> anyhow::Result> { .map(|t| t.test) } -fn start_vm(test_setup: TestSetup) -> anyhow::Result<()> { +fn start_vm(mut test_setup: TestSetup) -> anyhow::Result<()> { // Raise soft fd limit up to the hard limit let (_soft_limit, hard_limit) = getrlimit(Resource::RLIMIT_NOFILE).context("getrlimit RLIMIT_NOFILE")?; @@ -40,6 +40,8 @@ fn start_vm(test_setup: TestSetup) -> anyhow::Result<()> { .map(|t| t.requires_namespace) .unwrap_or(false); + test_setup.requires_namespace = requires_namespace; + if requires_namespace { setup_namespace_and_run(test_setup)?; } else { @@ -52,7 +54,7 @@ fn start_vm(test_setup: TestSetup) -> anyhow::Result<()> { fn setup_namespace_and_run(test_setup: TestSetup) -> anyhow::Result<()> { use nix::sched::{unshare, CloneFlags}; - use nix::unistd::{fork, Gid, Uid, ForkResult}; + use nix::unistd::{fork, ForkResult, Gid, Uid}; use std::fs; // Get our current uid/gid before entering the namespace @@ -65,17 +67,14 @@ fn setup_namespace_and_run(test_setup: TestSetup) -> anyhow::Result<()> { // Set up uid_map to map our uid to root (0) in the namespace let uid_map = format!("0 {} 1", uid); - fs::write("/proc/self/uid_map", uid_map) - .context("Failed to write uid_map")?; + fs::write("/proc/self/uid_map", uid_map).context("Failed to write uid_map")?; // Disable setgroups (required before writing gid_map as non-root) - fs::write("/proc/self/setgroups", "deny") - .context("Failed to write setgroups")?; + fs::write("/proc/self/setgroups", "deny").context("Failed to write setgroups")?; // Set up gid_map to map our gid to root (0) in the namespace let gid_map = format!("0 {} 1", gid); - fs::write("/proc/self/gid_map", gid_map) - .context("Failed to write gid_map")?; + fs::write("/proc/self/gid_map", gid_map).context("Failed to write gid_map")?; // Fork so the child becomes PID 1 in the new PID namespace // This is necessary to be able to mount procfs @@ -92,7 +91,47 @@ fn setup_namespace_and_run(test_setup: TestSetup) -> anyhow::Result<()> { } } ForkResult::Child => { + use nix::mount::{mount, MsFlags}; + use std::fs::create_dir; + // Child continues - we are now PID 1 in the PID namespace + // Set up the root directory structure (but don't chroot yet - that happens after krun loads libraries) + let root_dir = test_setup.tmp_dir.join("root"); + create_dir(&root_dir).context("Failed to create root directory")?; + + // Create necessary directories + create_dir(root_dir.join("tmp")).context("Failed to create tmp directory")?; + create_dir(root_dir.join("dev")).context("Failed to create dev directory")?; + create_dir(root_dir.join("proc")).context("Failed to create proc directory")?; + create_dir(root_dir.join("sys")).context("Failed to create sys directory")?; + + // Copy guest agent + let guest_agent_path = env::var_os("KRUN_TEST_GUEST_AGENT_PATH") + .context("KRUN_TEST_GUEST_AGENT_PATH env variable not set")?; + fs::copy(&guest_agent_path, root_dir.join("guest-agent")) + .context("Failed to copy guest agent")?; + + // Make mounts private so they don't affect parent namespace + mount( + None::<&str>, + "/", + None::<&str>, + MsFlags::MS_REC | MsFlags::MS_PRIVATE, + None::<&str>, + ) + .context("Failed to make / private")?; + + // Bind mount /dev + mount( + Some("/dev"), + root_dir.join("dev").as_path(), + None::<&str>, + MsFlags::MS_BIND | MsFlags::MS_REC, + None::<&str>, + ) + .context("Failed to bind mount /dev")?; + + // The test's start_vm will handle chroot after loading libraries let test = get_test(&test_setup.test_case)?; test.start_vm(test_setup.clone()) .with_context(|| format!("testcase: {test_setup:?}"))?; @@ -305,7 +344,11 @@ fn main() -> anyhow::Result<()> { let command = cli.command.unwrap_or_default(); match command { - CliCommand::StartVm { test_case, tmp_dir } => start_vm(TestSetup { test_case, tmp_dir }), + CliCommand::StartVm { test_case, tmp_dir } => start_vm(TestSetup { + test_case, + tmp_dir, + requires_namespace: false, // Will be set by start_vm based on test case + }), CliCommand::Test { test_case, base_dir, diff --git a/tests/test_cases/Cargo.toml b/tests/test_cases/Cargo.toml index 0e6da173c..8b9a37b9f 100644 --- a/tests/test_cases/Cargo.toml +++ b/tests/test_cases/Cargo.toml @@ -12,6 +12,6 @@ name = "test_cases" [dependencies] krun-sys = { path = "../../krun-sys", optional = true } macros = { path = "../macros" } -nix = { version = "0.29.0", features = ["socket", "sched", "user", "mount"] } +nix = { version = "0.29.0", features = ["socket", "sched", "user", "mount", "fs"] } anyhow = "1.0.95" tempdir = "0.3.7" \ No newline at end of file diff --git a/tests/test_cases/src/common.rs b/tests/test_cases/src/common.rs index e78dcc261..90f250ab9 100644 --- a/tests/test_cases/src/common.rs +++ b/tests/test_cases/src/common.rs @@ -1,8 +1,7 @@ -//! Common utilities used by multiple test +//! Common utilities used by multiple tests use anyhow::Context; use std::ffi::CString; -use std::fs; use std::fs::create_dir; use std::os::unix::ffi::OsStrExt; use std::path::Path; @@ -11,125 +10,57 @@ use std::ptr::null; use crate::{krun_call, TestSetup}; use krun_sys::*; -use nix::unistd::{chroot, chdir}; -use std::path::PathBuf; - fn copy_guest_agent(dir: &Path) -> anyhow::Result<()> { let path = std::env::var_os("KRUN_TEST_GUEST_AGENT_PATH") .context("KRUN_TEST_GUEST_AGENT_PATH env variable not set")?; let output_path = dir.join("guest-agent"); - fs::copy(path, output_path).context("Failed to copy executable into vm")?; + std::fs::copy(path, output_path).context("Failed to copy executable into vm")?; Ok(()) } -/// Common part of most test. This setups an empty root filesystem, copies the guest agent there -/// and runs the guest agent in the VM. -/// Note that some tests might want to use a different root file system (perhaps a qcow image), -/// in which case the test can implement the equivalent functionality itself, or better if there -/// are more test doing that, add another utility method in this file. +/// Common setup for most tests. Sets up the root filesystem and runs the guest agent in the VM. /// -/// The returned object is used for deleting the temporary files. -pub fn setup_fs_and_enter(ctx: u32, test_setup: TestSetup) -> anyhow::Result<()> { - let root_dir = test_setup.tmp_dir.join("root"); - create_dir(&root_dir).context("Failed to create root directory")?; - - let path_str = CString::new(root_dir.as_os_str().as_bytes()).context("CString::new")?; - copy_guest_agent(&root_dir)?; - unsafe { - krun_call!(krun_set_root(ctx, path_str.as_ptr()))?; - krun_call!(krun_set_workdir(ctx, c"/".as_ptr()))?; - let test_case_cstr = CString::new(test_setup.test_case).context("CString::new")?; - let argv = [test_case_cstr.as_ptr(), null()]; - //let envp = [c"RUST_BACKTRACE=1".as_ptr(), null()]; - let envp = [null()]; - krun_call!(krun_set_exec( - ctx, - c"/guest-agent".as_ptr(), - argv.as_ptr(), - envp.as_ptr(), - ))?; - krun_call!(krun_start_enter(ctx))?; - } - unreachable!() -} - -/// Like setup_fs_and_enter, but changes the host process's root to the guest's root -/// before entering the VM. This is needed for Unix domain socket TSI tests where the -/// host process needs to access socket paths in the guest filesystem. +/// If `requires_namespace` is true, the runner has already created the root directory structure +/// with /dev, /tmp, /sys, guest-agent. After krun_create_ctx loads libraries, we chroot there. /// -/// This function: -/// 1. Creates a new user namespace and mount namespace (unshare CLONE_NEWUSER | CLONE_NEWNS) -/// 2. Sets up uid/gid mappings to become root in the namespace -/// 3. Changes root to the guest's root directory (chroot) -/// 4. Then calls krun_start_enter -/// -/// The before_enter callback is called after chroot but before krun_start_enter, allowing -/// setup of host-side resources (like Unix domain socket servers) that need to be accessible -/// at the same paths as the guest will use. -/// -/// Note: This uses rootless namespaces (user namespaces) so it doesn't require root. -pub fn setup_fs_and_enter_with_namespace( - ctx: u32, - test_setup: TestSetup, - before_enter: F, -) -> anyhow::Result<()> -where - F: FnOnce() -> anyhow::Result<()>, -{ - let root_dir = test_setup.tmp_dir.join("root"); - create_dir(&root_dir).context("Failed to create root directory")?; - - // Create necessary directories in the guest root - create_dir(root_dir.join("tmp")).context("Failed to create tmp directory")?; - create_dir(root_dir.join("dev")).context("Failed to create dev directory")?; - create_dir(root_dir.join("proc")).context("Failed to create proc directory")?; - create_dir(root_dir.join("sys")).context("Failed to create sys directory")?; - - copy_guest_agent(&root_dir)?; - - // The runner has already set up the namespace for us (user+mount+pid) - // We are now root in the user namespace and PID 1 in the PID namespace - // Make our mounts private so they don't affect the parent namespace - use nix::mount::{mount, MsFlags}; - mount( - None::<&str>, - "/", - None::<&str>, - MsFlags::MS_REC | MsFlags::MS_PRIVATE, - None::<&str>, - ).context("Failed to make / private")?; - - // Bind mount /dev into the guest root so /dev/kvm is accessible - // (we're root in the namespace now) - mount( - Some("/dev"), - root_dir.join("dev").as_path(), - None::<&str>, - MsFlags::MS_BIND | MsFlags::MS_REC, - None::<&str>, - ).context("Failed to bind mount /dev")?; - - // Now we can chroot - let root_path = PathBuf::from(&root_dir); - chroot(&root_path).context("Failed to chroot to guest root")?; - chdir("/").context("Failed to chdir to /")?; - - // Mount procfs after chroot with standard proc mount flags - mount( - Some("proc"), - "/proc", - Some("proc"), - MsFlags::MS_NOSUID | MsFlags::MS_NODEV | MsFlags::MS_NOEXEC, - None::<&str>, - ).context("Failed to mount procfs")?; - - // Call the before_enter callback to set up host-side resources - before_enter().context("before_enter callback failed")?; +/// If `requires_namespace` is false, this function creates a root directory, copies the +/// guest agent there, and sets it as the VM root. +pub fn setup_fs_and_enter(ctx: u32, test_setup: TestSetup) -> anyhow::Result<()> { + let root_path = if test_setup.requires_namespace { + // Runner set up the root dir structure, now we chroot after libraries are loaded + use nix::mount::{mount, MsFlags}; + use nix::unistd::{chdir, chroot}; + + let root_dir = test_setup.tmp_dir.join("root"); + + // Chroot into the prepared root + chroot(&root_dir).context("Failed to chroot")?; + chdir("/").context("Failed to chdir to /")?; + + // Mount procfs after chroot + mount( + Some("proc"), + "/proc", + Some("proc"), + MsFlags::MS_NOSUID | MsFlags::MS_NODEV | MsFlags::MS_NOEXEC, + None::<&str>, + ) + .context("Failed to mount procfs")?; + + CString::new("/").context("CString::new")? + } else { + // Create root directory and copy guest agent + let root_dir = test_setup.tmp_dir.join("root"); + create_dir(&root_dir).context("Failed to create root directory")?; + // Create /tmp for tests that use Unix sockets + let _ = create_dir(root_dir.join("tmp")); + copy_guest_agent(&root_dir)?; + CString::new(root_dir.as_os_str().as_bytes()).context("CString::new")? + }; - let path_str = CString::new("/").context("CString::new")?; unsafe { - krun_call!(krun_set_root(ctx, path_str.as_ptr()))?; + krun_call!(krun_set_root(ctx, root_path.as_ptr()))?; krun_call!(krun_set_workdir(ctx, c"/".as_ptr()))?; let test_case_cstr = CString::new(test_setup.test_case).context("CString::new")?; let argv = [test_case_cstr.as_ptr(), null()]; diff --git a/tests/test_cases/src/datagram_tester.rs b/tests/test_cases/src/datagram_tester.rs new file mode 100644 index 000000000..9fe1829fc --- /dev/null +++ b/tests/test_cases/src/datagram_tester.rs @@ -0,0 +1,120 @@ +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr, UdpSocket}; +use std::os::unix::io::AsRawFd; +use std::os::unix::net::UnixDatagram; +use std::path::Path; +use std::thread::{self, JoinHandle}; +use std::time::Duration; + +use nix::sys::socket::{recvfrom, sendto, MsgFlags, UnixAddr}; + +use crate::IpVersion; + +trait DatagramSocket: Send + Sized + 'static { + type Addr: Clone + Send; + fn send_to(&self, buf: &[u8], addr: Self::Addr) -> std::io::Result; + fn recv_from(&self, buf: &mut [u8]) -> std::io::Result<(usize, Self::Addr)>; + + fn run_server(self) { + let mut buf = [0u8; 64]; + + let (len, client_addr) = self.recv_from(&mut buf).expect("recv_from failed"); + assert_eq!(len, 5); + assert_eq!(&buf[..len], b"ping!"); + + assert_eq!(self.send_to(b"pong!", client_addr.clone()).unwrap(), 5); + + let (len, _) = self.recv_from(&mut buf).expect("recv_from failed"); + assert_eq!(len, 4); + assert_eq!(&buf[..len], b"bye!"); + } + + fn run_client(self, server_addr: Self::Addr) { + let mut buf = [0u8; 64]; + + assert_eq!(self.send_to(b"ping!", server_addr.clone()).unwrap(), 5); + + let (len, _) = self.recv_from(&mut buf).expect("recv_from failed"); + assert_eq!(len, 5); + assert_eq!(&buf[..len], b"pong!"); + + assert_eq!(self.send_to(b"bye!", server_addr).unwrap(), 4); + } +} + +impl DatagramSocket for UdpSocket { + type Addr = SocketAddr; + fn send_to(&self, buf: &[u8], addr: SocketAddr) -> std::io::Result { + UdpSocket::send_to(self, buf, addr) + } + fn recv_from(&self, buf: &mut [u8]) -> std::io::Result<(usize, SocketAddr)> { + UdpSocket::recv_from(self, buf) + } +} + +impl DatagramSocket for UnixDatagram { + type Addr = UnixAddr; + fn send_to(&self, buf: &[u8], addr: UnixAddr) -> std::io::Result { + sendto(self.as_raw_fd(), buf, &addr, MsgFlags::empty()) + .map_err(|e| std::io::Error::from_raw_os_error(e as i32)) + } + fn recv_from(&self, buf: &mut [u8]) -> std::io::Result<(usize, UnixAddr)> { + let (size, addr_opt) = recvfrom::(self.as_raw_fd(), buf) + .map_err(|e| std::io::Error::from_raw_os_error(e as i32))?; + let addr = addr_opt.ok_or_else(|| std::io::Error::other("No source address"))?; + Ok((size, addr)) + } +} + +fn udp_bind(ip_version: IpVersion, port: u16) -> UdpSocket { + let addr = match ip_version { + IpVersion::V4 => SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), port), + IpVersion::V6 => SocketAddr::new(IpAddr::V6(Ipv6Addr::LOCALHOST), port), + }; + let socket = UdpSocket::bind(addr).expect("Failed to bind socket"); + socket + .set_read_timeout(Some(Duration::from_secs(5))) + .unwrap(); + socket +} + +fn unix_bind(path: &Path) -> UnixDatagram { + let _ = std::fs::remove_file(path); + let socket = UnixDatagram::bind(path).expect("Failed to bind socket"); + socket + .set_read_timeout(Some(Duration::from_secs(10))) + .unwrap(); + socket +} + +pub fn spawn_server_udp(ip_version: IpVersion, port: u16) -> JoinHandle<()> { + let socket = udp_bind(ip_version, port); + thread::spawn(move || socket.run_server()) +} + +pub fn spawn_client_udp(ip_version: IpVersion, port: u16) -> JoinHandle<()> { + thread::spawn(move || { + let socket = udp_bind(ip_version, 0); + let server_addr = match ip_version { + IpVersion::V4 => SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), port), + IpVersion::V6 => SocketAddr::new(IpAddr::V6(Ipv6Addr::LOCALHOST), port), + }; + socket.run_client(server_addr); + }) +} + +pub fn spawn_server_unix(path: impl AsRef + Send + 'static) -> JoinHandle<()> { + let socket = unix_bind(path.as_ref()); + thread::spawn(move || socket.run_server()) +} + +pub fn spawn_client_unix( + server_path: impl AsRef + Send + 'static, + client_path: impl AsRef + Send + 'static, +) -> JoinHandle<()> { + let server_path = server_path.as_ref().to_path_buf(); + let client_path = client_path.as_ref().to_path_buf(); + thread::spawn(move || { + let socket = unix_bind(&client_path); + socket.run_client(UnixAddr::new(&server_path).expect("Invalid server path")); + }) +} diff --git a/tests/test_cases/src/lib.rs b/tests/test_cases/src/lib.rs index 9eff3505d..505bc58d1 100644 --- a/tests/test_cases/src/lib.rs +++ b/tests/test_cases/src/lib.rs @@ -4,18 +4,22 @@ use test_vm_config::TestVmConfig; mod test_vsock_guest_connect; use test_vsock_guest_connect::TestVsockGuestConnect; -mod test_tsi_tcp_guest_connect; -use test_tsi_tcp_guest_connect::TestTsiTcpGuestConnect; +mod test_tsi_unix_dgram_setsockopt; +use test_tsi_unix_dgram_setsockopt::TestTsiUnixDgramSetsockopt; -mod test_tsi_tcp_guest_listen; -use test_tsi_tcp_guest_listen::TestTsiTcpGuestListen; +mod test_tsi; +use test_tsi::{At::*, TestTsi, Transport::*}; + +const UNIX_STREAM_PATH: &str = "/tmp/test-unix-stream.sock"; +const UNIX_DGRAM_SERVER: &str = "/tmp/test-unix-dgram.sock"; +const UNIX_DGRAM_CLIENT: &str = "/tmp/test-unix-dgram-client.sock"; mod test_multiport_console; use test_multiport_console::TestMultiportConsole; pub fn test_cases() -> Vec { - // Register your test here: vec![ + // VM config tests TestCase::new( "configure-vm-1cpu-256MiB", Box::new(TestVmConfig { @@ -30,14 +34,161 @@ pub fn test_cases() -> Vec { ram_mib: 1024, }), ), + // Vsock connecting to unix socket (non TSI) TestCase::new("vsock-guest-connect", Box::new(TestVsockGuestConnect)), + // Regression test for NULL pointer deref in tsi_dgram_setsockopt + // With unfixed kernel: guest kernel panics + // With fixed kernel: test passes + TestCase::new( + "tsi-unix-dgram-setsockopt", + Box::new(TestTsiUnixDgramSetsockopt), + ), + TestCase::new( + "tsi-tcp-host-guest-ipv4", + Box::new(TestTsi::new( + Tcp { + ip: IpVersion::V4, + port: 8000, + }, + Host, + Guest, + )), + ), + TestCase::new( + "tsi-tcp-host-guest-ipv6", + Box::new(TestTsi::new( + Tcp { + ip: IpVersion::V6, + port: 8001, + }, + Host, + Guest, + )), + ), + TestCase::new( + "tsi-tcp-guest-host-ipv4", + Box::new(TestTsi::new( + Tcp { + ip: IpVersion::V4, + port: 8002, + }, + Guest, + Host, + )), + ), + TestCase::new( + "tsi-tcp-guest-host-ipv6", + Box::new(TestTsi::new( + Tcp { + ip: IpVersion::V6, + port: 8003, + }, + Guest, + Host, + )), + ), + TestCase::new( + "tsi-tcp-guest-guest-ipv4", + Box::new(TestTsi::new( + Tcp { + ip: IpVersion::V4, + port: 8004, + }, + Guest, + Guest, + )), + ), + TestCase::new( + "tsi-tcp-guest-guest-ipv6", + Box::new(TestTsi::new( + Tcp { + ip: IpVersion::V6, + port: 8005, + }, + Guest, + Guest, + )), + ), TestCase::new( - "tsi-tcp-guest-connect", - Box::new(TestTsiTcpGuestConnect::new()), + "tsi-udp-host-guest-ipv4", + Box::new(TestTsi::new( + Udp { + ip: IpVersion::V4, + port: 8006, + }, + Host, + Guest, + )), ), TestCase::new( - "tsi-tcp-guest-listen", - Box::new(TestTsiTcpGuestListen::new()), + "tsi-udp-host-guest-ipv6", + Box::new(TestTsi::new( + Udp { + ip: IpVersion::V6, + port: 8007, + }, + Host, + Guest, + )), + ), + TestCase::new( + "tsi-udp-guest-guest-ipv4", + Box::new(TestTsi::new( + Udp { + ip: IpVersion::V4, + port: 8008, + }, + Guest, + Guest, + )), + ), + TestCase::new( + "tsi-udp-guest-guest-ipv6", + Box::new(TestTsi::new( + Udp { + ip: IpVersion::V6, + port: 8009, + }, + Guest, + Guest, + )), + ), + TestCase::new_with_namespace( + "tsi-unix-stream-host-guest", + Box::new(TestTsi::new( + UnixStream { + path: UNIX_STREAM_PATH, + }, + Host, + Guest, + )), + ), + // Unix stream: both in guest + TestCase::new( + "tsi-unix-stream-guest-guest", + Box::new(TestTsi::new( + UnixStream { + path: UNIX_STREAM_PATH, + }, + Guest, + Guest, + )), + ), + // TODO: this is probably still broken on the kernel side + // TestCase::new("tsi-unix-dgram-guest-guest", Box::new(TestTsi::new( + // UnixDgram { server_path: UNIX_DGRAM_SERVER, client_path: UNIX_DGRAM_CLIENT }, Guest, Guest))), + + // Unix dgram: server on host (requires namespace) + TestCase::new_with_namespace( + "tsi-unix-dgram-host-guest", + Box::new(TestTsi::new( + UnixDgram { + server_path: UNIX_DGRAM_SERVER, + client_path: UNIX_DGRAM_CLIENT, + }, + Host, + Guest, + )), ), TestCase::new("multiport-console", Box::new(TestMultiportConsole)), ] @@ -58,9 +209,10 @@ compile_error!("Cannot enable both guest and host in the same binary!"); #[cfg(feature = "host")] mod common; +mod datagram_tester; #[cfg(feature = "host")] mod krun; -mod tcp_tester; +mod stream_tester; #[host] #[derive(Clone, Debug)] @@ -68,6 +220,8 @@ pub struct TestSetup { pub test_case: String, // A tmp directory for misc. artifacts used be the test (e.g. sockets) pub tmp_dir: PathBuf, + // If true, runner has already set up namespace with chroot - root is "/" + pub requires_namespace: bool, } #[host] @@ -98,11 +252,19 @@ impl TestCase { // Your test can be parametrized, so you can add the same test multiple times constructed with // different parameters with and specify a different name here. pub fn new(name: &'static str, test: Box) -> Self { - Self { name, test, requires_namespace: false } + Self { + name, + test, + requires_namespace: false, + } } pub fn new_with_namespace(name: &'static str, test: Box) -> Self { - Self { name, test, requires_namespace: true } + Self { + name, + test, + requires_namespace: true, + } } #[allow(dead_code)] @@ -111,6 +273,12 @@ impl TestCase { } } +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub enum IpVersion { + V4, + V6, +} + #[cfg(test)] mod tests { use super::*; diff --git a/tests/test_cases/src/stream_tester.rs b/tests/test_cases/src/stream_tester.rs new file mode 100644 index 000000000..ec4c2691b --- /dev/null +++ b/tests/test_cases/src/stream_tester.rs @@ -0,0 +1,111 @@ +use std::io::{ErrorKind, Read, Write}; +use std::net::{ + IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr, SocketAddrV4, SocketAddrV6, TcpListener, TcpStream, +}; +use std::os::unix::net::{UnixListener, UnixStream}; +use std::path::Path; +use std::thread::{self, JoinHandle}; +use std::time::Duration; + +use crate::IpVersion; + +trait StreamSocket: Read + Write + Send + Sized + 'static { + fn set_nonblocking(&self, nonblocking: bool) -> std::io::Result<()>; + + fn expect_msg(&mut self, expected: &[u8]) { + let mut buf = vec![0; expected.len()]; + self.read_exact(&mut buf[..]).unwrap(); + assert_eq!(&buf[..], expected); + } + + fn expect_wouldblock(&mut self) { + self.set_nonblocking(true).unwrap(); + let err = self.read(&mut [0u8; 1]).unwrap_err(); + self.set_nonblocking(false).unwrap(); + assert_eq!(err.kind(), ErrorKind::WouldBlock); + } + + fn run_server(mut self) { + self.expect_msg(b"ping!"); + self.expect_wouldblock(); + self.write_all(b"pong!").unwrap(); + self.flush().unwrap(); + self.expect_msg(b"bye!"); + std::mem::forget(self); + } + + fn run_client(mut self) { + self.write_all(b"ping!").unwrap(); + self.flush().unwrap(); + self.expect_msg(b"pong!"); + self.expect_wouldblock(); + self.write_all(b"bye!").unwrap(); + self.flush().unwrap(); + } +} + +impl StreamSocket for TcpStream { + fn set_nonblocking(&self, nonblocking: bool) -> std::io::Result<()> { + TcpStream::set_nonblocking(self, nonblocking) + } +} + +impl StreamSocket for UnixStream { + fn set_nonblocking(&self, nonblocking: bool) -> std::io::Result<()> { + UnixStream::set_nonblocking(self, nonblocking) + } +} + +fn tcp_bind(ip_version: IpVersion, port: u16) -> TcpListener { + match ip_version { + IpVersion::V4 => TcpListener::bind(SocketAddrV4::new(Ipv4Addr::LOCALHOST, port)), + IpVersion::V6 => TcpListener::bind(SocketAddrV6::new(Ipv6Addr::LOCALHOST, port, 0, 0)), + } + .expect("Failed to bind server socket") +} + +fn connect_with_retry(mut connect: impl FnMut() -> std::io::Result) -> T { + for attempt in 1..=5 { + match connect() { + Ok(stream) => return stream, + Err(err) if attempt == 5 => panic!("Couldn't connect after 5 attempts: {err}"), + Err(_) => thread::sleep(Duration::from_secs(1)), + } + } + unreachable!() +} + +pub fn spawn_server_tcp(ip_version: IpVersion, port: u16) -> JoinHandle<()> { + let listener = tcp_bind(ip_version, port); + thread::spawn(move || { + let (stream, _) = listener.accept().unwrap(); + stream.run_server(); + }) +} + +pub fn spawn_client_tcp(ip_version: IpVersion, port: u16) -> JoinHandle<()> { + thread::spawn(move || { + let addr = match ip_version { + IpVersion::V4 => SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), port), + IpVersion::V6 => SocketAddr::new(IpAddr::V6(Ipv6Addr::LOCALHOST), port), + }; + connect_with_retry(|| TcpStream::connect(addr)).run_client(); + }) +} + +pub fn spawn_server_unix(path: impl AsRef + Send + 'static) -> JoinHandle<()> { + let path = path.as_ref().to_path_buf(); + let _ = std::fs::remove_file(&path); + let listener = UnixListener::bind(&path).expect("Failed to bind server socket"); + thread::spawn(move || { + let (stream, _) = listener.accept().unwrap(); + stream.run_server(); + }) +} + +pub fn spawn_client_unix(path: impl AsRef + Send + 'static) -> JoinHandle<()> { + let path = path.as_ref().to_path_buf(); + thread::spawn(move || { + connect_with_retry(|| UnixStream::connect(&path)).run_client(); + }) +} diff --git a/tests/test_cases/src/tcp_tester.rs b/tests/test_cases/src/tcp_tester.rs deleted file mode 100644 index c90f12c3b..000000000 --- a/tests/test_cases/src/tcp_tester.rs +++ /dev/null @@ -1,79 +0,0 @@ -use std::io::{ErrorKind, Read, Write}; -use std::mem; -use std::net::{IpAddr, Ipv4Addr, SocketAddr, SocketAddrV4, TcpListener, TcpStream}; -use std::thread; -use std::time::Duration; - -fn expect_msg(stream: &mut TcpStream, expected: &[u8]) { - let mut buf = vec![0; expected.len()]; - stream.read_exact(&mut buf[..]).unwrap(); - assert_eq!(&buf[..], expected); -} - -fn expect_wouldblock(stream: &mut TcpStream) { - stream.set_nonblocking(true).unwrap(); - let err = stream.read(&mut [0u8; 1]).unwrap_err(); - stream.set_nonblocking(false).unwrap(); - assert_eq!(err.kind(), ErrorKind::WouldBlock); -} - -fn set_timeouts(stream: &mut TcpStream) { - stream - .set_read_timeout(Some(Duration::from_millis(500))) - .unwrap(); - stream - .set_write_timeout(Some(Duration::from_millis(500))) - .unwrap(); -} - -fn connect(port: u16) -> TcpStream { - let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), port); - let mut tries = 0; - loop { - match TcpStream::connect(addr) { - Ok(stream) => return stream, - Err(err) => { - if tries == 5 { - panic!("Couldn't connect to server after 5 attempts: {err}"); - } - tries += 1; - thread::sleep(Duration::from_secs(1)); - } - } - } -} - -#[derive(Debug, Copy, Clone)] -pub struct TcpTester { - port: u16, -} - -impl TcpTester { - pub const fn new(port: u16) -> Self { - Self { port } - } - - pub fn create_server_socket(&self) -> TcpListener { - TcpListener::bind(SocketAddrV4::new(Ipv4Addr::new(0, 0, 0, 0), self.port)).unwrap() - } - - pub fn run_server(&self, listener: TcpListener) { - let (mut stream, _addr) = listener.accept().unwrap(); - set_timeouts(&mut stream); - stream.write_all(b"ping!").unwrap(); - expect_msg(&mut stream, b"pong!"); - expect_wouldblock(&mut stream); - stream.write_all(b"bye!").unwrap(); - // We leak the file descriptor for now, since there is no easy way to close it on libkrun exit - mem::forget(listener); - } - - pub fn run_client(&self) { - let mut stream = connect(self.port); - set_timeouts(&mut stream); - expect_msg(&mut stream, b"ping!"); - expect_wouldblock(&mut stream); - stream.write_all(b"pong!").unwrap(); - expect_msg(&mut stream, b"bye!"); - } -} diff --git a/tests/test_cases/src/test_tsi.rs b/tests/test_cases/src/test_tsi.rs new file mode 100644 index 000000000..002256d92 --- /dev/null +++ b/tests/test_cases/src/test_tsi.rs @@ -0,0 +1,147 @@ +use crate::IpVersion; +use macros::{guest, host}; +use std::thread::JoinHandle; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Transport { + Tcp { + ip: IpVersion, + port: u16, + }, + Udp { + ip: IpVersion, + port: u16, + }, + UnixStream { + path: &'static str, + }, + UnixDgram { + server_path: &'static str, + client_path: &'static str, + }, +} + +impl Transport { + fn spawn_server(&self, root: impl AsRef) -> JoinHandle<()> { + use crate::{datagram_tester, stream_tester}; + let root = root.as_ref(); + match self { + Transport::Tcp { ip, port } => stream_tester::spawn_server_tcp(*ip, *port), + Transport::Udp { ip, port } => datagram_tester::spawn_server_udp(*ip, *port), + Transport::UnixStream { path } => { + stream_tester::spawn_server_unix(root.join(&path[1..])) + } + Transport::UnixDgram { server_path, .. } => { + datagram_tester::spawn_server_unix(root.join(&server_path[1..])) + } + } + } + + fn spawn_client(&self, root: impl AsRef) -> JoinHandle<()> { + use crate::{datagram_tester, stream_tester}; + let root = root.as_ref(); + match self { + Transport::Tcp { ip, port } => stream_tester::spawn_client_tcp(*ip, *port), + Transport::Udp { ip, port } => datagram_tester::spawn_client_udp(*ip, *port), + Transport::UnixStream { path } => { + stream_tester::spawn_client_unix(root.join(&path[1..])) + } + Transport::UnixDgram { + server_path, + client_path, + } => datagram_tester::spawn_client_unix( + root.join(&server_path[1..]), + root.join(&client_path[1..]), + ), + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum At { + Host, + Guest, +} + +pub struct TestTsi { + transport: Transport, + server_at: At, + client_at: At, +} + +impl TestTsi { + pub fn new(transport: Transport, server_at: At, client_at: At) -> Self { + Self { + transport, + server_at, + client_at, + } + } +} + +#[host] +mod host { + use super::*; + use crate::common::setup_fs_and_enter; + use crate::{krun_call, krun_call_u32}; + use crate::{Test, TestSetup}; + use krun_sys::*; + use std::ffi::CString; + use std::ptr::null; + + impl Test for TestTsi { + fn start_vm(self: Box, test_setup: TestSetup) -> anyhow::Result<()> { + let root = test_setup.tmp_dir.join("root"); + + if self.server_at == At::Host { + self.transport.spawn_server(&root); + } + if self.client_at == At::Host { + self.transport.spawn_client(&root); + } + + unsafe { + krun_call!(krun_set_log_level(KRUN_LOG_LEVEL_WARN))?; + let ctx = krun_call_u32!(krun_create_ctx())?; + + // TCP with server in guest and client on host needs port mapping + if let Transport::Tcp { port, .. } = self.transport { + if self.server_at == At::Guest && self.client_at == At::Host { + let port_mapping = format!("{port}:{port}"); + let port_mapping = CString::new(port_mapping).unwrap(); + let port_map = [port_mapping.as_ptr(), null()]; + krun_call!(krun_set_port_map(ctx, port_map.as_ptr()))?; + } + } + + krun_call!(krun_set_vm_config(ctx, 1, 512))?; + setup_fs_and_enter(ctx, test_setup)?; + } + Ok(()) + } + } +} + +#[guest] +mod guest { + use super::*; + use crate::Test; + + impl Test for TestTsi { + fn in_guest(self: Box) { + let server_handle = + (self.server_at == At::Guest).then(|| self.transport.spawn_server("/")); + let client_handle = + (self.client_at == At::Guest).then(|| self.transport.spawn_client("/")); + + // Wait for whichever side runs in guest to complete + if let Some(handle) = client_handle { + handle.join().unwrap(); + } else if let Some(handle) = server_handle { + handle.join().unwrap(); + } + + println!("OK"); + } + } +} diff --git a/tests/test_cases/src/test_tsi_tcp_guest_connect.rs b/tests/test_cases/src/test_tsi_tcp_guest_connect.rs deleted file mode 100644 index 9ec07a7f8..000000000 --- a/tests/test_cases/src/test_tsi_tcp_guest_connect.rs +++ /dev/null @@ -1,54 +0,0 @@ -use crate::tcp_tester::TcpTester; -use macros::{guest, host}; - -const PORT: u16 = 8000; - -pub struct TestTsiTcpGuestConnect { - tcp_tester: TcpTester, -} - -impl TestTsiTcpGuestConnect { - pub fn new() -> TestTsiTcpGuestConnect { - Self { - tcp_tester: TcpTester::new(PORT), - } - } -} - -#[host] -mod host { - use super::*; - - use crate::common::setup_fs_and_enter; - use crate::{krun_call, krun_call_u32}; - use crate::{Test, TestSetup}; - use krun_sys::*; - use std::thread; - - impl Test for TestTsiTcpGuestConnect { - fn start_vm(self: Box, test_setup: TestSetup) -> anyhow::Result<()> { - let listener = self.tcp_tester.create_server_socket(); - thread::spawn(move || self.tcp_tester.run_server(listener)); - unsafe { - krun_call!(krun_set_log_level(KRUN_LOG_LEVEL_WARN))?; - let ctx = krun_call_u32!(krun_create_ctx())?; - krun_call!(krun_set_vm_config(ctx, 1, 512))?; - setup_fs_and_enter(ctx, test_setup)?; - } - Ok(()) - } - } -} - -#[guest] -mod guest { - use super::*; - use crate::Test; - - impl Test for TestTsiTcpGuestConnect { - fn in_guest(self: Box) { - self.tcp_tester.run_client(); - println!("OK"); - } - } -} diff --git a/tests/test_cases/src/test_tsi_tcp_guest_listen.rs b/tests/test_cases/src/test_tsi_tcp_guest_listen.rs deleted file mode 100644 index fa4d108bf..000000000 --- a/tests/test_cases/src/test_tsi_tcp_guest_listen.rs +++ /dev/null @@ -1,65 +0,0 @@ -use crate::tcp_tester::TcpTester; -use macros::{guest, host}; - -const PORT: u16 = 8001; - -pub struct TestTsiTcpGuestListen { - tcp_tester: TcpTester, -} - -impl TestTsiTcpGuestListen { - pub fn new() -> Self { - Self { - tcp_tester: TcpTester::new(PORT), - } - } -} - -#[host] -mod host { - use super::*; - use crate::common::setup_fs_and_enter; - use crate::{krun_call, krun_call_u32, Test, TestSetup}; - use krun_sys::*; - use std::ffi::CString; - use std::ptr::null; - use std::thread; - use std::time::Duration; - - impl Test for TestTsiTcpGuestListen { - fn start_vm(self: Box, test_setup: TestSetup) -> anyhow::Result<()> { - unsafe { - thread::spawn(move || { - thread::sleep(Duration::from_secs(1)); - self.tcp_tester.run_client(); - }); - - krun_call!(krun_set_log_level(KRUN_LOG_LEVEL_WARN))?; - let ctx = krun_call_u32!(krun_create_ctx())?; - let port_mapping = format!("{PORT}:{PORT}"); - let port_mapping = CString::new(port_mapping).unwrap(); - let port_map = [port_mapping.as_ptr(), null()]; - - krun_call!(krun_set_port_map(ctx, port_map.as_ptr()))?; - krun_call!(krun_set_vm_config(ctx, 1, 512))?; - setup_fs_and_enter(ctx, test_setup)?; - println!("OK"); - } - Ok(()) - } - } -} - -#[guest] -mod guest { - use super::*; - use crate::Test; - - impl Test for TestTsiTcpGuestListen { - fn in_guest(self: Box) { - let listener = self.tcp_tester.create_server_socket(); - self.tcp_tester.run_server(listener); - println!("OK"); - } - } -} diff --git a/tests/test_cases/src/test_tsi_unix_dgram_setsockopt.rs b/tests/test_cases/src/test_tsi_unix_dgram_setsockopt.rs new file mode 100644 index 000000000..87f76b352 --- /dev/null +++ b/tests/test_cases/src/test_tsi_unix_dgram_setsockopt.rs @@ -0,0 +1,81 @@ +use macros::{guest, host}; + +/// Test that setsockopt works on TSI-intercepted Unix DGRAM sockets. +/// This is a regression test for a kernel NULL pointer dereference bug +/// in tsi_dgram_setsockopt that occurred when setting socket options. +/// +/// The bug: tsi_dgram_setsockopt calls isocket->ops->setsockopt() for +/// SOL_SOCKET level options, but Unix sockets don't have a setsockopt +/// function in their proto_ops (it's NULL), causing a NULL pointer +/// dereference and kernel panic. +/// +/// The fix uses sock_setsockopt() for SOL_SOCKET level options: +/// if (level == SOL_SOCKET) { sock_setsockopt(...) } else { isocket->ops->setsockopt(...) } +/// +/// With an unfixed kernel, this test will cause the guest to kernel panic/hang. +/// With the fixed kernel, this test passes. +pub struct TestTsiUnixDgramSetsockopt; + +#[host] +mod host { + use super::*; + use crate::common::setup_fs_and_enter; + use crate::{krun_call, krun_call_u32}; + use crate::{Test, TestSetup}; + use krun_sys::*; + + impl Test for TestTsiUnixDgramSetsockopt { + fn start_vm(self: Box, test_setup: TestSetup) -> anyhow::Result<()> { + unsafe { + krun_call!(krun_set_log_level(KRUN_LOG_LEVEL_DEBUG))?; + let ctx = krun_call_u32!(krun_create_ctx())?; + krun_call!(krun_set_vm_config(ctx, 1, 512))?; + setup_fs_and_enter(ctx, test_setup)?; + } + Ok(()) + } + } +} + +#[guest] +mod guest { + use super::*; + use crate::Test; + use nix::libc; + use nix::sys::socket::{self, AddressFamily, SockFlag, SockType}; + use std::os::fd::AsRawFd; + + impl Test for TestTsiUnixDgramSetsockopt { + fn in_guest(self: Box) { + // Create a Unix DGRAM socket - this will be intercepted by TSI. + // Unix sockets don't have proto_ops->setsockopt defined (it's NULL), + // so calling isocket->ops->setsockopt() causes a NULL pointer deref. + let socket = + socket::socket(AddressFamily::Unix, SockType::Datagram, SockFlag::empty(), None) + .expect("Failed to create Unix DGRAM socket"); + + // SOL_SOCKET level setsockopt calls trigger a kernel NULL pointer + // dereference in the unfixed TSI code because Unix sockets don't + // have a setsockopt function in their proto_ops. + let optval: libc::c_int = 1; + let ret = unsafe { + libc::setsockopt( + socket.as_raw_fd(), + libc::SOL_SOCKET, + libc::SO_REUSEADDR, + &optval as *const _ as *const libc::c_void, + std::mem::size_of_val(&optval) as libc::socklen_t, + ) + }; + if ret != 0 { + panic!( + "setsockopt SOL_SOCKET/SO_REUSEADDR failed: {}", + std::io::Error::last_os_error() + ); + } + + // If we get here without a kernel panic, the test passed + println!("OK"); + } + } +} From 7250f7479573d36519c1ce83911ee7a68a4dc465 Mon Sep 17 00:00:00 2001 From: Matej Hrica Date: Wed, 26 Nov 2025 14:43:23 +0100 Subject: [PATCH 9/9] vsock/tsi_dgram: Make send_to bind to correct socket adress structs We need to bind to the correct socket types (IPv6, Unix) instead of only IPv4. This fixes UDP and unix dgram tests hanging when waiting for reply. Signed-off-by: Matej Hrica --- src/devices/src/virtio/vsock/tsi_dgram.rs | 26 ++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/src/devices/src/virtio/vsock/tsi_dgram.rs b/src/devices/src/virtio/vsock/tsi_dgram.rs index 896f539b8..71b3b197f 100644 --- a/src/devices/src/virtio/vsock/tsi_dgram.rs +++ b/src/devices/src/virtio/vsock/tsi_dgram.rs @@ -1,5 +1,5 @@ use std::collections::HashMap; -use std::net::{Ipv4Addr, SocketAddrV4}; +use std::net::{Ipv4Addr, Ipv6Addr, SocketAddrV4, SocketAddrV6}; use std::num::Wrapping; use std::os::fd::OwnedFd; use std::os::unix::io::{AsRawFd, RawFd}; @@ -8,7 +8,7 @@ use std::sync::{Arc, Mutex}; use nix::fcntl::{fcntl, FcntlArg, OFlag}; use nix::sys::socket::{ bind, connect, getpeername, recv, send, sendto, socket, AddressFamily, MsgFlags, SockFlag, - SockType, SockaddrIn, SockaddrLike, SockaddrStorage, + SockType, SockaddrIn, SockaddrLike, SockaddrStorage, UnixAddr, }; #[cfg(target_os = "macos")] @@ -35,6 +35,7 @@ pub struct TsiDgramProxy { pub status: ProxyStatus, sendto_addr: Option, listening: bool, + family: AddressFamily, mem: GuestMemoryMmap, queue: Arc>, rxq: Arc>, @@ -102,6 +103,7 @@ impl TsiDgramProxy { status: ProxyStatus::Idle, sendto_addr: None, listening: false, + family, mem, queue, rxq, @@ -339,7 +341,25 @@ impl Proxy for TsiDgramProxy { self.sendto_addr = Some(req.addr); if !self.listening { - match bind(self.fd.as_raw_fd(), &SockaddrIn::new(0, 0, 0, 0, 0)) { + let bind_result = match self.family { + AddressFamily::Inet => bind(self.fd.as_raw_fd(), &SockaddrIn::new(0, 0, 0, 0, 0)), + AddressFamily::Inet6 => { + let addr6: SockaddrStorage = + SocketAddrV6::new(Ipv6Addr::UNSPECIFIED, 0, 0, 0).into(); + bind(self.fd.as_raw_fd(), &addr6) + } + #[cfg(target_os = "linux")] + AddressFamily::Unix => { + let addr = UnixAddr::new_unnamed(); + bind(self.fd.as_raw_fd(), &addr) + } + _ => { + warn!("sendto_addr: unsupported address family: {:?}", self.family); + return update; + } + }; + + match bind_result { Ok(_) => { self.listening = true; update.polling = Some((self.id, self.fd.as_raw_fd(), EventSet::IN));