From eb775481ea4e247b0c98e6ef60d1a3534db66873 Mon Sep 17 00:00:00 2001 From: Jakob Naucke Date: Fri, 16 Jan 2026 13:03:13 +0100 Subject: [PATCH 1/6] register-server: Switch to Axum from Warp Warp is outsourcing multiple features with its move to Hyper v1 that we used or were soon going to use: - Client info like IP - Native TLS support These features are more broadly integrated in Axum, which kube-rs also recently moved to. Signed-off-by: Jakob Naucke Assisted-by: Claude --- Cargo.lock | 144 +++++++++++++++++++++++++--- Cargo.toml | 3 +- attestation-key-register/Cargo.toml | 2 +- register-server/Cargo.toml | 3 +- register-server/src/main.rs | 43 ++++----- 5 files changed, 156 insertions(+), 39 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b90e8696..d1da9c81 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -252,6 +252,75 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "axum" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b52af3cb4058c895d37317bb27508dccc8e5f2d39454016b297bf4a400597b8" +dependencies = [ + "axum-core", + "bytes", + "form_urlencoded", + "futures-util", + "http 1.4.0", + "http-body 1.0.1", + "http-body-util", + "hyper 1.8.1", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper 1.0.2", + "tokio", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +dependencies = [ + "bytes", + "futures-core", + "http 1.4.0", + "http-body 1.0.1", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper 1.0.2", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-server" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1df331683d982a0b9492b38127151e6453639cd34926eb9c07d4cd8c6d22bfc" +dependencies = [ + "bytes", + "either", + "fs-err", + "http 1.4.0", + "http-body 1.0.1", + "hyper 1.8.1", + "hyper-util", + "tokio", + "tower-service", +] + [[package]] name = "backon" version = "1.6.0" @@ -1038,6 +1107,16 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs-err" +version = "3.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf68cef89750956493a66a10f512b9e58d9db21f2a573c079c0bdf1207a54a7" +dependencies = [ + "autocfg", + "tokio", +] + [[package]] name = "fs_extra" version = "1.3.0" @@ -1241,6 +1320,25 @@ dependencies = [ "tracing", ] +[[package]] +name = "h2" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http 1.4.0", + "indexmap 2.12.0", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -1421,7 +1519,7 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "h2", + "h2 0.3.27", "http 0.2.12", "http-body 0.4.6", "httparse", @@ -1437,17 +1535,19 @@ dependencies = [ [[package]] name = "hyper" -version = "1.7.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb3aa54a13a0dfe7fbe3a59e0c76093041720fdc77b110cc0fc260fafb4dc51e" +checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11" dependencies = [ "atomic-waker", "bytes", "futures-channel", "futures-core", + "h2 0.4.13", "http 1.4.0", "http-body 1.0.1", "httparse", + "httpdate", "itoa", "pin-project-lite", "pin-utils", @@ -1463,7 +1563,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "527d4d619ca2c2aafa31ec139a3d1d60bf557bf7578a1f20f743637eccd9ca19" dependencies = [ "http 1.4.0", - "hyper 1.7.0", + "hyper 1.8.1", "hyper-util", "linked_hash_set", "once_cell", @@ -1495,7 +1595,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" dependencies = [ - "hyper 1.7.0", + "hyper 1.8.1", "hyper-util", "pin-project-lite", "tokio", @@ -1510,7 +1610,7 @@ checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" dependencies = [ "bytes", "http-body-util", - "hyper 1.7.0", + "hyper 1.8.1", "hyper-util", "native-tls", "tokio", @@ -1520,9 +1620,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.17" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c6995591a8f1380fcb4ba966a252a4b29188d51d2b89e3a252f5305be65aea8" +checksum = "727805d60e7938b76b826a6ef209eb70eaa1812794f9424d4a4e2d740662df5f" dependencies = [ "base64 0.22.1", "bytes", @@ -1531,7 +1631,7 @@ dependencies = [ "futures-util", "http 1.4.0", "http-body 1.0.1", - "hyper 1.7.0", + "hyper 1.8.1", "ipnet", "libc", "percent-encoding", @@ -1910,7 +2010,7 @@ dependencies = [ "http 1.4.0", "http-body 1.0.1", "http-body-util", - "hyper 1.7.0", + "hyper 1.8.1", "hyper-openssl", "hyper-timeout", "hyper-util", @@ -2104,6 +2204,12 @@ version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + [[package]] name = "memchr" version = "2.7.6" @@ -2846,6 +2952,8 @@ name = "register-server" version = "0.1.0" dependencies = [ "anyhow", + "axum", + "axum-server", "clap", "clevis-pin-trustee-lib", "env_logger", @@ -2860,7 +2968,6 @@ dependencies = [ "trusted-cluster-operator-lib", "trusted-cluster-operator-test-utils", "uuid", - "warp", ] [[package]] @@ -2874,7 +2981,7 @@ dependencies = [ "encoding_rs", "futures-core", "futures-util", - "h2", + "h2 0.3.27", "http 0.2.12", "http-body 0.4.6", "hyper 0.14.32", @@ -2917,7 +3024,7 @@ dependencies = [ "http 1.4.0", "http-body 1.0.1", "http-body-util", - "hyper 1.7.0", + "hyper 1.8.1", "hyper-tls", "hyper-util", "js-sys", @@ -3250,6 +3357,17 @@ dependencies = [ "zmij", ] +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" diff --git a/Cargo.toml b/Cargo.toml index 50df09aa..8acb7c5f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,5 +26,4 @@ log = "0.4.29" serde = { version = "1.0.228", features = ["derive"] } serde_json = "1.0.148" tokio = { version = "1.49.0", features = ["macros", "rt-multi-thread"] } -uuid = { version = "1.20", features = ["v4", "serde"] } -warp = { version = "0.3", default-features = false } +uuid = { version = "1.19", features = ["v4", "serde"] } diff --git a/attestation-key-register/Cargo.toml b/attestation-key-register/Cargo.toml index 1978bcf4..bcbe108c 100644 --- a/attestation-key-register/Cargo.toml +++ b/attestation-key-register/Cargo.toml @@ -20,4 +20,4 @@ serde.workspace = true serde_json.workspace = true tokio.workspace = true uuid.workspace = true -warp.workspace = true +warp = { version = "0.3", default-features = false } diff --git a/register-server/Cargo.toml b/register-server/Cargo.toml index 70760fe5..44151414 100644 --- a/register-server/Cargo.toml +++ b/register-server/Cargo.toml @@ -10,6 +10,8 @@ rust-version.workspace = true [dependencies] anyhow.workspace = true +axum = "0.8.8" +axum-server = "0.8.0" clap.workspace = true clevis-pin-trustee-lib.workspace = true trusted-cluster-operator-lib = { path = "../lib" } @@ -22,7 +24,6 @@ serde.workspace = true serde_json.workspace = true tokio.workspace = true uuid.workspace = true -warp.workspace = true [dev-dependencies] http.workspace = true diff --git a/register-server/src/main.rs b/register-server/src/main.rs index 14e14208..f33ab2ff 100644 --- a/register-server/src/main.rs +++ b/register-server/src/main.rs @@ -3,7 +3,10 @@ // // SPDX-License-Identifier: MIT -use anyhow::Context; +use anyhow::{anyhow, Context}; +use axum::response::{IntoResponse, Json}; +use axum::{extract::ConnectInfo, http::StatusCode}; +use axum::{routing::get, Router}; use clap::Parser; use clevis_pin_trustee_lib::{Config as ClevisConfig, Server as ClevisServer}; use env_logger::Env; @@ -13,10 +16,8 @@ use ignition_config::v3_5::{ use k8s_openapi::apimachinery::pkg::apis::meta::v1::{ObjectMeta, OwnerReference}; use kube::{Api, Client}; use log::{error, info}; -use std::convert::Infallible; use std::net::SocketAddr; use uuid::Uuid; -use warp::{http::StatusCode, reply, Filter}; use trusted_cluster_operator_lib::endpoints::REGISTER_SERVER_RESOURCE; use trusted_cluster_operator_lib::{ @@ -91,11 +92,9 @@ async fn get_public_trustee_addr(client: Client) -> anyhow::Result { )) } -async fn register_handler(remote_addr: Option) -> Result { +async fn register_handler(ConnectInfo(addr): ConnectInfo) -> impl IntoResponse { let id = Uuid::new_v4().to_string(); - let client_ip = remote_addr - .map(|addr| addr.ip().to_string()) - .unwrap_or_else(|| "unknown".to_string()); + let client_ip = addr.ip().to_string(); info!("Registration request from IP: {client_ip}"); @@ -106,7 +105,7 @@ async fn register_handler(remote_addr: Option) -> Result) -> Result return internal_error(e.context("Failed to get Trustee address")), }; - Ok(reply::with_status( - reply::json(&generate_ignition(&id, &public_addr)), - StatusCode::OK, - )) + let ignition = generate_ignition(&id, &public_addr); + let json = match serde_json::to_value(ignition) { + Ok(json) => json, + Err(e) => return internal_error(anyhow!("Failed to serialise Ignition: {e}")), + }; + (StatusCode::OK, Json(json)) } async fn create_machine( @@ -171,16 +172,14 @@ async fn main() { env_logger::Builder::from_env(Env::default().default_filter_or("info")).init(); let args = Args::parse(); - - let register_route = warp::path(REGISTER_SERVER_RESOURCE) - .and(warp::get()) - .and(warp::addr::remote()) - .and_then(register_handler); - - let routes = register_route; - - info!("Starting server on http://localhost:{}", args.port); - warp::serve(routes).run(([0, 0, 0, 0], args.port)).await; + let endpoint = format!("/{REGISTER_SERVER_RESOURCE}"); + let app = Router::new().route(&endpoint, get(register_handler)); + let addr = SocketAddr::from(([0, 0, 0, 0], args.port)); + let service = app.into_make_service_with_connect_info::(); + info!("Starting server on http://{}", addr); + + let run = axum_server::bind(addr).serve(service).await; + run.expect("Server failed"); } #[cfg(test)] From b5df209586ce3c119fe0ebaf114e68953e7dcbf8 Mon Sep 17 00:00:00 2001 From: Jakob Naucke Date: Thu, 12 Feb 2026 22:39:55 +0100 Subject: [PATCH 2/6] att-key-reg: Switch to Axum from Warp Signed-off-by: Jakob Naucke Assisted-by: Claude --- .github/dependabot.yml | 4 - Cargo.lock | 81 +--------------- Cargo.toml | 2 + attestation-key-register/Cargo.toml | 3 +- attestation-key-register/src/main.rs | 139 ++++++++++----------------- register-server/Cargo.toml | 4 +- 6 files changed, 57 insertions(+), 176 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index c599c0b5..2478c929 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -32,10 +32,6 @@ updates: patterns: - "*" ignore: - - dependency-name: "warp" - # Block newer versions of Warp so we can continue logging the client address, - # cf. https://github.com/seanmonstar/warp/issues/1127 - versions: [">=0.4"] - dependency-name: "rand_core" # Block newer versions of rand_core, incompatible versions: [">=0.7"] diff --git a/Cargo.lock b/Cargo.lock index d1da9c81..d249bd5e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -179,6 +179,8 @@ name = "attestation-key-register" version = "0.1.0" dependencies = [ "anyhow", + "axum", + "axum-server", "clap", "env_logger", "k8s-openapi", @@ -189,7 +191,6 @@ dependencies = [ "tokio", "trusted-cluster-operator-lib", "uuid", - "warp", ] [[package]] @@ -1356,30 +1357,6 @@ dependencies = [ "foldhash", ] -[[package]] -name = "headers" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06683b93020a07e3dbcf5f8c0f6d40080d725bea7936fc01ad345c01b97dc270" -dependencies = [ - "base64 0.21.7", - "bytes", - "headers-core", - "http 0.2.12", - "httpdate", - "mime", - "sha1", -] - -[[package]] -name = "headers-core" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7f66481bfee273957b1f20485a4ff3362987f85b2c236580d81b4eb7a326429" -dependencies = [ - "http 0.2.12", -] - [[package]] name = "heck" version = "0.5.0" @@ -2254,16 +2231,6 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" -[[package]] -name = "mime_guess" -version = "2.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e" -dependencies = [ - "mime", - "unicase", -] - [[package]] name = "miniz_oxide" version = "0.8.9" @@ -3209,12 +3176,6 @@ dependencies = [ "syn 2.0.110", ] -[[package]] -name = "scoped-tls" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" - [[package]] name = "scopeguard" version = "1.2.0" @@ -3393,17 +3354,6 @@ dependencies = [ "unsafe-libyaml", ] -[[package]] -name = "sha1" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest", -] - [[package]] name = "sha2" version = "0.10.9" @@ -4120,33 +4070,6 @@ dependencies = [ "try-lock", ] -[[package]] -name = "warp" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4378d202ff965b011c64817db11d5829506d3404edeadb61f190d111da3f231c" -dependencies = [ - "bytes", - "futures-channel", - "futures-util", - "headers", - "http 0.2.12", - "hyper 0.14.32", - "log", - "mime", - "mime_guess", - "percent-encoding", - "pin-project", - "scoped-tls", - "serde", - "serde_json", - "serde_urlencoded", - "tokio", - "tokio-util", - "tower-service", - "tracing", -] - [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" diff --git a/Cargo.toml b/Cargo.toml index 8acb7c5f..76657b3b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,6 +13,8 @@ rust-version = "1.88" [workspace.dependencies] anyhow = "1.0.101" +axum = "0.8.8" +axum-server = "0.8.0" chrono = "0.4.43" clap = { version = "4.5.58", features = ["derive"] } clevis-pin-trustee-lib = { git = "https://github.com/latchset/clevis-pin-trustee" } diff --git a/attestation-key-register/Cargo.toml b/attestation-key-register/Cargo.toml index bcbe108c..05d82467 100644 --- a/attestation-key-register/Cargo.toml +++ b/attestation-key-register/Cargo.toml @@ -10,6 +10,8 @@ rust-version.workspace = true [dependencies] anyhow.workspace = true +axum.workspace = true +axum-server.workspace = true clap.workspace = true trusted-cluster-operator-lib = { path = "../lib" } env_logger.workspace = true @@ -20,4 +22,3 @@ serde.workspace = true serde_json.workspace = true tokio.workspace = true uuid.workspace = true -warp = { version = "0.3", default-features = false } diff --git a/attestation-key-register/src/main.rs b/attestation-key-register/src/main.rs index ebe58f6a..87546bcf 100644 --- a/attestation-key-register/src/main.rs +++ b/attestation-key-register/src/main.rs @@ -2,16 +2,17 @@ // // SPDX-License-Identifier: MIT -use anyhow::Context; +use axum::response::{IntoResponse, Json}; +use axum::{extract::ConnectInfo, http::StatusCode}; +use axum::{routing::put, Router}; use clap::Parser; +use env_logger::Env; use k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta; use kube::{Api, Client}; use log::{error, info}; use serde::{Deserialize, Serialize}; -use std::convert::Infallible; use std::net::SocketAddr; use uuid::Uuid; -use warp::{http::StatusCode, reply, Filter}; use trusted_cluster_operator_lib::endpoints::ATTESTATION_KEY_REGISTER_RESOURCE; use trusted_cluster_operator_lib::{ @@ -40,41 +41,37 @@ struct AttestationKeyRegistration { } async fn handle_registration( - registration: AttestationKeyRegistration, - client: Client, - addr: Option, -) -> Result { + ConnectInfo(addr): ConnectInfo, + Json(registration): Json, +) -> impl IntoResponse { info!("Received registration request: {registration:?}"); + let internal_error = |e: anyhow::Error| { + let code = StatusCode::INTERNAL_SERVER_ERROR; + error!("{e:?}"); + let msg = serde_json::json!({ + "status": "error", + "message": format!("{e:#}"), + }); + (code, Json(msg)) + }; + + let client = match Client::try_default().await { + Ok(c) => c, + Err(e) => return internal_error(e.into()), + }; + let api: Api = Api::default_namespaced(client.clone()); // Get the TrustedExecutionCluster to use as owner reference let cluster = match get_trusted_execution_cluster(client.clone()).await { Ok(c) => c, - Err(e) => { - error!("Failed to get TrustedExecutionCluster: {e}"); - return Ok(reply::with_status( - reply::json(&serde_json::json!({ - "status": "error", - "message": format!("Failed to get TrustedExecutionCluster: {e}"), - })), - StatusCode::INTERNAL_SERVER_ERROR, - )); - } + Err(e) => return internal_error(e.context("Failed to get TrustedExecutionCluster")), }; let owner_reference = match generate_owner_reference(&cluster) { Ok(o) => o, - Err(e) => { - error!("Failed to generate owner reference: {e}"); - return Ok(reply::with_status( - reply::json(&serde_json::json!({ - "status": "error", - "message": format!("Failed to generate owner reference: {e}"), - })), - StatusCode::INTERNAL_SERVER_ERROR, - )); - } + Err(e) => return internal_error(e.context("Failed to generate owner reference")), }; match api.list(&Default::default()).await { @@ -85,31 +82,25 @@ async fn handle_registration( error!( "Duplicate public key detected: already exists in AttestationKey '{existing_name}'" ); - return Ok(reply::with_status( - reply::json(&serde_json::json!({ + return ( + StatusCode::CONFLICT, + Json(serde_json::json!({ "status": "error", - "message": format!("Public key already registered"), + "message": "Public key already registered", })), - StatusCode::CONFLICT, - )); + ); } } } Err(e) => { - error!("Failed to list AttestationKeys: {e}"); - return Ok(reply::with_status( - reply::json(&serde_json::json!({ - "status": "error", - "message": format!("Failed to check for existing keys: {e}"), - })), - StatusCode::INTERNAL_SERVER_ERROR, - )); + return internal_error( + anyhow::Error::from(e).context("Failed to check for existing keys"), + ) } } - let address = registration - .address - .or_else(|| addr.map(|socket_addr| socket_addr.ip().to_string())); + let or = || addr.ip().to_string(); + let address = Some(registration.address.unwrap_or_else(or)); let name = format!("ak-{}", Uuid::new_v4()); let attestation_key = AttestationKey { @@ -127,60 +118,28 @@ async fn handle_registration( match api.create(&Default::default(), &attestation_key).await { Ok(created) => { - info!( - "Successfully created AttestationKey: {}", - created.metadata.name.unwrap_or_default() - ); - Ok(reply::with_status( - reply::json(&serde_json::json!({ - "status": "success", - })), - StatusCode::CREATED, - )) - } - Err(e) => { - error!("Failed to create AttestationKey: {e}"); - Ok(reply::with_status( - reply::json(&serde_json::json!({ - "status": "error", - "message": format!("Failed to create AttestationKey: {e}"), - })), - StatusCode::INTERNAL_SERVER_ERROR, - )) + let name = created.metadata.name.unwrap_or_default(); + info!("Successfully created AttestationKey: {name}",); + let json = Json(serde_json::json!({ + "status": "success", + })); + (StatusCode::CREATED, json) } + Err(e) => internal_error(anyhow::Error::from(e).context("Failed to create AttestationKey")), } } -fn with_client(client: Client) -> impl Filter + Clone { - warp::any().map(move || client.clone()) -} - #[tokio::main] -async fn main() -> anyhow::Result<()> { - env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).init(); +async fn main() { + env_logger::Builder::from_env(Env::default().default_filter_or("info")).init(); let args = Args::parse(); - - info!( - "Starting attestation key registration server on port {}", - args.port - ); - - let client = Client::try_default() - .await - .context("Failed to create Kubernetes client")?; - - let register = warp::put() - .and(warp::path(ATTESTATION_KEY_REGISTER_RESOURCE)) - .and(warp::body::json()) - .and(with_client(client)) - .and(warp::addr::remote()) - .and_then(handle_registration); - + let endpoint = format!("/{ATTESTATION_KEY_REGISTER_RESOURCE}"); + let app = Router::new().route(&endpoint, put(handle_registration)); let addr = SocketAddr::from(([0, 0, 0, 0], args.port)); - info!("Listening on {addr}"); - - warp::serve(register).run(addr).await; + let service = app.into_make_service_with_connect_info::(); + info!("Starting attestation key registration server on http://{addr}",); - Ok(()) + let run = axum_server::bind(addr).serve(service).await; + run.expect("Server failed"); } diff --git a/register-server/Cargo.toml b/register-server/Cargo.toml index 44151414..6da85b1d 100644 --- a/register-server/Cargo.toml +++ b/register-server/Cargo.toml @@ -10,8 +10,8 @@ rust-version.workspace = true [dependencies] anyhow.workspace = true -axum = "0.8.8" -axum-server = "0.8.0" +axum.workspace = true +axum-server.workspace = true clap.workspace = true clevis-pin-trustee-lib.workspace = true trusted-cluster-operator-lib = { path = "../lib" } From 4ea4c4bb21e00624543d2002c8871d5bd16c1c0b Mon Sep 17 00:00:00 2001 From: Jakob Naucke Date: Tue, 5 Aug 2025 15:48:24 +0200 Subject: [PATCH 3/6] Install cert-manager Signed-off-by: Jakob Naucke --- scripts/create-cluster-kind.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/create-cluster-kind.sh b/scripts/create-cluster-kind.sh index d60c0c18..eed218fa 100755 --- a/scripts/create-cluster-kind.sh +++ b/scripts/create-cluster-kind.sh @@ -8,6 +8,7 @@ set -xo errexit . scripts/common.sh +CRT_MGR_VERSION=1.19.2 if [ "$(kind get clusters 2>/dev/null)" != "kind" ]; then kind create cluster --config kind/config.yaml @@ -55,3 +56,5 @@ CALICO_FILE=/tmp/calico.yaml curl -Lo $CALICO_FILE https://raw.githubusercontent.com/projectcalico/calico/v3.29.1/manifests/calico.yaml sed -i 's|docker.io/calico|quay.io/calico|g' $CALICO_FILE kubectl apply -f $CALICO_FILE + +kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v${CRT_MGR_VERSION}/cert-manager.yaml From 71978cd786272bf5309e2df724998156db79b9f1 Mon Sep 17 00:00:00 2001 From: Jakob Naucke Date: Thu, 15 Jan 2026 13:33:04 +0100 Subject: [PATCH 4/6] reg-srv: Add certificate support In the trustedexecutioncluster CRD, add optional secret name for a cert-manager shaped certificate. In testing, use cert-manager to create a certificate. Signed-off-by: Jakob Naucke --- Cargo.lock | 23 ++++ Cargo.toml | 2 +- Makefile | 4 +- api/trusted-cluster-gen.go | 3 + api/v1alpha1/crds.go | 5 + go.mod | 8 +- go.sum | 20 +++- lib/src/kopium.rs | 4 + lib/src/lib.rs | 5 + operator/src/lib.rs | 42 ++++++- operator/src/main.rs | 1 + operator/src/register_server.rs | 38 +++--- register-server/src/main.rs | 13 ++- test_utils/Cargo.toml | 1 + test_utils/src/lib.rs | 198 +++++++++++++++++++++++++++----- test_utils/src/mock_client.rs | 1 + test_utils/src/virt/mod.rs | 30 ++++- tests/attestation.rs | 6 +- 18 files changed, 339 insertions(+), 65 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d249bd5e..3d77ebb2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -123,6 +123,12 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "arc-swap" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" + [[package]] name = "async-broadcast" version = "0.7.2" @@ -311,6 +317,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1df331683d982a0b9492b38127151e6453639cd34926eb9c07d4cd8c6d22bfc" dependencies = [ + "arc-swap", "bytes", "either", "fs-err", @@ -318,7 +325,11 @@ dependencies = [ "http-body 1.0.1", "hyper 1.8.1", "hyper-util", + "openssl", + "openssl-sys", + "pin-project-lite", "tokio", + "tokio-openssl", "tower-service", ] @@ -3780,6 +3791,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-openssl" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59df6849caa43bb7567f9a36f863c447d95a11d5903c9cc334ba32576a27eadd" +dependencies = [ + "openssl", + "openssl-sys", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.24.1" @@ -3915,6 +3937,7 @@ dependencies = [ "k8s-openapi", "kube", "log", + "percent-encoding", "rand_core", "serde", "serde_json", diff --git a/Cargo.toml b/Cargo.toml index 76657b3b..50bdbdbf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,7 +14,7 @@ rust-version = "1.88" [workspace.dependencies] anyhow = "1.0.101" axum = "0.8.8" -axum-server = "0.8.0" +axum-server = { version = "0.8.0", features = ["tls-openssl"] } chrono = "0.4.43" clap = { version = "4.5.58", features = ["derive"] } clevis-pin-trustee-lib = { git = "https://github.com/latchset/clevis-pin-trustee" } diff --git a/Makefile b/Makefile index 472ffb17..f2d16e7b 100644 --- a/Makefile +++ b/Makefile @@ -56,6 +56,7 @@ generate: $(CONTROLLER_GEN) $(call controller-gen,./...,*) $(call controller-gen,github.com/openshift/api/route/v1,*) $(call controller-gen,github.com/openshift/api/config/v1,*_ingresses.yaml) + $(call controller-gen,github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1,*) RS_LIB_PATH = lib/src CRD_RS_PATH = $(RS_LIB_PATH)/kopium @@ -63,7 +64,8 @@ $(CRD_RS_PATH): mkdir $(CRD_RS_PATH) $(CRD_RS_PATH)/%.rs: $(CRD_YAML_PATH)/*_%.yaml $(KOPIUM) $(CRD_RS_PATH) - $(KOPIUM) -f $< > $@ + $(KOPIUM) -f $< $$(grep -Eq '(certificates|issuers)' <<< $< && echo --derive Default) > $@ + sed -i 'N; s/, Default)\]\n\(pub struct CertificateAdditionalOutputFormats\)/)]\n\1/; P; D' $@ rustfmt $@ crds-rs: generate $(KOPIUM) $(CRD_RS_PATH) diff --git a/api/trusted-cluster-gen.go b/api/trusted-cluster-gen.go index 2b590b51..854d5d5c 100644 --- a/api/trusted-cluster-gen.go +++ b/api/trusted-cluster-gen.go @@ -28,6 +28,7 @@ type Args struct { trusteeImage string pcrsComputeImage string registerServerImage string + registerServerSecret string attestationKeyRegisterImage string approvedImage string } @@ -40,6 +41,7 @@ func main() { flag.StringVar(&args.trusteeImage, "trustee-image", "operators", "Container image with all-in-one Trustee") flag.StringVar(&args.pcrsComputeImage, "pcrs-compute-image", "quay.io/trusted-execution-clusters/compute-pcrs:latest", "Container image with the Trusted Execution Clusters compute-pcrs binary") flag.StringVar(&args.registerServerImage, "register-server-image", "quay.io/trusted-execution-clusters/register-server:latest", "Register server image to use in the deployment") + flag.StringVar(&args.registerServerSecret, "register-server-secret", "", "When set, k8s secret for register server including tls.{crt,key}.") flag.StringVar(&args.attestationKeyRegisterImage, "attestation-key-register-image", "quay.io/trusted-execution-clusters/attestation-key-register:latest", "Attestation key register image to use in the deployment") flag.StringVar(&args.approvedImage, "approved-image", "", "When set, defines an initial approved image. Must be a bootable container image with SHA reference.") flag.Parse() @@ -144,6 +146,7 @@ func generateTrustedExecutionClusterCR(args *Args) error { PcrsComputeImage: args.pcrsComputeImage, RegisterServerImage: args.registerServerImage, AttestationKeyRegisterImage: &args.attestationKeyRegisterImage, + RegisterServerSecret: &args.registerServerSecret, PublicTrusteeAddr: nil, TrusteeKbsPort: 0, RegisterServerPort: 0, diff --git a/api/v1alpha1/crds.go b/api/v1alpha1/crds.go index 53419df2..b8b0e794 100644 --- a/api/v1alpha1/crds.go +++ b/api/v1alpha1/crds.go @@ -52,6 +52,11 @@ type TrustedExecutionClusterSpec struct { // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="Value is immutable" AttestationKeyRegisterImage *string `json:"attestationKeyRegisterImage"` + // Secret with tls.{crt,key} for register-server + // +optional + // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="Value is immutable" + RegisterServerSecret *string `json:"registerServerSecret"` + // Address where attester can connect to Trustee // +optional // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="Value is immutable" diff --git a/go.mod b/go.mod index 9b757efe..7196998b 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,8 @@ require ( ) require ( + github.com/cert-manager/cert-manager v1.19.3 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/fxamacker/cbor/v2 v2.9.0 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/gogo/protobuf v1.3.2 // indirect @@ -21,15 +23,19 @@ require ( github.com/kr/text v0.2.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect - github.com/openshift/api v0.0.0-20260128000234-c16ec2bcf089 // indirect + github.com/openshift/api v0.0.0-20260209232644-126cbbe24427 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/spf13/pflag v1.0.10 // indirect github.com/x448/float16 v0.8.4 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect golang.org/x/net v0.48.0 // indirect golang.org/x/text v0.33.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect + k8s.io/apiextensions-apiserver v0.35.0 // indirect k8s.io/klog/v2 v2.130.1 // indirect k8s.io/kube-openapi v0.0.0-20251125145642-4e65d59e963e // indirect k8s.io/utils v0.0.0-20260108192941-914a6e750570 // indirect + sigs.k8s.io/gateway-api v1.4.0 // indirect sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect sigs.k8s.io/randfill v1.0.0 // indirect sigs.k8s.io/structured-merge-diff/v6 v6.3.2-0.20260122202528-d9cc6641c482 // indirect diff --git a/go.sum b/go.sum index 9dfcd89c..3e448a67 100644 --- a/go.sum +++ b/go.sum @@ -1,9 +1,12 @@ github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/cert-manager/cert-manager v1.19.3 h1:3d0Nk/HO3BOmAdBJNaBh+6YgaO3Ciey3xCpOjiX5Obs= +github.com/cert-manager/cert-manager v1.19.3/go.mod h1:e9NzLtOKxTw7y99qLyWGmPo6mrC1Nh0EKKcMkRfK+GE= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= @@ -35,14 +38,15 @@ github.com/onsi/ginkgo/v2 v2.27.2 h1:LzwLj0b89qtIy6SSASkzlNvX6WktqurSHwkk2ipF/Ns github.com/onsi/ginkgo/v2 v2.27.2/go.mod h1:ArE1D/XhNXBXCBkKOLkbsb2c81dQHCRcF5zwn/ykDRo= github.com/onsi/gomega v1.38.2 h1:eZCjf2xjZAqe+LeWvKb5weQ+NcPwX84kqJ0cZNxok2A= github.com/onsi/gomega v1.38.2/go.mod h1:W2MJcYxRGV63b418Ai34Ud0hEdTVXq9NW9+Sx6uXf3k= -github.com/openshift/api v0.0.0-20260128000234-c16ec2bcf089 h1:qcKLN7H1dh2wt59Knpc1J5XzCCStSeaaFyEHHilFypg= -github.com/openshift/api v0.0.0-20260128000234-c16ec2bcf089/go.mod h1:d5uzF0YN2nQQFA0jIEWzzOZ+edmo6wzlGLvx5Fhz4uY= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/openshift/api v0.0.0-20260209232644-126cbbe24427 h1:MExw+yvWGmbwlTpsO8sk16n3YQeeE2QxLmLpQouIGeE= +github.com/openshift/api v0.0.0-20260209232644-126cbbe24427/go.mod h1:d5uzF0YN2nQQFA0jIEWzzOZ+edmo6wzlGLvx5Fhz4uY= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= -github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= -github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= +github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= @@ -101,6 +105,8 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= k8s.io/api v0.35.0 h1:iBAU5LTyBI9vw3L5glmat1njFK34srdLmktWwLTprlY= k8s.io/api v0.35.0/go.mod h1:AQ0SNTzm4ZAczM03QH42c7l3bih1TbAXYo0DkF8ktnA= +k8s.io/apiextensions-apiserver v0.35.0 h1:3xHk2rTOdWXXJM+RDQZJvdx0yEOgC0FgQ1PlJatA5T4= +k8s.io/apiextensions-apiserver v0.35.0/go.mod h1:E1Ahk9SADaLQ4qtzYFkwUqusXTcaV2uw3l14aqpL2LU= k8s.io/apimachinery v0.35.0 h1:Z2L3IHvPVv/MJ7xRxHEtk6GoJElaAqDCCU0S6ncYok8= k8s.io/apimachinery v0.35.0/go.mod h1:jQCgFZFR1F4Ik7hvr2g84RTJSZegBc8yHgFWKn//hns= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= @@ -111,6 +117,8 @@ k8s.io/utils v0.0.0-20260108192941-914a6e750570 h1:JT4W8lsdrGENg9W+YwwdLJxklIuKW k8s.io/utils v0.0.0-20260108192941-914a6e750570/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk= sigs.k8s.io/controller-runtime v0.23.1 h1:TjJSM80Nf43Mg21+RCy3J70aj/W6KyvDtOlpKf+PupE= sigs.k8s.io/controller-runtime v0.23.1/go.mod h1:B6COOxKptp+YaUT5q4l6LqUJTRpizbgf9KSRNdQGns0= +sigs.k8s.io/gateway-api v1.4.0 h1:ZwlNM6zOHq0h3WUX2gfByPs2yAEsy/EenYJB78jpQfQ= +sigs.k8s.io/gateway-api v1.4.0/go.mod h1:AR5RSqciWP98OPckEjOjh2XJhAe2Na4LHyXD2FUY7Qk= sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= diff --git a/lib/src/kopium.rs b/lib/src/kopium.rs index 4588066e..91ecd4d5 100644 --- a/lib/src/kopium.rs +++ b/lib/src/kopium.rs @@ -4,7 +4,11 @@ pub mod approvedimages; pub mod attestationkeys; +pub mod certificaterequests; +pub mod certificates; +pub mod clusterissuers; pub mod ingresses; +pub mod issuers; pub mod machines; pub mod routes; pub mod trustedexecutionclusters; diff --git a/lib/src/lib.rs b/lib/src/lib.rs index 690f506e..7d23f598 100644 --- a/lib/src/lib.rs +++ b/lib/src/lib.rs @@ -16,6 +16,11 @@ pub use kopium::ingresses as openshift_ingresses; pub use kopium::machines::*; pub use kopium::routes; pub use kopium::trustedexecutionclusters::*; + +pub use kopium::certificaterequests; +pub use kopium::certificates; +pub use kopium::clusterissuers; +pub use kopium::issuers; pub use vendor_kopium::virtualmachineinstances; pub use vendor_kopium::virtualmachines; diff --git a/operator/src/lib.rs b/operator/src/lib.rs index bd7e4a09..3dcb0968 100644 --- a/operator/src/lib.rs +++ b/operator/src/lib.rs @@ -8,9 +8,11 @@ // // Use in other crates is not an intended purpose. +use anyhow::Result; +use k8s_openapi::api::core::v1::{Secret, SecretVolumeSource, Volume, VolumeMount}; use k8s_openapi::apimachinery::pkg::apis::meta::v1::OwnerReference; -use kube::{Client, runtime::controller::Action}; -use log::info; +use kube::{Api, Client, runtime::controller::Action}; +use log::{info, warn}; use std::fmt::{Debug, Display}; use std::{sync::Arc, time::Duration}; @@ -56,3 +58,39 @@ macro_rules! create_or_info_if_exists { } }; } + +pub const TLS_DIR: &str = "/etc/tls"; + +/// Reads a TLS certificate secret and returns the Volume and VolumeMount for it. +/// Returns None if the secret name is not provided or the secret does not exist. +pub async fn read_certificate( + client: Client, + secret_name: &Option, +) -> Result> { + let secrets: Api = Api::default_namespaced(client.clone()); + if secret_name.is_none() { + return Ok(None); + } + let secret_name = secret_name.as_ref().unwrap(); + let secret = secrets.get(secret_name).await; + + if secret.is_err() { + warn!("Certificate secret {secret_name} was provided, but could not be retrieved"); + return Ok(None); + } + + let volume = Volume { + name: secret_name.clone(), + secret: Some(SecretVolumeSource { + secret_name: Some(secret_name.clone()), + ..Default::default() + }), + ..Default::default() + }; + let volume_mount = VolumeMount { + name: secret_name.clone(), + mount_path: TLS_DIR.to_string(), + ..Default::default() + }; + Ok(Some((volume, volume_mount))) +} diff --git a/operator/src/main.rs b/operator/src/main.rs index 09c79d43..2ee7dc2d 100644 --- a/operator/src/main.rs +++ b/operator/src/main.rs @@ -183,6 +183,7 @@ async fn install_register_server(client: Client, cluster: &TrustedExecutionClust client.clone(), owner_reference.clone(), &cluster.spec.register_server_image, + &cluster.spec.register_server_secret, ) .await { diff --git a/operator/src/register_server.rs b/operator/src/register_server.rs index c4e7ccc8..a4a71765 100644 --- a/operator/src/register_server.rs +++ b/operator/src/register_server.rs @@ -5,17 +5,13 @@ use anyhow::{Result, anyhow}; use futures_util::StreamExt; -use k8s_openapi::{ - api::{ - apps::v1::{Deployment, DeploymentSpec}, - core::v1::{ - Container, ContainerPort, PodSpec, PodTemplateSpec, Service, ServicePort, ServiceSpec, - }, - }, - apimachinery::pkg::{ - apis::meta::v1::{LabelSelector, ObjectMeta, OwnerReference}, - util::intstr::IntOrString, - }, +use k8s_openapi::api::apps::v1::{Deployment, DeploymentSpec}; +use k8s_openapi::api::core::v1::{ + Container, ContainerPort, PodSpec, PodTemplateSpec, Service, ServicePort, ServiceSpec, +}; +use k8s_openapi::apimachinery::pkg::{ + apis::meta::v1::{LabelSelector, ObjectMeta, OwnerReference}, + util::intstr::IntOrString, }; use kube::runtime::{ controller::{Action, Controller}, @@ -37,10 +33,20 @@ pub async fn create_register_server_deployment( client: Client, owner_reference: OwnerReference, image: &str, + secret: &Option, ) -> Result<()> { let app_label = "register-server"; let labels = BTreeMap::from([("app".to_string(), app_label.to_string())]); + let mut args = vec!["--port".to_string(), REGISTER_SERVER_PORT.to_string()]; + let volumes = read_certificate(client.clone(), secret).await?; + if volumes.is_some() { + args.push("--cert-path".to_string()); + args.push(format!("{TLS_DIR}/tls.crt")); + args.push("--key-path".to_string()); + args.push(format!("{TLS_DIR}/tls.key")); + } + let deployment = Deployment { metadata: ObjectMeta { name: Some(REGISTER_SERVER_DEPLOYMENT.to_string()), @@ -67,9 +73,11 @@ pub async fn create_register_server_deployment( container_port: REGISTER_SERVER_PORT, ..Default::default() }]), - args: Some(vec!["--port".to_string(), REGISTER_SERVER_PORT.to_string()]), + args: Some(args), + volume_mounts: volumes.as_ref().map(|(_, vm)| vec![vm.clone()]), ..Default::default() }], + volumes: volumes.as_ref().map(|(v, _)| vec![v.clone()]), ..Default::default() }), }, @@ -205,13 +213,15 @@ mod tests { #[tokio::test] async fn test_create_reg_server_depl_success() { - let clos = |client| create_register_server_deployment(client, Default::default(), "image"); + let clos = + |client| create_register_server_deployment(client, Default::default(), "image", &None); test_create_success::<_, _, Deployment>(clos).await; } #[tokio::test] async fn test_create_reg_server_depl_error() { - let clos = |client| create_register_server_deployment(client, Default::default(), "image"); + let clos = + |client| create_register_server_deployment(client, Default::default(), "image", &None); test_create_error(clos).await; } diff --git a/register-server/src/main.rs b/register-server/src/main.rs index f33ab2ff..8d57664b 100644 --- a/register-server/src/main.rs +++ b/register-server/src/main.rs @@ -7,6 +7,7 @@ use anyhow::{anyhow, Context}; use axum::response::{IntoResponse, Json}; use axum::{extract::ConnectInfo, http::StatusCode}; use axum::{routing::get, Router}; +use axum_server::tls_openssl::OpenSSLConfig; use clap::Parser; use clevis_pin_trustee_lib::{Config as ClevisConfig, Server as ClevisServer}; use env_logger::Env; @@ -30,6 +31,10 @@ use trusted_cluster_operator_lib::{ struct Args { #[arg(short, long, default_value = "8000")] port: u16, + #[arg(long)] + cert_path: Option, + #[arg(long)] + key_path: Option, } fn generate_ignition(id: &str, public_addr: &str) -> IgnitionConfig { @@ -178,7 +183,13 @@ async fn main() { let service = app.into_make_service_with_connect_info::(); info!("Starting server on http://{}", addr); - let run = axum_server::bind(addr).serve(service).await; + let run = if args.cert_path.is_some() && args.key_path.is_some() { + let config = OpenSSLConfig::from_pem_file(args.cert_path.unwrap(), args.key_path.unwrap()) + .expect("invalid PEM files"); + axum_server::bind_openssl(addr, config).serve(service).await + } else { + axum_server::bind(addr).serve(service).await + }; run.expect("Server failed"); } diff --git a/test_utils/Cargo.toml b/test_utils/Cargo.toml index ac8addba..8dcd5cc4 100644 --- a/test_utils/Cargo.toml +++ b/test_utils/Cargo.toml @@ -25,6 +25,7 @@ ignition-config.workspace = true k8s-openapi.workspace = true kube = { workspace = true } log.workspace = true +percent-encoding = "2.3.2" rand_core = "0.6" serde.workspace = true serde_json.workspace = true diff --git a/test_utils/src/lib.rs b/test_utils/src/lib.rs index df8879d7..77df9c6e 100644 --- a/test_utils/src/lib.rs +++ b/test_utils/src/lib.rs @@ -6,12 +6,16 @@ use anyhow::{Result, anyhow}; use fs_extra::dir; use k8s_openapi::api::apps::v1::Deployment; -use k8s_openapi::api::core::v1::{ConfigMap, Namespace}; -use kube::api::DeleteParams; +use k8s_openapi::api::core::v1::{ConfigMap, Namespace, Secret}; +use kube::api::{DeleteParams, ObjectMeta}; use kube::{Api, Client}; use std::path::{Path, PathBuf}; use std::{collections::BTreeMap, env, sync::Once, time::Duration}; use tokio::process::Command; +use trusted_cluster_operator_lib::certificates::{ + Certificate, CertificateIssuerRef, CertificateSpec, +}; +use trusted_cluster_operator_lib::issuers::{Issuer, IssuerCa, IssuerSpec}; use trusted_cluster_operator_lib::TrustedExecutionCluster; use trusted_cluster_operator_lib::endpoints::*; @@ -32,6 +36,9 @@ const CLUSTER_URL_ENV: &str = "CLUSTER_URL"; const YELLOW: &str = "\x1b[33m"; const ANSI_RESET: &str = "\x1b[0m"; +const ROOT_SECRET: &str = "root-secret"; +const REG_SECRET: &str = "reg-srv-secret"; + pub fn compare_pcrs(actual: &[Pcr], expected: &[Pcr]) -> bool { if actual.len() != expected.len() { return false; @@ -113,7 +120,7 @@ trait K8sPlatform: Send + Sync { client: Client, namespace: &str, service: &str, - port: i32, + port: Option, ) -> Result; } @@ -141,9 +148,13 @@ impl K8sPlatform for Kind { _: Client, namespace: &str, service: &str, - port: i32, + port: Option, ) -> Result { - Ok(format!("{service}.{namespace}.svc.cluster.local:{port}")) + let url = format!("{service}.{namespace}.svc.cluster.local"); + Ok(match port { + Some(port) => format!("{url}:{port}"), + None => url, + }) } } @@ -172,7 +183,7 @@ impl K8sPlatform for OpenShift { client: Client, namespace: &str, service: &str, - _: i32, + _: Option, ) -> Result { let routes: Api = Api::namespaced(client.clone(), namespace); if let Ok(route) = routes.get(service).await { @@ -197,7 +208,7 @@ impl K8sPlatform for OtherK8s { Ok(()) } - async fn get_cluster_url(&self, _: Client, _: &str, _: &str, _: i32) -> Result { + async fn get_cluster_url(&self, _: Client, _: &str, _: &str, _: Option) -> Result { Err(anyhow!( "Set {CLUSTER_URL_ENV} when {PLATFORM_ENV} is not one of: kind, openshift" )) @@ -208,10 +219,14 @@ pub async fn get_cluster_url( client: Client, namespace: &str, service: &str, - port: i32, + port: Option, ) -> Result { if let Ok(url) = env::var(CLUSTER_URL_ENV) { - return Ok(format!("{service}.{namespace}.{url}:{port}")); + let full_url = format!("{service}.{namespace}.{url}"); + return Ok(match port { + Some(port) => format!("{full_url}:{port}"), + None => full_url, + }); } get_k8s_platform() .get_cluster_url(client, namespace, service, port) @@ -417,6 +432,97 @@ impl TestContext { .await } + async fn create_certificate( + &self, + service_name: &str, + cert_name: &str, + secret_name: &str, + issuer_name: &str, + ) -> Result<()> { + let ns = &self.test_namespace; + let domain = get_cluster_url(self.client.clone(), ns, service_name, None).await?; + let certs: Api = Api::namespaced(self.client.clone(), ns); + let cert = Certificate { + metadata: ObjectMeta { + name: Some(cert_name.to_string()), + ..Default::default() + }, + spec: CertificateSpec { + secret_name: secret_name.to_string(), + issuer_ref: CertificateIssuerRef { + name: issuer_name.to_string(), + ..Default::default() + }, + dns_names: Some(vec![domain]), + ..Default::default() + }, + ..Default::default() + }; + certs.create(&Default::default(), &cert).await?; + Ok(()) + } + + async fn set_certificates(&self) -> anyhow::Result<()> { + let ns = &self.test_namespace; + let root_issuer_name = "root-issuer"; + let root_issuer = Issuer { + metadata: ObjectMeta { + name: Some(root_issuer_name.to_string()), + ..Default::default() + }, + spec: IssuerSpec { + self_signed: Some(Default::default()), + ..Default::default() + }, + ..Default::default() + }; + let issuers: Api = Api::namespaced(self.client.clone(), ns); + issuers.create(&Default::default(), &root_issuer).await?; + let root_cert = Certificate { + metadata: ObjectMeta { + name: Some("root-cert".to_string()), + ..Default::default() + }, + spec: CertificateSpec { + secret_name: ROOT_SECRET.to_string(), + is_ca: Some(true), + issuer_ref: CertificateIssuerRef { + name: root_issuer_name.to_string(), + ..Default::default() + }, + common_name: Some("selfsigned-ca".to_string()), + ..Default::default() + }, + ..Default::default() + }; + let certs: Api = Api::namespaced(self.client.clone(), ns); + certs.create(&Default::default(), &root_cert).await?; + let issuer_name = "issuer"; + let issuer = Issuer { + metadata: ObjectMeta { + name: Some(issuer_name.to_string()), + ..Default::default() + }, + spec: IssuerSpec { + ca: Some(IssuerCa { + secret_name: ROOT_SECRET.to_string(), + ..Default::default() + }), + ..Default::default() + }, + ..Default::default() + }; + issuers.create(&Default::default(), &issuer).await?; + + let svc = REGISTER_SERVER_SERVICE; + self.create_certificate(svc, "reg-srv-cert", REG_SECRET, issuer_name) + .await?; + + let secrets: Api = Api::namespaced(self.client.clone(), &self.test_namespace); + wait_for_resource_created(&secrets, REG_SECRET, 15, 1).await?; + Ok(()) + } + async fn generate_manifests(&self, workspace_root: &PathBuf) -> Result<(PathBuf, PathBuf)> { let ns = self.test_namespace.clone(); let controller_gen_path = workspace_root.join("bin/controller-gen-v0.19.0"); @@ -474,6 +580,7 @@ impl TestContext { args.extend(&["-pcrs-compute-image", &compute_pcrs_img]); args.extend(&["-trustee-image", &trustee_image]); args.extend(&["-register-server-image", ®_srv_img]); + args.extend(&["-register-server-secret", REG_SECRET]); args.extend(&["-attestation-key-register-image", &att_reg_img]); args.extend(&["-approved-image", &approved_image]); let manifest_gen = Command::new(&trusted_cluster_gen_path).args(args).output(); @@ -492,6 +599,7 @@ impl TestContext { let (crd_temp_dir, rbac_temp_dir) = self.generate_manifests(&workspace_root).await?; test_info!(&self.test_name, "Manifests generated successfully"); + self.set_certificates().await?; let tec = "trustedexecutionclusters.trusted-execution-clusters.io"; let args = ["get", "crd", tec]; let crd_check_output = Command::new("kubectl").args(args).output().await?; @@ -592,9 +700,9 @@ impl TestContext { } async fn apply_operator_manifest(&self, manifests_path: &Path) -> Result<()> { - let ns = self.test_namespace.clone(); + let ns = &self.test_namespace; let trustee_addr = - get_cluster_url(self.client.clone(), &ns, TRUSTEE_SERVICE, TRUSTEE_PORT).await?; + get_cluster_url(self.client.clone(), ns, TRUSTEE_SERVICE, Some(TRUSTEE_PORT)).await?; let cr_manifest_path = manifests_path.join("trusted_execution_cluster_cr.yaml"); let cr_content = std::fs::read_to_string(&cr_manifest_path)?; @@ -628,7 +736,7 @@ impl TestContext { "Applying ApprovedImage manifest" ); - let deployments_api: Api = Api::namespaced(self.client.clone(), &ns); + let deployments_api: Api = Api::namespaced(self.client.clone(), ns); self.wait_for_deployment_ready(&deployments_api, "trusted-cluster-operator", 120) .await?; @@ -641,14 +749,14 @@ impl TestContext { let platform = get_k8s_platform(); for svc in ["kbs-service", "attestation-key-register", "register-server"] { - platform.expose(&ns, svc, &self.test_name).await?; + platform.expose(ns, svc, &self.test_name).await?; } test_info!( &self.test_name, "Waiting for image-pcrs ConfigMap to be created" ); - let configmap_api: Api = Api::namespaced(self.client.clone(), &ns); + let configmap_api: Api = Api::namespaced(self.client.clone(), ns); let err = format!("image-pcrs ConfigMap in the namespace {ns} not found"); let poller = Poller::new() @@ -714,12 +822,39 @@ fn test_namespace_name() -> String { format!("{namespace_prefix}test-{uuid}") } +pub async fn wait_for_resource_created( + api: &Api, + resource_name: &str, + timeout_secs: u64, + interval_secs: u64, +) -> anyhow::Result<()> +where + K: kube::Resource + Clone + std::fmt::Debug, + K: k8s_openapi::serde::de::DeserializeOwned, +{ + wait_for_resource_state(api, resource_name, timeout_secs, interval_secs, true).await +} + pub async fn wait_for_resource_deleted( api: &Api, resource_name: &str, timeout_secs: u64, interval_secs: u64, ) -> Result<()> +where + K: kube::Resource + Clone + std::fmt::Debug, + K: k8s_openapi::serde::de::DeserializeOwned, +{ + wait_for_resource_state(api, resource_name, timeout_secs, interval_secs, false).await +} + +async fn wait_for_resource_state( + api: &Api, + resource_name: &str, + timeout_secs: u64, + interval_secs: u64, + state: bool, +) -> Result<()> where K: kube::Resource + Clone + std::fmt::Debug, K: k8s_openapi::serde::de::DeserializeOwned, @@ -727,21 +862,24 @@ where let poller = Poller::new() .with_timeout(Duration::from_secs(timeout_secs)) .with_interval(Duration::from_secs(interval_secs)) - .with_error_message(format!("waiting for {resource_name} to be deleted")); - - poller - .poll_async(|| { - let api = api.clone(); - let name = resource_name.to_string(); - async move { - match api.get(&name).await { - Ok(_) => Err("{name} still exists, retrying..."), - Err(kube::Error::Api(ae)) if ae.code == 404 => Ok(()), - Err(e) => { - panic!("Unexpected error while fetching {name}: {e:?}"); - } - } + .with_error_message(format!( + "{resource_name} did not reach state {} after {timeout_secs} seconds", + if state { "created" } else { "deleted" } + )); + + let check = || { + let api = api.clone(); + let name = resource_name.to_string(); + async move { + let result = api.get(&name).await; + if let Err(kube::Error::Api(ae)) = &result + && ae.code != 404 + { + panic!("Unexpected error while fetching {name}: {ae:?}"); } - }) - .await + let err = anyhow!("{name} not in desired state: {result:?}"); + (result.is_err() ^ state).then_some(()).ok_or(err) + } + }; + poller.poll_async(check).await } diff --git a/test_utils/src/mock_client.rs b/test_utils/src/mock_client.rs index 068eeb92..57e21b5a 100644 --- a/test_utils/src/mock_client.rs +++ b/test_utils/src/mock_client.rs @@ -189,6 +189,7 @@ pub fn dummy_cluster() -> TrustedExecutionCluster { pcrs_compute_image: "".to_string(), register_server_image: "".to_string(), public_trustee_addr: Some("::".to_string()), + register_server_secret: None, register_server_port: None, trustee_kbs_port: None, attestation_key_register_image: "".to_string(), diff --git a/test_utils/src/virt/mod.rs b/test_utils/src/virt/mod.rs index d3e109c0..13b28668 100644 --- a/test_utils/src/virt/mod.rs +++ b/test_utils/src/virt/mod.rs @@ -10,6 +10,7 @@ use anyhow::{Context, Result, anyhow}; use clevis_pin_trustee_lib::Key as ClevisKey; use k8s_openapi::api::core::v1::Secret; use kube::{Api, Client}; +use percent_encoding::{NON_ALPHANUMERIC, utf8_percent_encode}; use std::{env, path::PathBuf, time::Duration}; use tokio::process::Command; @@ -17,7 +18,7 @@ use endpoints::*; use trusted_cluster_operator_lib::*; use super::Poller; -use crate::{get_cluster_url, get_env}; +use crate::{ROOT_SECRET, get_cluster_url, get_env}; /// Environment variable name for selecting the VM provider pub const VIRT_PROVIDER_ENV: &str = "VIRT_PROVIDER"; @@ -30,6 +31,7 @@ pub struct VmConfig { pub ssh_public_key: String, pub ssh_private_key: PathBuf, pub image: String, + pub ca_pem: String, } impl VmConfig { @@ -79,19 +81,28 @@ pub async fn generate_ignition(config: &VmConfig, with_ak: bool) -> Result Result { let svc = ATTESTATION_KEY_REGISTER_SERVICE; let port = ATTESTATION_KEY_REGISTER_PORT; - let attestation_url = get_cluster_url(client, namespace, svc, port).await?; + let attestation_url = get_cluster_url(client, namespace, svc, Some(port)).await?; let ign_json = serde_json::json!({ "attestation_key": { "registration": { @@ -197,11 +208,17 @@ fn get_virt_provider() -> Result { } } -pub fn create_backend( +pub async fn create_backend( client: Client, namespace: &str, vm_name: &str, ) -> Result> { + let secrets: Api = Api::namespaced(client.clone(), namespace); + let root_secret = secrets.get(ROOT_SECRET).await?; + let root_secret_data = root_secret.data.unwrap(); + let ca_pem_bytes = root_secret_data.get("ca.crt").unwrap(); + let ca_pem = String::from_utf8(ca_pem_bytes.0.clone())?; + let provider = get_virt_provider()?; let (public_key, key_path) = generate_ssh_key_pair()?; let image = get_env("TEST_IMAGE")?; @@ -212,6 +229,7 @@ pub fn create_backend( ssh_public_key: public_key, ssh_private_key: key_path, image, + ca_pem, }; match provider { VirtProvider::Kubevirt => Ok(Box::new(kubevirt::KubevirtBackend(config))), diff --git a/tests/attestation.rs b/tests/attestation.rs index 608f61a5..118a1dea 100644 --- a/tests/attestation.rs +++ b/tests/attestation.rs @@ -39,7 +39,7 @@ impl SingleAttestationContext { let client = test_ctx.client(); let namespace = test_ctx.namespace(); - let backend = virt::create_backend(client.clone(), namespace, vm_name)?; + let backend = virt::create_backend(client.clone(), namespace, vm_name).await?; test_ctx.info(format!("Creating VM: {vm_name}")); backend.create_vm().await?; @@ -90,8 +90,8 @@ async fn test_parallel_vm_attestation() -> anyhow::Result<()> { // Launch both VMs in parallel let vm1_name = "test-coreos-vm1"; let vm2_name = "test-coreos-vm2"; - let backend1 = virt::create_backend(client.clone(), namespace, vm1_name)?; - let backend2 = virt::create_backend(client.clone(), namespace, vm2_name)?; + let backend1 = virt::create_backend(client.clone(), namespace, vm1_name).await?; + let backend2 = virt::create_backend(client.clone(), namespace, vm2_name).await?; test_ctx.info("Creating VM1 and VM2 in parallel"); let (vm1_result, vm2_result) = tokio::join!(backend1.create_vm(), backend2.create_vm()); From 808064c900d603a371bf696994cde85fb5a91cda Mon Sep 17 00:00:00 2001 From: Jakob Naucke Date: Wed, 11 Feb 2026 16:05:42 +0100 Subject: [PATCH 5/6] trustee: Add certificate support Signed-off-by: Jakob Naucke Assisted-by: Claude --- Cargo.lock | 55 +++++++++++++++++++ api/trusted-cluster-gen.go | 3 ++ api/v1alpha1/crds.go | 5 ++ operator/Cargo.toml | 1 + operator/src/kbs-config.toml | 1 - operator/src/main.rs | 9 +++- operator/src/trustee.rs | 99 ++++++++++++++++++++++++----------- register-server/src/main.rs | 75 ++++++++++++++++++-------- test_utils/src/lib.rs | 5 ++ test_utils/src/mock_client.rs | 1 + 10 files changed, 197 insertions(+), 57 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3d77ebb2..b3e48e8f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2506,6 +2506,7 @@ dependencies = [ "serde_json", "thiserror 2.0.18", "tokio", + "toml", "trusted-cluster-operator-lib", "trusted-cluster-operator-test-utils", ] @@ -3340,6 +3341,15 @@ dependencies = [ "serde_core", ] +[[package]] +name = "serde_spanned" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8bbf91e5a4d6315eee45e704372590b30e260ee83af6639d64557f51b067776" +dependencies = [ + "serde_core", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -3826,6 +3836,45 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml" +version = "0.9.12+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf92845e79fc2e2def6a5d828f0801e29a2f8acc037becc5ab08595c7d5e9863" +dependencies = [ + "indexmap 2.12.0", + "serde_core", + "serde_spanned", + "toml_datetime", + "toml_parser", + "toml_writer", + "winnow", +] + +[[package]] +name = "toml_datetime" +version = "0.7.5+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92e1cfed4a3038bc5a127e35a2d360f145e1f4b971b551a2ba5fd7aedf7e1347" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_parser" +version = "1.0.7+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "247eaa3197818b831697600aadf81514e577e0cba5eab10f7e064e78ae154df1" +dependencies = [ + "winnow", +] + +[[package]] +name = "toml_writer" +version = "1.0.6+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab16f14aed21ee8bfd8ec22513f7287cd4a91aa92e44edfe2c17ddd004e92607" + [[package]] name = "tower" version = "0.5.3" @@ -4516,6 +4565,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" +[[package]] +name = "winnow" +version = "0.7.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a5364e9d77fcdeeaa6062ced926ee3381faa2ee02d3eb83a5c27a8825540829" + [[package]] name = "winreg" version = "0.50.0" diff --git a/api/trusted-cluster-gen.go b/api/trusted-cluster-gen.go index 854d5d5c..d39c1fc4 100644 --- a/api/trusted-cluster-gen.go +++ b/api/trusted-cluster-gen.go @@ -29,6 +29,7 @@ type Args struct { pcrsComputeImage string registerServerImage string registerServerSecret string + trusteeSecret string attestationKeyRegisterImage string approvedImage string } @@ -42,6 +43,7 @@ func main() { flag.StringVar(&args.pcrsComputeImage, "pcrs-compute-image", "quay.io/trusted-execution-clusters/compute-pcrs:latest", "Container image with the Trusted Execution Clusters compute-pcrs binary") flag.StringVar(&args.registerServerImage, "register-server-image", "quay.io/trusted-execution-clusters/register-server:latest", "Register server image to use in the deployment") flag.StringVar(&args.registerServerSecret, "register-server-secret", "", "When set, k8s secret for register server including tls.{crt,key}.") + flag.StringVar(&args.trusteeSecret, "trustee-secret", "", "When set, k8s secret for Trustee including tls.{crt,key}.") flag.StringVar(&args.attestationKeyRegisterImage, "attestation-key-register-image", "quay.io/trusted-execution-clusters/attestation-key-register:latest", "Attestation key register image to use in the deployment") flag.StringVar(&args.approvedImage, "approved-image", "", "When set, defines an initial approved image. Must be a bootable container image with SHA reference.") flag.Parse() @@ -147,6 +149,7 @@ func generateTrustedExecutionClusterCR(args *Args) error { RegisterServerImage: args.registerServerImage, AttestationKeyRegisterImage: &args.attestationKeyRegisterImage, RegisterServerSecret: &args.registerServerSecret, + TrusteeSecret: &args.trusteeSecret, PublicTrusteeAddr: nil, TrusteeKbsPort: 0, RegisterServerPort: 0, diff --git a/api/v1alpha1/crds.go b/api/v1alpha1/crds.go index b8b0e794..96a20923 100644 --- a/api/v1alpha1/crds.go +++ b/api/v1alpha1/crds.go @@ -57,6 +57,11 @@ type TrustedExecutionClusterSpec struct { // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="Value is immutable" RegisterServerSecret *string `json:"registerServerSecret"` + // Secret with tls.{crt,key} for Trustee + // +optional + // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="Value is immutable" + TrusteeSecret *string `json:"trusteeSecret"` + // Address where attester can connect to Trustee // +optional // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="Value is immutable" diff --git a/operator/Cargo.toml b/operator/Cargo.toml index 97fcf308..a22f49a8 100644 --- a/operator/Cargo.toml +++ b/operator/Cargo.toml @@ -30,6 +30,7 @@ serde.workspace = true serde_json.workspace = true thiserror = "2.0.18" tokio.workspace = true +toml = "0.9.11" [dev-dependencies] http.workspace = true diff --git a/operator/src/kbs-config.toml b/operator/src/kbs-config.toml index 275e93c0..d14aa16a 100644 --- a/operator/src/kbs-config.toml +++ b/operator/src/kbs-config.toml @@ -1,6 +1,5 @@ [http_server] sockets = ["0.0.0.0:8080"] -insecure_http = true [admin] insecure_api = true diff --git a/operator/src/main.rs b/operator/src/main.rs index 2ee7dc2d..68ef2278 100644 --- a/operator/src/main.rs +++ b/operator/src/main.rs @@ -146,7 +146,10 @@ async fn install_trustee_configuration( ) -> Result<()> { let owner_reference = generate_owner_reference(cluster)?; - match trustee::generate_trustee_data(client.clone(), owner_reference.clone()).await { + let trustee_secret = &cluster.spec.trustee_secret; + match trustee::generate_trustee_data(client.clone(), owner_reference.clone(), trustee_secret) + .await + { Ok(_) => info!("Generate configmap for the KBS configuration",), Err(e) => error!("Failed to create the KBS configuration configmap: {e}"), } @@ -168,7 +171,9 @@ async fn install_trustee_configuration( } let trustee_image = &cluster.spec.trustee_image; - match trustee::generate_kbs_deployment(client, owner_reference, trustee_image).await { + match trustee::generate_kbs_deployment(client, owner_reference, trustee_image, trustee_secret) + .await + { Ok(_) => info!("Generate the KBS deployment"), Err(e) => error!("Failed to create the KBS deployment: {e}"), } diff --git a/operator/src/trustee.rs b/operator/src/trustee.rs index 849a0cf6..8f7a3819 100644 --- a/operator/src/trustee.rs +++ b/operator/src/trustee.rs @@ -23,7 +23,7 @@ use kube::{ api::{ObjectMeta, Patch, PatchParams}, }; use log::info; -use operator::{RvContextData, create_or_info_if_exists}; +use operator::{RvContextData, TLS_DIR, create_or_info_if_exists, read_certificate}; use serde::{Serialize, Serializer}; use serde_json::{Value::String as JsonString, json}; use std::collections::BTreeMap; @@ -370,12 +370,38 @@ pub async fn generate_attestation_policy( Ok(()) } -pub async fn generate_trustee_data(client: Client, owner_reference: OwnerReference) -> Result<()> { - let kbs_config = include_str!("kbs-config.toml"); +fn generate_kbs_config(has_certificate: bool) -> Result { + let kbs_config_template = include_str!("kbs-config.toml"); + let mut config: toml::Table = toml::from_str(kbs_config_template)?; + + let section_err = "kbs-config.toml missing http_server section"; + let http_section = config.get_mut("http_server").context(section_err)?; + let server_err = "http_server is not a table"; + let http_server = http_section.as_table_mut().context(server_err)?; + + if has_certificate { + let tls_key = toml::Value::String(format!("{TLS_DIR}/tls.key")); + http_server.insert("private_key".to_string(), tls_key); + let tls_cert = toml::Value::String(format!("{TLS_DIR}/tls.crt")); + http_server.insert("certificate".to_string(), tls_cert); + } else { + http_server.insert("insecure_http".to_string(), toml::Value::Boolean(true)); + } + + Ok(toml::to_string(&config)?) +} + +pub async fn generate_trustee_data( + client: Client, + owner_reference: OwnerReference, + secret: &Option, +) -> Result<()> { + let has_certificate = read_certificate(client.clone(), secret).await?.is_some(); + let kbs_config = generate_kbs_config(has_certificate)?; let policy_rego = include_str!("resource.rego"); let data = BTreeMap::from([ - ("kbs-config.toml".to_string(), kbs_config.to_string()), + ("kbs-config.toml".to_string(), kbs_config), ("policy.rego".to_string(), policy_rego.to_string()), (REFERENCE_VALUES_FILE.to_string(), "[]".to_string()), ]); @@ -460,8 +486,33 @@ fn generate_kbs_volume_templates() -> [(&'static str, &'static str, Volume); 3] ] } -fn generate_kbs_pod_spec(image: &str) -> PodSpec { - let volumes = generate_kbs_volume_templates(); +fn generate_kbs_pod_spec( + image: &str, + tls_volumes: Option<(Volume, VolumeMount)>, +) -> PodSpec { + let volume_templates = generate_kbs_volume_templates(); + let mut volumes: Vec = volume_templates + .iter() + .map(|(name, _, volume)| { + let mut volume = volume.clone(); + volume.name = name.to_string(); + volume + }) + .collect(); + let mut volume_mounts: Vec = volume_templates + .iter() + .map(|(name, mount_path, _)| VolumeMount { + name: name.to_string(), + mount_path: mount_path.to_string(), + ..Default::default() + }) + .collect(); + + if let Some((volume, volume_mount)) = tls_volumes { + volumes.push(volume); + volume_mounts.push(volume_mount); + } + PodSpec { containers: vec![Container { command: Some(vec![ @@ -480,28 +531,10 @@ fn generate_kbs_pod_spec(image: &str) -> PodSpec { container_port: TRUSTEE_PORT, ..Default::default() }]), - volume_mounts: Some( - volumes - .iter() - .map(|(name, mount_path, _)| VolumeMount { - name: name.to_string(), - mount_path: mount_path.to_string(), - ..Default::default() - }) - .collect(), - ), + volume_mounts: Some(volume_mounts), ..Default::default() }], - volumes: Some( - volumes - .iter() - .map(|(name, _, volume)| { - let mut volume = volume.clone(); - volume.name = name.to_string(); - volume.clone() - }) - .collect(), - ), + volumes: Some(volumes), ..Default::default() } } @@ -510,9 +543,11 @@ pub async fn generate_kbs_deployment( client: Client, owner_reference: OwnerReference, image: &str, + secret: &Option, ) -> Result<()> { let selector = Some(BTreeMap::from([("app".to_string(), "kbs".to_string())])); - let pod_spec = generate_kbs_pod_spec(image); + let tls_volumes = read_certificate(client.clone(), secret).await?; + let pod_spec = generate_kbs_pod_spec(image, tls_volumes); // Inspired by trustee-operator let deployment = Deployment { @@ -814,19 +849,19 @@ mod tests { #[tokio::test] async fn test_generate_trustee_data_success() { - let clos = |client| generate_trustee_data(client, Default::default()); + let clos = |client| generate_trustee_data(client, Default::default(), &None); test_create_success::<_, _, ConfigMap>(clos).await; } #[tokio::test] async fn test_generate_trustee_data_already_exists() { - let clos = |client| generate_trustee_data(client, Default::default()); + let clos = |client| generate_trustee_data(client, Default::default(), &None); test_create_already_exists(clos).await; } #[tokio::test] async fn test_generate_trustee_data_error() { - let clos = |client| generate_trustee_data(client, Default::default()); + let clos = |client| generate_trustee_data(client, Default::default(), &None); test_create_error(clos).await; } @@ -844,13 +879,13 @@ mod tests { #[tokio::test] async fn test_generate_kbs_depl_success() { - let clos = |client| generate_kbs_deployment(client, Default::default(), "image"); + let clos = |client| generate_kbs_deployment(client, Default::default(), "image", &None); test_create_success::<_, _, Deployment>(clos).await; } #[tokio::test] async fn test_generate_kbs_depl_error() { - let clos = |client| generate_kbs_deployment(client, Default::default(), "image"); + let clos = |client| generate_kbs_deployment(client, Default::default(), "image", &None); test_create_error(clos).await; } } diff --git a/register-server/src/main.rs b/register-server/src/main.rs index 8d57664b..97d78db9 100644 --- a/register-server/src/main.rs +++ b/register-server/src/main.rs @@ -14,6 +14,7 @@ use env_logger::Env; use ignition_config::v3_5::{ Clevis, ClevisCustom, Config as IgnitionConfig, Filesystem, Luks, Storage, }; +use k8s_openapi::api::core::v1::Secret; use k8s_openapi::apimachinery::pkg::apis::meta::v1::{ObjectMeta, OwnerReference}; use kube::{Api, Client}; use log::{error, info}; @@ -25,6 +26,14 @@ use trusted_cluster_operator_lib::{ generate_owner_reference, get_trusted_execution_cluster, Machine, MachineSpec, }; +/// Information about the Trustee server for clevis configuration +struct TrusteeInfo { + /// The public address of the Trustee server + public_addr: String, + /// The CA certificate (PEM-encoded) if TLS is enabled, None otherwise + ca_cert: Option, +} + #[derive(Parser)] #[command(name = "register-server")] #[command(about = "HTTP server that generates Clevis PINs with random UUIDs")] @@ -37,11 +46,15 @@ struct Args { key_path: Option, } -fn generate_ignition(id: &str, public_addr: &str) -> IgnitionConfig { +fn generate_ignition(id: &str, trustee_info: &TrusteeInfo) -> IgnitionConfig { + let (scheme, cert) = match &trustee_info.ca_cert { + Some(ca_cert) => ("https", ca_cert.clone()), + None => ("http", String::new()), + }; let clevis_conf = ClevisConfig { servers: vec![ClevisServer { - url: format!("http://{public_addr}"), - cert: "".to_string(), + url: format!("{scheme}://{}", trustee_info.public_addr), + cert, }], path: format!("default/{id}/root"), num_retries: None, @@ -88,13 +101,30 @@ fn generate_ignition(id: &str, public_addr: &str) -> IgnitionConfig { } } -async fn get_public_trustee_addr(client: Client) -> anyhow::Result { - let cluster = get_trusted_execution_cluster(client).await?; +async fn get_trustee_info(client: Client) -> anyhow::Result { + let cluster = get_trusted_execution_cluster(client.clone()).await?; let name = cluster.metadata.name.as_deref().unwrap_or(""); - cluster.spec.public_trustee_addr.context(format!( + let public_addr = cluster.spec.public_trustee_addr.context(format!( "TrustedExecutionCluster {name} did not specify a public Trustee address. \ Add an address and re-register the node." - )) + ))?; + + let ca_cert = if let Some(secret_name) = &cluster.spec.trustee_secret { + let secrets: Api = Api::default_namespaced(client); + let secret = secrets.get(secret_name).await?; + let err = format!("Trustee secret {secret_name} does not contain ca.crt"); + let ca_data = secret.data.as_ref(); + let ca_bytes = ca_data.and_then(|data| data.get("ca.crt")).expect(&err); + let ca_pem = String::from_utf8(ca_bytes.0.clone())?; + Some(ca_pem) + } else { + None + }; + + Ok(TrusteeInfo { + public_addr, + ca_cert, + }) } async fn register_handler(ConnectInfo(addr): ConnectInfo) -> impl IntoResponse { @@ -133,12 +163,12 @@ async fn register_handler(ConnectInfo(addr): ConnectInfo) -> impl In Ok(_) => info!("Machine created successfully: machine-{id}"), Err(e) => return internal_error(e.context("Failed to create machine")), } - let public_addr = match get_public_trustee_addr(kube_client).await { - Ok(a) => a, - Err(e) => return internal_error(e.context("Failed to get Trustee address")), + let trustee_info = match get_trustee_info(kube_client).await { + Ok(info) => info, + Err(e) => return internal_error(e.context("Failed to get Trustee info")), }; - let ignition = generate_ignition(&id, &public_addr); + let ignition = generate_ignition(&id, &trustee_info); let json = match serde_json::to_value(ignition) { Ok(json) => json, Err(e) => return internal_error(anyhow!("Failed to serialise Ignition: {e}")), @@ -211,57 +241,58 @@ mod tests { } #[tokio::test] - async fn test_get_public_trustee_addr() { + async fn test_get_trustee_info() { let clos = async |_, _| Ok(serde_json::to_string(&dummy_clusters()).unwrap()); count_check!(1, clos, |client| { - let addr = get_public_trustee_addr(client).await.unwrap(); - assert_eq!(addr, "::".to_string()); + let info = get_trustee_info(client).await.unwrap(); + assert_eq!(info.public_addr, "::".to_string()); + assert!(info.ca_cert.is_none()); }); } #[tokio::test] - async fn test_get_public_trustee_addr_none() { + async fn test_get_trustee_info_no_cluster() { let clos = async |_, _| { let mut clusters = dummy_clusters(); clusters.items.clear(); Ok(serde_json::to_string(&clusters).unwrap()) }; count_check!(1, clos, |client| { - let err = get_public_trustee_addr(client).await.err().unwrap(); + let err = get_trustee_info(client).await.err().unwrap(); assert!(err.to_string().contains("No TrustedExecutionCluster found")); }); } #[tokio::test] - async fn test_get_public_trustee_addr_multiple() { + async fn test_get_trustee_info_multiple() { let clos = async |_, _| { let mut clusters = dummy_clusters(); clusters.items.push(clusters.items[0].clone()); Ok(serde_json::to_string(&clusters).unwrap()) }; count_check!(1, clos, |client| { - let err = get_public_trustee_addr(client).await.err().unwrap(); + let err = get_trustee_info(client).await.err().unwrap(); assert!(err.to_string().contains("More than one")); }); } #[tokio::test] - async fn test_get_public_trustee_no_addr() { + async fn test_get_trustee_info_no_addr() { let clos = async |_, _| { let mut clusters = dummy_clusters(); clusters.items[0].spec.public_trustee_addr = None; Ok(serde_json::to_string(&clusters).unwrap()) }; count_check!(1, clos, |client| { - let err = get_public_trustee_addr(client).await.err().unwrap(); + let err = get_trustee_info(client).await.err().unwrap(); let contains = "did not specify a public Trustee address"; assert!(err.to_string().contains(contains)); }); } #[tokio::test] - async fn test_get_public_trustee_error() { - test_get_error(async |c| get_public_trustee_addr(c).await.map(|_| ())).await; + async fn test_get_trustee_info_error() { + test_get_error(async |c| get_trustee_info(c).await.map(|_| ())).await; } fn dummy_machine() -> Machine { diff --git a/test_utils/src/lib.rs b/test_utils/src/lib.rs index 77df9c6e..b846a04b 100644 --- a/test_utils/src/lib.rs +++ b/test_utils/src/lib.rs @@ -38,6 +38,7 @@ const ANSI_RESET: &str = "\x1b[0m"; const ROOT_SECRET: &str = "root-secret"; const REG_SECRET: &str = "reg-srv-secret"; +const TRUSTEE_SECRET: &str = "trustee-secret"; pub fn compare_pcrs(actual: &[Pcr], expected: &[Pcr]) -> bool { if actual.len() != expected.len() { @@ -517,9 +518,12 @@ impl TestContext { let svc = REGISTER_SERVER_SERVICE; self.create_certificate(svc, "reg-srv-cert", REG_SECRET, issuer_name) .await?; + self.create_certificate(TRUSTEE_SERVICE, "trustee-cert", TRUSTEE_SECRET, issuer_name) + .await?; let secrets: Api = Api::namespaced(self.client.clone(), &self.test_namespace); wait_for_resource_created(&secrets, REG_SECRET, 15, 1).await?; + wait_for_resource_created(&secrets, TRUSTEE_SECRET, 15, 1).await?; Ok(()) } @@ -581,6 +585,7 @@ impl TestContext { args.extend(&["-trustee-image", &trustee_image]); args.extend(&["-register-server-image", ®_srv_img]); args.extend(&["-register-server-secret", REG_SECRET]); + args.extend(&["-trustee-secret", TRUSTEE_SECRET]); args.extend(&["-attestation-key-register-image", &att_reg_img]); args.extend(&["-approved-image", &approved_image]); let manifest_gen = Command::new(&trusted_cluster_gen_path).args(args).output(); diff --git a/test_utils/src/mock_client.rs b/test_utils/src/mock_client.rs index 57e21b5a..ff14ec21 100644 --- a/test_utils/src/mock_client.rs +++ b/test_utils/src/mock_client.rs @@ -190,6 +190,7 @@ pub fn dummy_cluster() -> TrustedExecutionCluster { register_server_image: "".to_string(), public_trustee_addr: Some("::".to_string()), register_server_secret: None, + trustee_secret: None, register_server_port: None, trustee_kbs_port: None, attestation_key_register_image: "".to_string(), From 80516f5b1e8e8b2463eb5fcd90bbef618ea676e8 Mon Sep 17 00:00:00 2001 From: Jakob Naucke Date: Fri, 13 Feb 2026 11:45:41 +0100 Subject: [PATCH 6/6] trustee: Disallow admin access Signed-off-by: Jakob Naucke --- operator/src/kbs-config.toml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/operator/src/kbs-config.toml b/operator/src/kbs-config.toml index d14aa16a..fc1005d5 100644 --- a/operator/src/kbs-config.toml +++ b/operator/src/kbs-config.toml @@ -2,8 +2,7 @@ sockets = ["0.0.0.0:8080"] [admin] -insecure_api = true -type = "InsecureAllowAll" +type = "DenyAll" [attestation_token] insecure_key = true