From e891e6caaf190e8cf1062608f1b7c8bfb710b375 Mon Sep 17 00:00:00 2001 From: Joseph Marrero Corchado Date: Tue, 9 Sep 2025 11:53:31 -0400 Subject: [PATCH] WIP: Use own container-storage for host images if the refspec Signed-off-by: Joseph Marrero Corchado --- crates/lib/src/cli.rs | 34 ++++++++-- crates/lib/src/deploy.rs | 135 +++++++++++++++++++++++++++++++++++---- 2 files changed, 152 insertions(+), 17 deletions(-) diff --git a/crates/lib/src/cli.rs b/crates/lib/src/cli.rs index 3412a6b03..5f47ca895 100644 --- a/crates/lib/src/cli.rs +++ b/crates/lib/src/cli.rs @@ -7,7 +7,7 @@ use std::io::Seek; use std::os::unix::process::CommandExt; use std::process::Command; -use anyhow::{ensure, Context, Result}; +use anyhow::{Context, Result, ensure}; use camino::Utf8PathBuf; use cap_std_ext::cap_std; use cap_std_ext::cap_std::fs::Dir; @@ -928,7 +928,19 @@ async fn upgrade(opts: UpgradeOpts) -> Result<()> { } } } else { - let fetched = crate::deploy::pull(repo, imgref, None, opts.quiet, prog.clone()).await?; + // Check if image exists in bootc storage (/usr/lib/bootc/storage) + let imgstore = sysroot.get_ensure_imgstore()?; + let use_unified = imgstore + .exists(&format!("{imgref:#}")) + .await + .unwrap_or(false); + + let fetched = if use_unified { + crate::deploy::pull_unified(repo, imgref, None, opts.quiet, prog.clone(), sysroot) + .await? + } else { + crate::deploy::pull(repo, imgref, None, opts.quiet, prog.clone()).await? + }; let staged_digest = staged_image.map(|s| s.digest().expect("valid digest in status")); let fetched_digest = &fetched.manifest_digest; tracing::debug!("staged: {staged_digest:?}"); @@ -1056,7 +1068,18 @@ async fn switch(opts: SwitchOpts) -> Result<()> { let new_spec = RequiredHostSpec::from_spec(&new_spec)?; - let fetched = crate::deploy::pull(repo, &target, None, opts.quiet, prog.clone()).await?; + // Check if image exists in bootc storage (/usr/lib/bootc/storage) + let imgstore = sysroot.get_ensure_imgstore()?; + let use_unified = imgstore + .exists(&format!("{target:#}")) + .await + .unwrap_or(false); + + let fetched = if use_unified { + crate::deploy::pull_unified(repo, &target, None, opts.quiet, prog.clone(), sysroot).await? + } else { + crate::deploy::pull(repo, &target, None, opts.quiet, prog.clone()).await? + }; if !opts.retain { // By default, we prune the previous ostree ref so it will go away after later upgrades @@ -1422,7 +1445,10 @@ async fn run_from_opt(opt: Opt) -> Result<()> { let mut w = SplitStreamWriter::new(&cfs, None, Some(testdata_digest)); w.write_inline(testdata); let object = cfs.write_stream(w, Some("testobject"))?.to_hex(); - assert_eq!(object, "5d94ceb0b2bb3a78237e0a74bc030a262239ab5f47754a5eb2e42941056b64cb21035d64a8f7c2f156e34b820802fa51884de2b1f7dc3a41b9878fc543cd9b07"); + assert_eq!( + object, + "5d94ceb0b2bb3a78237e0a74bc030a262239ab5f47754a5eb2e42941056b64cb21035d64a8f7c2f156e34b820802fa51884de2b1f7dc3a41b9878fc543cd9b07" + ); Ok(()) } // We don't depend on fsverity-utils today, so re-expose some helpful CLI tools. diff --git a/crates/lib/src/deploy.rs b/crates/lib/src/deploy.rs index 7b7ff59e1..726a46403 100644 --- a/crates/lib/src/deploy.rs +++ b/crates/lib/src/deploy.rs @@ -6,7 +6,7 @@ use std::collections::HashSet; use std::io::{BufRead, Write}; use anyhow::Ok; -use anyhow::{anyhow, Context, Result}; +use anyhow::{Context, Result, anyhow}; use cap_std::fs::{Dir, MetadataExt}; use cap_std_ext::cap_std; use cap_std_ext::dirext::CapStdExtDirExt; @@ -380,6 +380,112 @@ pub(crate) async fn prepare_for_pull( Ok(PreparedPullResult::Ready(Box::new(prepared_image))) } +/// Unified approach: Use bootc's CStorage to pull the image, then prepare from containers-storage. +/// This reuses the same infrastructure as LBIs. +pub(crate) async fn prepare_for_pull_unified( + repo: &ostree::Repo, + imgref: &ImageReference, + target_imgref: Option<&OstreeImageReference>, + store: &Storage, +) -> Result { + // Get or initialize the bootc container storage (same as used for LBIs) + let imgstore = store.get_ensure_imgstore()?; + + let image_ref_str = format!("{imgref:#}"); + + // Log the original transport being used for the pull + tracing::info!( + "Unified pull: pulling from transport '{}' to bootc storage", + &imgref.transport + ); + + // Pull the image to bootc storage using the same method as LBIs + imgstore + .pull(&image_ref_str, crate::podstorage::PullMode::Always) + .await?; + + // Now create a containers-storage reference to read from bootc storage + tracing::info!("Unified pull: now importing from containers-storage transport"); + let containers_storage_imgref = ImageReference { + transport: "containers-storage".to_string(), + image: imgref.image.clone(), + signature: imgref.signature.clone(), + }; + let ostree_imgref = OstreeImageReference::from(containers_storage_imgref); + + // Use the standard preparation flow but reading from containers-storage + let mut imp = new_importer(repo, &ostree_imgref).await?; + if let Some(target) = target_imgref { + imp.set_target(target); + } + let prep = match imp.prepare().await? { + PrepareResult::AlreadyPresent(c) => { + println!("No changes in {imgref:#} => {}", c.manifest_digest); + return Ok(PreparedPullResult::AlreadyPresent(Box::new((*c).into()))); + } + PrepareResult::Ready(p) => p, + }; + check_bootc_label(&prep.config); + if let Some(warning) = prep.deprecated_warning() { + ostree_ext::cli::print_deprecated_warning(warning).await; + } + ostree_ext::cli::print_layer_status(&prep); + let layers_to_fetch = prep.layers_to_fetch().collect::>>()?; + + // Log that we're importing a new image from containers-storage + const PULLING_NEW_IMAGE_ID: &str = "6d5e4f3a2b1c0d9e8f7a6b5c4d3e2f1a0"; + tracing::info!( + message_id = PULLING_NEW_IMAGE_ID, + bootc.image.reference = &imgref.image, + bootc.image.transport = "containers-storage", + bootc.original_transport = &imgref.transport, + bootc.status = "importing_from_storage", + "Importing image from bootc storage: {}", + ostree_imgref + ); + + let prepared_image = PreparedImportMeta { + imp, + n_layers_to_fetch: layers_to_fetch.len(), + layers_total: prep.all_layers().count(), + bytes_to_fetch: layers_to_fetch.iter().map(|(l, _)| l.layer.size()).sum(), + bytes_total: prep.all_layers().map(|l| l.layer.size()).sum(), + digest: prep.manifest_digest.clone(), + prep, + }; + + Ok(PreparedPullResult::Ready(Box::new(prepared_image))) +} + +/// Unified pull: Use podman to pull to containers-storage, then read from there +pub(crate) async fn pull_unified( + repo: &ostree::Repo, + imgref: &ImageReference, + target_imgref: Option<&OstreeImageReference>, + quiet: bool, + prog: ProgressWriter, + store: &Storage, +) -> Result> { + match prepare_for_pull_unified(repo, imgref, target_imgref, store).await? { + PreparedPullResult::AlreadyPresent(existing) => { + // Log that the image was already present (Debug level since it's not actionable) + const IMAGE_ALREADY_PRESENT_ID: &str = "5c4d3e2f1a0b9c8d7e6f5a4b3c2d1e0f9"; + tracing::debug!( + message_id = IMAGE_ALREADY_PRESENT_ID, + bootc.image.reference = &imgref.image, + bootc.image.transport = &imgref.transport, + bootc.status = "already_present", + "Image already present: {}", + imgref + ); + Ok(existing) + } + PreparedPullResult::Ready(prepared_image_meta) => { + pull_from_prepared(imgref, quiet, prog, *prepared_image_meta).await + } + } +} + #[context("Pulling")] pub(crate) async fn pull_from_prepared( imgref: &ImageReference, @@ -429,18 +535,21 @@ pub(crate) async fn pull_from_prepared( let imgref_canonicalized = imgref.clone().canonicalize()?; tracing::debug!("Canonicalized image reference: {imgref_canonicalized:#}"); - // Log successful import completion - const IMPORT_COMPLETE_JOURNAL_ID: &str = "4d3e2f1a0b9c8d7e6f5a4b3c2d1e0f9a8"; - - tracing::info!( - message_id = IMPORT_COMPLETE_JOURNAL_ID, - bootc.image.reference = &imgref.image, - bootc.image.transport = &imgref.transport, - bootc.manifest_digest = import.manifest_digest.as_ref(), - bootc.ostree_commit = &import.merge_commit, - "Successfully imported image: {}", - imgref - ); + // Log successful import completion (skip if using unified storage to avoid double logging) + let is_unified_path = imgref.transport == "containers-storage"; + if !is_unified_path { + const IMPORT_COMPLETE_JOURNAL_ID: &str = "4d3e2f1a0b9c8d7e6f5a4b3c2d1e0f9a8"; + + tracing::info!( + message_id = IMPORT_COMPLETE_JOURNAL_ID, + bootc.image.reference = &imgref.image, + bootc.image.transport = &imgref.transport, + bootc.manifest_digest = import.manifest_digest.as_ref(), + bootc.ostree_commit = &import.merge_commit, + "Successfully imported image: {}", + imgref + ); + } if let Some(msg) = ostree_container::store::image_filtered_content_warning(&import.filtered_files)