Add composefs-ostree and some basic CLI tools

Cargo.toml

@@ -19,6 +19,7 @@ composefs = { version = "0.3.0", path = "crates/composefs", default-features = f
 composefs-oci = { version = "0.3.0", path = "crates/composefs-oci", default-features = false }
 composefs-boot = { version = "0.3.0", path = "crates/composefs-boot", default-features = false }
 composefs-http = { version = "0.3.0", path = "crates/composefs-http", default-features = false }
+composefs-ostree = { version = "0.3.0", path = "crates/composefs-ostree", default-features = false }
 
 [profile.dev.package.sha2]
 # this is *really* slow otherwise

crates/cfsctl/Cargo.toml

@@ -11,9 +11,10 @@ rust-version.workspace = true
 version.workspace = true
 
 [features]
-default = ['pre-6.15', 'oci']
+default = ['pre-6.15', 'oci','ostree']
 http = ['composefs-http']
 oci = ['composefs-oci']
+ostree = ['composefs-ostree']
 rhel9 = ['composefs/rhel9']
 'pre-6.15' = ['composefs/pre-6.15']
 
@@ -24,6 +25,7 @@ composefs = { workspace = true }
 composefs-boot = { workspace = true }
 composefs-oci = { workspace = true, optional = true }
 composefs-http = { workspace = true, optional = true }
+composefs-ostree = { workspace = true, optional = true }
 env_logger = { version = "0.11.0", default-features = false }
 hex = { version = "0.4.0", default-features = false }
 rustix = { version = "1.0.0", default-features = false, features = ["fs", "process"] }

crates/cfsctl/src/main.rs

@@ -86,6 +86,30 @@ enum OciCommand {
     },
 }
 
+#[cfg(feature = "ostree")]
+#[derive(Debug, Subcommand)]
+enum OstreeCommand {
+    PullLocal {
+        repo_path: PathBuf,
+        ostree_ref: String,
+        name: Option<String>,
+        #[clap(long)]
+        base_name: Option<String>,
+    },
+    Pull {
+        repo_url: String,
+        ostree_ref: String,
+        name: Option<String>,
+        #[clap(long)]
+        base_name: Option<String>,
+    },
+    CreateImage {
+        commit_name: String,
+        #[clap(long)]
+        image_name: Option<String>,
+    },
+}
+
 #[derive(Debug, Subcommand)]
 enum Command {
     /// Take a transaction lock on the repository.
@@ -108,6 +132,12 @@ enum Command {
         #[clap(subcommand)]
         cmd: OciCommand,
     },
+    /// Commands for dealing with OSTree commits
+    #[cfg(feature = "ostree")]
+    Ostree {
+        #[clap(subcommand)]
+        cmd: OstreeCommand,
+    },
     /// Mounts a composefs, possibly enforcing fsverity of the image
     Mount {
         /// the name of the image to mount, either a sha256 digest or prefixed with 'ref/'
@@ -180,7 +210,7 @@ async fn main() -> Result<()> {
             }
         }
         Command::Cat { name } => {
-            repo.merge_splitstream(&name, None, &mut std::io::stdout())?;
+            repo.merge_splitstream(&name, None, None, &mut std::io::stdout())?;
         }
         Command::ImportImage { reference } => {
             let image_id = repo.import_image(&reference, &mut std::io::stdin())?;
@@ -302,6 +332,51 @@ async fn main() -> Result<()> {
                 create_dir_all(state.join("etc/work"))?;
             }
         },
+        #[cfg(feature = "ostree")]
+        Command::Ostree { cmd: ostree_cmd } => match ostree_cmd {
+            OstreeCommand::PullLocal {
+                ref repo_path,
+                ref ostree_ref,
+                name,
+                base_name,
+            } => {
+                let verity = composefs_ostree::pull_local(
+                    &Arc::new(repo),
+                    repo_path,
+                    ostree_ref,
+                    name.as_deref(),
+                    base_name.as_deref(),
+                )
+                .await?;
+
+                println!("verity {}", verity.to_hex());
+            }
+            OstreeCommand::Pull {
+                ref repo_url,
+                ref ostree_ref,
+                name,
+                base_name,
+            } => {
+                let verity = composefs_ostree::pull(
+                    &Arc::new(repo),
+                    repo_url,
+                    ostree_ref,
+                    name.as_deref(),
+                    base_name.as_deref(),
+                )
+                .await?;
+
+                println!("verity {}", verity.to_hex());
+            }
+            OstreeCommand::CreateImage {
+                ref commit_name,
+                ref image_name,
+            } => {
+                let mut fs = composefs_ostree::create_filesystem(&repo, commit_name)?;
+                let image_id = fs.commit_image(&repo, image_name.as_deref())?;
+                println!("{}", image_id.to_id());
+            }
+        },
         Command::ComputeId {
             ref path,
             bootable,

crates/composefs-http/src/lib.rs

@@ -13,9 +13,7 @@ use sha2::{Digest, Sha256};
 use tokio::task::JoinSet;
 
 use composefs::{
-    fsverity::FsVerityHashValue,
-    repository::Repository,
-    splitstream::{DigestMapEntry, SplitStreamReader},
+    fsverity::FsVerityHashValue, repository::Repository, splitstream::SplitStreamReader,
     util::Sha256Digest,
 };
 
@@ -61,7 +59,7 @@ impl<ObjectID: FsVerityHashValue> Downloader<ObjectID> {
     }
 
     fn open_splitstream(&self, id: &ObjectID) -> Result<SplitStreamReader<File, ObjectID>> {
-        SplitStreamReader::new(File::from(self.repo.open_object(id)?))
+        SplitStreamReader::new(File::from(self.repo.open_object(id)?), None)
     }
 
     fn read_object(&self, id: &ObjectID) -> Result<Vec<u8>> {
@@ -107,7 +105,7 @@ impl<ObjectID: FsVerityHashValue> Downloader<ObjectID> {
 
             // this part is fast: it only touches the header
             let mut reader = self.open_splitstream(&id)?;
-            for DigestMapEntry { verity, body } in &reader.refs.map {
+            for (body, verity) in reader.iter_mappings() {
                 match splitstreams.insert(verity.clone(), Some(*body)) {
                     // This is the (normal) case if we encounter a splitstream we didn't see yet...
                     None => {

crates/composefs-oci/src/image.rs

@@ -9,6 +9,7 @@ use composefs::{
     tree::{Directory, FileSystem, Inode, Leaf},
 };
 
+use crate::skopeo::{OCI_CONFIG_CONTENT_TYPE, TAR_LAYER_CONTENT_TYPE};
 use crate::tar::{TarEntry, TarItem};
 
 pub fn process_entry<ObjectID: FsVerityHashValue>(
@@ -74,14 +75,19 @@ pub fn create_filesystem<ObjectID: FsVerityHashValue>(
 ) -> Result<FileSystem<ObjectID>> {
     let mut filesystem = FileSystem::default();
 
-    let mut config_stream = repo.open_stream(config_name, config_verity)?;
+    let mut config_stream =
+        repo.open_stream(config_name, config_verity, Some(OCI_CONFIG_CONTENT_TYPE))?;
     let config = ImageConfiguration::from_reader(&mut config_stream)?;
 
     for diff_id in config.rootfs().diff_ids() {
         let layer_sha256 = super::sha256_from_digest(diff_id)?;
         let layer_verity = config_stream.lookup(&layer_sha256)?;
 
-        let mut layer_stream = repo.open_stream(&hex::encode(layer_sha256), Some(layer_verity))?;
+        let mut layer_stream = repo.open_stream(
+            &hex::encode(layer_sha256),
+            Some(layer_verity),
+            Some(TAR_LAYER_CONTENT_TYPE),
+        )?;
         while let Some(entry) = crate::tar::get_entry(&mut layer_stream)? {
             process_entry(&mut filesystem, entry)?;
         }

crates/composefs-oci/src/lib.rs

@@ -15,6 +15,7 @@ use composefs::{
     util::{parse_sha256, Sha256Digest},
 };
 
+use crate::skopeo::{OCI_CONFIG_CONTENT_TYPE, TAR_LAYER_CONTENT_TYPE};
 use crate::tar::get_entry;
 
 type ContentAndVerity<ObjectID> = (Sha256Digest, ObjectID);
@@ -39,14 +40,19 @@ pub fn import_layer<ObjectID: FsVerityHashValue>(
     name: Option<&str>,
     tar_stream: &mut impl Read,
 ) -> Result<ObjectID> {
-    repo.ensure_stream(sha256, |writer| tar::split(tar_stream, writer), name)
+    repo.ensure_stream(
+        sha256,
+        TAR_LAYER_CONTENT_TYPE,
+        |writer| tar::split(tar_stream, writer),
+        name,
+    )
 }
 
 pub fn ls_layer<ObjectID: FsVerityHashValue>(
     repo: &Repository<ObjectID>,
     name: &str,
 ) -> Result<()> {
-    let mut split_stream = repo.open_stream(name, None)?;
+    let mut split_stream = repo.open_stream(name, None, Some(TAR_LAYER_CONTENT_TYPE))?;
 
     while let Some(entry) = get_entry(&mut split_stream)? {
         println!("{entry}");
@@ -81,9 +87,9 @@ pub fn open_config<ObjectID: FsVerityHashValue>(
                 .with_context(|| format!("Object {name} is unknown to us"))?
         }
     };
-    let mut stream = repo.open_stream(name, Some(id))?;
+    let mut stream = repo.open_stream(name, Some(id), Some(OCI_CONFIG_CONTENT_TYPE))?;
     let config = ImageConfiguration::from_reader(&mut stream)?;
-    Ok((config, stream.refs))
+    Ok((config, stream.get_mappings()))
 }
 
 fn hash(bytes: &[u8]) -> Sha256Digest {
@@ -104,7 +110,7 @@ pub fn open_config_shallow<ObjectID: FsVerityHashValue>(
             // we need to manually check the content digest
             let expected_hash = parse_sha256(name)
                 .context("Containers must be referred to by sha256 if verity is missing")?;
-            let mut stream = repo.open_stream(name, None)?;
+            let mut stream = repo.open_stream(name, None, Some(OCI_CONFIG_CONTENT_TYPE))?;
             let mut raw_config = vec![];
             stream.read_to_end(&mut raw_config)?;
             ensure!(hash(&raw_config) == expected_hash, "Data integrity issue");
@@ -121,7 +127,8 @@ pub fn write_config<ObjectID: FsVerityHashValue>(
     let json = config.to_string()?;
     let json_bytes = json.as_bytes();
     let sha256 = hash(json_bytes);
-    let mut stream = repo.create_stream(Some(sha256), Some(refs));
+    let mut stream = repo.create_stream(OCI_CONFIG_CONTENT_TYPE, Some(sha256));
+    stream.add_sha256_mappings(refs);
     stream.write_inline(json_bytes);
     let id = repo.write_stream(stream, None)?;
     Ok((sha256, id))
@@ -199,7 +206,7 @@ mod test {
         let id = import_layer(&repo, &layer_id, Some("name"), &mut layer.as_slice()).unwrap();
 
         let mut dump = String::new();
-        let mut split_stream = repo.open_stream("refs/name", Some(&id)).unwrap();
+        let mut split_stream = repo.open_stream("refs/name", Some(&id), None).unwrap();
         while let Some(entry) = tar::get_entry(&mut split_stream).unwrap() {
             writeln!(dump, "{entry}").unwrap();
         }

crates/composefs-oci/src/skopeo.rs

@@ -10,12 +10,13 @@ use oci_spec::image::{Descriptor, ImageConfiguration, ImageManifest, MediaType};
 use rustix::process::geteuid;
 use tokio::{io::AsyncReadExt, sync::Semaphore};
 
-use composefs::{
-    fsverity::FsVerityHashValue, repository::Repository, splitstream::DigestMap, util::Sha256Digest,
-};
+use composefs::{fsverity::FsVerityHashValue, repository::Repository, util::Sha256Digest};
 
 use crate::{sha256_from_descriptor, sha256_from_digest, tar::split_async, ContentAndVerity};
 
+pub const TAR_LAYER_CONTENT_TYPE: u64 = 0x2a037edfcae1ffea;
+pub const OCI_CONFIG_CONTENT_TYPE: u64 = 0x44218c839727a80b;
+
 struct ImageOp<ObjectID: FsVerityHashValue> {
     repo: Arc<Repository<ObjectID>>,
     proxy: ImageProxy,
@@ -78,7 +79,9 @@ impl<ObjectID: FsVerityHashValue> ImageOp<ObjectID> {
             self.progress
                 .println(format!("Fetching layer {}", hex::encode(layer_sha256)))?;
 
-            let mut splitstream = self.repo.create_stream(Some(layer_sha256), None);
+            let mut splitstream = self
+                .repo
+                .create_stream(TAR_LAYER_CONTENT_TYPE, Some(layer_sha256));
             match descriptor.media_type() {
                 MediaType::ImageLayer => {
                     split_async(progress, &mut splitstream).await?;
@@ -155,15 +158,15 @@ impl<ObjectID: FsVerityHashValue> ImageOp<ObjectID> {
                 entries.push((layer_sha256, future));
             }
 
+            let mut splitstream = self
+                .repo
+                .create_stream(OCI_CONFIG_CONTENT_TYPE, Some(config_sha256));
+
             // Collect the results.
-            let mut config_maps = DigestMap::new();
             for (layer_sha256, future) in entries {
-                config_maps.insert(&layer_sha256, &future.await??);
+                splitstream.add_sha256_mapping(&layer_sha256, &future.await??);
             }
 
-            let mut splitstream = self
-                .repo
-                .create_stream(Some(config_sha256), Some(config_maps));
             splitstream.write_inline(&raw_config);
             let config_id = self.repo.write_stream(splitstream, None)?;
 

crates/composefs-ostree/Cargo.toml

@@ -0,0 +1,29 @@
+[package]
+name = "composefs-ostree"
+description = "ostree support for composefs"
+keywords = ["composefs", "ostree"]
+
+edition.workspace = true
+license.workspace = true
+readme.workspace = true
+repository.workspace = true
+rust-version.workspace = true
+version.workspace = true
+
+[dependencies]
+anyhow = { version = "1.0.87", default-features = false }
+composefs = { workspace = true }
+configparser = { version = "3.1.0", features = [] }
+flate2 = { version = "1.1.2", default-features = true }
+gvariant = { version = "0.5.0", default-features = true}
+hex = { version = "0.4.0", default-features = false, features = ["std"] }
+rustix = { version = "1.0.0", default-features = false, features = ["fs", "mount", "process", "std"] }
+sha2 = { version = "0.10.1", default-features = false }
+zerocopy = { version = "0.8.0", default-features = false, features = ["derive", "std"] }
+reqwest = { version = "0.12.15", features = ["zstd"] }
+
+[dev-dependencies]
+similar-asserts = "1.7.0"
+
+[lints]
+workspace = true