From 6cd568b3916f00fd3fb3deae4e869d7ede3e433d Mon Sep 17 00:00:00 2001 From: Greg Schoeninger Date: Sat, 28 Mar 2026 11:42:57 -0700 Subject: [PATCH 1/2] Expose Content-Length for CORS headers to send through hub --- crates/lib/src/storage/local.rs | 17 ++++ crates/lib/src/storage/s3.rs | 23 +++++ crates/lib/src/storage/version_store.rs | 11 +++ crates/lib/src/util/fs.rs | 24 ++++- crates/server/src/controllers/file.rs | 72 +++++++++++--- crates/server/src/controllers/versions.rs | 94 ++++++++++++++++--- .../src/controllers/workspaces/files.rs | 73 +++++++++++--- crates/server/src/helpers.rs | 66 +++++++++++++ 8 files changed, 337 insertions(+), 43 deletions(-) diff --git a/crates/lib/src/storage/local.rs b/crates/lib/src/storage/local.rs index 34c3ed055..8d0268af5 100644 --- a/crates/lib/src/storage/local.rs +++ b/crates/lib/src/storage/local.rs @@ -148,6 +148,16 @@ impl VersionStore for LocalVersionStore { Ok(data) } + async fn get_version_derived_size( + &self, + orig_hash: &str, + derived_filename: &str, + ) -> Result { + let path = self.version_dir(orig_hash).join(derived_filename); + let metadata = fs::metadata(&path).await?; + Ok(metadata.len()) + } + async fn get_version_stream( &self, hash: &str, @@ -773,6 +783,13 @@ mod tests { collected.extend_from_slice(&chunk.unwrap()); } assert_eq!(collected, derived_data); + assert_eq!( + store + .get_version_derived_size(orig_hash, derived_filename) + .await + .unwrap(), + derived_data.len() as u64 + ); } #[tokio::test] diff --git a/crates/lib/src/storage/s3.rs b/crates/lib/src/storage/s3.rs index b9e71fcb8..478adbe2c 100644 --- a/crates/lib/src/storage/s3.rs +++ b/crates/lib/src/storage/s3.rs @@ -264,6 +264,29 @@ impl VersionStore for S3VersionStore { Ok(data) } + async fn get_version_derived_size( + &self, + orig_hash: &str, + derived_filename: &str, + ) -> Result { + let client = self.init_client().await?; + let key = format!("{}/{}", self.version_dir(orig_hash), derived_filename); + + let resp = client + .head_object() + .bucket(&self.bucket) + .key(&key) + .send() + .await + .map_err(|e| OxenError::basic_str(format!("S3 head_object failed: {e}")))?; + + let size = resp + .content_length() + .ok_or_else(|| OxenError::basic_str("S3 object missing content_length"))? + as u64; + Ok(size) + } + async fn get_version_stream( &self, hash: &str, diff --git a/crates/lib/src/storage/version_store.rs b/crates/lib/src/storage/version_store.rs index 156a788d2..567c1abe4 100644 --- a/crates/lib/src/storage/version_store.rs +++ b/crates/lib/src/storage/version_store.rs @@ -202,6 +202,17 @@ pub trait VersionStore: Debug + Send + Sync + 'static { hash: &str, ) -> Result> + Send + Unpin>, OxenError>; + /// Get a stream of a derived file (resized, video thumbnail, etc.) + /// + /// # Arguments + /// * `orig_hash` - The content hash of the parent version + /// * `derived_filename` - Filename for the derived artifact + async fn get_version_derived_size( + &self, + orig_hash: &str, + derived_filename: &str, + ) -> Result; + /// Get a stream of a derived file (resized, video thumbnail, etc.) /// /// # Arguments diff --git a/crates/lib/src/util/fs.rs b/crates/lib/src/util/fs.rs index 27faefba5..d98b683d3 100644 --- a/crates/lib/src/util/fs.rs +++ b/crates/lib/src/util/fs.rs @@ -94,7 +94,13 @@ pub async fn handle_image_resize( file_hash: String, file_path: &Path, img_resize: ImgResize, -) -> Result> + Send>>, OxenError> { +) -> Result< + ( + Pin> + Send>>, + u64, + ), + OxenError, +> { log::debug!("img_resize {img_resize:?}"); let derived_filename = resized_filename(file_path, img_resize.width, img_resize.height); @@ -1579,16 +1585,25 @@ pub async fn resize_cache_image_version_store( img_hash: &str, derived_filename: &str, resize: ImgResize, -) -> Result> + Send>>, OxenError> { +) -> Result< + ( + Pin> + Send>>, + u64, + ), + OxenError, +> { if version_store .derived_version_exists(img_hash, derived_filename) .await? { log::debug!("In the resize cache! {derived_filename}"); + let content_length = version_store + .get_version_derived_size(img_hash, derived_filename) + .await?; let stream = version_store .get_version_derived_stream(img_hash, derived_filename) .await?; - return Ok(stream.boxed()); + return Ok((stream.boxed(), content_length)); } log::debug!("create resized image {derived_filename} from hash {img_hash}"); @@ -1638,11 +1653,12 @@ pub async fn resize_cache_image_version_store( version_store .store_version_derived(img_hash, derived_filename, &buf) .await?; + let content_length = buf.len() as u64; let stream: Pin> + Send>> = futures::stream::once(async move { Ok(Bytes::from(buf)) }).boxed(); - Ok(stream) + Ok((stream, content_length)) } /// Generate a video thumbnail using thumbnails crate. diff --git a/crates/server/src/controllers/file.rs b/crates/server/src/controllers/file.rs index e21c6177d..1cf0b2850 100644 --- a/crates/server/src/controllers/file.rs +++ b/crates/server/src/controllers/file.rs @@ -1,5 +1,5 @@ use crate::errors::OxenHttpError; -use crate::helpers::{create_user_from_options, get_repo}; +use crate::helpers::{create_user_from_options, file_stream_response, get_repo}; use crate::params::{app_data, parse_resource, path_param}; use actix_multipart::form::text::Text; @@ -177,6 +177,7 @@ pub async fn get( let file_hash = entry.hash(); let hash_str = file_hash.to_string(); let mime_type = entry.mime_type(); + let num_bytes = entry.num_bytes(); let last_commit_id = entry.last_commit_id().to_string(); let query_params = query.into_inner(); @@ -190,7 +191,7 @@ pub async fn get( }; log::debug!("img_resize {img_resize:?}"); - let file_stream = util::fs::handle_image_resize( + let (file_stream, content_length) = util::fs::handle_image_resize( version_store.clone(), hash_str.clone(), &path, @@ -198,10 +199,10 @@ pub async fn get( ) .await?; - return Ok(HttpResponse::Ok() - .content_type(mime_type) - .insert_header(("oxen-revision-id", last_commit_id.as_str())) - .streaming(file_stream)); + return Ok( + file_stream_response(mime_type, &last_commit_id, Some(content_length)) + .streaming(file_stream), + ); } // Handle video thumbnail - requires thumbnail=true parameter @@ -218,10 +219,7 @@ pub async fn get( util::fs::handle_video_thumbnail(Arc::clone(&version_store), hash_str, video_thumbnail) .await?; - return Ok(HttpResponse::Ok() - .content_type("image/jpeg") - .insert_header(("oxen-revision-id", last_commit_id.as_str())) - .streaming(stream)); + return Ok(file_stream_response("image/jpeg", &last_commit_id, None).streaming(stream)); } log::debug!("did not hit the resize or thumbnail cache"); @@ -229,10 +227,7 @@ pub async fn get( // Stream the file let stream = version_store.get_version_stream(&hash_str).await?; - Ok(HttpResponse::Ok() - .content_type(mime_type) - .insert_header(("oxen-revision-id", last_commit_id.as_str())) - .streaming(stream)) + Ok(file_stream_response(mime_type, &last_commit_id, Some(num_bytes)).streaming(stream)) } /// Upload files @@ -793,6 +788,7 @@ mod tests { use std::path::{Path, PathBuf}; use actix_multipart_test::MultiPartFormDataBuilder; + use actix_web::http::header; use actix_web::{App, body, web}; use liboxen::view::CommitResponse; @@ -871,6 +867,54 @@ mod tests { Ok(()) } + #[actix_web::test] + async fn test_controllers_file_get_exposes_content_length() -> Result<(), OxenError> { + liboxen::test::init_test_env(); + let sync_dir = test::get_sync_dir()?; + let namespace = "Testing-Namespace"; + let repo_name = "Testing-Get-Headers"; + let repo = test::create_local_repo(&sync_dir, namespace, repo_name)?; + + util::fs::create_dir_all(repo.path.join("data"))?; + let hello_file = repo.path.join("data/hello.txt"); + let file_content = "Hello"; + util::fs::write_to_path(&hello_file, file_content)?; + repositories::add(&repo, &hello_file).await?; + let _commit = repositories::commit(&repo, "First commit")?; + + let uri = format!("/oxen/{namespace}/{repo_name}/file/main/data/hello.txt"); + let req = actix_web::test::TestRequest::get() + .uri(&uri) + .app_data(OxenAppData::new(sync_dir.to_path_buf())) + .to_request(); + + let app = actix_web::test::init_service( + App::new() + .app_data(OxenAppData::new(sync_dir.clone())) + .route( + "/oxen/{namespace}/{repo_name}/file/{resource:.*}", + web::get().to(controllers::file::get), + ), + ) + .await; + + let resp = actix_web::test::call_service(&app, req).await; + assert_eq!(resp.status(), actix_web::http::StatusCode::OK); + assert_eq!( + resp.headers().get(header::CONTENT_LENGTH).unwrap(), + file_content.len().to_string().as_str() + ); + assert_eq!( + resp.headers() + .get(header::ACCESS_CONTROL_EXPOSE_HEADERS) + .unwrap(), + header::CONTENT_LENGTH.as_str() + ); + + test::cleanup_sync_dir(&sync_dir)?; + Ok(()) + } + #[actix_web::test] async fn test_controllers_file_put_single_file_to_full_resource_path() -> Result<(), OxenError> { diff --git a/crates/server/src/controllers/versions.rs b/crates/server/src/controllers/versions.rs index 89cb01bfd..0557bcf4c 100644 --- a/crates/server/src/controllers/versions.rs +++ b/crates/server/src/controllers/versions.rs @@ -1,7 +1,7 @@ pub mod chunks; use crate::errors::OxenHttpError; -use crate::helpers::get_repo; +use crate::helpers::{file_stream_response, get_repo}; use crate::params::{app_data, parse_resource, path_param}; use actix_multipart::Multipart; @@ -130,6 +130,7 @@ pub async fn download( let file_hash = entry.hash(); let hash_str = file_hash.to_string(); let mime_type = entry.mime_type(); + let num_bytes = entry.num_bytes(); let last_commit_id = entry.last_commit_id().to_string(); // TODO: refactor out of here and check for type, // but seeing if it works to resize the image and cache it to disk if we have a resize query @@ -139,23 +140,20 @@ pub async fn download( { log::debug!("img_resize {img_resize:?}"); - let file_stream = + let (file_stream, content_length) = util::fs::handle_image_resize(Arc::clone(&version_store), hash_str, &path, img_resize) .await?; - return Ok(HttpResponse::Ok() - .content_type(mime_type) - .insert_header(("oxen-revision-id", last_commit_id.as_str())) - .streaming(file_stream)); + return Ok( + file_stream_response(mime_type, &last_commit_id, Some(content_length)) + .streaming(file_stream), + ); } else { log::debug!("did not hit the resize cache"); } let stream = version_store.get_version_stream(&hash_str).await?; - Ok(HttpResponse::Ok() - .content_type(mime_type) - .insert_header(("oxen-revision-id", last_commit_id.as_str())) - .streaming(stream)) + Ok(file_stream_response(mime_type, &last_commit_id, Some(num_bytes)).streaming(stream)) } /// Batch download version files @@ -598,7 +596,8 @@ mod tests { use crate::controllers; use crate::test; use actix_multipart::test::create_form_data_payload_and_headers; - use actix_web::{App, web, web::Bytes}; + use actix_web::http::header; + use actix_web::{App, body, web, web::Bytes}; use flate2::Compression; use flate2::write::GzEncoder; use liboxen::error::OxenError; @@ -644,6 +643,16 @@ mod tests { let resp = actix_web::test::call_service(&app, req).await; assert_eq!(resp.status(), actix_web::http::StatusCode::OK); + assert_eq!( + resp.headers().get(header::CONTENT_LENGTH).unwrap(), + file_content.len().to_string().as_str() + ); + assert_eq!( + resp.headers() + .get(header::ACCESS_CONTROL_EXPOSE_HEADERS) + .unwrap(), + header::CONTENT_LENGTH.as_str() + ); let bytes = actix_http::body::to_bytes(resp.into_body()).await.unwrap(); assert_eq!(bytes, "Hello"); @@ -652,6 +661,69 @@ mod tests { Ok(()) } + #[actix_web::test] + async fn test_controllers_versions_download_resize_sets_content_length() -> Result<(), OxenError> + { + liboxen::test::init_test_env(); + let sync_dir = test::get_sync_dir()?; + let namespace = "Testing-Namespace"; + let repo_name = "Testing-Resize-Content-Length"; + let repo = test::create_local_repo(&sync_dir, namespace, repo_name)?; + + util::fs::create_dir_all(repo.path.join("data"))?; + let relative_path = "data/pixel.png"; + let image_file = repo.path.join(relative_path); + let png_bytes = [ + 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d, 0x49, 0x48, + 0x44, 0x52, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x08, 0x06, 0x00, 0x00, + 0x00, 0x1f, 0x15, 0xc4, 0x89, 0x00, 0x00, 0x00, 0x0d, 0x49, 0x44, 0x41, 0x54, 0x78, + 0x9c, 0x63, 0xf8, 0xcf, 0xc0, 0xf0, 0x1f, 0x00, 0x05, 0x00, 0x01, 0xff, 0x89, 0x99, + 0x3d, 0x1d, 0x00, 0x00, 0x00, 0x00, 0x49, 0x45, 0x4e, 0x44, 0xae, 0x42, 0x60, 0x82, + ]; + util::fs::write_data(&image_file, &png_bytes)?; + repositories::add(&repo, &image_file).await?; + repositories::commit(&repo, "First commit")?; + + let uri = + format!("/oxen/{namespace}/{repo_name}/versions/main/{relative_path}?width=2&height=2"); + let req = actix_web::test::TestRequest::get() + .uri(&uri) + .app_data(OxenAppData::new(sync_dir.to_path_buf())) + .to_request(); + + let app = actix_web::test::init_service( + App::new() + .app_data(OxenAppData::new(sync_dir.clone())) + .route( + "/oxen/{namespace}/{repo_name}/versions/{resource:.*}", + web::get().to(controllers::versions::download), + ), + ) + .await; + + let resp = actix_web::test::call_service(&app, req).await; + assert_eq!(resp.status(), actix_web::http::StatusCode::OK); + assert_eq!( + resp.headers() + .get(header::ACCESS_CONTROL_EXPOSE_HEADERS) + .unwrap(), + header::CONTENT_LENGTH.as_str() + ); + let content_length = resp + .headers() + .get(header::CONTENT_LENGTH) + .unwrap() + .to_str() + .unwrap() + .parse::() + .unwrap(); + let bytes = body::to_bytes(resp.into_body()).await.unwrap(); + assert_eq!(content_length, bytes.len()); + + test::cleanup_sync_dir(&sync_dir)?; + Ok(()) + } + #[actix_web::test] async fn test_controllers_versions_download_bad_commit_returns_404() -> Result<(), OxenError> { liboxen::test::init_test_env(); diff --git a/crates/server/src/controllers/workspaces/files.rs b/crates/server/src/controllers/workspaces/files.rs index f88b61021..13ecb4b4e 100644 --- a/crates/server/src/controllers/workspaces/files.rs +++ b/crates/server/src/controllers/workspaces/files.rs @@ -1,5 +1,5 @@ use crate::errors::OxenHttpError; -use crate::helpers::get_repo; +use crate::helpers::{file_stream_response, get_repo}; use crate::params::{app_data, path_param}; use liboxen::core; @@ -118,6 +118,7 @@ pub async fn get( let file_hash = file_node.hash(); let hash_str = file_hash.to_string(); let mime_type = file_node.mime_type(); + let num_bytes = file_node.num_bytes(); let last_commit_id = file_node.last_commit_id().to_string(); let query_params = query.into_inner(); @@ -131,7 +132,7 @@ pub async fn get( }; log::debug!("img_resize {img_resize:?}"); - let file_stream = util::fs::handle_image_resize( + let (file_stream, content_length) = util::fs::handle_image_resize( Arc::clone(&version_store), hash_str.clone(), &PathBuf::from(&path), @@ -139,10 +140,10 @@ pub async fn get( ) .await?; - return Ok(HttpResponse::Ok() - .content_type(mime_type) - .insert_header(("oxen-revision-id", last_commit_id.as_str())) - .streaming(file_stream)); + return Ok( + file_stream_response(mime_type, &last_commit_id, Some(content_length)) + .streaming(file_stream), + ); } // Handle video thumbnail - requires thumbnail=true parameter @@ -159,10 +160,7 @@ pub async fn get( util::fs::handle_video_thumbnail(Arc::clone(&version_store), hash_str, video_thumbnail) .await?; - return Ok(HttpResponse::Ok() - .content_type("image/jpeg") - .insert_header(("oxen-revision-id", last_commit_id.as_str())) - .streaming(stream)); + return Ok(file_stream_response("image/jpeg", &last_commit_id, None).streaming(stream)); } log::debug!("did not hit the resize or thumbnail cache"); @@ -170,10 +168,7 @@ pub async fn get( // Stream the file let stream = version_store.get_version_stream(&hash_str).await?; - Ok(HttpResponse::Ok() - .content_type(mime_type) - .insert_header(("oxen-revision-id", last_commit_id.as_str())) - .streaming(stream)) + Ok(file_stream_response(mime_type, &last_commit_id, Some(num_bytes)).streaming(stream)) } /// Add files to workspace @@ -623,6 +618,7 @@ mod tests { use crate::app_data::OxenAppData; use crate::controllers; use crate::test; + use actix_web::http::header; use actix_web::{App, web}; use liboxen::error::OxenError; use liboxen::repositories; @@ -670,4 +666,53 @@ mod tests { test::cleanup_sync_dir(&sync_dir)?; Ok(()) } + + #[actix_web::test] + async fn test_workspace_file_get_exposes_content_length() -> Result<(), OxenError> { + liboxen::test::init_test_env(); + let sync_dir = test::get_sync_dir()?; + let namespace = "Testing-Namespace"; + let repo_name = "Testing-Workspace-Get-Headers"; + let repo = test::create_local_repo(&sync_dir, namespace, repo_name)?; + + let hello_file = repo.path.join("hello.txt"); + let file_content = "Hello"; + util::fs::write_to_path(&hello_file, file_content)?; + repositories::add(&repo, &hello_file).await?; + let commit = repositories::commit(&repo, "First commit")?; + + let workspace_id = uuid::Uuid::new_v4().to_string(); + repositories::workspaces::create(&repo, &commit, &workspace_id, false)?; + + let uri = + format!("/oxen/{namespace}/{repo_name}/workspaces/{workspace_id}/files/hello.txt"); + + let app = actix_web::test::init_service( + App::new() + .app_data(OxenAppData::new(sync_dir.clone())) + .route( + "/oxen/{namespace}/{repo_name}/workspaces/{workspace_id}/files/{path:.*}", + web::get().to(controllers::workspaces::files::get), + ), + ) + .await; + + let req = actix_web::test::TestRequest::get().uri(&uri).to_request(); + let resp = actix_web::test::call_service(&app, req).await; + + assert_eq!(resp.status(), actix_web::http::StatusCode::OK); + assert_eq!( + resp.headers().get(header::CONTENT_LENGTH).unwrap(), + file_content.len().to_string().as_str() + ); + assert_eq!( + resp.headers() + .get(header::ACCESS_CONTROL_EXPOSE_HEADERS) + .unwrap(), + header::CONTENT_LENGTH.as_str() + ); + + test::cleanup_sync_dir(&sync_dir)?; + Ok(()) + } } diff --git a/crates/server/src/helpers.rs b/crates/server/src/helpers.rs index 7c92528f9..c53da06df 100644 --- a/crates/server/src/helpers.rs +++ b/crates/server/src/helpers.rs @@ -1,6 +1,8 @@ use std::path::Path; // use liboxen::constants::DEFAULT_REDIS_URL; +use actix_web::http::header; +use actix_web::{HttpResponse, HttpResponseBuilder}; use liboxen::error::OxenError; use liboxen::model::{LocalRepository, RepoNew, User}; use liboxen::repositories; @@ -34,6 +36,70 @@ pub fn create_user_from_options( }) } +pub fn expose_response_header(builder: &mut HttpResponseBuilder, header_name: header::HeaderName) { + builder.append_header((header::ACCESS_CONTROL_EXPOSE_HEADERS, header_name.as_str())); +} + +pub fn expose_content_length(builder: &mut HttpResponseBuilder) { + expose_response_header(builder, header::CONTENT_LENGTH); +} + +pub fn file_stream_response( + mime_type: &str, + last_commit_id: &str, + content_length: Option, +) -> HttpResponseBuilder { + let mut response = HttpResponse::Ok(); + response.content_type(mime_type); + if let Some(content_length) = content_length { + response.no_chunking(content_length); + } + response.insert_header(("oxen-revision-id", last_commit_id)); + expose_content_length(&mut response); + response +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_expose_content_length_sets_header() { + let mut builder = HttpResponse::Ok(); + expose_content_length(&mut builder); + + let response = builder.finish(); + let value = response + .headers() + .get(header::ACCESS_CONTROL_EXPOSE_HEADERS) + .unwrap() + .to_str() + .unwrap(); + + assert_eq!(value, header::CONTENT_LENGTH.as_str()); + } + + #[test] + fn test_expose_content_length_appends_to_existing_headers() { + let mut builder = HttpResponse::Ok(); + builder.insert_header((header::ACCESS_CONTROL_EXPOSE_HEADERS, "X-Existing-Header")); + expose_content_length(&mut builder); + + let response = builder.finish(); + let values: Vec<&str> = response + .headers() + .get_all(header::ACCESS_CONTROL_EXPOSE_HEADERS) + .into_iter() + .map(|value| value.to_str().unwrap()) + .collect(); + + assert_eq!( + values, + vec!["X-Existing-Header", header::CONTENT_LENGTH.as_str()] + ); + } +} + // #[allow(dependency_on_unit_never_type_fallback)] // pub fn get_redis_connection() -> Result, OxenError> { // let redis_url = std::env::var("REDIS_URL").unwrap_or_else(|_| DEFAULT_REDIS_URL.to_string()); From 7c3df698420e58a0af489769e868b9dfb823524a Mon Sep 17 00:00:00 2001 From: Greg Schoeninger Date: Sat, 28 Mar 2026 12:39:45 -0700 Subject: [PATCH 2/2] fix doc string on get_version_derived_size --- crates/lib/src/storage/version_store.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/crates/lib/src/storage/version_store.rs b/crates/lib/src/storage/version_store.rs index 567c1abe4..67ad48d61 100644 --- a/crates/lib/src/storage/version_store.rs +++ b/crates/lib/src/storage/version_store.rs @@ -202,11 +202,16 @@ pub trait VersionStore: Debug + Send + Sync + 'static { hash: &str, ) -> Result> + Send + Unpin>, OxenError>; - /// Get a stream of a derived file (resized, video thumbnail, etc.) + /// Get the size in bytes of a derived file (e.g., resized image, video thumbnail). + /// + /// Returns the content length of the derived artifact on disk. /// /// # Arguments /// * `orig_hash` - The content hash of the parent version /// * `derived_filename` - Filename for the derived artifact + /// + /// # Errors + /// Returns `OxenError` if the derived file does not exist or cannot be read. async fn get_version_derived_size( &self, orig_hash: &str,