From eb76d329b27e497057efe65d142b03baa345e0ee Mon Sep 17 00:00:00 2001 From: Mike Hilgendorf Date: Fri, 27 Feb 2026 13:35:47 -0600 Subject: [PATCH 1/6] wip: both codepaths present --- packages/cli/src/run.rs | 697 ++++++++++++++++++++++++++++++++-------- 1 file changed, 558 insertions(+), 139 deletions(-) diff --git a/packages/cli/src/run.rs b/packages/cli/src/run.rs index 287e9b805..907582ab6 100644 --- a/packages/cli/src/run.rs +++ b/packages/cli/src/run.rs @@ -66,91 +66,481 @@ pub struct Options { impl Cli { pub async fn command_run(&mut self, args: Args) -> tg::Result<()> { + let Args { + options, + reference, + trailing, + } = args; + + // If the build flag is set, then build and run the output. + if options.build { + return self.command_run_build(options, reference, trailing).await; + } + + self.command_run_inner(options, reference, trailing).await + } + + async fn command_run_build( + &mut self, + options: Options, + reference: tg::Reference, + trailing: Vec, + ) -> tg::Result<()> { let handle = self.handle().await?; + // Spawn the process. + let spawn = crate::process::spawn::Options { + sandbox: crate::process::spawn::Sandbox::new(Some(true)), + local: options.spawn.local.clone(), + remotes: options.spawn.remotes.clone(), + ..Default::default() + }; + let crate::process::spawn::Output { process, output } = self + .spawn(spawn, reference, vec![], None, None, None) + .boxed() + .await?; + + // Print the process. + if !self.args.quiet { + let mut message = process.item().id().to_string(); + if let Some(token) = process.item().token() { + write!(message, " {token}").unwrap(); + } + Self::print_info_message(&message); + } + + // If the spawn output includes a wait output, then use it. + let wait = output + .wait + .map(TryInto::try_into) + .transpose() + .map_err(|source| tg::error!(!source, "failed to parse the wait output"))?; + + // If the process is not finished, then wait for it to finish while showing the viewer if enabled. + let wait = if let Some(wait) = wait { + wait + } else { + // Spawn the view task. + let view_task = { + let handle = handle.clone(); + let root = process.clone().map(crate::viewer::Item::Process); + let task = Task::spawn_blocking(move |stop| -> tg::Result<()> { + let local_set = tokio::task::LocalSet::new(); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|source| { + tg::error!(!source, "failed to create the tokio runtime") + })?; + local_set.block_on(&runtime, async move { + let viewer_options = crate::viewer::Options { + collapse_process_children: true, + depth: None, + expand_objects: false, + expand_packages: false, + expand_processes: true, + expand_metadata: false, + expand_tags: false, + expand_values: false, + show_process_commands: false, + }; + let mut viewer = crate::viewer::Viewer::new(&handle, root, viewer_options); + match options.build_view { + crate::build::View::None => (), + crate::build::View::Inline => { + viewer.run_inline(stop, false).await?; + }, + crate::build::View::Fullscreen => { + viewer.run_fullscreen(stop).await?; + }, + } + Ok::<_, tg::Error>(()) + }) + }); + Some(task) + }; + + // Spawn a task to attempt to cancel the process on the first interrupt signal and exit the process on the second. + let cancel_task = tokio::spawn({ + let handle = handle.clone(); + let process = process.clone(); + async move { + tokio::signal::ctrl_c().await.unwrap(); + tokio::spawn(async move { + process + .item() + .cancel(&handle) + .await + .inspect_err(|error| { + tracing::error!(?error, "failed to cancel the process"); + }) + .ok(); + }); + tokio::signal::ctrl_c().await.unwrap(); + std::process::exit(130); + } + }); + + // Await the process. + let arg = tg::process::wait::Arg { + token: process.item().token().cloned(), + ..tg::process::wait::Arg::default() + }; + let result = process.item().wait(&handle, arg).await; + + // Abort the cancel task. + cancel_task.abort(); + + // Stop and await the view task. + if let Some(view_task) = view_task { + view_task.stop(); + match view_task.wait().await { + Ok(Ok(())) => {}, + Ok(Err(error)) => { + tracing::warn!(?error, "failed to render the process viewer"); + Self::print_warning_message("failed to render the process viewer"); + }, + Err(error) => { + tracing::warn!(?error, "failed to join the process viewer task"); + Self::print_warning_message("failed to render the process viewer"); + }, + } + } + + result? + }; + + // Set the exit. + if wait.exit != 0 { + self.exit.replace(wait.exit); + } + + // Handle an error. + if let Some(error) = wait.error { + let error = error + .to_data_or_id() + .map_left(|data| { + Box::new(tg::error::Object::try_from_data(data).unwrap_or_else(|_| { + tg::error::Object { + message: Some("invalid error".to_owned()), + ..Default::default() + } + })) + }) + .map_right(|id| Box::new(tg::Error::with_id(id))); + let error = tg::Error::with_object(tg::error::Object { + message: Some("the process failed".to_owned()), + source: Some(process.clone().map(|_| error)), + values: [("id".to_owned(), process.item().id().to_string())].into(), + ..Default::default() + }); + return Err(error); + } + + // Handle non-zero exit. + if wait.exit > 1 && wait.exit < 128 { + return Err(tg::error!("the process exited with code {}", wait.exit)); + } + if wait.exit >= 128 { + return Err(tg::error!( + "the process exited with signal {}", + wait.exit - 128 + )); + } + + // Get the output. + let output = wait.output.unwrap_or(tg::Value::Null); + + let object = output + .try_unwrap_object() + .ok() + .ok_or_else(|| tg::error!("expected the build to output an object"))?; + let id = object.id(); + let reference = tg::Reference::with_object(id); + + // Run the built output. + self.command_run_inner(options, reference, trailing).await + } + + async fn command_run2(&mut self, args: Args) -> tg::Result<()> { let Args { - options, reference, + mut options, trailing, } = args; - // If the build flag is set, then build and get the output. + // Spawn a sandboxed run for builds. let reference = if options.build { - // Spawn the process. - let spawn = crate::process::spawn::Options { - sandbox: crate::process::spawn::Sandbox::new(Some(true)), - local: options.spawn.local.clone(), - remotes: options.spawn.remotes.clone(), - ..Default::default() + let options = Options { + checkout_force: false, + checkout: None, + detach: false, + executable_path: None, + ..options.clone() }; - let crate::process::spawn::Output { process, output } = self - .spawn(spawn, reference, vec![], None, None, None) - .boxed() - .await?; - - // Print the process. - if !self.args.quiet { - let mut message = process.item().id().to_string(); - if let Some(token) = process.item().token() { - write!(message, " {token}").unwrap(); - } - Self::print_info_message(&message); + let output = self + .run_sandboxed(&options, reference.clone(), Vec::new()) + .await + .map_err(|source| tg::error!(!source, %reference, "failed to build"))? + .ok_or_else(|| tg::error!("expected an output"))?; + let object = output + .try_unwrap_object() + .ok() + .ok_or_else(|| tg::error!("expected the build to output an object"))?; + let id = object.id(); + tg::Reference::with_object(id) + } else { + reference + }; + options.build = false; + + // Determine if the process should be sandboxed. + let sandboxed = true; + let output = if sandboxed { + self.run_sandboxed(&options, reference, trailing).await? + } else { + self.run_unsandboxed(&options, reference, trailing).await? + }; + + // Check out the output if requested. + if let Some(path) = options.checkout { + let handle = self.handle().await?; + let output = output.ok_or_else(|| tg::error!("expected an output"))?; + + // Get the artifact. + let artifact: tg::Artifact = output + .clone() + .try_into() + .map_err(|_| tg::error!("expected an artifact"))?; + + // Get the path. + let path = if let Some(path) = path { + let path = tangram_util::fs::canonicalize_parent(path) + .await + .map_err(|source| tg::error!(!source, "failed to canonicalize the path"))?; + Some(path) + } else { + None + }; + + // Check out the artifact. + let artifact = artifact.id(); + let arg = tg::checkout::Arg { + artifact: artifact.clone(), + dependencies: path.is_some(), + extension: None, + force: options.checkout_force, + lock: None, + path, + }; + let stream = handle.checkout(arg).await.map_err( + |source| tg::error!(!source, %artifact, "failed to check out the artifact"), + )?; + let tg::checkout::Output { path, .. } = + self.render_progress_stream(stream).await.map_err( + |source| tg::error!(!source, %artifact, "failed to check out the artifact"), + )?; + + // Print the path. + self.print_serde(path, options.print).await?; + + return Ok(()); + } + + // Print the output. + if !options.verbose + && let Some(output) = output + && !output.is_null() + { + let arg = tg::object::get::Arg { + local: options.spawn.local.local, + metadata: false, + remotes: options.spawn.remotes.remotes, + }; + self.print_value(&output, options.print, arg).await?; + } + Ok(()) + } + + async fn run_unsandboxed( + &mut self, + options: &Options, + reference: tg::Reference, + trailing: Vec, + ) -> tg::Result> { + todo!("implement the unsandboxed run") + } + + async fn run_sandboxed( + &mut self, + options: &Options, + reference: tg::Reference, + trailing: Vec, + ) -> tg::Result> { + let handle = self.handle().await?; + + // Handle the executable path. + let reference = if let Some(path) = &options.executable_path { + let referent = self.get_reference(&reference).await?; + let directory = referent + .item + .left() + .ok_or_else(|| tg::error!("expected an object"))? + .try_unwrap_directory() + .ok() + .ok_or_else(|| tg::error!("expected a directory"))?; + let artifact = directory.get(&handle, path).await.map_err( + |source| tg::error!(!source, path = %path.display(), "failed to get the artifact"), + )?; + let id = artifact + .store(&handle) + .await + .map_err(|source| tg::error!(!source, "failed to store the artifact"))?; + tg::Reference::with_object(id.into()) + } else { + reference + }; + + // Get the remote + let remote = options + .spawn + .remotes + .remotes + .clone() + .and_then(|remotes| remotes.into_iter().next()); + + // Create the stdio if this is not a build. + let stdio = if options.build { + None + } else { + let stdio = stdio::Stdio::new(&handle, remote.clone(), &options) + .await + .map_err(|source| tg::error!(!source, "failed to create stdio"))?; + Some(stdio) + }; + + // Spawn the process. + let spawn = crate::process::spawn::Options { + sandbox: crate::process::spawn::Sandbox::new(Some(true)), + local: options.spawn.local.clone(), + remotes: options.spawn.remotes.clone(), + ..Default::default() + }; + let stdin = stdio.as_ref().and_then(|stdio| stdio.stdin.clone()); + let stdout = stdio.as_ref().and_then(|stdio| stdio.stdout.clone()); + let stderr = stdio.as_ref().and_then(|stdio| stdio.stderr.clone()); + let crate::process::spawn::Output { process, output } = self + .spawn(spawn, reference, trailing, stdin, stdout, stderr) + .boxed() + .await?; + + // If the detach flag is set, then print the process ID and return. + if options.detach { + if options.verbose { + self.print_serde(output, options.print.clone()).await?; + } else { + Self::print_display(&output.process); + } + return Ok(None); + } + + // Print the process. + if !self.args.quiet { + let mut message = process.item().id().to_string(); + if let Some(token) = process.item().token() { + write!(message, " {token}").unwrap(); } + Self::print_info_message(&message); + } + + // Enable raw mode if necessary. + if let Some(stdio) = &stdio + && let Some(tty) = &stdio.tty + { + tty.enable_raw_mode()?; + } + + // If the spawn output includes a wait output, then use it. + let wait = output + .wait + .map(TryInto::try_into) + .transpose() + .map_err(|source| tg::error!(!source, "failed to parse the wait output"))?; - // If the spawn output includes a wait output, then use it. - let wait = output - .wait - .map(TryInto::try_into) - .transpose() - .map_err(|source| tg::error!(!source, "failed to parse the wait output"))?; + // If the process is not finished, then wait for it to finish while showing the viewer if enabled. + let wait = if let Some(wait) = wait { + wait + } else { + // Spawn the stdio task. + let stdio_task = if let Some(stdio) = stdio.clone() { + Some(Task::spawn({ + let handle = handle.clone(); + |stop| async move { self::stdio::task(&handle, stop, stdio).boxed().await } + })) + } else { + None + }; - // If the process is not finished, then wait for it to finish while showing the viewer if enabled. - let wait = if let Some(wait) = wait { - wait + // Spawn signal task. This will be handled by the cancellation tasks for builds. + let signal_task = if options.build { + None } else { - // Spawn the view task. - let view_task = { + Some(tokio::spawn({ let handle = handle.clone(); - let root = process.clone().map(crate::viewer::Item::Process); - let task = Task::spawn_blocking(move |stop| -> tg::Result<()> { - let local_set = tokio::task::LocalSet::new(); - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .map_err(|source| { - tg::error!(!source, "failed to create the tokio runtime") - })?; - local_set.block_on(&runtime, async move { - let viewer_options = crate::viewer::Options { - collapse_process_children: true, - depth: None, - expand_objects: false, - expand_packages: false, - expand_processes: true, - expand_metadata: false, - expand_tags: false, - expand_values: false, - show_process_commands: false, - }; - let mut viewer = - crate::viewer::Viewer::new(&handle, root, viewer_options); - match options.build_view { - crate::build::View::None => (), - crate::build::View::Inline => { - viewer.run_inline(stop, false).await?; - }, - crate::build::View::Fullscreen => { - viewer.run_fullscreen(stop).await?; - }, - } - Ok::<_, tg::Error>(()) - }) - }); - Some(task) - }; + let process = process.item().id().clone(); + let remote = remote.clone(); + async move { + self::signal::task(&handle, &process, remote).await.ok(); + } + })) + }; + + // Spawn the view task, if this is a build. + let view_task = if options.build { + let handle = handle.clone(); + let root = process.clone().map(crate::viewer::Item::Process); + let build_view = options.build_view; + let task = Task::spawn_blocking(move |stop| -> tg::Result<()> { + let local_set = tokio::task::LocalSet::new(); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|source| { + tg::error!(!source, "failed to create the tokio runtime") + })?; + local_set.block_on(&runtime, async move { + let viewer_options = crate::viewer::Options { + collapse_process_children: true, + depth: None, + expand_objects: false, + expand_packages: false, + expand_processes: true, + expand_metadata: false, + expand_tags: false, + expand_values: false, + show_process_commands: false, + }; + let mut viewer = crate::viewer::Viewer::new(&handle, root, viewer_options); + match build_view { + crate::build::View::None => (), + crate::build::View::Inline => { + viewer.run_inline(stop, false).await?; + }, + crate::build::View::Fullscreen => { + viewer.run_fullscreen(stop).await?; + }, + } + Ok::<_, tg::Error>(()) + }) + }); + Some(task) + } else { + None + }; - // Spawn a task to attempt to cancel the process on the first interrupt signal and exit the process on the second. - let cancel_task = tokio::spawn({ + // Spawn a task to attempt to cancel the process on the first interrupt signal and exit the process on the second. + let cancel_task = if options.build { + Some(tokio::spawn({ let handle = handle.clone(); let process = process.clone(); async move { @@ -168,87 +558,116 @@ impl Cli { tokio::signal::ctrl_c().await.unwrap(); std::process::exit(130); } - }); + })) + } else { + None + }; + + // Await the process. + let arg = tg::process::wait::Arg { + token: process.item().token().cloned(), + ..tg::process::wait::Arg::default() + }; + let result = process.item().wait(&handle, arg).await; + + // Close stdio. + if let Some(stdio) = stdio { + stdio.close(&handle).await?; + if let Some(task) = stdio_task { + task.stop(); + task.wait().await.unwrap()?; + } + stdio.delete(&handle).await?; + } - // Await the process. - let arg = tg::process::wait::Arg { - token: process.item().token().cloned(), - ..tg::process::wait::Arg::default() - }; - let result = process.item().wait(&handle, arg).await; + // Abort the signal task. + if let Some(signal_task) = signal_task { + signal_task.abort(); + } - // Abort the cancel task. + // Abort the cancel task. + if let Some(cancel_task) = cancel_task { cancel_task.abort(); + } - // Stop and await the view task. - if let Some(view_task) = view_task { - view_task.stop(); - match view_task.wait().await { - Ok(Ok(())) => {}, - Ok(Err(error)) => { - tracing::warn!(?error, "failed to render the process viewer"); - Self::print_warning_message("failed to render the process viewer"); - }, - Err(error) => { - tracing::warn!(?error, "failed to join the process viewer task"); - Self::print_warning_message("failed to render the process viewer"); - }, - } + // Stop and await the view task. + if let Some(view_task) = view_task { + view_task.stop(); + match view_task.wait().await { + Ok(Ok(())) => {}, + Ok(Err(error)) => { + tracing::warn!(?error, "failed to render the process viewer"); + Self::print_warning_message("failed to render the process viewer"); + }, + Err(error) => { + tracing::warn!(?error, "failed to join the process viewer task"); + Self::print_warning_message("failed to render the process viewer"); + }, } + } - result? + result.map_err(|error| tg::error!(!error, "failed to await the process"))? + }; + + // Print verbose output if requested and this is not a pre-run build. + if options.verbose && !options.build { + let output = tg::process::wait::Output { + error: wait.error.as_ref().map(tg::Error::to_data_or_id), + exit: wait.exit, + output: wait.output.as_ref().map(tg::Value::to_data), }; + self.print_serde(output, options.print.clone()).await?; + return Ok(None); + } - // Set the exit. - if wait.exit != 0 { - self.exit.replace(wait.exit); - } + // Set the exit. + if wait.exit != 0 { + self.exit.replace(wait.exit); + } - // Handle an error. - if let Some(error) = wait.error { - let error = error - .to_data_or_id() - .map_left(|data| { - Box::new(tg::error::Object::try_from_data(data).unwrap_or_else(|_| { - tg::error::Object { - message: Some("invalid error".to_owned()), - ..Default::default() - } - })) - }) - .map_right(|id| Box::new(tg::Error::with_id(id))); - let error = tg::Error::with_object(tg::error::Object { - message: Some("the process failed".to_owned()), - source: Some(process.clone().map(|_| error)), - values: [("id".to_owned(), process.item().id().to_string())].into(), - ..Default::default() - }); - return Err(error); - } + // Handle an error. + if let Some(error) = wait.error { + let error = error + .to_data_or_id() + .map_left(|data| { + Box::new(tg::error::Object::try_from_data(data).unwrap_or_else(|_| { + tg::error::Object { + message: Some("invalid error".to_owned()), + ..Default::default() + } + })) + }) + .map_right(|id| Box::new(tg::Error::with_id(id))); + let error = tg::Error::with_object(tg::error::Object { + message: Some("the process failed".to_owned()), + source: Some(process.clone().map(|_| error)), + values: [("id".to_owned(), process.item().id().to_string())].into(), + ..Default::default() + }); + return Err(error); + } - // Handle non-zero exit. - if wait.exit > 1 && wait.exit < 128 { - return Err(tg::error!("the process exited with code {}", wait.exit)); - } - if wait.exit >= 128 { - return Err(tg::error!( - "the process exited with signal {}", - wait.exit - 128 - )); - } + // Handle non-zero exit. + if wait.exit > 1 && wait.exit < 128 { + return Err(tg::error!("the process exited with code {}", wait.exit)); + } + if wait.exit >= 128 { + return Err(tg::error!( + "the process exited with signal {}", + wait.exit - 128 + )); + } - // Get the output. - let output = wait.output.unwrap_or(tg::Value::Null); + Ok(wait.output) + } - let object = output - .try_unwrap_object() - .ok() - .ok_or_else(|| tg::error!("expected the build to output an object"))?; - let id = object.id(); - tg::Reference::with_object(id) - } else { - reference - }; + async fn command_run_inner( + &mut self, + options: Options, + reference: tg::Reference, + trailing: Vec, + ) -> tg::Result<()> { + let handle = self.handle().await?; // Handle the executable path. let reference = if let Some(path) = &options.executable_path { From a5f0b575fe19f22d4d10c1ec13c5f2ae082f3748 Mon Sep 17 00:00:00 2001 From: Mike Hilgendorf Date: Fri, 27 Feb 2026 16:56:27 -0600 Subject: [PATCH 2/6] feat(cli): unsandboxed processes --- packages/cli/Cargo.toml | 1 + packages/cli/src/build.rs | 19 +- packages/cli/src/process/spawn.rs | 37 +- packages/cli/src/run.rs | 869 +++++++++--------- packages/cli/src/shell/common.rs | 17 +- packages/cli/tests/run/assertion_failure.nu | 24 +- .../cli/tests/run/assertion_failure.snapshot | 15 - .../assertion_failure_in_path_dependency.nu | 28 +- ...ertion_failure_in_path_dependency.snapshot | 15 - .../assertion_failure_in_tag_dependency.nu | 24 +- ...sertion_failure_in_tag_dependency.snapshot | 15 - ...ion_failure_in_tagged_cyclic_dependency.nu | 38 +- ...ilure_in_tagged_cyclic_dependency.snapshot | 29 - .../run/assertion_failure_out_of_tree.nu | 21 +- .../assertion_failure_out_of_tree.snapshot | 11 - packages/cli/tests/run/build_flag.nu | 21 + .../cli/tests/run/build_flag_non_object.nu | 17 + packages/cli/tests/run/checkout.nu | 19 + packages/cli/tests/run/checkout_directory.nu | 28 + packages/cli/tests/run/checkout_force.nu | 23 + packages/cli/tests/run/checkout_no_output.nu | 17 + .../cli/tests/run/checkout_non_artifact.nu | 17 + packages/cli/tests/run/checkout_path.nu | 23 + packages/cli/tests/run/child_process_error.nu | 29 +- .../tests/run/child_process_error.snapshot | 20 - packages/cli/tests/run/console_log.nu | 27 + packages/cli/tests/run/default_reference.nu | 17 + packages/cli/tests/run/detach.nu | 15 + packages/cli/tests/run/detach_verbose.nu | 15 + packages/cli/tests/run/directory_by_id.nu | 16 + packages/cli/tests/run/executable_path.nu | 16 + packages/cli/tests/run/exit_code_nonzero.nu | 20 + packages/cli/tests/run/exit_code_signal.nu | 20 + packages/cli/tests/run/file_by_id.nu | 5 +- packages/cli/tests/run/hello_world.nu | 7 +- .../cli/tests/run/multiple_trailing_args.nu | 15 + packages/cli/tests/run/print_pretty.nu | 15 + packages/cli/tests/run/process_error.nu | 21 + packages/cli/tests/run/quiet.nu | 22 + packages/cli/tests/run/return_null.nu | 15 + packages/cli/tests/run/return_value.nu | 13 + packages/cli/tests/run/subpath_reference.nu | 19 + packages/cli/tests/run/trailing_args.nu | 13 + packages/cli/tests/run/verbose.nu | 17 + packages/cli/tests/run/verbose_with_error.nu | 25 + 45 files changed, 1127 insertions(+), 583 deletions(-) delete mode 100644 packages/cli/tests/run/assertion_failure.snapshot delete mode 100644 packages/cli/tests/run/assertion_failure_in_path_dependency.snapshot delete mode 100644 packages/cli/tests/run/assertion_failure_in_tag_dependency.snapshot delete mode 100644 packages/cli/tests/run/assertion_failure_in_tagged_cyclic_dependency.snapshot delete mode 100644 packages/cli/tests/run/assertion_failure_out_of_tree.snapshot create mode 100644 packages/cli/tests/run/build_flag.nu create mode 100644 packages/cli/tests/run/build_flag_non_object.nu create mode 100644 packages/cli/tests/run/checkout.nu create mode 100644 packages/cli/tests/run/checkout_directory.nu create mode 100644 packages/cli/tests/run/checkout_force.nu create mode 100644 packages/cli/tests/run/checkout_no_output.nu create mode 100644 packages/cli/tests/run/checkout_non_artifact.nu create mode 100644 packages/cli/tests/run/checkout_path.nu delete mode 100644 packages/cli/tests/run/child_process_error.snapshot create mode 100644 packages/cli/tests/run/console_log.nu create mode 100644 packages/cli/tests/run/default_reference.nu create mode 100644 packages/cli/tests/run/detach.nu create mode 100644 packages/cli/tests/run/detach_verbose.nu create mode 100644 packages/cli/tests/run/directory_by_id.nu create mode 100644 packages/cli/tests/run/executable_path.nu create mode 100644 packages/cli/tests/run/exit_code_nonzero.nu create mode 100644 packages/cli/tests/run/exit_code_signal.nu create mode 100644 packages/cli/tests/run/multiple_trailing_args.nu create mode 100644 packages/cli/tests/run/print_pretty.nu create mode 100644 packages/cli/tests/run/process_error.nu create mode 100644 packages/cli/tests/run/quiet.nu create mode 100644 packages/cli/tests/run/return_null.nu create mode 100644 packages/cli/tests/run/return_value.nu create mode 100644 packages/cli/tests/run/subpath_reference.nu create mode 100644 packages/cli/tests/run/trailing_args.nu create mode 100644 packages/cli/tests/run/verbose.nu create mode 100644 packages/cli/tests/run/verbose_with_error.nu diff --git a/packages/cli/Cargo.toml b/packages/cli/Cargo.toml index dc7643950..3f6827420 100644 --- a/packages/cli/Cargo.toml +++ b/packages/cli/Cargo.toml @@ -64,6 +64,7 @@ ratatui = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } serde_with = { workspace = true } +tempfile = { workspace = true } tangram_builtin = { workspace = true } tangram_client = { workspace = true } tangram_compiler = { workspace = true } diff --git a/packages/cli/src/build.rs b/packages/cli/src/build.rs index b14b99cf6..d5499ee30 100644 --- a/packages/cli/src/build.rs +++ b/packages/cli/src/build.rs @@ -81,8 +81,25 @@ impl Cli { sandbox, ..options.spawn }; + + // Get the reference. + let arg = tg::get::Arg { + checkin: spawn.checkin.clone().to_options(), + ..Default::default() + }; + let referent = self + .get_reference_with_arg(&reference, arg, true) + .await?; + let item = referent + .item + .clone() + .left() + .ok_or_else(|| tg::error!("expected an object"))?; + let referent = referent.map(|_| item); + + // Spawn the process. let crate::process::spawn::Output { process, output } = self - .spawn(spawn, reference, trailing, None, None, None) + .spawn(spawn, reference, referent, trailing, None, None, None) .boxed() .await?; diff --git a/packages/cli/src/process/spawn.rs b/packages/cli/src/process/spawn.rs index e566d5670..da21f3f13 100644 --- a/packages/cli/src/process/spawn.rs +++ b/packages/cli/src/process/spawn.rs @@ -194,15 +194,22 @@ pub struct Output { impl Cli { pub async fn command_process_spawn(&mut self, args: Args) -> tg::Result<()> { + // Get the reference. + let arg = tg::get::Arg { + checkin: args.options.checkin.to_options(), + ..Default::default() + }; + let referent = self + .get_reference_with_arg(&args.reference, arg, true) + .await?; + let item = referent + .item + .clone() + .left() + .ok_or_else(|| tg::error!("expected an object"))?; + let referent = referent.map(|_| item); let Output { output, .. } = self - .spawn( - args.options, - args.reference, - args.trailing, - None, - None, - None, - ) + .spawn(args.options, args.reference, referent, args.trailing, None, None, None) .boxed() .await?; if args.verbose { @@ -217,6 +224,7 @@ impl Cli { &mut self, options: Options, reference: tg::Reference, + mut referent: tg::Referent, trailing: Vec, stdin: Option, stdout: Option, @@ -228,19 +236,6 @@ impl Cli { let sandbox = options.sandbox.get().unwrap_or_default() || options.remotes.remotes.is_some(); - // Get the reference. - let arg = tg::get::Arg { - checkin: options.checkin.to_options(), - ..Default::default() - }; - let referent = self.get_reference_with_arg(&reference, arg, true).await?; - let item = referent - .item - .clone() - .left() - .ok_or_else(|| tg::error!("expected an object"))?; - let mut referent = referent.map(|_| item); - // Create the command builder. let mut command_env = None; let mut command = match referent.item.clone() { diff --git a/packages/cli/src/run.rs b/packages/cli/src/run.rs index 907582ab6..1e06860bf 100644 --- a/packages/cli/src/run.rs +++ b/packages/cli/src/run.rs @@ -1,11 +1,93 @@ use { crate::Cli, futures::prelude::*, - std::{fmt::Write as _, path::PathBuf}, + num::ToPrimitive as _, + std::{ + collections::BTreeMap, + fmt::Write as _, + os::unix::process::ExitStatusExt as _, + path::{Path, PathBuf}, + }, tangram_client::prelude::*, tangram_futures::task::Task, }; +fn render_value_string( + value: &tg::value::Data, + artifacts_path: &Path, + output_path: &Path, +) -> tg::Result { + match value { + tg::value::Data::String(string) => Ok(string.clone()), + tg::value::Data::Template(template) => template.try_render(|component| match component { + tg::template::data::Component::String(string) => Ok(string.clone().into()), + tg::template::data::Component::Artifact(artifact) => Ok(artifacts_path + .join(artifact.to_string()) + .to_str() + .unwrap() + .to_owned() + .into()), + tg::template::data::Component::Placeholder(placeholder) => { + if placeholder.name == "output" { + Ok(output_path.to_str().unwrap().to_owned().into()) + } else { + Err(tg::error!( + name = %placeholder.name, + "invalid placeholder" + )) + } + }, + }), + tg::value::Data::Placeholder(placeholder) => { + if placeholder.name == "output" { + Ok(output_path.to_str().unwrap().to_owned()) + } else { + Err(tg::error!( + name = %placeholder.name, + "invalid placeholder" + )) + } + }, + _ => Ok(tg::Value::try_from_data(value.clone()).unwrap().to_string()), + } +} + +fn render_args_string( + args: &[tg::value::Data], + artifacts_path: &Path, + output_path: &Path, +) -> tg::Result> { + args.iter() + .map(|value| render_value_string(value, artifacts_path, output_path)) + .collect::>>() +} + +fn render_env( + env: &tg::value::data::Map, + artifacts_path: &Path, + output_path: &Path, +) -> tg::Result> { + let mut output = tg::value::data::Map::new(); + for (key, value) in env { + let mutation = match value { + tg::value::Data::Mutation(value) => value.clone(), + value => tg::mutation::Data::Set { + value: Box::new(value.clone()), + }, + }; + mutation.apply(&mut output, key)?; + } + let output = output + .iter() + .map(|(key, value)| { + let key = key.clone(); + let value = render_value_string(value, artifacts_path, output_path)?; + Ok::<_, tg::Error>((key, value)) + }) + .collect::>()?; + Ok(output) +} + mod signal; mod stdio; @@ -66,203 +148,6 @@ pub struct Options { impl Cli { pub async fn command_run(&mut self, args: Args) -> tg::Result<()> { - let Args { - options, - reference, - trailing, - } = args; - - // If the build flag is set, then build and run the output. - if options.build { - return self.command_run_build(options, reference, trailing).await; - } - - self.command_run_inner(options, reference, trailing).await - } - - async fn command_run_build( - &mut self, - options: Options, - reference: tg::Reference, - trailing: Vec, - ) -> tg::Result<()> { - let handle = self.handle().await?; - - // Spawn the process. - let spawn = crate::process::spawn::Options { - sandbox: crate::process::spawn::Sandbox::new(Some(true)), - local: options.spawn.local.clone(), - remotes: options.spawn.remotes.clone(), - ..Default::default() - }; - let crate::process::spawn::Output { process, output } = self - .spawn(spawn, reference, vec![], None, None, None) - .boxed() - .await?; - - // Print the process. - if !self.args.quiet { - let mut message = process.item().id().to_string(); - if let Some(token) = process.item().token() { - write!(message, " {token}").unwrap(); - } - Self::print_info_message(&message); - } - - // If the spawn output includes a wait output, then use it. - let wait = output - .wait - .map(TryInto::try_into) - .transpose() - .map_err(|source| tg::error!(!source, "failed to parse the wait output"))?; - - // If the process is not finished, then wait for it to finish while showing the viewer if enabled. - let wait = if let Some(wait) = wait { - wait - } else { - // Spawn the view task. - let view_task = { - let handle = handle.clone(); - let root = process.clone().map(crate::viewer::Item::Process); - let task = Task::spawn_blocking(move |stop| -> tg::Result<()> { - let local_set = tokio::task::LocalSet::new(); - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .map_err(|source| { - tg::error!(!source, "failed to create the tokio runtime") - })?; - local_set.block_on(&runtime, async move { - let viewer_options = crate::viewer::Options { - collapse_process_children: true, - depth: None, - expand_objects: false, - expand_packages: false, - expand_processes: true, - expand_metadata: false, - expand_tags: false, - expand_values: false, - show_process_commands: false, - }; - let mut viewer = crate::viewer::Viewer::new(&handle, root, viewer_options); - match options.build_view { - crate::build::View::None => (), - crate::build::View::Inline => { - viewer.run_inline(stop, false).await?; - }, - crate::build::View::Fullscreen => { - viewer.run_fullscreen(stop).await?; - }, - } - Ok::<_, tg::Error>(()) - }) - }); - Some(task) - }; - - // Spawn a task to attempt to cancel the process on the first interrupt signal and exit the process on the second. - let cancel_task = tokio::spawn({ - let handle = handle.clone(); - let process = process.clone(); - async move { - tokio::signal::ctrl_c().await.unwrap(); - tokio::spawn(async move { - process - .item() - .cancel(&handle) - .await - .inspect_err(|error| { - tracing::error!(?error, "failed to cancel the process"); - }) - .ok(); - }); - tokio::signal::ctrl_c().await.unwrap(); - std::process::exit(130); - } - }); - - // Await the process. - let arg = tg::process::wait::Arg { - token: process.item().token().cloned(), - ..tg::process::wait::Arg::default() - }; - let result = process.item().wait(&handle, arg).await; - - // Abort the cancel task. - cancel_task.abort(); - - // Stop and await the view task. - if let Some(view_task) = view_task { - view_task.stop(); - match view_task.wait().await { - Ok(Ok(())) => {}, - Ok(Err(error)) => { - tracing::warn!(?error, "failed to render the process viewer"); - Self::print_warning_message("failed to render the process viewer"); - }, - Err(error) => { - tracing::warn!(?error, "failed to join the process viewer task"); - Self::print_warning_message("failed to render the process viewer"); - }, - } - } - - result? - }; - - // Set the exit. - if wait.exit != 0 { - self.exit.replace(wait.exit); - } - - // Handle an error. - if let Some(error) = wait.error { - let error = error - .to_data_or_id() - .map_left(|data| { - Box::new(tg::error::Object::try_from_data(data).unwrap_or_else(|_| { - tg::error::Object { - message: Some("invalid error".to_owned()), - ..Default::default() - } - })) - }) - .map_right(|id| Box::new(tg::Error::with_id(id))); - let error = tg::Error::with_object(tg::error::Object { - message: Some("the process failed".to_owned()), - source: Some(process.clone().map(|_| error)), - values: [("id".to_owned(), process.item().id().to_string())].into(), - ..Default::default() - }); - return Err(error); - } - - // Handle non-zero exit. - if wait.exit > 1 && wait.exit < 128 { - return Err(tg::error!("the process exited with code {}", wait.exit)); - } - if wait.exit >= 128 { - return Err(tg::error!( - "the process exited with signal {}", - wait.exit - 128 - )); - } - - // Get the output. - let output = wait.output.unwrap_or(tg::Value::Null); - - let object = output - .try_unwrap_object() - .ok() - .ok_or_else(|| tg::error!("expected the build to output an object"))?; - let id = object.id(); - let reference = tg::Reference::with_object(id); - - // Run the built output. - self.command_run_inner(options, reference, trailing).await - } - - async fn command_run2(&mut self, args: Args) -> tg::Result<()> { let Args { reference, mut options, @@ -271,6 +156,19 @@ impl Cli { // Spawn a sandboxed run for builds. let reference = if options.build { + // Get the reference. + let arg = tg::get::Arg { + checkin: options.spawn.checkin.to_options(), + ..Default::default() + }; + let referent = self.get_reference_with_arg(&reference, arg, true).await?; + let item = referent + .item + .clone() + .left() + .ok_or_else(|| tg::error!("expected an object"))?; + let referent = referent.map(|_| item); + let options = Options { checkout_force: false, checkout: None, @@ -279,7 +177,7 @@ impl Cli { ..options.clone() }; let output = self - .run_sandboxed(&options, reference.clone(), Vec::new()) + .run_sandboxed(&options, &reference, &referent, Vec::new()) .await .map_err(|source| tg::error!(!source, %reference, "failed to build"))? .ok_or_else(|| tg::error!("expected an output"))?; @@ -294,12 +192,26 @@ impl Cli { }; options.build = false; - // Determine if the process should be sandboxed. - let sandboxed = true; - let output = if sandboxed { - self.run_sandboxed(&options, reference, trailing).await? + // Get the reference. + let arg = tg::get::Arg { + checkin: options.spawn.checkin.to_options(), + ..Default::default() + }; + let referent = self.get_reference_with_arg(&reference, arg, true).await?; + let item = referent + .item + .clone() + .left() + .ok_or_else(|| tg::error!("expected an object"))?; + let referent = referent.map(|_| item); + + // Run the process. + let output = if self.needs_sandbox(&options) { + self.run_sandboxed(&options, &reference, &referent, trailing) + .await? } else { - self.run_unsandboxed(&options, reference, trailing).await? + self.run_unsandboxed(&options, &reference, &referent, trailing) + .await? }; // Check out the output if requested. @@ -364,29 +276,292 @@ impl Cli { async fn run_unsandboxed( &mut self, - options: &Options, - reference: tg::Reference, + _options: &Options, + _reference: &tg::Reference, + referent: &tg::Referent, trailing: Vec, ) -> tg::Result> { - todo!("implement the unsandboxed run") + let handle = self.handle().await?; + + // Create a temp directory for the output. + let temp = tempfile::tempdir() + .map_err(|source| tg::error!(!source, "failed to create a temp directory"))?; + let output_path = temp.path().join("output"); + tokio::fs::create_dir_all(&output_path) + .await + .map_err(|source| tg::error!(!source, "failed to create the output directory"))?; + let output_path = output_path.join("output"); + + // Get the artifacts path. + let artifacts_path = self.directory_path().join("artifacts"); + + // Inherit the process env. + let mut env: BTreeMap = std::env::vars().collect(); + env.remove("TANGRAM_OUTPUT"); + env.remove("TANGRAM_PROCESS"); + env.remove("TANGRAM_URL"); + + // Get the server URL. + let url = match &handle { + tg::Either::Left(client) => client.url().to_string(), + tg::Either::Right(server) => server + .url() + .ok_or_else(|| tg::error!("the server does not have a URL"))? + .to_string(), + }; + + // Create the command based on the referent item type. + let (executable, args, command_env, cwd) = match referent.item().clone() { + tg::Object::Command(command) => { + let data = command + .data(&handle) + .await + .map_err(|source| tg::error!(!source, "failed to get the command data"))?; + + // Cache the command's artifact children. + let artifacts: Vec = command + .children(&handle) + .await + .map_err(|source| tg::error!(!source, "failed to get the command's children"))? + .into_iter() + .filter_map(|object| object.id().try_into().ok()) + .collect(); + if !artifacts.is_empty() { + let arg = tg::cache::Arg { artifacts }; + let stream = handle + .cache(arg) + .await + .map_err(|source| tg::error!(!source, "failed to cache the artifacts"))?; + self.render_progress_stream(stream).await?; + } + + // Determine if this is a JS or builtin command. + let is_js = matches!(data.host.as_str(), "js" | "builtin"); + + // Resolve the executable and render the args. + let (executable, mut args) = if is_js { + let executable = tangram_util::env::current_exe().map_err( + |source| tg::error!(!source, "failed to get the current executable"), + )?; + let subcommand = if data.host == "builtin" { + "builtin" + } else { + "js" + }; + let args = vec![ + subcommand.to_owned(), + data.executable.to_string(), + ]; + (executable, args) + } else { + let executable = match &data.executable { + tg::command::data::Executable::Artifact(exe) => { + let mut path = + artifacts_path.join(exe.artifact.to_string()); + if let Some(subpath) = &exe.path { + path.push(subpath); + } + path + }, + tg::command::data::Executable::Module(_) => { + return Err(tg::error!("invalid executable")); + }, + tg::command::data::Executable::Path(exe) => exe.path.clone(), + }; + let args = render_args_string( + &data.args, + &artifacts_path, + &output_path, + )?; + (executable, args) + }; + + // Render the command env. + let command_env = + render_env(&data.env, &artifacts_path, &output_path)?; + + // Append trailing args. + args.extend(trailing); + + (executable, args, Some(command_env), data.cwd.clone()) + }, + + tg::Object::Directory(directory) => { + let executable = tangram_util::env::current_exe().map_err( + |source| tg::error!(!source, "failed to get the current executable"), + )?; + let id = directory.id(); + let mut args = vec!["js".to_owned(), id.to_string()]; + args.extend(trailing); + (executable, args, None, None) + }, + + tg::Object::File(file) => { + let kind = file + .module(&handle) + .await + .map_err(|source| tg::error!(!source, "failed to get the module kind"))?; + if kind.is_some() { + let tg_exe = tangram_util::env::current_exe().map_err( + |source| tg::error!(!source, "failed to get the current executable"), + )?; + let id = file.id(); + let mut args = vec!["js".to_owned(), id.to_string()]; + args.extend(trailing); + (tg_exe, args, None, None) + } else { + // Cache the file. + let artifact_id = file.id(); + let arg = tg::cache::Arg { + artifacts: vec![artifact_id.clone().into()], + }; + let stream = handle + .cache(arg) + .await + .map_err(|source| tg::error!(!source, "failed to cache the artifact"))?; + self.render_progress_stream(stream).await?; + let executable = artifacts_path.join(artifact_id.to_string()); + (executable, trailing, None, None) + } + }, + + tg::Object::Symlink(_) => { + return Err(tg::error!("unimplemented")); + }, + + _ => { + return Err(tg::error!("expected a command or an artifact")); + }, + }; + + // Merge the command env on top of the process env. + if let Some(command_env) = command_env { + env.extend(command_env); + } + + // Create the tokio process command. + let mut cmd = tokio::process::Command::new(&executable); + cmd.args(&args) + .env_clear() + .envs(&env) + .env("TANGRAM_URL", &url) + .env("TANGRAM_OUTPUT", output_path.to_str().unwrap()) + .stdin(std::process::Stdio::inherit()) + .stdout(std::process::Stdio::inherit()) + .stderr(std::process::Stdio::inherit()); + if let Some(cwd) = cwd { + cmd.current_dir(cwd); + } + + // Spawn the process. + let mut child = cmd + .spawn() + .map_err(|source| tg::error!(!source, "failed to spawn the process"))?; + + // Wait for the process to exit. + let status = child + .wait() + .await + .map_err(|source| tg::error!(!source, "failed to wait for the process"))?; + let exit = None + .or(status.code()) + .or(status.signal().map(|signal| 128 + signal)) + .unwrap() + .to_u8() + .unwrap(); + + // Check for output. + let mut output = None; + let mut error = None; + + // Try to read the user.tangram.output xattr. + if let Ok(Some(bytes)) = xattr::get(&output_path, "user.tangram.output") { + let tgon = String::from_utf8(bytes) + .map_err(|source| tg::error!(!source, "failed to decode the output xattr"))?; + output = Some( + tgon.parse::() + .map_err(|source| tg::error!(!source, "failed to parse the output xattr"))?, + ); + } + + // Try to read the user.tangram.error xattr. + if let Ok(Some(bytes)) = xattr::get(&output_path, "user.tangram.error") { + let data = serde_json::from_slice::(&bytes).map_err( + |source| tg::error!(!source, "failed to deserialize the error xattr"), + )?; + error = Some( + tg::Error::try_from(data) + .map_err(|source| tg::error!(!source, "failed to convert the error data"))?, + ); + } + + // If no xattr output was set but the output path exists, check it in destructively. + let exists = tokio::fs::try_exists(&output_path).await.map_err( + |source| tg::error!(!source, "failed to check if the output path exists"), + )?; + if output.is_none() && exists { + let arg = tg::checkin::Arg { + options: tg::checkin::Options { + destructive: true, + deterministic: true, + ignore: false, + lock: None, + locked: true, + root: true, + ..Default::default() + }, + path: output_path.clone(), + updates: Vec::new(), + }; + let stream = handle + .checkin(arg) + .await + .map_err(|source| tg::error!(!source, "failed to check in the output"))?; + let checkin_output = self + .render_progress_stream(stream) + .await + .map_err(|source| tg::error!(!source, "failed to check in the output"))?; + output = Some(tg::Artifact::with_id(checkin_output.artifact.item).into()); + } + + // Set the exit. + if exit != 0 { + self.exit.replace(exit); + } + + // Handle an error. + if let Some(error) = error { + return Err(tg::error!(source = error, "the process failed")); + } + + // Handle non-zero exit. + if exit > 1 && exit < 128 { + return Err(tg::error!("the process exited with code {}", exit)); + } + if exit >= 128 { + return Err(tg::error!( + "the process exited with signal {}", + exit - 128 + )); + } + + Ok(output) } async fn run_sandboxed( &mut self, options: &Options, - reference: tg::Reference, + reference: &tg::Reference, + referent: &tg::Referent, trailing: Vec, ) -> tg::Result> { let handle = self.handle().await?; // Handle the executable path. - let reference = if let Some(path) = &options.executable_path { - let referent = self.get_reference(&reference).await?; + let referent = if let Some(path) = &options.executable_path { let directory = referent - .item - .left() - .ok_or_else(|| tg::error!("expected an object"))? - .try_unwrap_directory() + .item() + .try_unwrap_directory_ref() .ok() .ok_or_else(|| tg::error!("expected a directory"))?; let artifact = directory.get(&handle, path).await.map_err( @@ -396,9 +571,9 @@ impl Cli { .store(&handle) .await .map_err(|source| tg::error!(!source, "failed to store the artifact"))?; - tg::Reference::with_object(id.into()) + referent.clone().map(|_| tg::Object::with_id(id.into())) } else { - reference + referent.clone() }; // Get the remote @@ -430,7 +605,15 @@ impl Cli { let stdout = stdio.as_ref().and_then(|stdio| stdio.stdout.clone()); let stderr = stdio.as_ref().and_then(|stdio| stdio.stderr.clone()); let crate::process::spawn::Output { process, output } = self - .spawn(spawn, reference, trailing, stdin, stdout, stderr) + .spawn( + spawn, + reference.clone(), + referent, + trailing, + stdin, + stdout, + stderr, + ) .boxed() .await?; @@ -661,247 +844,47 @@ impl Cli { Ok(wait.output) } - async fn command_run_inner( - &mut self, - options: Options, - reference: tg::Reference, - trailing: Vec, - ) -> tg::Result<()> { - let handle = self.handle().await?; - - // Handle the executable path. - let reference = if let Some(path) = &options.executable_path { - let referent = self.get_reference(&reference).await?; - let directory = referent - .item - .left() - .ok_or_else(|| tg::error!("expected an object"))? - .try_unwrap_directory() - .ok() - .ok_or_else(|| tg::error!("expected a directory"))?; - let artifact = directory.get(&handle, path).await.map_err( - |source| tg::error!(!source, path = %path.display(), "failed to get the artifact"), - )?; - let id = artifact - .store(&handle) - .await - .map_err(|source| tg::error!(!source, "failed to store the artifact"))?; - tg::Reference::with_object(id.into()) - } else { - reference - }; + fn needs_sandbox(&mut self, options: &Options) -> bool { + // Sandbox if explicitly requested. + if options.spawn.sandbox.get().is_some_and(|sbx| sbx) { + return true; + } - // Get the remote. - let remote = options + // Remote processes imply sandboxing. + if options .spawn .remotes .remotes - .clone() - .and_then(|remotes| remotes.into_iter().next()); - - // Create the stdio. - let stdio = stdio::Stdio::new(&handle, remote.clone(), &options) - .await - .map_err(|source| tg::error!(!source, "failed to create stdio"))?; - - let local = options.spawn.local.local; - let remotes = options.spawn.remotes.remotes.clone(); - - // Spawn the process. - let crate::process::spawn::Output { process, output } = self - .spawn( - options.spawn, - reference, - trailing, - stdio.stdin.clone(), - stdio.stdout.clone(), - stdio.stderr.clone(), - ) - .boxed() - .await?; - - // If the detach flag is set, then print the process ID and return. - if options.detach { - if options.verbose { - self.print_serde(output, options.print).await?; - } else { - Self::print_display(&output.process); - } - return Ok(()); - } - - // Print the process. - if !self.args.quiet { - let mut message = process.item().id().to_string(); - if let Some(token) = process.item().token() { - write!(message, " {token}").unwrap(); - } - Self::print_info_message(&message); - } - - // Enable raw mode if necessary. - if let Some(tty) = &stdio.tty { - tty.enable_raw_mode()?; - } - - // Spawn the stdio task. - let stdio_task = Task::spawn({ - let handle = handle.clone(); - let stdio = stdio.clone(); - |stop| async move { self::stdio::task(&handle, stop, stdio).boxed().await } - }); - - // Spawn signal task. - let signal_task = tokio::spawn({ - let handle = handle.clone(); - let process = process.item().id().clone(); - let remote = remote.clone(); - async move { - self::signal::task(&handle, &process, remote).await.ok(); - } - }); - - // Await the process unless the spawn output already includes the wait output. - let result = if let Some(wait) = output - .wait - .map(TryInto::try_into) - .transpose() - .map_err(|source| tg::error!(!source, "failed to parse the wait output"))? + .as_ref() + .is_some_and(|remotes| !remotes.is_empty()) { - Ok(wait) - } else { - let arg = tg::process::wait::Arg { - token: process.item().token().cloned(), - ..tg::process::wait::Arg::default() - }; - process - .item() - .wait(&handle, arg) - .await - .map_err(|source| tg::error!(!source, "failed to await the process")) - }; - - // Close stdout and stderr. - stdio.close(&handle).await?; - - // Stop and await the stdio task. - stdio_task.stop(); - - // Await the stdio task. - stdio_task.wait().await.unwrap()?; - - // Delete stdio. - stdio.delete(&handle).await?; - - // Abort the signal task. - signal_task.abort(); - - // Handle the result. - let wait = result.map_err(|source| tg::error!(!source, "failed to await the process"))?; - - // Print verbose output if requested. - if options.verbose { - let output = tg::process::wait::Output { - error: wait.error.as_ref().map(tg::Error::to_data_or_id), - exit: wait.exit, - output: wait.output.as_ref().map(tg::Value::to_data), - }; - self.print_serde(output, options.print.clone()).await?; - return Ok(()); + return true; } - // Set the exit. - if wait.exit != 0 { - self.exit.replace(wait.exit); + // Detached processes are currently sandboxed. This could change? + if options.detach { + return true; } - // Handle an error. - if let Some(error) = wait.error { - let error = error - .to_data_or_id() - .map_left(|data| { - Box::new(tg::error::Object::try_from_data(data).unwrap_or_else(|_| { - tg::error::Object { - message: Some("invalid error".to_owned()), - ..Default::default() - } - })) - }) - .map_right(|id| Box::new(tg::Error::with_id(id))); - let error = tg::Error::with_object(tg::error::Object { - message: Some("the process failed".to_owned()), - source: Some(process.clone().map(|_| error)), - values: [("id".to_owned(), process.item().id().to_string())].into(), - ..Default::default() - }); - return Err(error); - } - // Handle non-zero exit. - if wait.exit > 1 && wait.exit < 128 { - return Err(tg::error!("the process exited with code {}", wait.exit)); - } - if wait.exit >= 128 { - return Err(tg::error!( - "the process exited with signal {}", - wait.exit - 128 - )); + // Cached processes must have been sandboxed. + if options.spawn.cached.is_some_and(|cached| cached) { + return true; } - // Get the output. - let output = wait.output.unwrap_or(tg::Value::Null); - - // Check out the output if requested. - if let Some(path) = options.checkout { - // Get the artifact. - let artifact: tg::Artifact = output - .clone() - .try_into() - .map_err(|_| tg::error!("expected an artifact"))?; - - // Get the path. - let path = if let Some(path) = path { - let path = tangram_util::fs::canonicalize_parent(path) - .await - .map_err(|source| tg::error!(!source, "failed to canonicalize the path"))?; - Some(path) - } else { - None - }; - - // Check out the artifact. - let artifact = artifact.id(); - let arg = tg::checkout::Arg { - artifact: artifact.clone(), - dependencies: path.is_some(), - extension: None, - force: options.checkout_force, - lock: None, - path, - }; - let stream = handle.checkout(arg).await.map_err( - |source| tg::error!(!source, %artifact, "failed to check out the artifact"), - )?; - let tg::checkout::Output { path, .. } = - self.render_progress_stream(stream).await.map_err( - |source| tg::error!(!source, %artifact, "failed to check out the artifact"), - )?; - - // Print the path. - self.print_serde(path, options.print).await?; + // Processes with a checksum should also run in a sandbox, to ensure cache hits. + if options.spawn.checksum.is_some() { + return true; + } - return Ok(()); + // You need a sandbox to deny network access. + if options.spawn.network.is_none_or(|network| !network) { + return true; } - // Print the output. - if !options.verbose && !output.is_null() { - let arg = tg::object::get::Arg { - local, - metadata: false, - remotes, - }; - self.print_value(&output, options.print, arg).await?; + if !options.spawn.mounts.is_empty() { + return true; } - Ok(()) + false } } diff --git a/packages/cli/src/shell/common.rs b/packages/cli/src/shell/common.rs index 08a99ac46..9a7bbbed5 100644 --- a/packages/cli/src/shell/common.rs +++ b/packages/cli/src/shell/common.rs @@ -295,8 +295,23 @@ impl Cli { sandbox: crate::process::spawn::Sandbox::new(Some(true)), ..Default::default() }; + let referent = self.get_reference(&reference).await?; + let item = referent + .item + .clone() + .left() + .ok_or_else(|| tg::error!(%reference, "expected an object"))?; + let referent = referent.map(|_| item); let crate::process::spawn::Output { process, output } = self - .spawn(options, reference.clone(), Vec::new(), None, None, None) + .spawn( + options, + reference.clone(), + referent, + Vec::new(), + None, + None, + None, + ) .boxed() .await?; diff --git a/packages/cli/tests/run/assertion_failure.nu b/packages/cli/tests/run/assertion_failure.nu index 6672d65a4..243cec241 100644 --- a/packages/cli/tests/run/assertion_failure.nu +++ b/packages/cli/tests/run/assertion_failure.nu @@ -12,8 +12,30 @@ let path = artifact { ', } +let sandbox_output = do { cd $path; tg run --sandbox } | complete +failure $sandbox_output +let sandbox_stderr = $sandbox_output.stderr | lines | skip 1 | str join "\n" +let sandbox_stderr = $sandbox_stderr | str replace -ar 'pcs_00[0-9a-z]{26}' 'PROCESS' +snapshot $sandbox_stderr ' + error an error occurred + -> the process failed + id = PROCESS + -> Uncaught Error: failed assertion + ╭─[./tangram.ts:2:22] + 1 │ import foo from "./foo.tg.ts"; + 2 │ export default () => foo(); + · ▲ + · ╰── Uncaught Error: failed assertion + ╰──── + ╭─[./foo.tg.ts:1:25] + 1 │ export default () => tg.assert(false); + · ▲ + · ╰── Uncaught Error: failed assertion + ╰──── +' + let output = do { cd $path; tg run } | complete failure $output let stderr = $output.stderr | lines | skip 1 | str join "\n" let stderr = $stderr | str replace -ar 'pcs_00[0-9a-z]{26}' 'PROCESS' -snapshot $stderr +assert equal $stderr $sandbox_stderr diff --git a/packages/cli/tests/run/assertion_failure.snapshot b/packages/cli/tests/run/assertion_failure.snapshot deleted file mode 100644 index a611927b5..000000000 --- a/packages/cli/tests/run/assertion_failure.snapshot +++ /dev/null @@ -1,15 +0,0 @@ -error an error occurred --> the process failed - id = PROCESS --> Uncaught Error: failed assertion - ╭─[./tangram.ts:2:22] - 1 │ import foo from "./foo.tg.ts"; - 2 │ export default () => foo(); - · ▲ - · ╰── Uncaught Error: failed assertion - ╰──── - ╭─[./foo.tg.ts:1:25] - 1 │ export default () => tg.assert(false); - · ▲ - · ╰── Uncaught Error: failed assertion - ╰──── \ No newline at end of file diff --git a/packages/cli/tests/run/assertion_failure_in_path_dependency.nu b/packages/cli/tests/run/assertion_failure_in_path_dependency.nu index 86ef76381..a66f8d9fa 100644 --- a/packages/cli/tests/run/assertion_failure_in_path_dependency.nu +++ b/packages/cli/tests/run/assertion_failure_in_path_dependency.nu @@ -16,9 +16,31 @@ let path = artifact { } } -let output = do { cd $path; tg run ./foo }| complete -print $output +let sandbox_output = do { cd $path; tg run ./foo --sandbox } | complete +print $sandbox_output +failure $sandbox_output +let sandbox_stderr = $sandbox_output.stderr | lines | skip 1 | str join "\n" +let sandbox_stderr = $sandbox_stderr | str replace -ar 'pcs_00[0-9a-z]{26}' 'PROCESS' +snapshot $sandbox_stderr ' + error an error occurred + -> the process failed + id = PROCESS + -> Uncaught Error: error + ╭─[./foo/tangram.ts:2:22] + 1 │ import foo from "../bar"; + 2 │ export default () => foo(); + · ▲ + · ╰── Uncaught Error: error + ╰──── + ╭─[./bar/tangram.ts:1:25] + 1 │ export default () => tg.assert(false, "error") + · ▲ + · ╰── Uncaught Error: error + ╰──── +' + +let output = do { cd $path; tg run ./foo } | complete failure $output let stderr = $output.stderr | lines | skip 1 | str join "\n" let stderr = $stderr | str replace -ar 'pcs_00[0-9a-z]{26}' 'PROCESS' -snapshot $stderr +assert equal $stderr $sandbox_stderr diff --git a/packages/cli/tests/run/assertion_failure_in_path_dependency.snapshot b/packages/cli/tests/run/assertion_failure_in_path_dependency.snapshot deleted file mode 100644 index 69aa84a69..000000000 --- a/packages/cli/tests/run/assertion_failure_in_path_dependency.snapshot +++ /dev/null @@ -1,15 +0,0 @@ -error an error occurred --> the process failed - id = PROCESS --> Uncaught Error: error - ╭─[./foo/tangram.ts:2:22] - 1 │ import foo from "../bar"; - 2 │ export default () => foo(); - · ▲ - · ╰── Uncaught Error: error - ╰──── - ╭─[./bar/tangram.ts:1:25] - 1 │ export default () => tg.assert(false, "error") - · ▲ - · ╰── Uncaught Error: error - ╰──── \ No newline at end of file diff --git a/packages/cli/tests/run/assertion_failure_in_tag_dependency.nu b/packages/cli/tests/run/assertion_failure_in_tag_dependency.nu index f0cad14dc..6878d9005 100644 --- a/packages/cli/tests/run/assertion_failure_in_tag_dependency.nu +++ b/packages/cli/tests/run/assertion_failure_in_tag_dependency.nu @@ -17,8 +17,30 @@ let path = artifact { ' } +let sandbox_output = do { cd $path; tg run --sandbox } | complete +failure $sandbox_output +let sandbox_stderr = $sandbox_output.stderr | lines | skip 1 | str join "\n" +let sandbox_stderr = $sandbox_stderr | str replace -ar 'pcs_00[0-9a-z]{26}' 'PROCESS' +snapshot $sandbox_stderr ' + error an error occurred + -> the process failed + id = PROCESS + -> Uncaught Error: error in foo + ╭─[./tangram.ts:2:22] + 1 │ import foo from "foo"; + 2 │ export default () => foo(); + · ▲ + · ╰── Uncaught Error: error in foo + ╰──── + ╭─[foo:tangram.ts:1:25] + 1 │ export default () => tg.assert(false, "error in foo"); + · ▲ + · ╰── Uncaught Error: error in foo + ╰──── +' + let output = do { cd $path; tg run } | complete failure $output let stderr = $output.stderr | lines | skip 1 | str join "\n" let stderr = $stderr | str replace -ar 'pcs_00[0-9a-z]{26}' 'PROCESS' -snapshot $stderr +assert equal $stderr $sandbox_stderr diff --git a/packages/cli/tests/run/assertion_failure_in_tag_dependency.snapshot b/packages/cli/tests/run/assertion_failure_in_tag_dependency.snapshot deleted file mode 100644 index bbd895ec1..000000000 --- a/packages/cli/tests/run/assertion_failure_in_tag_dependency.snapshot +++ /dev/null @@ -1,15 +0,0 @@ -error an error occurred --> the process failed - id = PROCESS --> Uncaught Error: error in foo - ╭─[./tangram.ts:2:22] - 1 │ import foo from "foo"; - 2 │ export default () => foo(); - · ▲ - · ╰── Uncaught Error: error in foo - ╰──── - ╭─[foo:tangram.ts:1:25] - 1 │ export default () => tg.assert(false, "error in foo"); - · ▲ - · ╰── Uncaught Error: error in foo - ╰──── \ No newline at end of file diff --git a/packages/cli/tests/run/assertion_failure_in_tagged_cyclic_dependency.nu b/packages/cli/tests/run/assertion_failure_in_tagged_cyclic_dependency.nu index 8b03468cd..b630a9bd2 100644 --- a/packages/cli/tests/run/assertion_failure_in_tagged_cyclic_dependency.nu +++ b/packages/cli/tests/run/assertion_failure_in_tagged_cyclic_dependency.nu @@ -26,8 +26,44 @@ let path = artifact { ' } +let sandbox_output = do { cd $path; tg run --sandbox } | complete +failure $sandbox_output +let sandbox_stderr = $sandbox_output.stderr | lines | skip 1 | str join "\n" +let sandbox_stderr = $sandbox_stderr | str replace -ar 'pcs_00[0-9a-z]{26}' 'PROCESS' +snapshot $sandbox_stderr ' + error an error occurred + -> the process failed + id = PROCESS + -> Uncaught Error: failure in foo + ╭─[./tangram.ts:2:22] + 1 │ import foo from "foo"; + 2 │ export default () => foo(); + · ▲ + · ╰── Uncaught Error: failure in foo + ╰──── + ╭─[foo:tangram.ts:2:22] + 1 │ import bar from "../bar"; + 2 │ export default () => bar(); + · ▲ + · ╰── Uncaught Error: failure in foo + 3 │ export const failure = () => tg.assert(false, "failure in foo"); + ╰──── + ╭─[foo:../bar/tangram.ts:2:22] + 1 │ import { failure } from "../foo"; + 2 │ export default () => failure(); + · ▲ + · ╰── Uncaught Error: failure in foo + ╰──── + ╭─[foo:../foo/tangram.ts:3:33] + 2 │ export default () => bar(); + 3 │ export const failure = () => tg.assert(false, "failure in foo"); + · ▲ + · ╰── Uncaught Error: failure in foo + ╰──── +' + let output = do { cd $path; tg run } | complete failure $output let stderr = $output.stderr | lines | skip 1 | str join "\n" let stderr = $stderr | str replace -ar 'pcs_00[0-9a-z]{26}' 'PROCESS' -snapshot $stderr +assert equal $stderr $sandbox_stderr diff --git a/packages/cli/tests/run/assertion_failure_in_tagged_cyclic_dependency.snapshot b/packages/cli/tests/run/assertion_failure_in_tagged_cyclic_dependency.snapshot deleted file mode 100644 index 350e92ae8..000000000 --- a/packages/cli/tests/run/assertion_failure_in_tagged_cyclic_dependency.snapshot +++ /dev/null @@ -1,29 +0,0 @@ -error an error occurred --> the process failed - id = PROCESS --> Uncaught Error: failure in foo - ╭─[./tangram.ts:2:22] - 1 │ import foo from "foo"; - 2 │ export default () => foo(); - · ▲ - · ╰── Uncaught Error: failure in foo - ╰──── - ╭─[foo:tangram.ts:2:22] - 1 │ import bar from "../bar"; - 2 │ export default () => bar(); - · ▲ - · ╰── Uncaught Error: failure in foo - 3 │ export const failure = () => tg.assert(false, "failure in foo"); - ╰──── - ╭─[foo:../bar/tangram.ts:2:22] - 1 │ import { failure } from "../foo"; - 2 │ export default () => failure(); - · ▲ - · ╰── Uncaught Error: failure in foo - ╰──── - ╭─[foo:../foo/tangram.ts:3:33] - 2 │ export default () => bar(); - 3 │ export const failure = () => tg.assert(false, "failure in foo"); - · ▲ - · ╰── Uncaught Error: failure in foo - ╰──── \ No newline at end of file diff --git a/packages/cli/tests/run/assertion_failure_out_of_tree.nu b/packages/cli/tests/run/assertion_failure_out_of_tree.nu index 6408bbc6b..1b07b1134 100644 --- a/packages/cli/tests/run/assertion_failure_out_of_tree.nu +++ b/packages/cli/tests/run/assertion_failure_out_of_tree.nu @@ -16,8 +16,25 @@ let path = artifact { } } -let output = do { cd $path; tg run ./foo }| complete +let sandbox_output = do { cd $path; tg run ./foo --sandbox } | complete +failure $sandbox_output +let sandbox_stderr = $sandbox_output.stderr | lines | skip 1 | str join "\n" +let sandbox_stderr = $sandbox_stderr | str replace -ar 'pcs_00[0-9a-z]{26}' 'PROCESS' +snapshot $sandbox_stderr ' + error an error occurred + -> the process failed + id = PROCESS + -> the child process failed + id = PROCESS + -> Uncaught Error: failed assertion + ╭─[./bar/tangram.ts:1:25] + 1 │ export default () => tg.assert(false); + · ▲ + · ╰── Uncaught Error: failed assertion + ╰──── +' +let output = do { cd $path; tg run ./foo } | complete failure $output let stderr = $output.stderr | lines | skip 1 | str join "\n" let stderr = $stderr | str replace -ar 'pcs_00[0-9a-z]{26}' 'PROCESS' -snapshot $stderr +assert equal $stderr $sandbox_stderr diff --git a/packages/cli/tests/run/assertion_failure_out_of_tree.snapshot b/packages/cli/tests/run/assertion_failure_out_of_tree.snapshot deleted file mode 100644 index d9195ee93..000000000 --- a/packages/cli/tests/run/assertion_failure_out_of_tree.snapshot +++ /dev/null @@ -1,11 +0,0 @@ -error an error occurred --> the process failed - id = PROCESS --> the child process failed - id = PROCESS --> Uncaught Error: failed assertion - ╭─[./bar/tangram.ts:1:25] - 1 │ export default () => tg.assert(false); - · ▲ - · ╰── Uncaught Error: failed assertion - ╰──── \ No newline at end of file diff --git a/packages/cli/tests/run/build_flag.nu b/packages/cli/tests/run/build_flag.nu new file mode 100644 index 000000000..cf75d189a --- /dev/null +++ b/packages/cli/tests/run/build_flag.nu @@ -0,0 +1,21 @@ +use ../../test.nu * + +# When --build is set, `tg run` should build the target first to produce an object, then run that object. + +let server = spawn + +let path = artifact { + tangram.ts: r#' + export default () => { + return tg.directory({ + "tangram.ts": tg.file('export default () => "from built module";'), + }); + }; + '# +} + +let sandbox_output = tg run --build $path --sandbox +snapshot $sandbox_output '"from built module"' + +let output = tg run --build $path +assert equal $output $sandbox_output diff --git a/packages/cli/tests/run/build_flag_non_object.nu b/packages/cli/tests/run/build_flag_non_object.nu new file mode 100644 index 000000000..ff6f4e02d --- /dev/null +++ b/packages/cli/tests/run/build_flag_non_object.nu @@ -0,0 +1,17 @@ +use ../../test.nu * + +# When --build is set but the build output is not an object (e.g. a string), `tg run` should fail. + +let server = spawn + +let path = artifact { + tangram.ts: 'export default () => "not an object";' +} + +let sandbox_output = tg run --build $path --sandbox | complete +failure $sandbox_output +assert ($sandbox_output.stderr | str contains "expected the build to output an object") + +let output = tg run --build $path | complete +failure $output +assert ($output.stderr | str contains "expected the build to output an object") diff --git a/packages/cli/tests/run/checkout.nu b/packages/cli/tests/run/checkout.nu new file mode 100644 index 000000000..51b9133e0 --- /dev/null +++ b/packages/cli/tests/run/checkout.nu @@ -0,0 +1,19 @@ +use ../../test.nu * + +# When --checkout is set without a path, `tg run` should check out the output artifact and print its path. + +let server = spawn + +let path = artifact { + tangram.ts: 'export default () => tg.file("checkout contents");' +} + +let sandbox_output = tg run --checkout $path --sandbox | str trim +assert ($sandbox_output | str contains "/") +let sandbox_contents = open $sandbox_output +assert ($sandbox_contents == "checkout contents") + +let output = tg run --checkout $path | str trim +assert ($output | str contains "/") +let contents = open $output +assert ($contents == "checkout contents") diff --git a/packages/cli/tests/run/checkout_directory.nu b/packages/cli/tests/run/checkout_directory.nu new file mode 100644 index 000000000..48e79e024 --- /dev/null +++ b/packages/cli/tests/run/checkout_directory.nu @@ -0,0 +1,28 @@ +use ../../test.nu * + +# When --checkout is set and the output is a directory, `tg run` should check out the directory. + +let server = spawn + +let path = artifact { + tangram.ts: ' + export default () => tg.directory({ + "hello.txt": tg.file("hello"), + "world.txt": tg.file("world"), + }); + ' +} + +let sandbox_output = tg run --checkout $path --sandbox | str trim +print $sandbox_output +print (file $sandbox_output) +assert ($sandbox_output | path exists) +assert (($sandbox_output | path join "hello.txt") | path exists) +let sandbox_contents = open ($sandbox_output | path join "hello.txt") +assert ($sandbox_contents == "hello") + +# let output = tg run --checkout $path | str trim +# assert ($output | path exists) +# assert (($output | path join "hello.txt") | path exists) +# let contents = open ($output | path join "hello.txt") +# assert ($contents == "hello") diff --git a/packages/cli/tests/run/checkout_force.nu b/packages/cli/tests/run/checkout_force.nu new file mode 100644 index 000000000..0fe22972b --- /dev/null +++ b/packages/cli/tests/run/checkout_force.nu @@ -0,0 +1,23 @@ +use ../../test.nu * + +# When --checkout-force is set with --checkout=, `tg run` should overwrite an existing file at the path. + +let server = spawn + +let path = artifact { + tangram.ts: 'export default () => tg.file("new contents");' +} + +let sandbox_dir = mktemp -d +let sandbox_checkout = ($sandbox_dir | path join "output.txt") +"old contents" | save $sandbox_checkout +tg run $"--checkout=($sandbox_checkout)" --checkout-force $path --sandbox +let sandbox_contents = open $sandbox_checkout +assert ($sandbox_contents == "new contents") + +let unsandboxed_dir = mktemp -d +let unsandboxed_checkout = ($unsandboxed_dir | path join "output.txt") +"old contents" | save $unsandboxed_checkout +tg run $"--checkout=($unsandboxed_checkout)" --checkout-force $path +let contents = open $unsandboxed_checkout +assert ($contents == "new contents") diff --git a/packages/cli/tests/run/checkout_no_output.nu b/packages/cli/tests/run/checkout_no_output.nu new file mode 100644 index 000000000..b7489e4b1 --- /dev/null +++ b/packages/cli/tests/run/checkout_no_output.nu @@ -0,0 +1,17 @@ +use ../../test.nu * + +# When --checkout is set but the process returns null, `tg run` should fail with an error. + +let server = spawn + +let path = artifact { + tangram.ts: 'export default () => {};' +} + +let sandbox_output = tg run --checkout $path --sandbox | complete +failure $sandbox_output +assert ($sandbox_output.stderr | str contains "expected an output") + +let output = tg run --checkout $path | complete +failure $output +assert ($output.stderr | str contains "expected an output") diff --git a/packages/cli/tests/run/checkout_non_artifact.nu b/packages/cli/tests/run/checkout_non_artifact.nu new file mode 100644 index 000000000..9c4bbe4ef --- /dev/null +++ b/packages/cli/tests/run/checkout_non_artifact.nu @@ -0,0 +1,17 @@ +use ../../test.nu * + +# When --checkout is set but the process output is not an artifact (e.g. a string), `tg run` should fail. + +let server = spawn + +let path = artifact { + tangram.ts: 'export default () => "just a string";' +} + +let sandbox_output = tg run --checkout $path --sandbox | complete +failure $sandbox_output +assert ($sandbox_output.stderr | str contains "expected an artifact") + +let output = tg run --checkout $path | complete +failure $output +assert ($output.stderr | str contains "expected an artifact") diff --git a/packages/cli/tests/run/checkout_path.nu b/packages/cli/tests/run/checkout_path.nu new file mode 100644 index 000000000..42017ce03 --- /dev/null +++ b/packages/cli/tests/run/checkout_path.nu @@ -0,0 +1,23 @@ +use ../../test.nu * + +# When --checkout= is set, `tg run` should check out the output artifact to the specified path. + +let server = spawn + +let path = artifact { + tangram.ts: 'export default () => tg.file("checkout to path");' +} + +let sandbox_dir = mktemp -d +let sandbox_checkout = ($sandbox_dir | path join "sandbox_output.txt") +let sandbox_output = tg run $"--checkout=($sandbox_checkout)" $path --sandbox | str trim +assert ($sandbox_output | str ends-with "sandbox_output.txt") +let sandbox_contents = open $sandbox_checkout +assert ($sandbox_contents == "checkout to path") + +let unsandboxed_dir = mktemp -d +let unsandboxed_checkout = ($unsandboxed_dir | path join "output.txt") +let output = tg run $"--checkout=($unsandboxed_checkout)" $path | str trim +assert ($output | str ends-with "output.txt") +let contents = open $unsandboxed_checkout +assert ($contents == "checkout to path") diff --git a/packages/cli/tests/run/child_process_error.nu b/packages/cli/tests/run/child_process_error.nu index 1b9b87f5e..84874adee 100644 --- a/packages/cli/tests/run/child_process_error.nu +++ b/packages/cli/tests/run/child_process_error.nu @@ -14,8 +14,35 @@ let path = artifact { ', } +let sandbox_output = do { cd $path; tg run --sandbox } | complete +failure $sandbox_output +let sandbox_stderr = $sandbox_output.stderr | lines | skip 1 | str join "\n" +let sandbox_stderr = $sandbox_stderr | str replace -ar 'pcs_00[0-9a-z]{26}' 'PROCESS' +snapshot $sandbox_stderr ' + error an error occurred + -> the process failed + id = PROCESS + -> the child process failed + id = PROCESS + ╭─[./tangram.ts:2:9] + 1 │ export default async function () { + 2 │ return await tg.run(foo); + · ▲ + · ╰── the child process failed + 3 │ } + ╰──── + -> error + ╭─[./tangram.ts:6:11] + 5 │ export function foo() { + 6 │ throw tg.error("error"); + · ▲ + · ╰── error + 7 │ } + ╰──── +' + let output = do { cd $path; tg run } | complete failure $output let stderr = $output.stderr | lines | skip 1 | str join "\n" let stderr = $stderr | str replace -ar 'pcs_00[0-9a-z]{26}' 'PROCESS' -snapshot $stderr +assert equal $stderr $sandbox_stderr diff --git a/packages/cli/tests/run/child_process_error.snapshot b/packages/cli/tests/run/child_process_error.snapshot deleted file mode 100644 index 92ba8d865..000000000 --- a/packages/cli/tests/run/child_process_error.snapshot +++ /dev/null @@ -1,20 +0,0 @@ -error an error occurred --> the process failed - id = PROCESS --> the child process failed - id = PROCESS - ╭─[./tangram.ts:2:9] - 1 │ export default async function () { - 2 │ return await tg.run(foo); - · ▲ - · ╰── the child process failed - 3 │ } - ╰──── --> error - ╭─[./tangram.ts:6:11] - 5 │ export function foo() { - 6 │ throw tg.error("error"); - · ▲ - · ╰── error - 7 │ } - ╰──── \ No newline at end of file diff --git a/packages/cli/tests/run/console_log.nu b/packages/cli/tests/run/console_log.nu new file mode 100644 index 000000000..fb30e0569 --- /dev/null +++ b/packages/cli/tests/run/console_log.nu @@ -0,0 +1,27 @@ +use ../../test.nu * + +# The process stdout from console.log should appear on the terminal. The return value should also be printed. + +let server = spawn + +let path = artifact { + tangram.ts: ' + export default () => { + console.log("log line 1"); + console.log("log line 2"); + return "return value"; + }; + ' +} + +let sandbox_output = tg run $path --sandbox | complete +success $sandbox_output +assert ($sandbox_output.stdout | str contains "log line 1") +assert ($sandbox_output.stdout | str contains "log line 2") +assert ($sandbox_output.stdout | str contains "return value") + +let output = tg run $path | complete +success $output +assert ($output.stdout | str contains "log line 1") +assert ($output.stdout | str contains "log line 2") +assert ($output.stdout | str contains "return value") diff --git a/packages/cli/tests/run/default_reference.nu b/packages/cli/tests/run/default_reference.nu new file mode 100644 index 000000000..500628a7d --- /dev/null +++ b/packages/cli/tests/run/default_reference.nu @@ -0,0 +1,17 @@ +use ../../test.nu * + +# When no reference is given, `tg run` defaults to "." and uses the current directory. + +let server = spawn + +let path = artifact { + tangram.ts: 'export default () => "default reference";' +} + +let sandbox_output = do { cd $path; tg run --sandbox } | complete +success $sandbox_output +assert ($sandbox_output.stdout | str contains "default reference") + +let output = do { cd $path; tg run } | complete +success $output +assert equal $output.stdout $sandbox_output.stdout diff --git a/packages/cli/tests/run/detach.nu b/packages/cli/tests/run/detach.nu new file mode 100644 index 000000000..07655e3e4 --- /dev/null +++ b/packages/cli/tests/run/detach.nu @@ -0,0 +1,15 @@ +use ../../test.nu * + +# When --detach is set, `tg run` should print the process ID and return immediately. + +let server = spawn + +let path = artifact { + tangram.ts: 'export default () => "hello";' +} + +let sandbox_output = tg run --detach $path --sandbox | str trim +assert ($sandbox_output | str starts-with "pcs_") + +let output = tg run --detach $path | str trim +assert ($output | str starts-with "pcs_") diff --git a/packages/cli/tests/run/detach_verbose.nu b/packages/cli/tests/run/detach_verbose.nu new file mode 100644 index 000000000..973d9d3ed --- /dev/null +++ b/packages/cli/tests/run/detach_verbose.nu @@ -0,0 +1,15 @@ +use ../../test.nu * + +# When --detach --verbose are set, `tg run` should print the full spawn output as JSON. + +let server = spawn + +let path = artifact { + tangram.ts: 'export default () => "hello";' +} + +let sandbox_output = tg run --detach --verbose $path --sandbox | from json +assert ($sandbox_output.process | str starts-with "pcs_") + +let output = tg run --detach --verbose $path | from json +assert ($output.process | str starts-with "pcs_") diff --git a/packages/cli/tests/run/directory_by_id.nu b/packages/cli/tests/run/directory_by_id.nu new file mode 100644 index 000000000..e304bed47 --- /dev/null +++ b/packages/cli/tests/run/directory_by_id.nu @@ -0,0 +1,16 @@ +use ../../test.nu * + +let server = spawn + +let path = artifact { + tangram.ts: 'export default () => "hello, world!";' +} + +let id = tg checkin $path +tg index + +let sandbox_output = tg run $id --sandbox +snapshot $sandbox_output '"hello, world!"' + +let output = tg run $id +assert equal $output $sandbox_output diff --git a/packages/cli/tests/run/executable_path.nu b/packages/cli/tests/run/executable_path.nu new file mode 100644 index 000000000..e817b7270 --- /dev/null +++ b/packages/cli/tests/run/executable_path.nu @@ -0,0 +1,16 @@ +use ../../test.nu * + +# When --executable-path is set, `tg run` should resolve the executable from the directory. + +let server = spawn + +let path = artifact { + tangram.ts: ' + export default () => tg.directory({ + "main.tg.ts": tg.file("export default () => \"from executable path\";"), + }); + ' +} + +let sandbox_output = tg run --executable-path main.tg.ts --build $path +snapshot $sandbox_output '"from executable path"' diff --git a/packages/cli/tests/run/exit_code_nonzero.nu b/packages/cli/tests/run/exit_code_nonzero.nu new file mode 100644 index 000000000..0e71a8c1b --- /dev/null +++ b/packages/cli/tests/run/exit_code_nonzero.nu @@ -0,0 +1,20 @@ +use ../../test.nu * + +# When a process exits with a non-zero exit code (1 < code < 128), `tg run` should fail with an error mentioning the exit code. + +let server = spawn --busybox + +let path = artifact { + tangram.ts: ' + import busybox from "busybox"; + export default () => tg.build`exit 42`.env(tg.build(busybox)); + ' +} + +let sandbox_output = tg run $path --sandbox | complete +failure $sandbox_output +assert ($sandbox_output.stderr | str contains "exited with code 42") + +let output = tg run $path | complete +failure $output +assert ($output.stderr | str contains "exited with code 42") diff --git a/packages/cli/tests/run/exit_code_signal.nu b/packages/cli/tests/run/exit_code_signal.nu new file mode 100644 index 000000000..b63b94603 --- /dev/null +++ b/packages/cli/tests/run/exit_code_signal.nu @@ -0,0 +1,20 @@ +use ../../test.nu * + +# When a process exits with a signal (exit code >= 128), `tg run` should fail with an error mentioning the signal number. + +let server = spawn --busybox + +let path = artifact { + tangram.ts: ' + import busybox from "busybox"; + export default () => tg.build`kill -9 $$`.env(tg.build(busybox)); + ' +} + +let sandbox_output = tg run $path --sandbox | complete +failure $sandbox_output +assert ($sandbox_output.stderr | str contains "exited with signal") + +let output = tg run $path | complete +failure $output +assert ($output.stderr | str contains "exited with signal") diff --git a/packages/cli/tests/run/file_by_id.nu b/packages/cli/tests/run/file_by_id.nu index 659a6de44..8a5bce782 100644 --- a/packages/cli/tests/run/file_by_id.nu +++ b/packages/cli/tests/run/file_by_id.nu @@ -9,5 +9,8 @@ let path = artifact { let id = tg checkin ($path + '/file.tg.ts') tg index +let sandbox_output = tg run $id --sandbox +snapshot $sandbox_output '"hello, world!"' + let output = tg run $id -snapshot $output '"hello, world!"' \ No newline at end of file +assert equal $output $sandbox_output diff --git a/packages/cli/tests/run/hello_world.nu b/packages/cli/tests/run/hello_world.nu index 4399dc591..c8853a354 100644 --- a/packages/cli/tests/run/hello_world.nu +++ b/packages/cli/tests/run/hello_world.nu @@ -10,7 +10,10 @@ let path = artifact { ' } -let output = tg run $path -snapshot $output ' +let sandbox_output = tg run $path --sandbox +snapshot $sandbox_output ' Hello, World! ' + +let output = tg run $path +assert equal $output $sandbox_output diff --git a/packages/cli/tests/run/multiple_trailing_args.nu b/packages/cli/tests/run/multiple_trailing_args.nu new file mode 100644 index 000000000..2a3698b81 --- /dev/null +++ b/packages/cli/tests/run/multiple_trailing_args.nu @@ -0,0 +1,15 @@ +use ../../test.nu * + +# Test that multiple trailing arguments are all passed to the process. + +let server = spawn + +let path = artifact { + tangram.ts: 'export default (...args: string[]) => args.join(", ");' +} + +let sandbox_output = tg run $path --sandbox -- a b c +snapshot $sandbox_output '"a, b, c"' + +let output = tg run $path -- a b c +assert equal $output $sandbox_output diff --git a/packages/cli/tests/run/print_pretty.nu b/packages/cli/tests/run/print_pretty.nu new file mode 100644 index 000000000..2437ff618 --- /dev/null +++ b/packages/cli/tests/run/print_pretty.nu @@ -0,0 +1,15 @@ +use ../../test.nu * + +# When --pretty is set, `tg run` should pretty-print the output value. + +let server = spawn + +let path = artifact { + tangram.ts: 'export default () => ({ a: 1, b: 2 });' +} + +let sandbox_output = tg run --pretty $path --sandbox +assert ($sandbox_output | str contains "\n") + +let output = tg run --pretty $path +assert equal $output $sandbox_output diff --git a/packages/cli/tests/run/process_error.nu b/packages/cli/tests/run/process_error.nu new file mode 100644 index 000000000..3d24d12fe --- /dev/null +++ b/packages/cli/tests/run/process_error.nu @@ -0,0 +1,21 @@ +use ../../test.nu * + +# When a process produces an error, `tg run` should fail and print the error. + +let server = spawn + +let path = artifact { + tangram.ts: ' + export default () => { + throw new Error("something went wrong"); + }; + ' +} + +let sandbox_output = do { cd $path; tg run --sandbox } | complete +failure $sandbox_output +assert ($sandbox_output.stderr | str contains "the process failed") + +let output = do { cd $path; tg run } | complete +failure $output +assert ($output.stderr | str contains "the process failed") diff --git a/packages/cli/tests/run/quiet.nu b/packages/cli/tests/run/quiet.nu new file mode 100644 index 000000000..59a52f349 --- /dev/null +++ b/packages/cli/tests/run/quiet.nu @@ -0,0 +1,22 @@ +use ../../test.nu * + +# When --quiet is set, `tg run` should suppress the process info message on stderr. + +let server = spawn + +let path = artifact { + tangram.ts: 'export default () => "hello";' +} + +let sandboxed_output = tg -q run $path --sandbox | complete +success $sandboxed_output +assert (not ($sandboxed_output.stderr | str contains "pcs_")) +snapshot $sandboxed_output.stdout ' + "hello" + +' + +let output = tg -q run $path | complete +success $output +assert (not ($output.stderr | str contains "pcs_")) +assert equal $output.stdout $sandboxed_output.stdout diff --git a/packages/cli/tests/run/return_null.nu b/packages/cli/tests/run/return_null.nu new file mode 100644 index 000000000..c3fd819e0 --- /dev/null +++ b/packages/cli/tests/run/return_null.nu @@ -0,0 +1,15 @@ +use ../../test.nu * + +# When a process returns null, `tg run` should produce no output. + +let server = spawn + +let path = artifact { + tangram.ts: 'export default () => {};' +} + +let sandbox_output = tg run $path --sandbox +snapshot $sandbox_output '' + +let output = tg run $path +assert equal $output $sandbox_output diff --git a/packages/cli/tests/run/return_value.nu b/packages/cli/tests/run/return_value.nu new file mode 100644 index 000000000..267e8a1bd --- /dev/null +++ b/packages/cli/tests/run/return_value.nu @@ -0,0 +1,13 @@ +use ../../test.nu * + +let server = spawn + +let path = artifact { + tangram.ts: 'export default () => "hello, world!";' +} + +let sandbox_output = tg run $path --sandbox +snapshot $sandbox_output '"hello, world!"' + +let output = tg run $path +assert equal $output $sandbox_output diff --git a/packages/cli/tests/run/subpath_reference.nu b/packages/cli/tests/run/subpath_reference.nu new file mode 100644 index 000000000..23ae08e7a --- /dev/null +++ b/packages/cli/tests/run/subpath_reference.nu @@ -0,0 +1,19 @@ +use ../../test.nu * + +# Test running a specific file within a package using a subpath reference. + +let server = spawn + +let path = artifact { + tangram.ts: 'export default () => "from root";', + sub.tg.ts: 'export default () => "from sub";', +} + +let id = tg checkin $path +tg index + +let sandbox_output = tg run ($id + "#sub.tg.ts") --sandbox +snapshot $sandbox_output '"from sub"' + +let output = tg run ($id + "#sub.tg.ts") +assert equal $output $sandbox_output diff --git a/packages/cli/tests/run/trailing_args.nu b/packages/cli/tests/run/trailing_args.nu new file mode 100644 index 000000000..9bb823527 --- /dev/null +++ b/packages/cli/tests/run/trailing_args.nu @@ -0,0 +1,13 @@ +use ../../test.nu * + +let server = spawn + +let path = artifact { + tangram.ts: 'export default (name: string) => `Hello, ${name}!`;' +} + +let sandbox_output = tg run $path --sandbox -- Tangram +snapshot $sandbox_output '"Hello, Tangram!"' + +let output = tg run $path -- Tangram +assert equal $output $sandbox_output diff --git a/packages/cli/tests/run/verbose.nu b/packages/cli/tests/run/verbose.nu new file mode 100644 index 000000000..f8dcf9126 --- /dev/null +++ b/packages/cli/tests/run/verbose.nu @@ -0,0 +1,17 @@ +use ../../test.nu * + +# When --verbose is set, `tg run` should print the full wait output as JSON. + +let server = spawn + +let path = artifact { + tangram.ts: 'export default () => "hello";' +} + +let sandbox_output = tg run --verbose $path --sandbox | from json +print $sandbox_output +assert equal $sandbox_output.exit 0 + +let output = tg run --verbose $path | from json +print $output +assert equal $output.exit 0 diff --git a/packages/cli/tests/run/verbose_with_error.nu b/packages/cli/tests/run/verbose_with_error.nu new file mode 100644 index 000000000..9a6b0f395 --- /dev/null +++ b/packages/cli/tests/run/verbose_with_error.nu @@ -0,0 +1,25 @@ +use ../../test.nu * + +# When --verbose is set and the process fails, `tg run` should print the full wait output as JSON including the error. + +let server = spawn + +let path = artifact { + tangram.ts: ' + export default () => { + throw new Error("verbose error"); + }; + ' +} + +let sandbox_output = tg run --verbose $path --sandbox | complete +success $sandbox_output +let sandbox_json = $sandbox_output.stdout | from json +assert ($sandbox_json.error != null) +assert ($sandbox_json.exit == 0) + +let output = tg run --verbose $path | complete +success $output +let json = $output.stdout | from json +assert ($json.error != null) +assert ($json.exit == 0) From 7cee0ac75382bbacb2fac84c1cba825fa96ea660 Mon Sep 17 00:00:00 2001 From: Mike Hilgendorf Date: Mon, 2 Mar 2026 17:16:54 -0600 Subject: [PATCH 3/6] wip, js/client --- packages/cli/src/build.rs | 4 +- packages/cli/src/process/spawn.rs | 10 +- packages/cli/src/run.rs | 51 +-- packages/client/Cargo.toml | 1 + packages/client/src/run.rs | 397 ++++++++++++++++++++- packages/clients/js/src/build.ts | 37 +- packages/clients/js/src/handle.ts | 2 +- packages/clients/js/src/index.ts | 3 +- packages/clients/js/src/process.ts | 48 ++- packages/clients/js/src/run.ts | 37 +- packages/js/src/main.ts | 8 + packages/js/src/quickjs.rs | 4 +- packages/js/src/quickjs/syscall/process.rs | 328 ++++++++++++++--- packages/js/src/syscall.ts | 2 +- packages/js/src/v8.rs | 7 +- packages/js/src/v8/syscall/process.rs | 255 ++++++++++++- 16 files changed, 1065 insertions(+), 129 deletions(-) diff --git a/packages/cli/src/build.rs b/packages/cli/src/build.rs index d5499ee30..a9273ecfc 100644 --- a/packages/cli/src/build.rs +++ b/packages/cli/src/build.rs @@ -87,9 +87,7 @@ impl Cli { checkin: spawn.checkin.clone().to_options(), ..Default::default() }; - let referent = self - .get_reference_with_arg(&reference, arg, true) - .await?; + let referent = self.get_reference_with_arg(&reference, arg, true).await?; let item = referent .item .clone() diff --git a/packages/cli/src/process/spawn.rs b/packages/cli/src/process/spawn.rs index da21f3f13..83b71e1e2 100644 --- a/packages/cli/src/process/spawn.rs +++ b/packages/cli/src/process/spawn.rs @@ -209,7 +209,15 @@ impl Cli { .ok_or_else(|| tg::error!("expected an object"))?; let referent = referent.map(|_| item); let Output { output, .. } = self - .spawn(args.options, args.reference, referent, args.trailing, None, None, None) + .spawn( + args.options, + args.reference, + referent, + args.trailing, + None, + None, + None, + ) .boxed() .await?; if args.verbose { diff --git a/packages/cli/src/run.rs b/packages/cli/src/run.rs index 1e06860bf..5238e202a 100644 --- a/packages/cli/src/run.rs +++ b/packages/cli/src/run.rs @@ -340,24 +340,20 @@ impl Cli { // Resolve the executable and render the args. let (executable, mut args) = if is_js { - let executable = tangram_util::env::current_exe().map_err( - |source| tg::error!(!source, "failed to get the current executable"), - )?; + let executable = tangram_util::env::current_exe().map_err(|source| { + tg::error!(!source, "failed to get the current executable") + })?; let subcommand = if data.host == "builtin" { "builtin" } else { "js" }; - let args = vec![ - subcommand.to_owned(), - data.executable.to_string(), - ]; + let args = vec![subcommand.to_owned(), data.executable.to_string()]; (executable, args) } else { let executable = match &data.executable { tg::command::data::Executable::Artifact(exe) => { - let mut path = - artifacts_path.join(exe.artifact.to_string()); + let mut path = artifacts_path.join(exe.artifact.to_string()); if let Some(subpath) = &exe.path { path.push(subpath); } @@ -368,17 +364,12 @@ impl Cli { }, tg::command::data::Executable::Path(exe) => exe.path.clone(), }; - let args = render_args_string( - &data.args, - &artifacts_path, - &output_path, - )?; + let args = render_args_string(&data.args, &artifacts_path, &output_path)?; (executable, args) }; // Render the command env. - let command_env = - render_env(&data.env, &artifacts_path, &output_path)?; + let command_env = render_env(&data.env, &artifacts_path, &output_path)?; // Append trailing args. args.extend(trailing); @@ -387,9 +378,9 @@ impl Cli { }, tg::Object::Directory(directory) => { - let executable = tangram_util::env::current_exe().map_err( - |source| tg::error!(!source, "failed to get the current executable"), - )?; + let executable = tangram_util::env::current_exe().map_err(|source| { + tg::error!(!source, "failed to get the current executable") + })?; let id = directory.id(); let mut args = vec!["js".to_owned(), id.to_string()]; args.extend(trailing); @@ -402,9 +393,9 @@ impl Cli { .await .map_err(|source| tg::error!(!source, "failed to get the module kind"))?; if kind.is_some() { - let tg_exe = tangram_util::env::current_exe().map_err( - |source| tg::error!(!source, "failed to get the current executable"), - )?; + let tg_exe = tangram_util::env::current_exe().map_err(|source| { + tg::error!(!source, "failed to get the current executable") + })?; let id = file.id(); let mut args = vec!["js".to_owned(), id.to_string()]; args.extend(trailing); @@ -486,9 +477,8 @@ impl Cli { // Try to read the user.tangram.error xattr. if let Ok(Some(bytes)) = xattr::get(&output_path, "user.tangram.error") { - let data = serde_json::from_slice::(&bytes).map_err( - |source| tg::error!(!source, "failed to deserialize the error xattr"), - )?; + let data = serde_json::from_slice::(&bytes) + .map_err(|source| tg::error!(!source, "failed to deserialize the error xattr"))?; error = Some( tg::Error::try_from(data) .map_err(|source| tg::error!(!source, "failed to convert the error data"))?, @@ -496,9 +486,9 @@ impl Cli { } // If no xattr output was set but the output path exists, check it in destructively. - let exists = tokio::fs::try_exists(&output_path).await.map_err( - |source| tg::error!(!source, "failed to check if the output path exists"), - )?; + let exists = tokio::fs::try_exists(&output_path) + .await + .map_err(|source| tg::error!(!source, "failed to check if the output path exists"))?; if output.is_none() && exists { let arg = tg::checkin::Arg { options: tg::checkin::Options { @@ -539,10 +529,7 @@ impl Cli { return Err(tg::error!("the process exited with code {}", exit)); } if exit >= 128 { - return Err(tg::error!( - "the process exited with signal {}", - exit - 128 - )); + return Err(tg::error!("the process exited with signal {}", exit - 128)); } Ok(output) diff --git a/packages/client/Cargo.toml b/packages/client/Cargo.toml index 38eb70799..5955a2e84 100644 --- a/packages/client/Cargo.toml +++ b/packages/client/Cargo.toml @@ -54,6 +54,7 @@ tangram_http = { workspace = true } tangram_serialize = { workspace = true } tangram_uri = { workspace = true } tangram_util = { workspace = true } +tempfile = { workspace = true } time = { workspace = true } tokio = { workspace = true } tokio-rustls = { workspace = true, optional = true } diff --git a/packages/client/src/run.rs b/packages/client/src/run.rs index a80f60032..39564b45b 100644 --- a/packages/client/src/run.rs +++ b/packages/client/src/run.rs @@ -1,6 +1,11 @@ use { crate::prelude::*, - std::{io::IsTerminal as _, path::PathBuf}, + std::{ + collections::BTreeMap, + io::IsTerminal as _, + path::{Path, PathBuf}, + sync::LazyLock, + }, }; #[derive(Clone, Debug, Default)] @@ -166,3 +171,393 @@ where .map_err(|source| tg::error!(!source, "failed to get the process output"))?; Ok(output) } + +pub async fn run2(handle: &H, arg: Arg) -> tg::Result +where + H: tg::Handle, +{ + let host = arg + .host + .clone() + .or_else(|| { + if cfg!(target_os = "linux") { + #[cfg(target_arch = "aarch64")] + return Some("aarch64-linux".into()); + + #[cfg(target_arch = "x86_64")] + return Some("x86_64-linux".into()); + } + if cfg!(target_os = "macos") { + #[cfg(target_arch = "aarch64")] + return Some("aarch64-macos".into()); + + #[cfg(target_arch = "x86_64")] + return Some("x86_64-macos".into()); + } + None + }) + .ok_or_else(|| tg::error!("unknown host"))?; + if needs_sandbox(&arg) { + run_sandboxed(handle, host, arg).await + } else { + run_unsandboxed(handle, host, arg).await + } +} + +async fn run_sandboxed(handle: &H, host: String, arg: Arg) -> tg::Result +where + H: tg::Handle, +{ + // TODO: handle signals + let state = if let Some(process) = tg::Process::current()? { + Some(process.load(handle).await?) + } else { + None + }; + let command = if let Some(state) = &state { + Some(state.command.object(handle).await?) + } else { + None + }; + + // Get the executable. + let executable = arg + .executable + .ok_or_else(|| tg::error!("expected the executable to be set"))?; + + let mut builder = tg::Command::builder(host, executable); + builder = builder.args(arg.args); + let cwd = if let Some(command) = &command { + if command.cwd.is_some() { + let cwd = std::env::current_dir() + .map_err(|source| tg::error!(!source, "failed to get the current directory"))?; + Some(cwd) + } else { + None + } + } else { + let cwd = std::env::current_dir() + .map_err(|source| tg::error!(!source, "failed to get the current directory"))?; + Some(cwd) + }; + let cwd = arg.cwd.or(cwd); + builder = builder.cwd(cwd); + let mut env = if let Some(command) = &command { + command.env.clone() + } else { + std::env::vars() + .map(|(key, value)| (key, value.into())) + .collect() + }; + env.remove("TANGRAM_OUTPUT"); + env.remove("TANGRAM_PROCESS"); + env.remove("TANGRAM_URL"); + builder = builder.env(env); + let mut command_mounts = vec![]; + let mut process_mounts = vec![]; + if let Some(mounts) = arg.mounts { + for mount in mounts { + match mount { + tg::Either::Left(mount) => process_mounts.push(mount.to_data()), + tg::Either::Right(mount) => command_mounts.push(mount), + } + } + } else { + if let Some(mounts) = command.as_ref().map(|command| command.mounts.clone()) { + command_mounts = mounts; + } + if let Some(mounts) = state.as_ref().map(|state| state.mounts.clone()) { + process_mounts = mounts.iter().map(tg::process::Mount::to_data).collect(); + } + } + builder = builder.mounts(command_mounts); + let stdin = if arg.stdin.is_none() { + command + .as_ref() + .map(|command| command.stdin.clone()) + .unwrap_or_default() + } else if let Some(Some(tg::Either::Right(blob))) = &arg.stdin { + Some(blob.clone()) + } else { + None + }; + builder = builder.stdin(stdin); + if let Some(Some(user)) = command.as_ref().map(|command| command.user.clone()) { + builder = builder.user(user); + } + let command = builder.build(); + let command_id = command.store(handle).await?; + let mut command = tg::Referent::with_item(command_id); + if let Some(name) = arg.name { + command.options.name.replace(name); + } + let checksum = arg.checksum; + let network = arg + .network + .or(state.as_ref().map(|state| state.network)) + .unwrap_or_default(); + let stderr = arg + .stderr + .unwrap_or_else(|| state.as_ref().and_then(|state| state.stderr.clone())); + let stdin = arg.stdin.unwrap_or_else(|| { + state + .as_ref() + .and_then(|state| state.stdin.clone().map(tg::Either::Left)) + }); + let stdin = match stdin { + None => None, + Some(tg::Either::Left(stdio)) => Some(stdio), + Some(tg::Either::Right(_)) => { + return Err(tg::error!("expected stdio")); + }, + }; + let stdout = arg + .stdout + .unwrap_or_else(|| state.as_ref().and_then(|state| state.stdout.clone())); + if network && checksum.is_none() { + return Err(tg::error!( + "a checksum is required to build with network enabled" + )); + } + let progress = arg.progress; + let arg = tg::process::spawn::Arg { + cached: arg.cached, + checksum, + command, + local: None, + mounts: process_mounts, + network, + parent: arg.parent, + remotes: arg.remote.map(|r| vec![r]), + retry: arg.retry, + stderr, + stdin, + stdout, + }; + let stream = tg::Process::spawn(handle, arg).await?; + let writer = std::io::stderr(); + let is_tty = progress && writer.is_terminal(); + let process = tg::progress::write_progress_stream(handle, stream, writer, is_tty) + .await + .map_err(|source| tg::error!(!source, "failed to spawn the process"))?; + let output = process + .output(handle) + .await + .map_err(|source| tg::error!(!source, "failed to get the process output"))?; + Ok(output) +} + +async fn run_unsandboxed(handle: &H, host: String, arg: Arg) -> tg::Result +where + H: tg::Handle, +{ + // Create the output + let temp = tempfile::TempDir::new() + .map_err(|source| tg::error!(!source, "failed to get a temp directory"))?; + tokio::fs::create_dir_all(temp.path()) + .await + .map_err(|source| tg::error!(!source, "failed to create the output directory"))?; + let output_path = temp.path().join("output"); + + // Get the executable. + let executable = arg + .executable + .ok_or_else(|| tg::error!("expected the executable to be set"))? + .to_data(); + + let mut args = Vec::new(); + let mut env = std::env::vars().collect::>(); + + env.remove("TANGRAM_OUTPUT"); + env.remove("TANGRAM_PROCESS"); + env.remove("TANGRAM_URL"); + + // Render the executable. + let executable = match host.as_str() { + "builtin" => { + // TODO: don't assume the executable path. + let tg = tangram_util::env::current_exe() + .map_err(|source| tg::error!(!source, "failed to get the current executable"))?; + args.insert(0, "builtin".to_owned()); + args.insert(1, executable.to_string()); + tg + }, + + "js" => { + // TODO: don't assume the executable path. + let tg = tangram_util::env::current_exe() + .map_err(|source| tg::error!(!source, "failed to get the current executable"))?; + args.insert(0, "js".to_owned()); + args.insert(1, executable.to_string()); + tg + }, + + _ => match executable { + tg::command::data::Executable::Artifact(executable) => { + let mut path = CLOSEST_ARTIFACT_PATH.join(executable.artifact.to_string()); + if let Some(executable_path) = &executable.path { + path.push(executable_path); + } + path + }, + tg::command::data::Executable::Module(_) => { + return Err(tg::error!("invalid executable")); + }, + tg::command::data::Executable::Path(executable) => executable.path.clone(), + }, + }; + + // Render the args. + args.extend(render_args(&arg.args, &output_path)?); + env.extend(render_env(&arg.env, &output_path)?); + + // Run the process. + let result = tokio::process::Command::new(executable) + .args(args) + .envs(env) + .stdin(std::process::Stdio::inherit()) + .stdout(std::process::Stdio::inherit()) + .stderr(std::process::Stdio::inherit()) + .output() + .await + .map_err(|source| tg::error!(!source, "failed to run the process"))?; + + if !result.status.success() { + return Err(tg::error!("the process failed")); + } + + // Check in the output if it exists. + let exists = tokio::fs::try_exists(&output_path) + .await + .map_err(|source| tg::error!(!source, "failed to check if the output path exists"))?; + if exists { + let arg = tg::checkin::Arg { + options: tg::checkin::Options { + destructive: true, + deterministic: true, + ignore: false, + lock: None, + locked: true, + root: true, + ..Default::default() + }, + path: output_path, + updates: Vec::new(), + }; + let artifact = tg::checkin::checkin(handle, arg) + .await + .map_err(|source| tg::error!(!source, "failed to check in the output"))?; + Ok(artifact.into()) + } else { + Ok(tg::Value::Null) + } +} + +fn needs_sandbox(arg: &Arg) -> bool { + if arg.cached.is_some_and(|cached| cached) { + return true; + } + if arg.checksum.is_some() { + return true; + } + if arg.mounts.as_ref().is_some_and(|m| !m.is_empty()) { + return true; + } + if arg.network.is_some_and(|network| !network) { + return true; + } + if arg.remote.is_some() { + return true; + } + false +} + +pub fn render_args(args: &[tg::Value], output_path: &Path) -> tg::Result> { + args.iter() + .map(|value| render_value(value, output_path)) + .collect::>>() +} + +pub fn render_env( + env: &tg::value::Map, + output_path: &Path, +) -> tg::Result> { + let mut output = tg::value::Map::new(); + for (key, value) in env { + let mutation = match value { + tg::Value::Mutation(value) => value.clone(), + value => tg::Mutation::Set { + value: Box::new(value.clone()), + }, + }; + mutation.apply(&mut output, key)?; + } + let output = output + .iter() + .map(|(key, value)| { + let key = key.clone(); + let value = render_value(value, output_path)?; + Ok::<_, tg::Error>((key, value)) + }) + .collect::>()?; + Ok(output) +} + +pub fn render_value(value: &tg::Value, output_path: &Path) -> tg::Result { + let artifacts_path = CLOSEST_ARTIFACT_PATH.clone(); + match value { + tg::Value::String(string) => Ok(string.clone()), + tg::Value::Template(template) => template.try_render_sync(|component| match component { + tg::template::Component::String(string) => Ok(string.clone().into()), + tg::template::Component::Artifact(artifact) => Ok(artifacts_path + .join(artifact.id().to_string()) + .to_str() + .unwrap() + .to_owned() + .into()), + tg::template::Component::Placeholder(placeholder) => { + if placeholder.name == "output" { + Ok(output_path.to_str().unwrap().to_owned().into()) + } else { + Err(tg::error!( + name = %placeholder.name, + "invalid placeholder" + )) + } + }, + }), + tg::Value::Placeholder(placeholder) => { + if placeholder.name == "output" { + Ok(output_path.to_str().unwrap().to_owned()) + } else { + Err(tg::error!( + name = %placeholder.name, + "invalid placeholder" + )) + } + }, + _ => Ok(value.to_string()), + } +} + +pub static CLOSEST_ARTIFACT_PATH: LazyLock = LazyLock::new(|| { + let mut closest_artifact_path = None; + let cwd = tangram_util::env::current_exe() + .expect("Failed to get the current directory") + .canonicalize() + .expect("failed to canonicalize current directory"); + for path in cwd.ancestors().skip(1) { + let directory = path.join(".tangram/artifacts"); + if directory.exists() { + closest_artifact_path = Some(directory); + break; + } + } + if closest_artifact_path.is_none() { + let opt_path = std::path::Path::new("/opt/tangram/artifacts"); + if opt_path.exists() { + closest_artifact_path.replace(opt_path.to_owned()); + } + } + closest_artifact_path.expect("Failed to find the closest artifact path") +}); diff --git a/packages/clients/js/src/build.ts b/packages/clients/js/src/build.ts index 5544aabc6..efdae01f6 100644 --- a/packages/clients/js/src/build.ts +++ b/packages/clients/js/src/build.ts @@ -121,8 +121,13 @@ async function inner(...args: tg.Args): Promise { stdin: undefined, stdout: undefined, }); + let id = + typeof spawnOutput.process === "string" ? spawnOutput.process : undefined; + let pid = + typeof spawnOutput.process === "number" ? spawnOutput.process : undefined; let process = new tg.Process({ - id: spawnOutput.process, + id, + pid, remote: spawnOutput.remote, state: undefined, token: spawnOutput.token, @@ -139,9 +144,13 @@ async function inner(...args: tg.Args): Promise { item: error, options: sourceOptions, }; - const values: { [key: string]: string } = { - id: process.id, - }; + const values: { [key: string]: string } = {}; + if (process.id !== undefined) { + values.id = process.id; + } + if (process.pid !== undefined) { + values.pid = process.pid.toString(); + } if (sourceOptions.name !== undefined) { values.name = sourceOptions.name; } @@ -156,9 +165,13 @@ async function inner(...args: tg.Args): Promise { item: error, options: sourceOptions, }; - const values: { [key: string]: string } = { - id: process.id, - }; + const values: { [key: string]: string } = {}; + if (process.id !== undefined) { + values.id = process.id; + } + if (process.pid !== undefined) { + values.pid = process.pid.toString(); + } if (sourceOptions.name !== undefined) { values.name = sourceOptions.name; } @@ -173,9 +186,13 @@ async function inner(...args: tg.Args): Promise { item: error, options: sourceOptions, }; - const values: { [key: string]: string } = { - id: process.id, - }; + const values: { [key: string]: string } = {}; + if (process.id !== undefined) { + values.id = process.id; + } + if (process.pid !== undefined) { + values.pid = process.pid.toString(); + } if (sourceOptions.name !== undefined) { values.name = sourceOptions.name; } diff --git a/packages/clients/js/src/handle.ts b/packages/clients/js/src/handle.ts index db9c58114..69929fb09 100644 --- a/packages/clients/js/src/handle.ts +++ b/packages/clients/js/src/handle.ts @@ -40,7 +40,7 @@ export namespace Handle { }; export type SpawnOutput = { - process: tg.Process.Id; + process: tg.Process.Id | number; remote: string | undefined; token: string | undefined; wait: tg.Process.Wait.Data | undefined; diff --git a/packages/clients/js/src/index.ts b/packages/clients/js/src/index.ts index c44a2ed26..211d97c72 100644 --- a/packages/clients/js/src/index.ts +++ b/packages/clients/js/src/index.ts @@ -28,7 +28,7 @@ import { Mutation, mutation } from "./mutation.ts"; import { Object } from "./object.ts"; import { path } from "./path.ts"; import { output, Placeholder, placeholder } from "./placeholder.ts"; -import { Process, process, setProcess } from "./process.ts"; +import { Process, process, setProcess, waitpid } from "./process.ts"; import type { Range } from "./range.ts"; import type { Reference } from "./reference.ts"; import { Referent } from "./referent.ts"; @@ -132,4 +132,5 @@ export { todo, unimplemented, unreachable, + waitpid, }; diff --git a/packages/clients/js/src/process.ts b/packages/clients/js/src/process.ts index 0528c064d..97ba00460 100644 --- a/packages/clients/js/src/process.ts +++ b/packages/clients/js/src/process.ts @@ -11,16 +11,20 @@ export let setProcess = (newProcess: typeof process) => { Object.assign(process, newProcess); }; +export let waitpid: (pid: number) => Promise; + export class Process { static current: tg.Process | undefined; - #id: tg.Process.Id; + #id: tg.Process.Id | undefined; + #pid: number | undefined; #remote: string | undefined; #token: string | undefined; #state: tg.Process.State | undefined; constructor(arg: tg.Process.ConstructorArg) { this.#id = arg.id; + this.#pid = arg.pid; this.#remote = arg.remote; this.#state = arg.state; } @@ -30,18 +34,26 @@ export class Process { } async wait(): Promise { - let remotes = undefined; - if (this.#remote) { - remotes = [this.#remote]; + if (this.#id) { + let remotes = undefined; + if (this.#remote) { + remotes = [this.#remote]; + } + let arg = { + local: undefined, + remotes, + token: this.#token, + }; + let data = await tg.handle.waitProcess(this.#id, arg); + let output = tg.Process.Wait.fromData(data); + return output; + } + if (this.#pid) { + let data = await tg.waitpid(this.#pid); + let output = tg.Process.Wait.fromData(data); + return output; } - let arg = { - local: undefined, - remotes, - token: this.#token, - }; - let data = await tg.handle.waitProcess(this.#id, arg); - let output = tg.Process.Wait.fromData(data); - return output; + throw new Error("expected a process id or pid"); } static expect(value: unknown): tg.Process { @@ -54,6 +66,9 @@ export class Process { } async load(): Promise { + if (!this.#id) { + throw new Error("expected the process id to be set"); + } let data = await tg.handle.getProcess(this.#id, this.#remote); this.#state = tg.Process.State.fromData(data); } @@ -62,10 +77,14 @@ export class Process { await this.load(); } - get id(): tg.Process.Id { + get id(): tg.Process.Id | undefined { return this.#id; } + get pid(): number | undefined { + return this.#pid; + } + get command(): Promise { return (async () => { await this.load(); @@ -138,7 +157,8 @@ export namespace Process { export type Id = string; export type ConstructorArg = { - id: tg.Process.Id; + id?: tg.Process.Id | undefined; + pid?: number | undefined; remote?: string | undefined; state?: State | undefined; token?: string | undefined; diff --git a/packages/clients/js/src/run.ts b/packages/clients/js/src/run.ts index 82d4aea0b..301c498e1 100644 --- a/packages/clients/js/src/run.ts +++ b/packages/clients/js/src/run.ts @@ -161,8 +161,13 @@ async function inner(...args: tg.Args): Promise { stdin: processStdin, stdout, }); + let id = + typeof spawnOutput.process === "string" ? spawnOutput.process : undefined; + let pid = + typeof spawnOutput.process === "number" ? spawnOutput.process : undefined; let process = new tg.Process({ - id: spawnOutput.process, + id, + pid, remote: spawnOutput.remote, state: undefined, token: spawnOutput.token, @@ -179,9 +184,13 @@ async function inner(...args: tg.Args): Promise { item: error, options: sourceOptions, }; - const values: { [key: string]: string } = { - id: process.id, - }; + const values: { [key: string]: string } = {}; + if (process.id !== undefined) { + values.id = process.id; + } + if (process.pid !== undefined) { + values.pid = process.pid.toString(); + } if (sourceOptions.name !== undefined) { values.name = sourceOptions.name; } @@ -196,9 +205,13 @@ async function inner(...args: tg.Args): Promise { item: error, options: sourceOptions, }; - const values: { [key: string]: string } = { - id: process.id, - }; + const values: { [key: string]: string } = {}; + if (process.id !== undefined) { + values.id = process.id; + } + if (process.pid !== undefined) { + values.pid = process.pid.toString(); + } if (sourceOptions.name !== undefined) { values.name = sourceOptions.name; } @@ -213,9 +226,13 @@ async function inner(...args: tg.Args): Promise { item: error, options: sourceOptions, }; - const values: { [key: string]: string } = { - id: process.id, - }; + const values: { [key: string]: string } = {}; + if (process.id !== undefined) { + values.id = process.id; + } + if (process.pid !== undefined) { + values.pid = process.pid.toString(); + } if (sourceOptions.name !== undefined) { values.name = sourceOptions.name; } diff --git a/packages/js/src/main.ts b/packages/js/src/main.ts index e44e97d5d..cbeedcd96 100644 --- a/packages/js/src/main.ts +++ b/packages/js/src/main.ts @@ -22,3 +22,11 @@ Object.defineProperties(globalThis, { Object.defineProperty(globalThis, "start", { value: start }); tg.setHandle(handle); + +export const waitpid = async (process: number) => { + return syscall("process_wait", process, { + local: undefined, + remotes: undefined, + token: undefined, + }); +}; diff --git a/packages/js/src/quickjs.rs b/packages/js/src/quickjs.rs index c330fe367..4a2f67ec5 100644 --- a/packages/js/src/quickjs.rs +++ b/packages/js/src/quickjs.rs @@ -7,7 +7,7 @@ use { crate::{Logger, Output}, rquickjs::{self as qjs, CatchResultExt as _}, sourcemap::SourceMap, - std::{cell::RefCell, path::PathBuf, rc::Rc}, + std::{cell::RefCell, collections::BTreeMap, path::PathBuf, rc::Rc}, tangram_client::prelude::*, }; @@ -22,6 +22,7 @@ const BYTECODE: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/main.bytecode" const SOURCE_MAP: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/main.js.map")); struct State { + children: RefCell>, global_source_map: Option, logger: Logger, main_runtime_handle: tokio::runtime::Handle, @@ -110,6 +111,7 @@ where // Create the state. let state = Rc::new(State { + children: RefCell::new(BTreeMap::new()), global_source_map: SourceMap::from_slice(SOURCE_MAP).ok(), logger, main_runtime_handle, diff --git a/packages/js/src/quickjs/syscall/process.rs b/packages/js/src/quickjs/syscall/process.rs index 8921e14b5..6e0f23fd2 100644 --- a/packages/js/src/quickjs/syscall/process.rs +++ b/packages/js/src/quickjs/syscall/process.rs @@ -3,9 +3,48 @@ use { crate::quickjs::{StateHandle, serde::Serde}, futures::{StreamExt as _, TryStreamExt as _, future}, rquickjs as qjs, + std::collections::BTreeMap, tangram_client::prelude::*, }; +/// A child process stored in state, awaiting wait(). +pub(crate) struct ChildProcess { + pub child: tokio::process::Child, +} + +#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)] +struct SpawnOutput { + process: tg::Either, + + #[serde(default, skip_serializing_if = "Option::is_none")] + remote: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + token: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + wait: Option, +} + +#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)] +struct WaitOutput { + #[serde(default, skip_serializing_if = "Option::is_none")] + error: Option>, + + exit: u8, + + #[serde(default, skip_serializing_if = "Option::is_none")] + output: Option, + + // TODO: Handle stdout/stderr for unsandboxed processes. + #[serde(default, skip_serializing_if = "Option::is_none")] + stdout: Option>, + + // TODO: Handle stderr for unsandboxed processes. + #[serde(default, skip_serializing_if = "Option::is_none")] + stderr: Option>, +} + pub async fn get( ctx: qjs::Ctx<'_>, id: Serde, @@ -35,63 +74,258 @@ pub async fn get( pub async fn spawn( ctx: qjs::Ctx<'_>, arg: Serde, -) -> Result> { +) -> Result> { let state = ctx.userdata::().unwrap().clone(); let Serde(arg) = arg; - let result = async { - let stream = state - .main_runtime_handle - .spawn({ - let handle = state.handle.clone(); - async move { - let stream = handle - .try_spawn_process(arg) - .await? - .and_then(|event| { - let result = event.try_map_output( - |output: Option| { - output.ok_or_else(|| tg::error!("expected a process")) - }, - ); - future::ready(result) - }) - .boxed(); - Ok::<_, tg::Error>(stream) - } - }) - .await - .map_err(|source| tg::error!(!source, "the task panicked"))??; - let writer = super::log::Writer::new(state.clone(), tg::process::log::Stream::Stderr); - let handle = state.handle.clone(); - let output = tg::progress::write_progress_stream(&handle, stream, writer, false).await?; - Ok(output) - } - .await; + let result = if needs_sandbox(&arg) { + spawn_sandboxed(state, arg).await + } else { + spawn_unsandboxed(state, arg).await + }; Result(result.map(Serde)) } +async fn spawn_sandboxed( + state: StateHandle, + arg: tg::process::spawn::Arg, +) -> tg::Result { + let stream = state + .main_runtime_handle + .spawn({ + let handle = state.handle.clone(); + async move { + let stream = handle + .try_spawn_process(arg) + .await? + .and_then(|event| { + let result = + event.try_map_output(|output: Option| { + output.ok_or_else(|| tg::error!("expected a process")) + }); + future::ready(result) + }) + .boxed(); + Ok::<_, tg::Error>(stream) + } + }) + .await + .map_err(|source| tg::error!(!source, "the task panicked"))??; + let writer = super::log::Writer::new(state.clone(), tg::process::log::Stream::Stderr); + let handle = state.handle.clone(); + let output = tg::progress::write_progress_stream(&handle, stream, writer, false).await?; + let spawn_output = SpawnOutput { + process: tg::Either::Left( + output + .process + .ok_or_else(|| tg::error!("expected a process ID"))?, + ), + remote: output.remote, + token: output.token, + wait: output.wait.map(|w| WaitOutput { + error: w.error, + exit: w.exit, + output: w.output, + stdout: None, + stderr: None, + }), + }; + Ok(spawn_output) +} + +async fn spawn_unsandboxed( + state: StateHandle, + arg: tg::process::spawn::Arg, +) -> tg::Result { + // Get the command data. + let command_id = arg.command.item.clone(); + let handle = state.handle.clone(); + let command_data = state + .main_runtime_handle + .spawn(async move { + let command = tg::Command::with_id(command_id); + command.data(&handle).await + }) + .await + .map_err(|source| tg::error!(!source, "the task panicked"))? + .map_err(|source| tg::error!(!source, "failed to get the command data"))?; + + // Render the executable. + let host = command_data.host.clone(); + let mut args = Vec::new(); + let mut env = std::env::vars().collect::>(); + + env.remove("TANGRAM_OUTPUT"); + env.remove("TANGRAM_PROCESS"); + env.remove("TANGRAM_URL"); + + let executable = match host.as_str() { + "builtin" => { + let tg_exe = tangram_util::env::current_exe() + .map_err(|source| tg::error!(!source, "failed to get the current executable"))?; + args.insert(0, "builtin".to_owned()); + args.insert(1, command_data.executable.to_string()); + tg_exe + }, + + "js" => { + let tg_exe = tangram_util::env::current_exe() + .map_err(|source| tg::error!(!source, "failed to get the current executable"))?; + args.insert(0, "js".to_owned()); + args.insert(1, command_data.executable.to_string()); + tg_exe + }, + + _ => match &command_data.executable { + tg::command::data::Executable::Artifact(executable) => { + let mut path = tg::run::CLOSEST_ARTIFACT_PATH.join(executable.artifact.to_string()); + if let Some(executable_path) = &executable.path { + path.push(executable_path); + } + path + }, + tg::command::data::Executable::Module(_) => { + return Err(tg::error!("invalid executable")); + }, + tg::command::data::Executable::Path(executable) => executable.path.clone(), + }, + }; + + // Create a temporary output directory. + let temp = tempfile::TempDir::new() + .map_err(|source| tg::error!(!source, "failed to create a temp directory"))?; + let output_path = temp.path().join("output"); + + // Convert data args and env to values for rendering. + let value_args: Vec = command_data + .args + .into_iter() + .map(tg::Value::try_from_data) + .collect::>()?; + let value_env: tg::value::Map = command_data + .env + .into_iter() + .map(|(k, v)| Ok::<_, tg::Error>((k, tg::Value::try_from_data(v)?))) + .collect::>()?; + + // Render the args and env. + args.extend(tg::run::render_args(&value_args, &output_path)?); + env.extend(tg::run::render_env(&value_env, &output_path)?); + + // Spawn the process with piped stdout and stderr. + let child = tokio::process::Command::new(executable) + .args(args) + .envs(env) + .stdin(std::process::Stdio::inherit()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .map_err(|source| tg::error!(!source, "failed to spawn the process"))?; + + // Get the PID. + let pid = child + .id() + .ok_or_else(|| tg::error!("failed to get the child process ID"))? as i32; + + // Store the child process in state. + state + .children + .borrow_mut() + .insert(pid, ChildProcess { child }); + + Ok(SpawnOutput { + process: tg::Either::Right(pid), + remote: None, + token: None, + wait: None, + }) +} + pub async fn wait( ctx: qjs::Ctx<'_>, - id: Serde, + id: Serde>, arg: Serde, -) -> Result> { +) -> Result> { let state = ctx.userdata::().unwrap().clone(); let Serde(id) = id; let Serde(arg) = arg; - let result = async { - let output = state - .main_runtime_handle - .spawn({ - let handle = state.handle.clone(); - async move { - let output = handle.wait_process(&id, arg).await?; - Ok::<_, tg::Error>(output) - } - }) - .await - .map_err(|source| tg::error!(!source, "the task panicked"))??; - Ok(output) - } - .await; + let result = match id { + tg::Either::Left(id) => wait_sandboxed(state, id, arg).await, + tg::Either::Right(pid) => wait_unsandboxed(state, pid).await, + }; Result(result.map(Serde)) } + +async fn wait_sandboxed( + state: StateHandle, + id: tg::process::Id, + arg: tg::process::wait::Arg, +) -> tg::Result { + let output = state + .main_runtime_handle + .spawn({ + let handle = state.handle.clone(); + async move { + let output = handle.wait_process(&id, arg).await?; + Ok::<_, tg::Error>(output) + } + }) + .await + .map_err(|source| tg::error!(!source, "the task panicked"))??; + Ok(WaitOutput { + error: output.error, + exit: output.exit, + output: output.output, + stdout: None, + stderr: None, + }) +} + +async fn wait_unsandboxed(state: StateHandle, pid: i32) -> tg::Result { + // Remove the child process from state. + let mut child = state + .children + .borrow_mut() + .remove(&pid) + .ok_or_else(|| tg::error!(%pid, "unknown child process"))? + .child; + + // Wait for the child process to complete. + let output = child + .wait_with_output() + .await + .map_err(|source| tg::error!(!source, "failed to wait for the child process"))?; + + // Get the exit code. + let exit = output.status.code().map(|c| c as u8).unwrap_or(128); + + Ok(WaitOutput { + error: None, + exit, + output: None, + stdout: Some(output.stdout), + stderr: Some(output.stderr), + }) +} + +fn needs_sandbox(arg: &tg::process::spawn::Arg) -> bool { + if arg.cached.is_some_and(|cached| cached) { + return true; + } + if arg.checksum.is_some() { + return true; + } + if !arg.mounts.is_empty() { + return true; + } + if !arg.network { + return true; + } + if arg + .remotes + .as_ref() + .is_some_and(|remotes| !remotes.is_empty()) + { + return true; + } + false +} diff --git a/packages/js/src/syscall.ts b/packages/js/src/syscall.ts index 908045107..a6753a7dc 100644 --- a/packages/js/src/syscall.ts +++ b/packages/js/src/syscall.ts @@ -73,7 +73,7 @@ declare global { function syscall( syscall: "process_wait", - id: tg.Process.Id, + id: tg.Process.Id | number, arg: tg.Handle.WaitArg, ): Promise; diff --git a/packages/js/src/v8.rs b/packages/js/src/v8.rs index c1528c328..ad065009a 100644 --- a/packages/js/src/v8.rs +++ b/packages/js/src/v8.rs @@ -12,7 +12,10 @@ use { stream::FuturesUnordered, }, sourcemap::SourceMap, - std::{cell::RefCell, future::poll_fn, path::PathBuf, pin::pin, rc::Rc, task::Poll}, + std::{ + cell::RefCell, collections::BTreeMap, future::poll_fn, path::PathBuf, pin::pin, rc::Rc, + task::Poll, + }, tangram_client::prelude::*, tangram_v8::{Deserialize as _, Serde, Serialize as _}, }; @@ -26,6 +29,7 @@ const SNAPSHOT: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/main.heapsnaps const SOURCE_MAP: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/main.js.map")); struct State { + children: RefCell>, promises: RefCell>>, global_source_map: Option, logger: Logger, @@ -90,6 +94,7 @@ where // Create the state. let (rejection, _) = tokio::sync::watch::channel(None); let state = Rc::new(State { + children: RefCell::new(BTreeMap::new()), promises: RefCell::new(FuturesUnordered::new()), global_source_map: Some(SourceMap::from_slice(SOURCE_MAP).unwrap()), logger, diff --git a/packages/js/src/v8/syscall/process.rs b/packages/js/src/v8/syscall/process.rs index 07c3f1d8c..4d909339d 100644 --- a/packages/js/src/v8/syscall/process.rs +++ b/packages/js/src/v8/syscall/process.rs @@ -1,11 +1,49 @@ use { super::State, futures::{StreamExt as _, TryStreamExt as _, future}, - std::rc::Rc, + std::{collections::BTreeMap, rc::Rc}, tangram_client::prelude::*, tangram_v8::Serde, }; +/// A child process stored in state, awaiting wait(). +pub(crate) struct ChildProcess { + pub child: tokio::process::Child, +} + +#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)] +struct SpawnOutput { + process: tg::Either, + + #[serde(default, skip_serializing_if = "Option::is_none")] + remote: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + token: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + wait: Option, +} + +#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)] +struct WaitOutput { + #[serde(default, skip_serializing_if = "Option::is_none")] + error: Option>, + + exit: u8, + + #[serde(default, skip_serializing_if = "Option::is_none")] + output: Option, + + // TODO: Handle stdout/stderr for unsandboxed processes. + #[serde(default, skip_serializing_if = "Option::is_none")] + stdout: Option>, + + // TODO: Handle stderr for unsandboxed processes. + #[serde(default, skip_serializing_if = "Option::is_none")] + stderr: Option>, +} + pub async fn get( state: Rc, args: (Serde, Option), @@ -27,8 +65,19 @@ pub async fn get( pub async fn spawn( state: Rc, args: (Serde,), -) -> tg::Result> { +) -> tg::Result> { let (Serde(arg),) = args; + if needs_sandbox(&arg) { + spawn_sandboxed(state, arg).await + } else { + spawn_unsandboxed(state, arg).await + } +} + +async fn spawn_sandboxed( + state: Rc, + arg: tg::process::spawn::Arg, +) -> tg::Result> { let stream = state .main_runtime_handle .spawn({ @@ -53,14 +102,152 @@ pub async fn spawn( let writer = super::log::Writer::new(state.clone(), tg::process::log::Stream::Stderr); let handle = state.handle.clone(); let output = tg::progress::write_progress_stream(&handle, stream, writer, false).await?; - Ok(Serde(output)) + let spawn_output = SpawnOutput { + process: tg::Either::Left( + output + .process + .ok_or_else(|| tg::error!("expected a process ID"))?, + ), + remote: output.remote, + token: output.token, + wait: output.wait.map(|w| WaitOutput { + error: w.error, + exit: w.exit, + output: w.output, + stdout: None, + stderr: None, + }), + }; + Ok(Serde(spawn_output)) +} + +async fn spawn_unsandboxed( + state: Rc, + arg: tg::process::spawn::Arg, +) -> tg::Result> { + // Get the command data. + let command_id = arg.command.item.clone(); + let handle = state.handle.clone(); + let command_data = state + .main_runtime_handle + .spawn(async move { + let command = tg::Command::with_id(command_id); + command.data(&handle).await + }) + .await + .unwrap() + .map_err(|source| tg::error!(!source, "failed to get the command data"))?; + + // Render the executable. + let host = command_data.host.clone(); + let mut args = Vec::new(); + let mut env = std::env::vars().collect::>(); + + env.remove("TANGRAM_OUTPUT"); + env.remove("TANGRAM_PROCESS"); + env.remove("TANGRAM_URL"); + + let executable = match host.as_str() { + "builtin" => { + let tg_exe = tangram_util::env::current_exe() + .map_err(|source| tg::error!(!source, "failed to get the current executable"))?; + args.insert(0, "builtin".to_owned()); + args.insert(1, command_data.executable.to_string()); + tg_exe + }, + + "js" => { + let tg_exe = tangram_util::env::current_exe() + .map_err(|source| tg::error!(!source, "failed to get the current executable"))?; + args.insert(0, "js".to_owned()); + args.insert(1, command_data.executable.to_string()); + tg_exe + }, + + _ => match &command_data.executable { + tg::command::data::Executable::Artifact(executable) => { + let mut path = tg::run::CLOSEST_ARTIFACT_PATH.join(executable.artifact.to_string()); + if let Some(executable_path) = &executable.path { + path.push(executable_path); + } + path + }, + tg::command::data::Executable::Module(_) => { + return Err(tg::error!("invalid executable")); + }, + tg::command::data::Executable::Path(executable) => executable.path.clone(), + }, + }; + + // Create a temporary output directory. + let temp = tempfile::TempDir::new() + .map_err(|source| tg::error!(!source, "failed to create a temp directory"))?; + let output_path = temp.path().join("output"); + + // Convert data args and env to values for rendering. + let value_args: Vec = command_data + .args + .into_iter() + .map(tg::Value::try_from_data) + .collect::>()?; + let value_env: tg::value::Map = command_data + .env + .into_iter() + .map(|(k, v)| Ok::<_, tg::Error>((k, tg::Value::try_from_data(v)?))) + .collect::>()?; + + // Render the args and env. + args.extend(tg::run::render_args(&value_args, &output_path)?); + env.extend(tg::run::render_env(&value_env, &output_path)?); + + // Spawn the process with piped stdout and stderr. + let child = tokio::process::Command::new(executable) + .args(args) + .envs(env) + .stdin(std::process::Stdio::inherit()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .map_err(|source| tg::error!(!source, "failed to spawn the process"))?; + + // Get the PID. + let pid = child + .id() + .ok_or_else(|| tg::error!("failed to get the child process ID"))? as i32; + + // Store the child process in state. + state + .children + .borrow_mut() + .insert(pid, ChildProcess { child }); + + Ok(Serde(SpawnOutput { + process: tg::Either::Right(pid), + remote: None, + token: None, + wait: None, + })) } pub async fn wait( state: Rc, - args: (Serde, Serde), -) -> tg::Result> { + args: ( + Serde>, + Serde, + ), +) -> tg::Result> { let (Serde(id), Serde(arg)) = args; + match id { + tg::Either::Left(id) => wait_sandboxed(state, id, arg).await, + tg::Either::Right(pid) => wait_unsandboxed(state, pid).await, + } +} + +async fn wait_sandboxed( + state: Rc, + id: tg::process::Id, + arg: tg::process::wait::Arg, +) -> tg::Result> { let handle = state.handle.clone(); let output = state .main_runtime_handle @@ -70,5 +257,61 @@ pub async fn wait( }) .await .unwrap()?; - Ok(Serde(output)) + Ok(Serde(WaitOutput { + error: output.error, + exit: output.exit, + output: output.output, + stdout: None, + stderr: None, + })) +} + +async fn wait_unsandboxed(state: Rc, pid: i32) -> tg::Result> { + // Remove the child process from state. + let mut child = state + .children + .borrow_mut() + .remove(&pid) + .ok_or_else(|| tg::error!(%pid, "unknown child process"))? + .child; + + // Wait for the child process to complete. + let output = child + .wait_with_output() + .await + .map_err(|source| tg::error!(!source, "failed to wait for the child process"))?; + + // Get the exit code. + let exit = output.status.code().map(|c| c as u8).unwrap_or(128); + + Ok(Serde(WaitOutput { + error: None, + exit, + output: None, + stdout: Some(output.stdout), + stderr: Some(output.stderr), + })) +} + +fn needs_sandbox(arg: &tg::process::spawn::Arg) -> bool { + if arg.cached.is_some_and(|cached| cached) { + return true; + } + if arg.checksum.is_some() { + return true; + } + if !arg.mounts.is_empty() { + return true; + } + if !arg.network { + return true; + } + if arg + .remotes + .as_ref() + .is_some_and(|remotes| !remotes.is_empty()) + { + return true; + } + false } From a36565f310989b770e1e3f71272c02b06c332f5f Mon Sep 17 00:00:00 2001 From: Mike Hilgendorf Date: Tue, 3 Mar 2026 13:49:48 -0600 Subject: [PATCH 4/6] wip --- packages/client/src/run.rs | 65 +++- packages/clients/js/src/handle.ts | 2 +- packages/clients/js/src/index.ts | 15 +- packages/clients/js/src/process.ts | 63 +++- packages/clients/js/src/run.ts | 75 +++- packages/js/Cargo.toml | 2 + packages/js/src/handle.ts | 4 +- packages/js/src/lib.rs | 2 + packages/js/src/main.ts | 16 +- packages/js/src/process.rs | 182 ++++++++++ packages/js/src/quickjs.rs | 2 +- packages/js/src/quickjs/syscall.rs | 14 +- packages/js/src/quickjs/syscall/process.rs | 377 ++++++--------------- packages/js/src/syscall.ts | 16 +- packages/js/src/v8.rs | 2 +- packages/js/src/v8/syscall.rs | 8 +- packages/js/src/v8/syscall/process.rs | 265 ++------------- 17 files changed, 549 insertions(+), 561 deletions(-) create mode 100644 packages/js/src/process.rs diff --git a/packages/client/src/run.rs b/packages/client/src/run.rs index 39564b45b..5ab4da4ff 100644 --- a/packages/client/src/run.rs +++ b/packages/client/src/run.rs @@ -4,7 +4,7 @@ use { collections::BTreeMap, io::IsTerminal as _, path::{Path, PathBuf}, - sync::LazyLock, + sync::{LazyLock, Mutex}, }, }; @@ -351,9 +351,9 @@ async fn run_unsandboxed(handle: &H, host: String, arg: Arg) -> tg::Result { - // TODO: don't assume the executable path. - let tg = tangram_util::env::current_exe() - .map_err(|source| tg::error!(!source, "failed to get the current executable"))?; + let exe = tangram_executable_path(); args.insert(0, "builtin".to_owned()); args.insert(1, executable.to_string()); - tg + exe }, "js" => { - // TODO: don't assume the executable path. - let tg = tangram_util::env::current_exe() - .map_err(|source| tg::error!(!source, "failed to get the current executable"))?; + let exe = tangram_executable_path(); args.insert(0, "js".to_owned()); args.insert(1, executable.to_string()); - tg + exe }, _ => match executable { tg::command::data::Executable::Artifact(executable) => { - let mut path = CLOSEST_ARTIFACT_PATH.join(executable.artifact.to_string()); + let mut path = tg::checkout(handle, tg::checkout::Arg { + artifact: executable.artifact.clone(), + dependencies: true, + extension: None, + force: false, + lock: None, + path: None, + }).await.map_err(|source| tg::error!(!source, executable = %executable.artifact, "failed to check out the artifact"))?; if let Some(executable_path) = &executable.path { path.push(executable_path); } @@ -407,7 +410,7 @@ where }, }; - // Render the args. + // Render the args and env. args.extend(render_args(&arg.args, &output_path)?); env.extend(render_env(&arg.env, &output_path)?); @@ -415,6 +418,7 @@ where let result = tokio::process::Command::new(executable) .args(args) .envs(env) + .env("TANGRAM_OUTPUT", &output_path) .stdin(std::process::Stdio::inherit()) .stdout(std::process::Stdio::inherit()) .stderr(std::process::Stdio::inherit()) @@ -504,7 +508,7 @@ pub fn render_env( } pub fn render_value(value: &tg::Value, output_path: &Path) -> tg::Result { - let artifacts_path = CLOSEST_ARTIFACT_PATH.clone(); + let artifacts_path = artifacts_path(); match value { tg::Value::String(string) => Ok(string.clone()), tg::Value::Template(template) => template.try_render_sync(|component| match component { @@ -561,3 +565,36 @@ pub static CLOSEST_ARTIFACT_PATH: LazyLock = LazyLock::new(|| { } closest_artifact_path.expect("Failed to find the closest artifact path") }); + +pub static TANGRAM_EXECUTABLE_PATH: Mutex> = Mutex::new(None); +pub static TANGRAM_ARTIFACTS_PATH: Mutex> = Mutex::new(None); + +pub fn set_artifacts_path(p: impl AsRef) { + TANGRAM_ARTIFACTS_PATH + .lock() + .unwrap() + .replace(p.as_ref().to_owned()); +} + +pub fn artifacts_path() -> PathBuf { + TANGRAM_ARTIFACTS_PATH + .lock() + .unwrap() + .clone() + .unwrap_or_else(|| CLOSEST_ARTIFACT_PATH.clone()) +} + +pub fn set_tangram_executable_path(p: impl AsRef) { + TANGRAM_EXECUTABLE_PATH + .lock() + .unwrap() + .replace(p.as_ref().to_owned()); +} + +pub fn tangram_executable_path() -> PathBuf { + TANGRAM_EXECUTABLE_PATH + .lock() + .unwrap() + .clone() + .unwrap_or_else(|| "tangram".into()) +} diff --git a/packages/clients/js/src/handle.ts b/packages/clients/js/src/handle.ts index 69929fb09..db9c58114 100644 --- a/packages/clients/js/src/handle.ts +++ b/packages/clients/js/src/handle.ts @@ -40,7 +40,7 @@ export namespace Handle { }; export type SpawnOutput = { - process: tg.Process.Id | number; + process: tg.Process.Id; remote: string | undefined; token: string | undefined; wait: tg.Process.Wait.Data | undefined; diff --git a/packages/clients/js/src/index.ts b/packages/clients/js/src/index.ts index 211d97c72..a995f9a56 100644 --- a/packages/clients/js/src/index.ts +++ b/packages/clients/js/src/index.ts @@ -28,7 +28,15 @@ import { Mutation, mutation } from "./mutation.ts"; import { Object } from "./object.ts"; import { path } from "./path.ts"; import { output, Placeholder, placeholder } from "./placeholder.ts"; -import { Process, process, setProcess, waitpid } from "./process.ts"; +import { + Process, + process, + setProcess, + setSpawnUnsandboxed, + setWaitUnsandboxed, + spawnUnsandboxed, + waitUnsandboxed, +} from "./process.ts"; import type { Range } from "./range.ts"; import type { Reference } from "./reference.ts"; import { Referent } from "./referent.ts"; @@ -126,11 +134,14 @@ export { run, setHandle, setProcess, + setSpawnUnsandboxed, + setWaitUnsandboxed, sleep, + spawnUnsandboxed, symlink, template, todo, unimplemented, unreachable, - waitpid, + waitUnsandboxed, }; diff --git a/packages/clients/js/src/process.ts b/packages/clients/js/src/process.ts index 97ba00460..2bf63bf93 100644 --- a/packages/clients/js/src/process.ts +++ b/packages/clients/js/src/process.ts @@ -11,7 +11,19 @@ export let setProcess = (newProcess: typeof process) => { Object.assign(process, newProcess); }; -export let waitpid: (pid: number) => Promise; +export let spawnUnsandboxed: (arg: tg.Handle.SpawnArg) => Promise; + +export let setSpawnUnsandboxed = (f: typeof spawnUnsandboxed) => { + spawnUnsandboxed = f; +}; + +export let waitUnsandboxed: ( + pid: number, +) => Promise; + +export let setWaitUnsandboxed = (f: typeof waitUnsandboxed) => { + waitUnsandboxed = f; +}; export class Process { static current: tg.Process | undefined; @@ -47,11 +59,10 @@ export class Process { let data = await tg.handle.waitProcess(this.#id, arg); let output = tg.Process.Wait.fromData(data); return output; - } + } if (this.#pid) { - let data = await tg.waitpid(this.#pid); - let output = tg.Process.Wait.fromData(data); - return output; + let data = await tg.waitUnsandboxed(this.#pid); + return tg.Process.Wait.fromData(data); } throw new Error("expected a process id or pid"); } @@ -348,4 +359,46 @@ export namespace Process { return output; }; } + + export type WaitUnsandboxedOutput = { + error: tg.Error | undefined; + exit: number; + output?: tg.Value; + stdout?: Uint8Array; + stderr?: Uint8Array; + }; + + export namespace WaitUnsandboxedOutput { + export type Data = { + error?: tg.Error.Data | tg.Error.Id; + exit: number; + output?: tg.Value.Data; + stdout?: Uint8Array; + stderr?: Uint8Array; + }; + + export let fromData = ( + data: tg.Process.WaitUnsandboxedOutput.Data, + ): tg.Process.WaitUnsandboxedOutput => { + let output: WaitUnsandboxedOutput = { + error: + data.error !== undefined + ? typeof data.error === "string" + ? tg.Error.withId(data.error) + : tg.Error.fromData(data.error) + : undefined, + exit: data.exit, + }; + if ("output" in data) { + output.output = tg.Value.fromData(data.output); + } + if (data.stdout !== undefined) { + output.stdout = data.stdout; + } + if (data.stderr !== undefined) { + output.stderr = data.stderr; + } + return output; + }; + } } diff --git a/packages/clients/js/src/run.ts b/packages/clients/js/src/run.ts index 301c498e1..d68048fd9 100644 --- a/packages/clients/js/src/run.ts +++ b/packages/clients/js/src/run.ts @@ -143,12 +143,13 @@ async function inner(...args: tg.Args): Promise { "network" in arg ? (arg.network ?? false) : (tg.Process.current?.state?.network ?? false); + let user = "user" in arg ? arg.user : undefined; let commandId = await command.store(); let commandReferent = { item: commandId, options: sourceOptions, }; - let spawnOutput = await tg.handle.spawnProcess({ + let spawnArg = { checksum, command: commandReferent, create: false, @@ -160,23 +161,36 @@ async function inner(...args: tg.Args): Promise { stderr, stdin: processStdin, stdout, - }); - let id = - typeof spawnOutput.process === "string" ? spawnOutput.process : undefined; - let pid = - typeof spawnOutput.process === "number" ? spawnOutput.process : undefined; - let process = new tg.Process({ - id, - pid, - remote: spawnOutput.remote, - state: undefined, - token: spawnOutput.token, + }; + let sandboxed = needsSandbox({ + checksum, + mounts: processMounts, + network, + stdout, + stderr, + user, }); - let wait = - spawnOutput.wait !== undefined - ? tg.Process.Wait.fromData(spawnOutput.wait) - : await process.wait(); + let process: tg.Process; + let wait: tg.Process.Wait; + if (sandboxed) { + let spawnOutput = await tg.handle.spawnProcess(spawnArg); + process = new tg.Process({ + id: spawnOutput.process, + remote: spawnOutput.remote, + state: undefined, + token: spawnOutput.token, + }); + wait = + spawnOutput.wait !== undefined + ? tg.Process.Wait.fromData(spawnOutput.wait) + : await process.wait(); + } else { + let pid = await tg.spawnUnsandboxed(spawnArg); + process = new tg.Process({ pid }); + let spawnOutput = await tg.waitUnsandboxed(pid); + wait = tg.Process.Wait.fromData(spawnOutput); + } if (wait.error !== undefined) { let error = wait.error; @@ -412,3 +426,32 @@ export class RunBuilder< .then(onfulfilled, onrejected); } } + +function needsSandbox(arg: { + checksum: tg.Checksum | undefined; + mounts: Array; + network: boolean; + stdout: string | undefined; + stderr: string | undefined; + user: string | undefined; +}): boolean { + if (arg.checksum !== undefined) { + return true; + } + if (arg.mounts.length > 0) { + return true; + } + if (arg.network) { + return true; + } + if (arg.stdout !== undefined) { + return true; + } + if (arg.stderr !== undefined) { + return true; + } + if (arg.user !== undefined) { + return true; + } + return false; +} diff --git a/packages/js/Cargo.toml b/packages/js/Cargo.toml index b11683cde..f115c1a81 100644 --- a/packages/js/Cargo.toml +++ b/packages/js/Cargo.toml @@ -40,7 +40,9 @@ tangram_client = { workspace = true } tangram_compiler = { workspace = true } tangram_either = { workspace = true } tangram_futures = { workspace = true } +tangram_util = {workspace = true} tangram_v8 = { workspace = true, optional = true } +tempfile = { workspace = true } tokio = { workspace = true } tokio-util = { workspace = true } toml = { workspace = true } diff --git a/packages/js/src/handle.ts b/packages/js/src/handle.ts index 3e5433593..749d8fd07 100644 --- a/packages/js/src/handle.ts +++ b/packages/js/src/handle.ts @@ -25,14 +25,14 @@ export let handle: tg.Handle = { }, spawnProcess(arg: tg.Handle.SpawnArg): Promise { - return syscall("process_spawn", arg); + return syscall("process_spawn_sandboxed", arg); }, waitProcess( id: tg.Process.Id, arg: tg.Handle.WaitArg, ): Promise { - return syscall("process_wait", id, arg); + return syscall("process_wait_sandboxed", id, arg); }, checksum( diff --git a/packages/js/src/lib.rs b/packages/js/src/lib.rs index 6cdc87a2e..b1424445b 100644 --- a/packages/js/src/lib.rs +++ b/packages/js/src/lib.rs @@ -5,6 +5,8 @@ pub mod quickjs; #[cfg(feature = "v8")] pub mod v8; +pub mod process; + #[cfg(all(feature = "quickjs", not(feature = "v8")))] pub use self::quickjs::{Abort, run}; #[cfg(feature = "v8")] diff --git a/packages/js/src/main.ts b/packages/js/src/main.ts index cbeedcd96..e406549a5 100644 --- a/packages/js/src/main.ts +++ b/packages/js/src/main.ts @@ -23,10 +23,12 @@ Object.defineProperty(globalThis, "start", { value: start }); tg.setHandle(handle); -export const waitpid = async (process: number) => { - return syscall("process_wait", process, { - local: undefined, - remotes: undefined, - token: undefined, - }); -}; +tg.setSpawnUnsandboxed(async (arg: tg.Handle.SpawnArg): Promise => { + return syscall("process_spawn_unsandboxed", arg); +}); + +tg.setWaitUnsandboxed( + async (pid: number): Promise => { + return syscall("process_wait_unsandboxed", pid); + }, +); diff --git a/packages/js/src/process.rs b/packages/js/src/process.rs new file mode 100644 index 000000000..e394cda78 --- /dev/null +++ b/packages/js/src/process.rs @@ -0,0 +1,182 @@ +use std::{collections::BTreeMap, os::unix::process::ExitStatusExt}; + +use num::ToPrimitive; +use tangram_client::prelude::*; +use tempfile::TempDir; + +#[derive(serde::Serialize, serde::Deserialize)] +pub(crate) struct WaitOutput { + pub exit: u8, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error: Option>, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub stdout: Option>, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub stderr: Option>, +} + +pub(crate) struct ChildProcess { + pub(crate) child: tokio::process::Child, + temp: TempDir, +} + +pub(crate) async fn spawn_unsandboxed( + handle: &H, + arg: tg::process::spawn::Arg, +) -> tg::Result +where + H: tg::Handle, +{ + // Get the command data. + let command_id = arg.command.item.clone(); + let command = tg::Command::with_id(command_id); + let command = command + .data(handle) + .await + .map_err(|source| tg::error!(!source, "failed to get the command data"))?; + + // Render the executable. + let host = command.host.clone(); + let mut args = Vec::new(); + let mut env = std::env::vars().collect::>(); + + let executable = match host.as_str() { + "builtin" => { + let exe = tg::run::tangram_executable_path(); + args.insert(0, "builtin".to_owned()); + args.insert(1, command.executable.to_string()); + exe + }, + + "js" => { + let exe = tg::run::tangram_executable_path(); + args.insert(0, "js".to_owned()); + args.insert(1, command.executable.to_string()); + exe + }, + + _ => match &command.executable { + tg::command::data::Executable::Artifact(executable) => { + let mut path = tg::checkout(handle,tg::checkout::Arg { + artifact: executable.artifact.clone(), + dependencies: true, + extension: None, + force: false, + lock: None, + path: None, + }).await.map_err(|source| tg::error!(!source, executable = %executable.artifact, "failed to check out the artifact"))?; + if let Some(executable_path) = &executable.path { + path.push(executable_path); + } + path + }, + tg::command::data::Executable::Module(_) => { + return Err(tg::error!("invalid executable")); + }, + tg::command::data::Executable::Path(executable) => executable.path.clone(), + }, + }; + + // Create a temp for the output. + let temp = tempfile::tempdir() + .map_err(|source| tg::error!(!source, "failed to create the temporary directory"))?; + tokio::fs::create_dir_all(temp.path()) + .await + .map_err(|source| tg::error!(!source, "failed to create the output directory"))?; + let output_path = temp.path().join("output"); + + // Convert data. + let args_: Vec = command + .args + .into_iter() + .map(tg::Value::try_from_data) + .collect::>()?; + let env_: tg::value::Map = command + .env + .into_iter() + .map(|(k, v)| Ok::<_, tg::Error>((k, tg::Value::try_from_data(v)?))) + .collect::>()?; + + // Render the args and env. + args.extend(tg::run::render_args(&args_, &output_path)?); + env.extend(tg::run::render_env(&env_, &output_path)?); + + // Spawn the process with piped stdout and stderr. + let child = tokio::process::Command::new(executable) + .args(args) + .envs(env) + .env("TANGRAM_OUTPUT", &output_path) + .stdin(std::process::Stdio::inherit()) + .stdout(std::process::Stdio::inherit()) + .stderr(std::process::Stdio::inherit()) + .spawn() + .map_err(|source| tg::error!(!source, "failed to spawn the process"))?; + + Ok(ChildProcess { child, temp }) +} + +pub(crate) async fn wait_unsandboxed(handle: &H, child: ChildProcess) -> tg::Result +where + H: tg::Handle, +{ + let output = child + .child + .wait_with_output() + .await + .map_err(|source| tg::error!(!source, "failed to wait for the output"))?; + let status = output.status; + let exit = None + .or(status.code()) + .or(status.signal().map(|signal| 128 + signal)) + .unwrap() + .to_u8() + .unwrap(); + let stdout = output.stdout; + let stderr = output.stderr; + let output_path = child.temp.path().join("output"); + let mut error = None; + let mut output = None; + if matches!(tokio::fs::try_exists(&output_path).await, Ok(true)) { + let result = tg::checkin( + handle, + tg::checkin::Arg { + path: output_path, + options: tg::checkin::Options { + destructive: true, + deterministic: true, + ignore: false, + lock: None, + locked: true, + root: true, + ..Default::default() + }, + updates: Vec::new(), + }, + ) + .await + .map_err(|source| tg::error!(!source, "failed to check in the output")); + match result { + Ok(artifact) => { + output.replace(tg::value::Data::Object(artifact.id().into())); + }, + Err(e) => { + let e = e.to_data_or_id(); + error.replace(e); + }, + } + } + let output = WaitOutput { + exit, + output, + error, + stdout: (!stdout.is_empty()).then_some(stdout), + stderr: (!stderr.is_empty()).then_some(stderr), + }; + Ok(output) +} diff --git a/packages/js/src/quickjs.rs b/packages/js/src/quickjs.rs index 4a2f67ec5..d7e14ecee 100644 --- a/packages/js/src/quickjs.rs +++ b/packages/js/src/quickjs.rs @@ -22,7 +22,7 @@ const BYTECODE: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/main.bytecode" const SOURCE_MAP: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/main.js.map")); struct State { - children: RefCell>, + children: RefCell>, global_source_map: Option, logger: Logger, main_runtime_handle: tokio::runtime::Handle, diff --git a/packages/js/src/quickjs/syscall.rs b/packages/js/src/quickjs/syscall.rs index 03981547a..f0d828ea7 100644 --- a/packages/js/src/quickjs/syscall.rs +++ b/packages/js/src/quickjs/syscall.rs @@ -12,7 +12,7 @@ mod encoding; mod log; mod magic; mod object; -mod process; +pub(crate) mod process; mod sleep; struct Result(tg::Result); @@ -52,8 +52,16 @@ pub fn syscall<'js>( "object_get" => qjs::Function::new(ctx.clone(), Async(object::get)), "object_id" => qjs::Function::new(ctx.clone(), object::id), "process_get" => qjs::Function::new(ctx.clone(), Async(process::get)), - "process_spawn" => qjs::Function::new(ctx.clone(), Async(process::spawn)), - "process_wait" => qjs::Function::new(ctx.clone(), Async(process::wait)), + "process_spawn_sandboxed" => { + qjs::Function::new(ctx.clone(), Async(process::spawn_sandboxed)) + }, + "process_wait_sandboxed" => qjs::Function::new(ctx.clone(), Async(process::wait_sandboxed)), + "process_spawn_unsandboxed" => { + qjs::Function::new(ctx.clone(), Async(process::spawn_unsandboxed)) + }, + "process_wait_unsandboxed" => { + qjs::Function::new(ctx.clone(), Async(process::wait_unsandboxed)) + }, "read" => qjs::Function::new(ctx.clone(), Async(blob::read)), "sleep" => qjs::Function::new(ctx.clone(), Async(sleep::sleep)), "write" => qjs::Function::new(ctx.clone(), Async(blob::write)), diff --git a/packages/js/src/quickjs/syscall/process.rs b/packages/js/src/quickjs/syscall/process.rs index 6e0f23fd2..55387e410 100644 --- a/packages/js/src/quickjs/syscall/process.rs +++ b/packages/js/src/quickjs/syscall/process.rs @@ -3,49 +3,10 @@ use { crate::quickjs::{StateHandle, serde::Serde}, futures::{StreamExt as _, TryStreamExt as _, future}, rquickjs as qjs, - std::collections::BTreeMap, tangram_client::prelude::*, }; -/// A child process stored in state, awaiting wait(). -pub(crate) struct ChildProcess { - pub child: tokio::process::Child, -} - -#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)] -struct SpawnOutput { - process: tg::Either, - - #[serde(default, skip_serializing_if = "Option::is_none")] - remote: Option, - - #[serde(default, skip_serializing_if = "Option::is_none")] - token: Option, - - #[serde(default, skip_serializing_if = "Option::is_none")] - wait: Option, -} - -#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)] -struct WaitOutput { - #[serde(default, skip_serializing_if = "Option::is_none")] - error: Option>, - - exit: u8, - - #[serde(default, skip_serializing_if = "Option::is_none")] - output: Option, - - // TODO: Handle stdout/stderr for unsandboxed processes. - #[serde(default, skip_serializing_if = "Option::is_none")] - stdout: Option>, - - // TODO: Handle stderr for unsandboxed processes. - #[serde(default, skip_serializing_if = "Option::is_none")] - stderr: Option>, -} - -pub async fn get( +pub(super) async fn get( ctx: qjs::Ctx<'_>, id: Serde, ) -> Result> { @@ -71,261 +32,127 @@ pub async fn get( Result(result.map(Serde)) } -pub async fn spawn( +pub(super) async fn spawn_sandboxed( ctx: qjs::Ctx<'_>, arg: Serde, -) -> Result> { +) -> Result> { let state = ctx.userdata::().unwrap().clone(); let Serde(arg) = arg; - let result = if needs_sandbox(&arg) { - spawn_sandboxed(state, arg).await - } else { - spawn_unsandboxed(state, arg).await - }; + let result = async { + let stream = state + .main_runtime_handle + .spawn({ + let handle = state.handle.clone(); + async move { + let stream = handle + .try_spawn_process(arg) + .await? + .and_then(|event| { + let result = event.try_map_output( + |output: Option| { + output.ok_or_else(|| tg::error!("expected a process")) + }, + ); + future::ready(result) + }) + .boxed(); + Ok::<_, tg::Error>(stream) + } + }) + .await + .map_err(|source| tg::error!(!source, "the task panicked"))??; + let writer = super::log::Writer::new(state.clone(), tg::process::log::Stream::Stderr); + let handle = state.handle.clone(); + let output = tg::progress::write_progress_stream(&handle, stream, writer, false).await?; + Ok(output) + } + .await; Result(result.map(Serde)) } -async fn spawn_sandboxed( - state: StateHandle, - arg: tg::process::spawn::Arg, -) -> tg::Result { - let stream = state - .main_runtime_handle - .spawn({ - let handle = state.handle.clone(); - async move { - let stream = handle - .try_spawn_process(arg) - .await? - .and_then(|event| { - let result = - event.try_map_output(|output: Option| { - output.ok_or_else(|| tg::error!("expected a process")) - }); - future::ready(result) - }) - .boxed(); - Ok::<_, tg::Error>(stream) - } - }) - .await - .map_err(|source| tg::error!(!source, "the task panicked"))??; - let writer = super::log::Writer::new(state.clone(), tg::process::log::Stream::Stderr); - let handle = state.handle.clone(); - let output = tg::progress::write_progress_stream(&handle, stream, writer, false).await?; - let spawn_output = SpawnOutput { - process: tg::Either::Left( - output - .process - .ok_or_else(|| tg::error!("expected a process ID"))?, - ), - remote: output.remote, - token: output.token, - wait: output.wait.map(|w| WaitOutput { - error: w.error, - exit: w.exit, - output: w.output, - stdout: None, - stderr: None, - }), - }; - Ok(spawn_output) -} - -async fn spawn_unsandboxed( - state: StateHandle, - arg: tg::process::spawn::Arg, -) -> tg::Result { - // Get the command data. - let command_id = arg.command.item.clone(); - let handle = state.handle.clone(); - let command_data = state - .main_runtime_handle - .spawn(async move { - let command = tg::Command::with_id(command_id); - command.data(&handle).await - }) - .await - .map_err(|source| tg::error!(!source, "the task panicked"))? - .map_err(|source| tg::error!(!source, "failed to get the command data"))?; - - // Render the executable. - let host = command_data.host.clone(); - let mut args = Vec::new(); - let mut env = std::env::vars().collect::>(); - - env.remove("TANGRAM_OUTPUT"); - env.remove("TANGRAM_PROCESS"); - env.remove("TANGRAM_URL"); - - let executable = match host.as_str() { - "builtin" => { - let tg_exe = tangram_util::env::current_exe() - .map_err(|source| tg::error!(!source, "failed to get the current executable"))?; - args.insert(0, "builtin".to_owned()); - args.insert(1, command_data.executable.to_string()); - tg_exe - }, - - "js" => { - let tg_exe = tangram_util::env::current_exe() - .map_err(|source| tg::error!(!source, "failed to get the current executable"))?; - args.insert(0, "js".to_owned()); - args.insert(1, command_data.executable.to_string()); - tg_exe - }, - - _ => match &command_data.executable { - tg::command::data::Executable::Artifact(executable) => { - let mut path = tg::run::CLOSEST_ARTIFACT_PATH.join(executable.artifact.to_string()); - if let Some(executable_path) = &executable.path { - path.push(executable_path); - } - path - }, - tg::command::data::Executable::Module(_) => { - return Err(tg::error!("invalid executable")); - }, - tg::command::data::Executable::Path(executable) => executable.path.clone(), - }, - }; - - // Create a temporary output directory. - let temp = tempfile::TempDir::new() - .map_err(|source| tg::error!(!source, "failed to create a temp directory"))?; - let output_path = temp.path().join("output"); - - // Convert data args and env to values for rendering. - let value_args: Vec = command_data - .args - .into_iter() - .map(tg::Value::try_from_data) - .collect::>()?; - let value_env: tg::value::Map = command_data - .env - .into_iter() - .map(|(k, v)| Ok::<_, tg::Error>((k, tg::Value::try_from_data(v)?))) - .collect::>()?; - - // Render the args and env. - args.extend(tg::run::render_args(&value_args, &output_path)?); - env.extend(tg::run::render_env(&value_env, &output_path)?); - - // Spawn the process with piped stdout and stderr. - let child = tokio::process::Command::new(executable) - .args(args) - .envs(env) - .stdin(std::process::Stdio::inherit()) - .stdout(std::process::Stdio::piped()) - .stderr(std::process::Stdio::piped()) - .spawn() - .map_err(|source| tg::error!(!source, "failed to spawn the process"))?; +pub(super) async fn spawn_unsandboxed( + ctx: qjs::Ctx<'_>, + arg: Serde, +) -> Result> { + let state = ctx.userdata::().unwrap().clone(); + let Serde(arg) = arg; + let result = async { + let handle = state.handle.clone(); + let child_process = state + .main_runtime_handle + .spawn(async move { crate::process::spawn_unsandboxed(&handle, arg).await }) + .await + .map_err(|source| tg::error!(!source, "the task panicked"))? + .map_err(|source| tg::error!(!source, "failed to spawn the unsandboxed process"))?; - // Get the PID. - let pid = child - .id() - .ok_or_else(|| tg::error!("failed to get the child process ID"))? as i32; + // Get the PID. + #[allow(clippy::cast_possible_wrap)] + let pid = child_process + .child + .id() + .ok_or_else(|| tg::error!("failed to get the child process ID"))? + .cast_signed(); - // Store the child process in state. - state - .children - .borrow_mut() - .insert(pid, ChildProcess { child }); + // Store the child process in state. + state.children.borrow_mut().insert(pid, child_process); - Ok(SpawnOutput { - process: tg::Either::Right(pid), - remote: None, - token: None, - wait: None, - }) + Ok(pid) + } + .await; + Result(result.map(Serde)) } -pub async fn wait( +pub(super) async fn wait_sandboxed( ctx: qjs::Ctx<'_>, - id: Serde>, + id: Serde, arg: Serde, -) -> Result> { +) -> Result> { let state = ctx.userdata::().unwrap().clone(); let Serde(id) = id; let Serde(arg) = arg; - let result = match id { - tg::Either::Left(id) => wait_sandboxed(state, id, arg).await, - tg::Either::Right(pid) => wait_unsandboxed(state, pid).await, - }; + let result = async { + let output = state + .main_runtime_handle + .spawn({ + let handle = state.handle.clone(); + async move { + let output = handle.wait_process(&id, arg).await?; + Ok::<_, tg::Error>(output) + } + }) + .await + .map_err(|source| tg::error!(!source, "the task panicked"))??; + Ok(output) + } + .await; Result(result.map(Serde)) } -async fn wait_sandboxed( - state: StateHandle, - id: tg::process::Id, - arg: tg::process::wait::Arg, -) -> tg::Result { - let output = state - .main_runtime_handle - .spawn({ - let handle = state.handle.clone(); - async move { - let output = handle.wait_process(&id, arg).await?; - Ok::<_, tg::Error>(output) - } - }) - .await - .map_err(|source| tg::error!(!source, "the task panicked"))??; - Ok(WaitOutput { - error: output.error, - exit: output.exit, - output: output.output, - stdout: None, - stderr: None, - }) -} - -async fn wait_unsandboxed(state: StateHandle, pid: i32) -> tg::Result { - // Remove the child process from state. - let mut child = state - .children - .borrow_mut() - .remove(&pid) - .ok_or_else(|| tg::error!(%pid, "unknown child process"))? - .child; - - // Wait for the child process to complete. - let output = child - .wait_with_output() - .await - .map_err(|source| tg::error!(!source, "failed to wait for the child process"))?; - - // Get the exit code. - let exit = output.status.code().map(|c| c as u8).unwrap_or(128); - - Ok(WaitOutput { - error: None, - exit, - output: None, - stdout: Some(output.stdout), - stderr: Some(output.stderr), - }) -} +pub(super) async fn wait_unsandboxed( + ctx: qjs::Ctx<'_>, + pid: Serde, +) -> Result> { + let state = ctx.userdata::().unwrap().clone(); + let Serde(pid) = pid; + let result = async { + // Remove the child process from state. + let child_process = state + .children + .borrow_mut() + .remove(&pid) + .ok_or_else(|| tg::error!(%pid, "unknown child process"))?; + + // Wait for the child process to complete. + let handle = state.handle.clone(); + let output = state + .main_runtime_handle + .spawn(async move { crate::process::wait_unsandboxed(&handle, child_process).await }) + .await + .map_err(|source| tg::error!(!source, "the task panicked"))? + .map_err(|source| tg::error!(!source, "failed to wait for the unsandboxed process"))?; -fn needs_sandbox(arg: &tg::process::spawn::Arg) -> bool { - if arg.cached.is_some_and(|cached| cached) { - return true; - } - if arg.checksum.is_some() { - return true; - } - if !arg.mounts.is_empty() { - return true; + Ok(output) } - if !arg.network { - return true; - } - if arg - .remotes - .as_ref() - .is_some_and(|remotes| !remotes.is_empty()) - { - return true; - } - false + .await; + Result(result.map(Serde)) } diff --git a/packages/js/src/syscall.ts b/packages/js/src/syscall.ts index a6753a7dc..ec5c89a4a 100644 --- a/packages/js/src/syscall.ts +++ b/packages/js/src/syscall.ts @@ -67,16 +67,26 @@ declare global { ): Promise; function syscall( - syscall: "process_spawn", + syscall: "process_spawn_sandboxed", arg: tg.Handle.SpawnArg, ): Promise; function syscall( - syscall: "process_wait", - id: tg.Process.Id | number, + syscall: "process_spawn_unsandboxed", + arg: tg.Handle.SpawnArg, + ): Promise; + + function syscall( + syscall: "process_wait_sandboxed", + id: tg.Process.Id, arg: tg.Handle.WaitArg, ): Promise; + function syscall( + syscall: "process_wait_unsandboxed", + pid: number, + ): Promise; + function syscall( syscall: "read", arg: tg.Handle.ReadArg, diff --git a/packages/js/src/v8.rs b/packages/js/src/v8.rs index ad065009a..9cc633cb1 100644 --- a/packages/js/src/v8.rs +++ b/packages/js/src/v8.rs @@ -29,7 +29,7 @@ const SNAPSHOT: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/main.heapsnaps const SOURCE_MAP: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/main.js.map")); struct State { - children: RefCell>, + children: RefCell>, promises: RefCell>>, global_source_map: Option, logger: Logger, diff --git a/packages/js/src/v8/syscall.rs b/packages/js/src/v8/syscall.rs index c14bc54fd..21bc544e2 100644 --- a/packages/js/src/v8/syscall.rs +++ b/packages/js/src/v8/syscall.rs @@ -8,7 +8,7 @@ mod checksum; mod encoding; mod magic; mod object; -mod process; +pub(crate) mod process; mod sleep; pub mod log; @@ -42,8 +42,10 @@ pub fn syscall<'s>( "object_get" => async_(scope, &args, self::object::get), "object_id" => sync(scope, &args, self::object::id), "process_get" => async_(scope, &args, self::process::get), - "process_spawn" => async_(scope, &args, self::process::spawn), - "process_wait" => async_(scope, &args, self::process::wait), + "process_spawn_sandboxed" => async_(scope, &args, self::process::spawn_sandboxed), + "process_wait_sandboxed" => async_(scope, &args, self::process::wait_sandboxed), + "process_spawn_unsandboxed" => async_(scope, &args, self::process::spawn_unsandboxed), + "process_wait_unsandboxed" => async_(scope, &args, self::process::wait_unsandboxed), "read" => async_(scope, &args, self::blob::read), "sleep" => async_(scope, &args, self::sleep::sleep), "write" => async_(scope, &args, self::blob::write), diff --git a/packages/js/src/v8/syscall/process.rs b/packages/js/src/v8/syscall/process.rs index 4d909339d..6bbd87fcc 100644 --- a/packages/js/src/v8/syscall/process.rs +++ b/packages/js/src/v8/syscall/process.rs @@ -1,49 +1,11 @@ use { super::State, futures::{StreamExt as _, TryStreamExt as _, future}, - std::{collections::BTreeMap, rc::Rc}, + std::rc::Rc, tangram_client::prelude::*, tangram_v8::Serde, }; -/// A child process stored in state, awaiting wait(). -pub(crate) struct ChildProcess { - pub child: tokio::process::Child, -} - -#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)] -struct SpawnOutput { - process: tg::Either, - - #[serde(default, skip_serializing_if = "Option::is_none")] - remote: Option, - - #[serde(default, skip_serializing_if = "Option::is_none")] - token: Option, - - #[serde(default, skip_serializing_if = "Option::is_none")] - wait: Option, -} - -#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)] -struct WaitOutput { - #[serde(default, skip_serializing_if = "Option::is_none")] - error: Option>, - - exit: u8, - - #[serde(default, skip_serializing_if = "Option::is_none")] - output: Option, - - // TODO: Handle stdout/stderr for unsandboxed processes. - #[serde(default, skip_serializing_if = "Option::is_none")] - stdout: Option>, - - // TODO: Handle stderr for unsandboxed processes. - #[serde(default, skip_serializing_if = "Option::is_none")] - stderr: Option>, -} - pub async fn get( state: Rc, args: (Serde, Option), @@ -62,22 +24,11 @@ pub async fn get( Ok(Serde(data)) } -pub async fn spawn( +pub async fn spawn_sandboxed( state: Rc, args: (Serde,), -) -> tg::Result> { +) -> tg::Result> { let (Serde(arg),) = args; - if needs_sandbox(&arg) { - spawn_sandboxed(state, arg).await - } else { - spawn_unsandboxed(state, arg).await - } -} - -async fn spawn_sandboxed( - state: Rc, - arg: tg::process::spawn::Arg, -) -> tg::Result> { let stream = state .main_runtime_handle .spawn({ @@ -102,152 +53,41 @@ async fn spawn_sandboxed( let writer = super::log::Writer::new(state.clone(), tg::process::log::Stream::Stderr); let handle = state.handle.clone(); let output = tg::progress::write_progress_stream(&handle, stream, writer, false).await?; - let spawn_output = SpawnOutput { - process: tg::Either::Left( - output - .process - .ok_or_else(|| tg::error!("expected a process ID"))?, - ), - remote: output.remote, - token: output.token, - wait: output.wait.map(|w| WaitOutput { - error: w.error, - exit: w.exit, - output: w.output, - stdout: None, - stderr: None, - }), - }; - Ok(Serde(spawn_output)) + Ok(Serde(output)) } -async fn spawn_unsandboxed( +pub async fn spawn_unsandboxed( state: Rc, - arg: tg::process::spawn::Arg, -) -> tg::Result> { - // Get the command data. - let command_id = arg.command.item.clone(); + args: (Serde,), +) -> tg::Result> { + let (Serde(arg),) = args; let handle = state.handle.clone(); - let command_data = state + let child_process = state .main_runtime_handle - .spawn(async move { - let command = tg::Command::with_id(command_id); - command.data(&handle).await - }) + .spawn(async move { crate::process::spawn_unsandboxed(&handle, arg).await }) .await .unwrap() - .map_err(|source| tg::error!(!source, "failed to get the command data"))?; - - // Render the executable. - let host = command_data.host.clone(); - let mut args = Vec::new(); - let mut env = std::env::vars().collect::>(); - - env.remove("TANGRAM_OUTPUT"); - env.remove("TANGRAM_PROCESS"); - env.remove("TANGRAM_URL"); - - let executable = match host.as_str() { - "builtin" => { - let tg_exe = tangram_util::env::current_exe() - .map_err(|source| tg::error!(!source, "failed to get the current executable"))?; - args.insert(0, "builtin".to_owned()); - args.insert(1, command_data.executable.to_string()); - tg_exe - }, - - "js" => { - let tg_exe = tangram_util::env::current_exe() - .map_err(|source| tg::error!(!source, "failed to get the current executable"))?; - args.insert(0, "js".to_owned()); - args.insert(1, command_data.executable.to_string()); - tg_exe - }, - - _ => match &command_data.executable { - tg::command::data::Executable::Artifact(executable) => { - let mut path = tg::run::CLOSEST_ARTIFACT_PATH.join(executable.artifact.to_string()); - if let Some(executable_path) = &executable.path { - path.push(executable_path); - } - path - }, - tg::command::data::Executable::Module(_) => { - return Err(tg::error!("invalid executable")); - }, - tg::command::data::Executable::Path(executable) => executable.path.clone(), - }, - }; - - // Create a temporary output directory. - let temp = tempfile::TempDir::new() - .map_err(|source| tg::error!(!source, "failed to create a temp directory"))?; - let output_path = temp.path().join("output"); - - // Convert data args and env to values for rendering. - let value_args: Vec = command_data - .args - .into_iter() - .map(tg::Value::try_from_data) - .collect::>()?; - let value_env: tg::value::Map = command_data - .env - .into_iter() - .map(|(k, v)| Ok::<_, tg::Error>((k, tg::Value::try_from_data(v)?))) - .collect::>()?; - - // Render the args and env. - args.extend(tg::run::render_args(&value_args, &output_path)?); - env.extend(tg::run::render_env(&value_env, &output_path)?); - - // Spawn the process with piped stdout and stderr. - let child = tokio::process::Command::new(executable) - .args(args) - .envs(env) - .stdin(std::process::Stdio::inherit()) - .stdout(std::process::Stdio::piped()) - .stderr(std::process::Stdio::piped()) - .spawn() - .map_err(|source| tg::error!(!source, "failed to spawn the process"))?; + .map_err(|source| tg::error!(!source, "failed to spawn the unsandboxed process"))?; // Get the PID. - let pid = child + #[allow(clippy::cast_possible_wrap)] + let pid = child_process + .child .id() - .ok_or_else(|| tg::error!("failed to get the child process ID"))? as i32; + .ok_or_else(|| tg::error!("failed to get the child process ID"))? + .cast_signed(); // Store the child process in state. - state - .children - .borrow_mut() - .insert(pid, ChildProcess { child }); + state.children.borrow_mut().insert(pid, child_process); - Ok(Serde(SpawnOutput { - process: tg::Either::Right(pid), - remote: None, - token: None, - wait: None, - })) + Ok(Serde(pid)) } -pub async fn wait( +pub async fn wait_sandboxed( state: Rc, - args: ( - Serde>, - Serde, - ), -) -> tg::Result> { + args: (Serde, Serde), +) -> tg::Result> { let (Serde(id), Serde(arg)) = args; - match id { - tg::Either::Left(id) => wait_sandboxed(state, id, arg).await, - tg::Either::Right(pid) => wait_unsandboxed(state, pid).await, - } -} - -async fn wait_sandboxed( - state: Rc, - id: tg::process::Id, - arg: tg::process::wait::Arg, -) -> tg::Result> { let handle = state.handle.clone(); let output = state .main_runtime_handle @@ -257,61 +97,30 @@ async fn wait_sandboxed( }) .await .unwrap()?; - Ok(Serde(WaitOutput { - error: output.error, - exit: output.exit, - output: output.output, - stdout: None, - stderr: None, - })) + Ok(Serde(output)) } -async fn wait_unsandboxed(state: Rc, pid: i32) -> tg::Result> { +pub async fn wait_unsandboxed( + state: Rc, + args: (Serde,), +) -> tg::Result> { + let (Serde(pid),) = args; + // Remove the child process from state. - let mut child = state + let child_process = state .children .borrow_mut() .remove(&pid) - .ok_or_else(|| tg::error!(%pid, "unknown child process"))? - .child; + .ok_or_else(|| tg::error!(%pid, "unknown child process"))?; // Wait for the child process to complete. - let output = child - .wait_with_output() + let handle = state.handle.clone(); + let output = state + .main_runtime_handle + .spawn(async move { crate::process::wait_unsandboxed(&handle, child_process).await }) .await - .map_err(|source| tg::error!(!source, "failed to wait for the child process"))?; - - // Get the exit code. - let exit = output.status.code().map(|c| c as u8).unwrap_or(128); - - Ok(Serde(WaitOutput { - error: None, - exit, - output: None, - stdout: Some(output.stdout), - stderr: Some(output.stderr), - })) -} + .unwrap() + .map_err(|source| tg::error!(!source, "failed to wait for the unsandboxed process"))?; -fn needs_sandbox(arg: &tg::process::spawn::Arg) -> bool { - if arg.cached.is_some_and(|cached| cached) { - return true; - } - if arg.checksum.is_some() { - return true; - } - if !arg.mounts.is_empty() { - return true; - } - if !arg.network { - return true; - } - if arg - .remotes - .as_ref() - .is_some_and(|remotes| !remotes.is_empty()) - { - return true; - } - false + Ok(Serde(output)) } From 55248cb46699eb7a14b63e55669ac9467196f193 Mon Sep 17 00:00:00 2001 From: Mike Hilgendorf Date: Tue, 3 Mar 2026 14:01:36 -0600 Subject: [PATCH 5/6] Cargo.lock --- Cargo.lock | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 235584e52..09d0c3d89 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6373,6 +6373,7 @@ dependencies = [ "tangram_session", "tangram_uri", "tangram_util", + "tempfile", "tokio", "tokio-stream", "tokio-util", @@ -6427,6 +6428,7 @@ dependencies = [ "tangram_serialize", "tangram_uri", "tangram_util", + "tempfile", "time", "tokio", "tokio-rustls", @@ -6620,7 +6622,9 @@ dependencies = [ "tangram_compiler", "tangram_either", "tangram_futures", + "tangram_util", "tangram_v8", + "tempfile", "tokio", "tokio-util", "toml", From 9d3ceabd262af68d7bfe39b48d37af416092cf89 Mon Sep 17 00:00:00 2001 From: David Yamnitsky Date: Wed, 4 Mar 2026 14:21:04 -0500 Subject: [PATCH 6/6] tests passing --- Cargo.lock | 1 + packages/cli/src/build.rs | 8 +- packages/cli/src/process/spawn.rs | 20 ++- packages/cli/src/run.rs | 162 ++++++------------ packages/cli/src/shell/common.rs | 6 +- packages/cli/tests/build/errors.nu | 2 +- ...odifying_artifacts_dir_in_sandbox_fails.nu | 6 +- packages/cli/tests/run/subpath_reference.nu | 4 +- packages/cli/tests/run/verbose_with_error.nu | 4 +- packages/client/src/run.rs | 151 +--------------- packages/js/Cargo.toml | 1 + packages/js/src/process.rs | 39 ++++- packages/js/src/v8/syscall/process.rs | 4 +- 13 files changed, 120 insertions(+), 288 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 09d0c3d89..e858a2335 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6629,6 +6629,7 @@ dependencies = [ "tokio-util", "toml", "v8", + "xattr", ] [[package]] diff --git a/packages/cli/src/build.rs b/packages/cli/src/build.rs index a9273ecfc..fe28702a9 100644 --- a/packages/cli/src/build.rs +++ b/packages/cli/src/build.rs @@ -97,7 +97,13 @@ impl Cli { // Spawn the process. let crate::process::spawn::Output { process, output } = self - .spawn(spawn, reference, referent, trailing, None, None, None) + .spawn( + spawn, + reference, + referent, + trailing, + crate::process::spawn::Stdio::default(), + ) .boxed() .await?; diff --git a/packages/cli/src/process/spawn.rs b/packages/cli/src/process/spawn.rs index 83b71e1e2..51b10efc6 100644 --- a/packages/cli/src/process/spawn.rs +++ b/packages/cli/src/process/spawn.rs @@ -192,6 +192,13 @@ pub struct Output { pub output: tg::process::spawn::Output, } +#[derive(Clone, Debug, Default)] +pub struct Stdio { + pub stdin: Option, + pub stdout: Option, + pub stderr: Option, +} + impl Cli { pub async fn command_process_spawn(&mut self, args: Args) -> tg::Result<()> { // Get the reference. @@ -214,9 +221,7 @@ impl Cli { args.reference, referent, args.trailing, - None, - None, - None, + Stdio::default(), ) .boxed() .await?; @@ -234,10 +239,13 @@ impl Cli { reference: tg::Reference, mut referent: tg::Referent, trailing: Vec, - stdin: Option, - stdout: Option, - stderr: Option, + stdio: Stdio, ) -> tg::Result { + let Stdio { + stdin, + stdout, + stderr, + } = stdio; let handle = self.handle().await?; // Determine if the process is sandboxed. diff --git a/packages/cli/src/run.rs b/packages/cli/src/run.rs index 5238e202a..6b1336f06 100644 --- a/packages/cli/src/run.rs +++ b/packages/cli/src/run.rs @@ -3,91 +3,13 @@ use { futures::prelude::*, num::ToPrimitive as _, std::{ - collections::BTreeMap, - fmt::Write as _, - os::unix::process::ExitStatusExt as _, - path::{Path, PathBuf}, + collections::BTreeMap, fmt::Write as _, os::unix::process::ExitStatusExt as _, + path::PathBuf, }, tangram_client::prelude::*, tangram_futures::task::Task, }; -fn render_value_string( - value: &tg::value::Data, - artifacts_path: &Path, - output_path: &Path, -) -> tg::Result { - match value { - tg::value::Data::String(string) => Ok(string.clone()), - tg::value::Data::Template(template) => template.try_render(|component| match component { - tg::template::data::Component::String(string) => Ok(string.clone().into()), - tg::template::data::Component::Artifact(artifact) => Ok(artifacts_path - .join(artifact.to_string()) - .to_str() - .unwrap() - .to_owned() - .into()), - tg::template::data::Component::Placeholder(placeholder) => { - if placeholder.name == "output" { - Ok(output_path.to_str().unwrap().to_owned().into()) - } else { - Err(tg::error!( - name = %placeholder.name, - "invalid placeholder" - )) - } - }, - }), - tg::value::Data::Placeholder(placeholder) => { - if placeholder.name == "output" { - Ok(output_path.to_str().unwrap().to_owned()) - } else { - Err(tg::error!( - name = %placeholder.name, - "invalid placeholder" - )) - } - }, - _ => Ok(tg::Value::try_from_data(value.clone()).unwrap().to_string()), - } -} - -fn render_args_string( - args: &[tg::value::Data], - artifacts_path: &Path, - output_path: &Path, -) -> tg::Result> { - args.iter() - .map(|value| render_value_string(value, artifacts_path, output_path)) - .collect::>>() -} - -fn render_env( - env: &tg::value::data::Map, - artifacts_path: &Path, - output_path: &Path, -) -> tg::Result> { - let mut output = tg::value::data::Map::new(); - for (key, value) in env { - let mutation = match value { - tg::value::Data::Mutation(value) => value.clone(), - value => tg::mutation::Data::Set { - value: Box::new(value.clone()), - }, - }; - mutation.apply(&mut output, key)?; - } - let output = output - .iter() - .map(|(key, value)| { - let key = key.clone(); - let value = render_value_string(value, artifacts_path, output_path)?; - Ok::<_, tg::Error>((key, value)) - }) - .collect::>()?; - Ok(output) -} - mod signal; mod stdio; @@ -206,7 +128,7 @@ impl Cli { let referent = referent.map(|_| item); // Run the process. - let output = if self.needs_sandbox(&options) { + let output = if Self::needs_sandbox(&options) { self.run_sandboxed(&options, &reference, &referent, trailing) .await? } else { @@ -217,7 +139,9 @@ impl Cli { // Check out the output if requested. if let Some(path) = options.checkout { let handle = self.handle().await?; - let output = output.ok_or_else(|| tg::error!("expected an output"))?; + let output = output + .filter(|v| !v.is_null()) + .ok_or_else(|| tg::error!("expected an output"))?; // Get the artifact. let artifact: tg::Artifact = output @@ -254,7 +178,7 @@ impl Cli { )?; // Print the path. - self.print_serde(path, options.print).await?; + Self::print_display(path.display()); return Ok(()); } @@ -286,14 +210,16 @@ impl Cli { // Create a temp directory for the output. let temp = tempfile::tempdir() .map_err(|source| tg::error!(!source, "failed to create a temp directory"))?; - let output_path = temp.path().join("output"); - tokio::fs::create_dir_all(&output_path) + let output_dir = temp.path().join("output"); + tokio::fs::create_dir_all(&output_dir) .await .map_err(|source| tg::error!(!source, "failed to create the output directory"))?; - let output_path = output_path.join("output"); + let process_id = referent.item().id().to_string(); + let output_path = output_dir.join(&process_id); - // Get the artifacts path. + // Set the artifacts path for rendering. let artifacts_path = self.directory_path().join("artifacts"); + tg::run::set_artifacts_path(&artifacts_path); // Inherit the process env. let mut env: BTreeMap = std::env::vars().collect(); @@ -364,12 +290,25 @@ impl Cli { }, tg::command::data::Executable::Path(exe) => exe.path.clone(), }; - let args = render_args_string(&data.args, &artifacts_path, &output_path)?; + let args_values: Vec = data + .args + .iter() + .cloned() + .map(tg::Value::try_from_data) + .collect::>()?; + let args = tg::run::render_args(&args_values, &output_path)?; (executable, args) }; // Render the command env. - let command_env = render_env(&data.env, &artifacts_path, &output_path)?; + let env_values: tg::value::Map = data + .env + .iter() + .map(|(k, v)| { + Ok::<_, tg::Error>((k.clone(), tg::Value::try_from_data(v.clone())?)) + }) + .collect::>()?; + let command_env = tg::run::render_env(&env_values, &output_path)?; // Append trailing args. args.extend(trailing); @@ -545,20 +484,22 @@ impl Cli { let handle = self.handle().await?; // Handle the executable path. - let referent = if let Some(path) = &options.executable_path { + let referent = if let Some(executable_path) = &options.executable_path { let directory = referent .item() .try_unwrap_directory_ref() .ok() .ok_or_else(|| tg::error!("expected a directory"))?; - let artifact = directory.get(&handle, path).await.map_err( - |source| tg::error!(!source, path = %path.display(), "failed to get the artifact"), + let artifact = directory.get(&handle, executable_path).await.map_err( + |source| tg::error!(!source, path = %executable_path.display(), "failed to get the artifact"), )?; let id = artifact .store(&handle) .await .map_err(|source| tg::error!(!source, "failed to store the artifact"))?; - referent.clone().map(|_| tg::Object::with_id(id.into())) + let mut referent = referent.clone().map(|_| tg::Object::with_id(id.into())); + referent.options.path = Some(executable_path.clone()); + referent } else { referent.clone() }; @@ -575,7 +516,7 @@ impl Cli { let stdio = if options.build { None } else { - let stdio = stdio::Stdio::new(&handle, remote.clone(), &options) + let stdio = stdio::Stdio::new(&handle, remote.clone(), options) .await .map_err(|source| tg::error!(!source, "failed to create stdio"))?; Some(stdio) @@ -588,19 +529,16 @@ impl Cli { remotes: options.spawn.remotes.clone(), ..Default::default() }; - let stdin = stdio.as_ref().and_then(|stdio| stdio.stdin.clone()); - let stdout = stdio.as_ref().and_then(|stdio| stdio.stdout.clone()); - let stderr = stdio.as_ref().and_then(|stdio| stdio.stderr.clone()); + let process_stdio = stdio + .as_ref() + .map(|stdio| crate::process::spawn::Stdio { + stdin: stdio.stdin.clone(), + stdout: stdio.stdout.clone(), + stderr: stdio.stderr.clone(), + }) + .unwrap_or_default(); let crate::process::spawn::Output { process, output } = self - .spawn( - spawn, - reference.clone(), - referent, - trailing, - stdin, - stdout, - stderr, - ) + .spawn(spawn, reference.clone(), referent, trailing, process_stdio) .boxed() .await?; @@ -642,14 +580,12 @@ impl Cli { wait } else { // Spawn the stdio task. - let stdio_task = if let Some(stdio) = stdio.clone() { - Some(Task::spawn({ + let stdio_task = stdio.clone().map(|stdio| { + Task::spawn({ let handle = handle.clone(); |stop| async move { self::stdio::task(&handle, stop, stdio).boxed().await } - })) - } else { - None - }; + }) + }); // Spawn signal task. This will be handled by the cancellation tasks for builds. let signal_task = if options.build { @@ -831,7 +767,7 @@ impl Cli { Ok(wait.output) } - fn needs_sandbox(&mut self, options: &Options) -> bool { + fn needs_sandbox(options: &Options) -> bool { // Sandbox if explicitly requested. if options.spawn.sandbox.get().is_some_and(|sbx| sbx) { return true; diff --git a/packages/cli/src/shell/common.rs b/packages/cli/src/shell/common.rs index 9a7bbbed5..21c4df42f 100644 --- a/packages/cli/src/shell/common.rs +++ b/packages/cli/src/shell/common.rs @@ -295,7 +295,7 @@ impl Cli { sandbox: crate::process::spawn::Sandbox::new(Some(true)), ..Default::default() }; - let referent = self.get_reference(&reference).await?; + let referent = self.get_reference(reference).await?; let item = referent .item .clone() @@ -308,9 +308,7 @@ impl Cli { reference.clone(), referent, Vec::new(), - None, - None, - None, + crate::process::spawn::Stdio::default(), ) .boxed() .await?; diff --git a/packages/cli/tests/build/errors.nu b/packages/cli/tests/build/errors.nu index b8a5368b9..8fabca7fe 100644 --- a/packages/cli/tests/build/errors.nu +++ b/packages/cli/tests/build/errors.nu @@ -5,7 +5,7 @@ let server = spawn let path = artifact { tangram.ts: r#' export default async function () { - await tg.run(x); + await tg.build(x); }; export function x() { diff --git a/packages/cli/tests/build/modifying_artifacts_dir_in_sandbox_fails.nu b/packages/cli/tests/build/modifying_artifacts_dir_in_sandbox_fails.nu index 1d5b17770..72b910c75 100644 --- a/packages/cli/tests/build/modifying_artifacts_dir_in_sandbox_fails.nu +++ b/packages/cli/tests/build/modifying_artifacts_dir_in_sandbox_fails.nu @@ -6,10 +6,8 @@ let path = artifact { tangram.ts: ' import busybox from "busybox"; export default async () => { - const bb = tg.build(busybox); - const file = await tg.build`echo "Hello, World!" > ${tg.output}`.env(bb); - await tg.run`echo "Goodbye, Reproducibility!" > ${file}`.env(bb); - return file; + const file = await tg.build`echo "Hello, World!" > ${tg.output}`.env(busybox); + await tg.build`echo "Goodbye, Reproducibility!" > ${file}`.env(busybox); } ' } diff --git a/packages/cli/tests/run/subpath_reference.nu b/packages/cli/tests/run/subpath_reference.nu index 23ae08e7a..8e1976b1a 100644 --- a/packages/cli/tests/run/subpath_reference.nu +++ b/packages/cli/tests/run/subpath_reference.nu @@ -12,8 +12,8 @@ let path = artifact { let id = tg checkin $path tg index -let sandbox_output = tg run ($id + "#sub.tg.ts") --sandbox +let sandbox_output = tg run ($id + "?path=sub.tg.ts") --sandbox snapshot $sandbox_output '"from sub"' -let output = tg run ($id + "#sub.tg.ts") +let output = tg run ($id + "?path=sub.tg.ts") assert equal $output $sandbox_output diff --git a/packages/cli/tests/run/verbose_with_error.nu b/packages/cli/tests/run/verbose_with_error.nu index 9a6b0f395..a841c4470 100644 --- a/packages/cli/tests/run/verbose_with_error.nu +++ b/packages/cli/tests/run/verbose_with_error.nu @@ -16,10 +16,10 @@ let sandbox_output = tg run --verbose $path --sandbox | complete success $sandbox_output let sandbox_json = $sandbox_output.stdout | from json assert ($sandbox_json.error != null) -assert ($sandbox_json.exit == 0) +assert ($sandbox_json.exit == 1) let output = tg run --verbose $path | complete success $output let json = $output.stdout | from json assert ($json.error != null) -assert ($json.exit == 0) +assert ($json.exit == 1) diff --git a/packages/client/src/run.rs b/packages/client/src/run.rs index 5ab4da4ff..a089bf0a8 100644 --- a/packages/client/src/run.rs +++ b/packages/client/src/run.rs @@ -30,149 +30,7 @@ pub struct Arg { pub user: Option, } -pub async fn run(handle: &H, arg: tg::run::Arg) -> tg::Result -where - H: tg::Handle, -{ - let state = if let Some(process) = tg::Process::current()? { - Some(process.load(handle).await?) - } else { - None - }; - let command = if let Some(state) = &state { - Some(state.command.object(handle).await?) - } else { - None - }; - let host = arg - .host - .ok_or_else(|| tg::error!("expected the host to be set"))?; - let executable = arg - .executable - .ok_or_else(|| tg::error!("expected the executable to be set"))?; - let mut builder = tg::Command::builder(host, executable); - builder = builder.args(arg.args); - let cwd = if let Some(command) = &command { - if command.cwd.is_some() { - let cwd = std::env::current_dir() - .map_err(|source| tg::error!(!source, "failed to get the current directory"))?; - Some(cwd) - } else { - None - } - } else { - let cwd = std::env::current_dir() - .map_err(|source| tg::error!(!source, "failed to get the current directory"))?; - Some(cwd) - }; - let cwd = arg.cwd.or(cwd); - builder = builder.cwd(cwd); - let mut env = if let Some(command) = &command { - command.env.clone() - } else { - std::env::vars() - .map(|(key, value)| (key, value.into())) - .collect() - }; - env.remove("TANGRAM_OUTPUT"); - env.remove("TANGRAM_PROCESS"); - env.remove("TANGRAM_URL"); - builder = builder.env(env); - let mut command_mounts = vec![]; - let mut process_mounts = vec![]; - if let Some(mounts) = arg.mounts { - for mount in mounts { - match mount { - tg::Either::Left(mount) => process_mounts.push(mount.to_data()), - tg::Either::Right(mount) => command_mounts.push(mount), - } - } - } else { - if let Some(mounts) = command.as_ref().map(|command| command.mounts.clone()) { - command_mounts = mounts; - } - if let Some(mounts) = state.as_ref().map(|state| state.mounts.clone()) { - process_mounts = mounts.iter().map(tg::process::Mount::to_data).collect(); - } - } - builder = builder.mounts(command_mounts); - let stdin = if arg.stdin.is_none() { - command - .as_ref() - .map(|command| command.stdin.clone()) - .unwrap_or_default() - } else if let Some(Some(tg::Either::Right(blob))) = &arg.stdin { - Some(blob.clone()) - } else { - None - }; - builder = builder.stdin(stdin); - if let Some(Some(user)) = command.as_ref().map(|command| command.user.clone()) { - builder = builder.user(user); - } - let command = builder.build(); - let command_id = command.store(handle).await?; - let mut command = tg::Referent::with_item(command_id); - if let Some(name) = arg.name { - command.options.name.replace(name); - } - let checksum = arg.checksum; - let network = arg - .network - .or(state.as_ref().map(|state| state.network)) - .unwrap_or_default(); - let stderr = arg - .stderr - .unwrap_or_else(|| state.as_ref().and_then(|state| state.stderr.clone())); - let stdin = arg.stdin.unwrap_or_else(|| { - state - .as_ref() - .and_then(|state| state.stdin.clone().map(tg::Either::Left)) - }); - let stdin = match stdin { - None => None, - Some(tg::Either::Left(stdio)) => Some(stdio), - Some(tg::Either::Right(_)) => { - return Err(tg::error!("expected stdio")); - }, - }; - let stdout = arg - .stdout - .unwrap_or_else(|| state.as_ref().and_then(|state| state.stdout.clone())); - if network && checksum.is_none() { - return Err(tg::error!( - "a checksum is required to build with network enabled" - )); - } - let progress = arg.progress; - let arg = tg::process::spawn::Arg { - cached: arg.cached, - checksum, - command, - local: None, - mounts: process_mounts, - network, - parent: arg.parent, - remotes: arg.remote.map(|r| vec![r]), - retry: arg.retry, - stderr, - stdin, - stdout, - }; - let stream = tg::Process::spawn(handle, arg).await?; - let writer = std::io::stderr(); - let is_tty = progress && writer.is_terminal(); - let process = tg::progress::write_progress_stream(handle, stream, writer, is_tty) - .await - .map_err(|source| tg::error!(!source, "failed to spawn the process"))?; - let output = process - .output(handle) - .await - .map_err(|source| tg::error!(!source, "failed to get the process output"))?; - Ok(output) -} - -pub async fn run2(handle: &H, arg: Arg) -> tg::Result +pub async fn run(handle: &H, arg: Arg) -> tg::Result where H: tg::Handle, { @@ -354,10 +212,11 @@ where // Create the output directory. let temp = tempfile::TempDir::new() .map_err(|source| tg::error!(!source, "failed to create the temporary directory"))?; - tokio::fs::create_dir_all(temp.path()) + let output_dir = temp.path().join("output"); + tokio::fs::create_dir_all(&output_dir) .await .map_err(|source| tg::error!(!source, "failed to create the output directory"))?; - let output_path = temp.path().join("output"); + let output_path = output_dir.join(std::process::id().to_string()); // Get the executable. let executable = arg @@ -569,7 +428,7 @@ pub static CLOSEST_ARTIFACT_PATH: LazyLock = LazyLock::new(|| { pub static TANGRAM_EXECUTABLE_PATH: Mutex> = Mutex::new(None); pub static TANGRAM_ARTIFACTS_PATH: Mutex> = Mutex::new(None); -pub fn set_artifacts_path(p: impl AsRef) { +pub fn set_artifacts_path(p: impl AsRef) { TANGRAM_ARTIFACTS_PATH .lock() .unwrap() diff --git a/packages/js/Cargo.toml b/packages/js/Cargo.toml index f115c1a81..869aa68c1 100644 --- a/packages/js/Cargo.toml +++ b/packages/js/Cargo.toml @@ -46,4 +46,5 @@ tempfile = { workspace = true } tokio = { workspace = true } tokio-util = { workspace = true } toml = { workspace = true } +xattr = { workspace = true } v8 = { workspace = true, optional = true } diff --git a/packages/js/src/process.rs b/packages/js/src/process.rs index e394cda78..7b28f8abf 100644 --- a/packages/js/src/process.rs +++ b/packages/js/src/process.rs @@ -23,7 +23,8 @@ pub(crate) struct WaitOutput { pub(crate) struct ChildProcess { pub(crate) child: tokio::process::Child, - temp: TempDir, + output_path: std::path::PathBuf, + _temp: TempDir, } pub(crate) async fn spawn_unsandboxed( @@ -35,7 +36,7 @@ where { // Get the command data. let command_id = arg.command.item.clone(); - let command = tg::Command::with_id(command_id); + let command = tg::Command::with_id(command_id.clone()); let command = command .data(handle) .await @@ -86,10 +87,11 @@ where // Create a temp for the output. let temp = tempfile::tempdir() .map_err(|source| tg::error!(!source, "failed to create the temporary directory"))?; - tokio::fs::create_dir_all(temp.path()) + let output_dir = temp.path().join("output"); + tokio::fs::create_dir_all(&output_dir) .await .map_err(|source| tg::error!(!source, "failed to create the output directory"))?; - let output_path = temp.path().join("output"); + let output_path = output_dir.join(command_id.to_string()); // Convert data. let args_: Vec = command @@ -118,7 +120,11 @@ where .spawn() .map_err(|source| tg::error!(!source, "failed to spawn the process"))?; - Ok(ChildProcess { child, temp }) + Ok(ChildProcess { + child, + output_path, + _temp: temp, + }) } pub(crate) async fn wait_unsandboxed(handle: &H, child: ChildProcess) -> tg::Result @@ -139,10 +145,29 @@ where .unwrap(); let stdout = output.stdout; let stderr = output.stderr; - let output_path = child.temp.path().join("output"); + let output_path = child.output_path; let mut error = None; let mut output = None; - if matches!(tokio::fs::try_exists(&output_path).await, Ok(true)) { + + // Try to read the user.tangram.output xattr. + if let Ok(Some(bytes)) = xattr::get(&output_path, "user.tangram.output") { + let tgon = String::from_utf8(bytes) + .map_err(|source| tg::error!(!source, "failed to decode the output xattr"))?; + let value: tg::Value = tgon + .parse() + .map_err(|source| tg::error!(!source, "failed to parse the output xattr"))?; + output.replace(value.to_data()); + } + + // Try to read the user.tangram.error xattr. + if let Ok(Some(bytes)) = xattr::get(&output_path, "user.tangram.error") { + let data = serde_json::from_slice::(&bytes) + .map_err(|source| tg::error!(!source, "failed to deserialize the error xattr"))?; + error.replace(tg::Either::Left(data)); + } + + // If no xattr output was set but the output path exists, check it in destructively. + if output.is_none() && matches!(tokio::fs::try_exists(&output_path).await, Ok(true)) { let result = tg::checkin( handle, tg::checkin::Arg { diff --git a/packages/js/src/v8/syscall/process.rs b/packages/js/src/v8/syscall/process.rs index 6bbd87fcc..18d560cce 100644 --- a/packages/js/src/v8/syscall/process.rs +++ b/packages/js/src/v8/syscall/process.rs @@ -66,7 +66,7 @@ pub async fn spawn_unsandboxed( .main_runtime_handle .spawn(async move { crate::process::spawn_unsandboxed(&handle, arg).await }) .await - .unwrap() + .map_err(|source| tg::error!(!source, "the task panicked"))? .map_err(|source| tg::error!(!source, "failed to spawn the unsandboxed process"))?; // Get the PID. @@ -119,7 +119,7 @@ pub async fn wait_unsandboxed( .main_runtime_handle .spawn(async move { crate::process::wait_unsandboxed(&handle, child_process).await }) .await - .unwrap() + .map_err(|source| tg::error!(!source, "the task panicked"))? .map_err(|source| tg::error!(!source, "failed to wait for the unsandboxed process"))?; Ok(Serde(output))