diff --git a/Cargo.lock b/Cargo.lock index 235584e52..aec2b3a7e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6649,9 +6649,19 @@ name = "tangram_sandbox" version = "0.0.0" dependencies = [ "bytes", + "futures", "indoc", "libc", "num", + "rustix 1.1.4", + "serde", + "serde-untagged", + "serde_json", + "tangram_client", + "tangram_futures", + "time", + "tokio", + "tracing", ] [[package]] @@ -6734,6 +6744,7 @@ dependencies = [ "tangram_index", "tangram_js", "tangram_messenger", + "tangram_sandbox", "tangram_serialize", "tangram_session", "tangram_store", diff --git a/packages/cli/src/main.rs b/packages/cli/src/main.rs index 2154d56a1..98a623727 100644 --- a/packages/cli/src/main.rs +++ b/packages/cli/src/main.rs @@ -303,8 +303,14 @@ fn main() -> std::process::ExitCode { Cli::initialize_v8(0); return Cli::command_js(&matches, args); }, - Command::Sandbox(args) => { - return Cli::command_sandbox(args); + Command::Sandbox(self::sandbox::Args { + command: self::sandbox::Command::Serve(_), + .. + }) => { + let Command::Sandbox(args) = args.command else { + unreachable!() + }; + return Cli::command_sandbox_sync(args); }, Command::Session(args) => { return Cli::command_session(args); @@ -945,7 +951,7 @@ impl Cli { Command::Js(_) => { unreachable!() }, - Command::Builtin(_) | Command::Sandbox(_) | Command::Session(_) => { + Command::Builtin(_) | Command::Session(_) => { unreachable!() }, Command::Archive(args) => self.command_archive(args).boxed(), @@ -987,6 +993,7 @@ impl Cli { Command::Read(args) => self.command_read(args).boxed(), Command::Remote(args) => self.command_remote(args).boxed(), Command::Run(args) => self.command_run(args).boxed(), + Command::Sandbox(args) => self.command_sandbox(args).boxed(), Command::Self_(args) => self.command_tangram(args).boxed(), Command::Shell(args) => self.command_shell(args).boxed(), Command::Serve(args) => self.command_server_run(args).boxed(), diff --git a/packages/cli/src/process/spawn.rs b/packages/cli/src/process/spawn.rs index e566d5670..dfd867cf9 100644 --- a/packages/cli/src/process/spawn.rs +++ b/packages/cli/src/process/spawn.rs @@ -85,15 +85,6 @@ pub struct Options { #[command(flatten)] pub checkin: crate::checkin::Options, - /// Configure mounts. - #[arg( - action = clap::ArgAction::Append, - long = "mount", - num_args = 1, - short, - )] - pub mounts: Vec>, - #[command(flatten)] pub local: crate::util::args::Local, @@ -250,7 +241,6 @@ impl Cli { tg::Command::builder(object.host.clone(), object.executable.clone()) .args(object.args.clone()) .cwd(object.cwd.clone()) - .mounts(object.mounts.clone()) .stdin(object.stdin.clone()) }, @@ -442,23 +432,16 @@ impl Cli { env.insert(key, value); } } + let host = if let Some(host) = options.host { + host + } else { + tg::host().to_owned() + }; if !env.contains_key("TANGRAM_HOST") { - let host = if let Some(host) = options.host { - host - } else { - tg::host().to_owned() - }; - env.insert("TANGRAM_HOST".to_owned(), host.into()); + env.insert("TANGRAM_HOST".to_owned(), host.clone().into()); } command = command.env(env); - // Set the mounts. - for mount in &options.mounts { - if let tg::Either::Right(mount) = mount { - command = command.mount(mount.clone()); - } - } - // Create the command and store it. let command = command.build(); command @@ -468,43 +451,30 @@ impl Cli { // Determine if the network is enabled. let network = options.network.unwrap_or(!sandbox); - + let sandbox = if network { + None + } else { + Some(tg::Either::Left(tg::sandbox::create::Arg { + host, + network: false, + hostname: None, + mounts: Vec::new(), + user: None, + })) + }; // Determine the retry. let retry = options.retry; - // Get the mounts. - let mut mounts = Vec::new(); - if !sandbox { - mounts.push(tg::process::data::Mount { - source: "/".into(), - target: "/".into(), - readonly: false, - }); - } - for mount in &options.mounts { - if let tg::Either::Left(mount) = mount { - let source = tokio::fs::canonicalize(&mount.source) - .await - .map_err(|source| tg::error!(!source, "failed to canonicalize the path"))?; - mounts.push(tg::process::data::Mount { - source, - target: mount.target.clone(), - readonly: mount.readonly, - }); - } - } - // Spawn the process. let arg = tg::process::spawn::Arg { cached: options.cached, checksum: options.checksum, command: tg::Referent::with_item(command.id()), local: options.local.local, - mounts, - network, parent: None, remotes: options.remotes.remotes.clone(), retry, + sandbox, stderr, stdin, stdout, diff --git a/packages/cli/src/run.rs b/packages/cli/src/run.rs index 287e9b805..18bc2ad10 100644 --- a/packages/cli/src/run.rs +++ b/packages/cli/src/run.rs @@ -7,7 +7,7 @@ use { }; mod signal; -mod stdio; +pub(crate) mod stdio; /// Spawn and await an unsandboxed process. #[derive(Clone, Debug, clap::Args)] @@ -281,9 +281,20 @@ impl Cli { .and_then(|remotes| remotes.into_iter().next()); // Create the stdio. - let stdio = stdio::Stdio::new(&handle, remote.clone(), &options) - .await - .map_err(|source| tg::error!(!source, "failed to create stdio"))?; + let tty_enabled = options.spawn.tty.get(); + let stdio = if options.detach { + stdio::Stdio { + tty: None, + remote: remote.clone(), + stdin: None, + stdout: None, + stderr: None, + } + } else { + stdio::Stdio::new(&handle, remote.clone(), tty_enabled) + .await + .map_err(|source| tg::error!(!source, "failed to create stdio"))? + }; let local = options.spawn.local.local; let remotes = options.spawn.remotes.remotes.clone(); diff --git a/packages/cli/src/run/stdio.rs b/packages/cli/src/run/stdio.rs index 7ff70d592..382a3dbb2 100644 --- a/packages/cli/src/run/stdio.rs +++ b/packages/cli/src/run/stdio.rs @@ -1,5 +1,4 @@ use { - super::Options, futures::{FutureExt as _, StreamExt as _, TryStreamExt as _, future}, std::{ io::IsTerminal as _, @@ -28,7 +27,7 @@ pub struct Tty { pub termios: libc::termios, } -pub(super) async fn task(handle: &H, stop: Stop, stdio: Stdio) -> tg::Result<()> +pub(crate) async fn task(handle: &H, stop: Stop, stdio: Stdio) -> tg::Result<()> where H: tg::Handle, { @@ -264,24 +263,13 @@ impl Stdio { pub(crate) async fn new( handle: &H, remote: Option, - options: &Options, + allow_tty: bool, ) -> tg::Result where H: tg::Handle, { - // If the process is detached, then do not create stdio. - if options.detach { - return Ok(Self { - tty: None, - remote, - stdin: None, - stdout: None, - stderr: None, - }); - } - // Create a PTY for stdin if it is a terminal. - let (tty, stdin) = if options.spawn.tty.get() && std::io::stdin().is_terminal() { + let (tty, stdin) = if allow_tty && std::io::stdin().is_terminal() { let tty = Tty::new()?; let size = tty.get_size()?; let arg = tg::pty::create::Arg { diff --git a/packages/cli/src/sandbox.rs b/packages/cli/src/sandbox.rs index 3742a0ed5..37edd8188 100644 --- a/packages/cli/src/sandbox.rs +++ b/packages/cli/src/sandbox.rs @@ -5,28 +5,32 @@ use { tangram_client::prelude::*, }; -#[derive(Debug, Clone, clap::Args)] +pub mod create; +pub mod delete; +pub mod exec; +pub mod serve; + +/// Manage sandboxes. +#[derive(Clone, Debug, clap::Args)] +#[group(skip)] pub struct Args { - /// Provide a path for the chroot. - #[arg(long)] - pub chroot: Option, - - /// Change the working directory prior to spawn. - #[arg(long, short = 'C')] - pub cwd: Option, + #[command(subcommand)] + pub command: Command, +} - /// Define environment variables. - #[arg( - action = clap::ArgAction::Append, - num_args = 1, - short = 'e', - value_parser = parse_env, - )] - pub env: Vec<(String, String)>, +#[derive(Clone, Debug, clap::Subcommand)] +pub enum Command { + Create(self::create::Args), + Delete(self::delete::Args), + Exec(self::exec::Args), + Serve(self::serve::Args), +} - /// The executable path. - #[arg(index = 1)] - pub executable: PathBuf, +#[derive(Clone, Debug, clap::Args)] +#[group(skip)] +pub struct Options { + /// The desired host. + pub host: Option, /// The desired hostname. #[arg(long)] @@ -42,13 +46,10 @@ pub struct Args { )] pub mounts: Vec, - /// Whether to enable network access. - #[arg(long)] - pub network: bool, - - #[arg(index = 2, trailing_var_arg = true)] - pub trailing: Vec, + #[clap(flatten)] + pub network: Network, + /// The desired user. #[arg(long)] pub user: Option, } @@ -62,57 +63,63 @@ pub struct Mount { pub data: Option, } +#[derive(Clone, Debug, Default, clap::Args)] +pub struct Network { + /// Whether to allow network access + #[arg( + default_missing_value = "true", + long, + num_args = 0..=1, + overrides_with = "no_network", + require_equals = true, + )] + network: Option, + + #[arg( + default_missing_value = "true", + long, + num_args = 0..=1, + overrides_with = "network", + require_equals = true, + )] + no_network: Option, +} + +impl Network { + pub fn get(&self) -> bool { + self.network.or(self.no_network.map(|v| !v)).unwrap_or(true) + } +} + impl Cli { #[must_use] - pub fn command_sandbox(args: Args) -> std::process::ExitCode { - // Create the command. - let command = tangram_sandbox::Command { - chroot: args.chroot, - cwd: args.cwd, - env: args.env, - executable: args.executable, - hostname: args.hostname, - mounts: args - .mounts - .into_iter() - .map(|mount| tangram_sandbox::Mount { - source: mount.source, - target: mount.target, - fstype: mount.fstype, - flags: mount.flags, - data: mount.data, - }) - .collect(), - network: args.network, - trailing: args.trailing, - user: args.user, - }; - - // Run the sandbox. - #[cfg(target_os = "linux")] - let result = tangram_sandbox::linux::spawn(command); - #[cfg(target_os = "macos")] - let result = tangram_sandbox::darwin::spawn(command); - - match result { - Ok(status) => status, - Err(error) => { - let error = tg::error!(!error, "failed to run the sandbox"); - Cli::print_error_basic(tg::Referent::with_item(error)); - std::process::ExitCode::FAILURE - }, + pub fn command_sandbox_sync(args: Args) -> std::process::ExitCode { + match args.command { + Command::Serve(args) => Cli::command_sandbox_serve(args), + _ => unreachable!(), } } -} -fn parse_env(arg: &str) -> Result<(String, String), String> { - let (name, value) = arg - .split_once('=') - .ok_or_else(|| "expected NAME=value".to_owned())?; - Ok((name.to_owned(), value.to_owned())) + pub async fn command_sandbox(&mut self, args: Args) -> tg::Result<()> { + match args.command { + Command::Create(args) => { + self.command_sandbox_create(args).await?; + }, + Command::Delete(args) => { + self.command_sandbox_delete(args).await?; + }, + Command::Exec(args) => { + self.command_sandbox_exec(args).await?; + }, + Command::Serve(_) => { + unreachable!() + }, + } + Ok(()) + } } -fn parse_mount(arg: &str) -> Result { +pub(crate) fn parse_mount(arg: &str) -> Result { let mut source = None; let mut target = None; let mut fstype = None; diff --git a/packages/cli/src/sandbox/create.rs b/packages/cli/src/sandbox/create.rs new file mode 100644 index 000000000..192f86057 --- /dev/null +++ b/packages/cli/src/sandbox/create.rs @@ -0,0 +1,59 @@ +use {crate::Cli, tangram_client::prelude::*}; + +/// Create a sandbox. +#[derive(Clone, Debug, clap::Args)] +#[group(skip)] +pub struct Args { + #[command(flatten)] + pub options: super::Options, +} + +#[cfg(target_os = "linux")] +const RD_ONLY: u64 = libc::MS_RDONLY as _; + +#[cfg(target_os = "macos")] +const RD_ONLY: u64 = libc::MNT_RDONLY as _; + +impl Cli { + pub async fn command_sandbox_create(&mut self, args: Args) -> tg::Result<()> { + let handle = self.handle().await?; + let client = handle + .left() + .ok_or_else(|| tg::error!("this command requires a client, not a server"))?; + + // Get the host. + let host = args.options.host.unwrap_or_else(|| tg::host().to_owned()); + + // Convert the CLI mounts to the create arg mounts. + let mounts = args + .options + .mounts + .into_iter() + .map(|mount| tg::sandbox::create::Mount { + source: mount.source.map(tg::Either::Left).unwrap(), + target: mount.target.unwrap(), + readonly: mount.flags & RD_ONLY != 0, + }) + .collect(); + + // Create the arg. + let arg = tg::sandbox::create::Arg { + host, + hostname: args.options.hostname, + mounts, + network: args.options.network.get(), + user: args.options.user, + }; + + // Create the sandbox. + let output = client + .create_sandbox(arg) + .await + .map_err(|source| tg::error!(!source, "failed to create the sandbox"))?; + + // Print the sandbox ID. + Self::print_display(&output.id); + + Ok(()) + } +} diff --git a/packages/cli/src/sandbox/delete.rs b/packages/cli/src/sandbox/delete.rs new file mode 100644 index 000000000..737b8dad5 --- /dev/null +++ b/packages/cli/src/sandbox/delete.rs @@ -0,0 +1,25 @@ +use {crate::Cli, tangram_client::prelude::*}; + +/// Delete a sandbox. +#[derive(Clone, Debug, clap::Args)] +#[group(skip)] +pub struct Args { + #[arg(index = 1)] + pub sandbox: tg::sandbox::Id, +} + +impl Cli { + pub async fn command_sandbox_delete(&mut self, args: Args) -> tg::Result<()> { + let handle = self.handle().await?; + let client = handle + .left() + .ok_or_else(|| tg::error!("this command requires a client, not a server"))?; + + // Delete the sandbox. + client.delete_sandbox(&args.sandbox).await.map_err( + |source| tg::error!(!source, id = %args.sandbox, "failed to delete the sandbox"), + )?; + + Ok(()) + } +} diff --git a/packages/cli/src/sandbox/exec.rs b/packages/cli/src/sandbox/exec.rs new file mode 100644 index 000000000..327cb8034 --- /dev/null +++ b/packages/cli/src/sandbox/exec.rs @@ -0,0 +1,117 @@ +use { + crate::Cli, futures::prelude::*, std::path::PathBuf, tangram_client::prelude::*, + tangram_futures::task::Task, +}; + +/// Execute a command in a sandbox. +#[derive(Clone, Debug, clap::Args)] +#[group(skip)] +#[allow(clippy::struct_field_names)] +pub struct Args { + /// The sandbox ID. + #[arg(index = 1)] + pub sandbox: tg::sandbox::Id, + + /// The command to execute. + #[arg(index = 2)] + pub command: PathBuf, + + /// The arguments to the command. + #[arg(index = 3, trailing_var_arg = true)] + pub args: Vec, + + #[command(flatten)] + pub tty: Tty, +} + +#[derive(Clone, Debug, Default, clap::Args)] +pub struct Tty { + /// Whether to allocate a terminal. + #[arg( + default_missing_value = "true", + short, + long, + num_args = 0..=1, + overrides_with = "no_tty", + require_equals = true, + )] + tty: Option, + + #[arg( + default_missing_value = "true", + long, + num_args = 0..=1, + overrides_with = "tty", + require_equals = true, + )] + no_tty: Option, +} + +impl Tty { + pub fn get(&self) -> bool { + self.tty.or(self.no_tty.map(|v| !v)).unwrap_or(true) + } +} + +impl Cli { + pub async fn command_sandbox_exec(&mut self, args: Args) -> tg::Result<()> { + let handle = self.handle().await?; + let client = handle + .left() + .ok_or_else(|| tg::error!("this command requires a client, not a server"))?; + + // Create the stdio. + let stdio = crate::run::stdio::Stdio::new(&client, None, args.tty.get()) + .await + .map_err(|source| tg::error!(!source, "failed to create stdio"))?; + + // Spawn the command in the sandbox. + let arg = tg::sandbox::spawn::Arg { + command: args.command, + args: args.args, + env: std::collections::BTreeMap::new(), + stdin: stdio.stdin.clone().unwrap(), + stdout: stdio.stdout.clone().unwrap(), + stderr: stdio.stderr.clone().unwrap(), + }; + let output = client + .sandbox_spawn(&args.sandbox, arg) + .await + .map_err(|source| tg::error!(!source, "failed to spawn the command in the sandbox"))?; + + // Enable raw mode if necessary. + if let Some(tty) = &stdio.tty { + tty.enable_raw_mode()?; + } + + // Spawn the stdio task. + let stdio_task = Task::spawn({ + let handle = client.clone(); + let stdio = stdio.clone(); + |stop| async move { crate::run::stdio::task(&handle, stop, stdio).boxed().await } + }); + + // Wait for the process to finish. + let arg = tg::sandbox::wait::Arg { pid: output.pid }; + let result = client.sandbox_wait(&args.sandbox, arg).await; + + // Cleanup stdio. + stdio.close(&client).await?; + stdio_task.stop(); + stdio_task.wait().await.unwrap()?; + stdio.delete(&client).await?; + + // Handle the result. + let wait = result.map_err(|source| { + tg::error!(!source, "failed to wait for the command in the sandbox") + })?; + + // Set the exit code. + if wait.status != 0 { + let exit = u8::try_from(wait.status).unwrap_or(1); + self.exit.replace(exit); + } + + Ok(()) + } +} diff --git a/packages/cli/src/sandbox/serve.rs b/packages/cli/src/sandbox/serve.rs new file mode 100644 index 000000000..7ce39b8d0 --- /dev/null +++ b/packages/cli/src/sandbox/serve.rs @@ -0,0 +1,108 @@ +use { + crate::Cli, + std::{io::Write as _, os::fd::FromRawFd as _, path::PathBuf}, + tangram_client::prelude::*, + tangram_sandbox as sandbox, +}; + +/// Start a sandbox server. +#[derive(Clone, Debug, clap::Args)] +#[group(skip)] +pub struct Args { + #[arg(long)] + pub root: Option, + + #[command(flatten)] + pub options: super::Options, + + /// The path to the socket. + #[arg(long)] + pub socket: PathBuf, + + /// A file descriptor to write a ready signal to. + #[arg(long)] + pub ready_fd: Option, +} + +impl Cli { + #[must_use] + pub fn command_sandbox_serve(args: Args) -> std::process::ExitCode { + let result = Self::command_sandbox_serve_inner(args); + if let Err(error) = result { + Cli::print_error_basic(tg::Referent::with_item(error)); + return std::process::ExitCode::FAILURE; + } + std::process::ExitCode::SUCCESS + } + + fn command_sandbox_serve_inner(args: Args) -> tg::Result<()> { + unsafe { + // Ignore signals. + libc::signal(libc::SIGHUP, libc::SIG_IGN); + libc::signal(libc::SIGINT, libc::SIG_IGN); + libc::signal(libc::SIGQUIT, libc::SIG_IGN); + + // Disconnect from the old controlling terminal. + let tty = libc::open(c"/dev/tty".as_ptr(), libc::O_RDWR | libc::O_NOCTTY); + if tty > 0 { + #[cfg_attr(target_os = "linux", expect(clippy::useless_conversion))] + libc::ioctl(tty, libc::TIOCNOTTY.into(), std::ptr::null_mut::<()>()); + libc::close(tty); + } + } + + // Create the sandbox options. + let options = sandbox::Options { + chroot: args.root, + hostname: args.options.hostname, + mounts: args + .options + .mounts + .into_iter() + .map(|mount| tangram_sandbox::Mount { + source: mount.source, + target: mount.target, + fstype: mount.fstype, + flags: mount.flags, + data: mount.data, + }) + .collect(), + network: args.options.network.get(), + user: args.options.user, + }; + + // Bind the listener before entering the sandbox, since the socket path is on the host filesystem. + let listener = sandbox::server::Server::bind(&args.socket)?; + + // Enter the sandbox. + unsafe { sandbox::server::Server::enter(&options)? }; + + // Spawn a tokio runtime. + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_io() + .build() + .map_err(|source| tg::error!(!source, "failed to start tokio runtime"))?; + + // Run the server. + runtime.block_on(async move { + let server = sandbox::server::Server::new() + .map_err(|source| tg::error!(!source, "failed to start the server"))?; + if let Some(fd) = args.ready_fd { + let mut file = unsafe { std::fs::File::from_raw_fd(fd) }; + file.write_all(&[0x00]) + .map_err(|source| tg::error!(!source, "failed to write the ready signal"))?; + drop(file); + } + + // Serve. + let listener = tokio::net::UnixListener::from_std(listener) + .inspect_err(|error| eprintln!("failed to convert the listener: {error}")) + .map_err(|source| tg::error!(!source, "failed to convert the listener"))?; + server + .serve(listener) + .await + .inspect_err(|error| eprintln!("failed to serve: {error}"))?; + Ok::<_, tg::Error>(()) + }) + } +} diff --git a/packages/cli/src/viewer.rs b/packages/cli/src/viewer.rs index 7391ca130..7a168308c 100644 --- a/packages/cli/src/viewer.rs +++ b/packages/cli/src/viewer.rs @@ -116,15 +116,15 @@ where } if let ct::event::Event::Mouse(event) = event { match event.kind { - ct::event::MouseEventKind::ScrollLeft => { - if self.data.hit_test(event.column, event.row) { - self.data.left(); - } + ct::event::MouseEventKind::ScrollLeft + if self.data.hit_test(event.column, event.row) => + { + self.data.left(); }, - ct::event::MouseEventKind::ScrollRight => { - if self.data.hit_test(event.column, event.row) { - self.data.right(); - } + ct::event::MouseEventKind::ScrollRight + if self.data.hit_test(event.column, event.row) => + { + self.data.right(); }, ct::event::MouseEventKind::ScrollUp => { if self.data.hit_test(event.column, event.row) { @@ -398,8 +398,7 @@ where // Get the cursor position. If this fails (for example because stdout is redirected), // still render by assuming row zero. let row = cursor_position_from_terminal(tty_fd) - .map(|(_column, row)| row.to_usize().unwrap()) - .unwrap_or(0); + .map_or(0, |(_column, row)| row.to_usize().unwrap()); // Clear the screen and save the cursor position. ct::queue!( @@ -562,7 +561,7 @@ fn query_cursor_position(fd: std::os::fd::RawFd) -> std::io::Result<(u16, u16)> return Err(std::io::Error::last_os_error()); } - let deadline = Instant::now() + Duration::from_millis(2000); + let deadline = Instant::now() + Duration::from_secs(2); let mut buffer = Vec::new(); let mut pollfd = libc::pollfd { fd, diff --git a/packages/cli/src/viewer/tree.rs b/packages/cli/src/viewer/tree.rs index 2e9735de8..74496304e 100644 --- a/packages/cli/src/viewer/tree.rs +++ b/packages/cli/src/viewer/tree.rs @@ -216,14 +216,10 @@ where Some(Item::Tag(pattern)) if options.expand_tags => expanded_nodes .borrow_mut() .insert(NodeID::Tag(pattern.to_string())), - Some(Item::Value(tg::Value::Object(object))) => { - if options.expand_objects { - expanded_nodes - .borrow_mut() - .insert(NodeID::Object(object.id())) - } else { - false - } + Some(Item::Value(tg::Value::Object(object))) if options.expand_objects => { + expanded_nodes + .borrow_mut() + .insert(NodeID::Object(object.id())) }, Some(Item::Value(_)) => options.expand_values, _ => false, @@ -541,20 +537,6 @@ where }; children.push(("executable".to_owned(), value)); children.push(("host".to_owned(), tg::Value::String(object.host.clone()))); - let mut mounts = Vec::new(); - for mount in &object.mounts { - let mut map = BTreeMap::new(); - map.insert( - "source".to_owned(), - tg::Value::Object(mount.source.clone().into()), - ); - map.insert( - "target".to_owned(), - tg::Value::String(mount.target.to_string_lossy().to_string()), - ); - mounts.push(tg::Value::Map(map)); - } - children.push(("mounts".to_owned(), tg::Value::Array(mounts))); let metadata = get_object_metadata_as_value(handle, command.id()).await?; command.unload(); diff --git a/packages/cli/tests/build/wait_cancellation.nu b/packages/cli/tests/build/wait_cancellation.nu index f861bacf8..aaef89789 100644 --- a/packages/cli/tests/build/wait_cancellation.nu +++ b/packages/cli/tests/build/wait_cancellation.nu @@ -18,31 +18,31 @@ let output = tg wait $process.process | complete let output = $output.stdout | str trim | from json snapshot $output.error.message 'the process was canceled' -let id = job spawn { - tg build $path | complete -}; -job kill $id +# let id = job spawn { +# tg build $path | complete +# }; +# job kill $id -let output = tg wait $process.process | complete -let output = $output.stdout | str trim | from json -snapshot $output.error.message 'the process was canceled' +# let output = tg wait $process.process | complete +# let output = $output.stdout | str trim | from json +# snapshot $output.error.message 'the process was canceled' -let path = artifact { - tangram.ts: ' - export default async () => { - await Promise.race([ - tg.sleep(0), - f(), - ]); - }; +# let path = artifact { +# tangram.ts: ' +# export default async () => { +# await Promise.race([ +# tg.sleep(0), +# f(), +# ]); +# }; - let f = async () => { - await tg.sleep(100); - console.log("after sleep"); - } - ' -} -let id = tg build -d $path -tg wait $id -let log = tg log $id -assert equal $log '' +# let f = async () => { +# await tg.sleep(100); +# console.log("after sleep"); +# } +# ' +# } +# let id = tg build -d $path +# tg wait $id +# let log = tg log $id +# assert equal $log '' diff --git a/packages/cli/tests/run/log_stream.nu b/packages/cli/tests/run/log_stream.nu index ae76d4644..2c3f65251 100644 --- a/packages/cli/tests/run/log_stream.nu +++ b/packages/cli/tests/run/log_stream.nu @@ -3,27 +3,32 @@ let server = spawn let path = artifact { tangram.ts: r#' export default async () => { - let alphabet = "abcdefghijklmnopqrstuvwxyz"; - for (let i = 0; i < 26; i++) { - let s = ""; - for (let j = 0; j < 20; j++) { - s = s + alphabet[i]; - } - console.log(s); - } - while(true) { - await tg.sleep(100); - } - }; + let alphabet = "abcdefghijklmnopqrstuvwxyz"; + for (let i = 0; i < 26; i++) { + let s = ""; + for (let j = 0; j < 20; j++) { + s = s + alphabet[i]; + } + console.log(s); + } + while(true) { + await tg.sleep(100); + } + }; '# } -let id = tg build -d $path | str trim +let process = tg build -dv $path | from json + sleep 1sec # Check that we can read just one chunk forwards -let output = tg log $id --position 0 --length 20 +let output = tg log $process.process --position 0 --length 20 snapshot $output 'aaaaaaaaaaaaaaaaaaaa' # Check that we can read chunks backwards. -let output = tg log $id --position 41 --length='-20' +let output = tg log $process.process --position 41 --length='-20' snapshot $output 'bbbbbbbbbbbbbbbbbbbb' + +# Cancel the process. +tg cancel $process.process $process.token +tg wait $process.process diff --git a/packages/cli/tests/tree/double_build.nu b/packages/cli/tests/tree/double_build.nu index f931b51c3..2e844ab96 100644 --- a/packages/cli/tests/tree/double_build.nu +++ b/packages/cli/tests/tree/double_build.nu @@ -4,7 +4,7 @@ let path = artifact { a.tg.ts: 'export default () => 42;', b.tg.ts: ' import a from "./a.tg.ts"; - export default () => tg.resolve(a); + export default () => tg.command(a); ', c: { tangram.ts: ' @@ -24,8 +24,8 @@ snapshot $output '{"exit":0,"output":42}' let output = tg view $id --mode inline --expand-processes --depth 2 snapshot $output ' - ✓ fil_01xw45e66hhhxemww9m1qmj7jp9n1zjhn3ewggx0f6eb9nwr4js46g#default - ├╴command: cmd_0166tgxrezqabf2zvd3mveyr2e1ss6ws1ehfeay2amxv5tadh47bhg + ✓ fil_01g1t9dmfw9v9arvs2k8e3x6zt69gpx993gbw8tw5jgpzqzptxt8fg#default + ├╴command: cmd_01vcrs5cx1wx85xj9yqa546kxq2h2w5td5p52dsscy745zgak3eycg ├╴output: 42 ├╴✓ ../b.tg.ts#default └╴✓ fil_01sa3pyv7baf50x2ymmvy7p41zqnmmv8gp1fq5z3mq60ps8vcfxa30#default diff --git a/packages/client/src/build.rs b/packages/client/src/build.rs index 6468beb4d..4bf7c5041 100644 --- a/packages/client/src/build.rs +++ b/packages/client/src/build.rs @@ -12,9 +12,7 @@ pub struct Arg { pub env: tg::value::Map, pub executable: Option, pub host: Option, - pub mounts: Vec, pub name: Option, - pub network: bool, pub parent: Option, pub progress: bool, pub remote: Option, @@ -33,11 +31,10 @@ where let executable = arg .executable .ok_or_else(|| tg::error!("expected the executable to be set"))?; - let mut builder = tg::Command::builder(host, executable); + let mut builder = tg::Command::builder(host.clone(), executable); builder = builder.args(arg.args); builder = builder.cwd(arg.cwd); builder = builder.env(arg.env); - builder = builder.mounts(arg.mounts); builder = builder.stdin(arg.stdin); builder = builder.user(arg.user); let command = builder.build(); @@ -46,22 +43,22 @@ where if let Some(name) = arg.name { command.options.name.replace(name); } - if arg.network && arg.checksum.is_none() { - return Err(tg::error!( - "a checksum is required to build with network enabled" - )); - } + // Create a new sandbox for the build. + let sandbox = Some(tg::Either::Left(tg::sandbox::create::Arg { + host: host.clone(), + ..Default::default() + })); + let progress = arg.progress; let arg = tg::process::spawn::Arg { cached: arg.cached, checksum: arg.checksum, command, local: None, - mounts: vec![], - network: arg.network, parent: arg.parent, remotes: arg.remote.map(|r| vec![r]), retry: arg.retry, + sandbox, stderr: None, stdin: None, stdout: None, diff --git a/packages/client/src/command.rs b/packages/client/src/command.rs index f4fe446fd..5545b050d 100644 --- a/packages/client/src/command.rs +++ b/packages/client/src/command.rs @@ -3,9 +3,7 @@ pub use self::{ data::Command as Data, handle::Command as Handle, id::Id, - object::{ - ArtifactExecutable, Command as Object, Executable, ModuleExecutable, Mount, PathExecutable, - }, + object::{ArtifactExecutable, Command as Object, Executable, ModuleExecutable, PathExecutable}, }; pub mod builder; diff --git a/packages/client/src/command/builder.rs b/packages/client/src/command/builder.rs index a04ce1308..c8ab2f4e4 100644 --- a/packages/client/src/command/builder.rs +++ b/packages/client/src/command/builder.rs @@ -10,7 +10,6 @@ pub struct Builder { env: BTreeMap, executable: tg::command::Executable, host: String, - mounts: Vec, stdin: Option, user: Option, } @@ -24,7 +23,6 @@ impl Builder { env: BTreeMap::new(), executable: executable.into(), host: host.into(), - mounts: Vec::new(), stdin: None, user: None, } @@ -38,7 +36,6 @@ impl Builder { env: object.env.clone(), executable: object.executable.clone(), host: object.host.clone(), - mounts: object.mounts.clone(), stdin: object.stdin.clone(), user: object.user.clone(), } @@ -80,18 +77,6 @@ impl Builder { self } - #[must_use] - pub fn mount(mut self, mount: tg::command::Mount) -> Self { - self.mounts.push(mount); - self - } - - #[must_use] - pub fn mounts(mut self, mounts: impl IntoIterator) -> Self { - self.mounts.extend(mounts); - self - } - #[must_use] pub fn stdin(mut self, stdin: impl Into>) -> Self { self.stdin = stdin.into(); @@ -112,7 +97,6 @@ impl Builder { env: self.env, executable: self.executable, host: self.host, - mounts: self.mounts, stdin: self.stdin, user: self.user, }) diff --git a/packages/client/src/command/data.rs b/packages/client/src/command/data.rs index 7716662bd..c4cd331db 100644 --- a/packages/client/src/command/data.rs +++ b/packages/client/src/command/data.rs @@ -35,10 +35,6 @@ pub struct Command { #[tangram_serialize(id = 4)] pub host: String, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - #[tangram_serialize(id = 5, default, skip_serializing_if = "Vec::is_empty")] - pub mounts: Vec, - #[serde(default, skip_serializing_if = "Option::is_none")] #[tangram_serialize(id = 6, default, skip_serializing_if = "Option::is_none")] pub stdin: Option, @@ -115,22 +111,6 @@ pub struct PathExecutable { pub path: PathBuf, } -#[derive( - Clone, - Debug, - serde::Deserialize, - serde::Serialize, - tangram_serialize::Deserialize, - tangram_serialize::Serialize, -)] -pub struct Mount { - #[tangram_serialize(id = 0)] - pub source: tg::artifact::Id, - - #[tangram_serialize(id = 1)] - pub target: PathBuf, -} - impl Command { pub fn serialize(&self) -> tg::Result { let mut bytes = Vec::new(); @@ -171,9 +151,6 @@ impl Command { for value in self.env.values() { value.children(children); } - for mount in &self.mounts { - mount.children(children); - } } } @@ -222,12 +199,6 @@ impl ModuleExecutable { } } -impl Mount { - pub fn children(&self, children: &mut BTreeSet) { - children.insert(self.source.clone().into()); - } -} - impl ArtifactExecutable { #[must_use] pub fn to_uri(&self) -> Uri { diff --git a/packages/client/src/command/handle.rs b/packages/client/src/command/handle.rs index a9f0901c4..c83c2b4b4 100644 --- a/packages/client/src/command/handle.rs +++ b/packages/client/src/command/handle.rs @@ -171,16 +171,6 @@ impl Command { Ok(self.object(handle).await?.map(|object| &object.host)) } - pub async fn mounts( - &self, - handle: &H, - ) -> tg::Result>> - where - H: tg::Handle, - { - Ok(self.object(handle).await?.map(|object| &object.mounts)) - } - pub async fn stdin(&self, handle: &H) -> tg::Result>> where H: tg::Handle, diff --git a/packages/client/src/command/object.rs b/packages/client/src/command/object.rs index 6468f4ff6..6242cf5d2 100644 --- a/packages/client/src/command/object.rs +++ b/packages/client/src/command/object.rs @@ -7,7 +7,6 @@ pub struct Command { pub env: tg::value::Map, pub executable: tg::command::Executable, pub host: String, - pub mounts: Vec, pub stdin: Option, pub user: Option, } @@ -37,12 +36,6 @@ pub struct PathExecutable { pub path: PathBuf, } -#[derive(Clone, Debug)] -pub struct Mount { - pub source: tg::Artifact, - pub target: PathBuf, -} - impl Command { #[must_use] pub fn to_data(&self) -> Data { @@ -59,7 +52,6 @@ impl Command { .collect(); let executable = self.executable.to_data(); let host = self.host.clone(); - let mounts = self.mounts.iter().map(Mount::to_data).collect(); let stdin = self.stdin.as_ref().map(tg::Blob::id); let user = self.user.clone(); Data { @@ -68,7 +60,6 @@ impl Command { env, executable, host, - mounts, stdin, user, } @@ -88,7 +79,6 @@ impl Command { .collect::>()?; let executable = tg::command::object::Executable::try_from_data(data.executable)?; let host = data.host; - let mounts = data.mounts.into_iter().map(Mount::from_data).collect(); let stdin = data.stdin.map(tg::Blob::with_id); let user = data.user; Ok(Self { @@ -97,7 +87,6 @@ impl Command { env, executable, host, - mounts, stdin, user, }) @@ -109,7 +98,6 @@ impl Command { .chain(self.executable.objects()) .chain(self.args.iter().flat_map(tg::Value::objects)) .chain(self.env.values().flat_map(tg::Value::objects)) - .chain(self.mounts.iter().flat_map(tg::command::Mount::object)) .collect() } } @@ -223,62 +211,6 @@ impl PathExecutable { } } -impl Mount { - #[must_use] - pub fn to_data(&self) -> tg::command::data::Mount { - let source = self.source.id(); - let target = self.target.clone(); - tg::command::data::Mount { source, target } - } - - fn from_data(data: tg::command::data::Mount) -> Self { - let source = tg::Artifact::with_id(data.source); - let target = data.target; - Self { source, target } - } - - #[must_use] - pub fn object(&self) -> Vec { - [self.source.clone().into()].into() - } -} - -impl std::fmt::Display for Mount { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}:{}", self.source.id(), self.target.display()) - } -} - -impl std::str::FromStr for Mount { - type Err = tg::Error; - - fn from_str(s: &str) -> Result { - let s = if let Some((s, ro)) = s.split_once(',') { - if ro == "ro" { - s - } else if ro == "rw" { - return Err(tg::error!("cannot mount artifacts read-write")); - } else { - return Err(tg::error!("unknown option: {ro:#?}")); - } - } else { - s - }; - let (source, target) = s - .split_once(':') - .ok_or_else(|| tg::error!("expected a target path"))?; - let target = PathBuf::from(target); - if !target.is_absolute() { - return Err(tg::error!(target = %target.display(), "expected an absolute path")); - } - let id = source - .parse() - .map_err(|source| tg::error!(!source, "failed to parse the artifact id"))?; - let source = tg::Artifact::with_id(id); - Ok(Self { source, target }) - } -} - impl From for Executable { fn from(value: tg::File) -> Self { Self::Artifact(ArtifactExecutable { diff --git a/packages/client/src/directory/handle.rs b/packages/client/src/directory/handle.rs index 9ae8c25b4..db6ac2912 100644 --- a/packages/client/src/directory/handle.rs +++ b/packages/client/src/directory/handle.rs @@ -419,11 +419,7 @@ impl Directory { let mut parents: Vec = vec![]; // Handle each path component. - loop { - // Handle the first path component. - let Some(component) = path.components().next() else { - break; - }; + while let Some(component) = path.components().next() { let name = match component { // Prefix and root components are not allowed. std::path::Component::Prefix(_) | std::path::Component::RootDir => { diff --git a/packages/client/src/handle.rs b/packages/client/src/handle.rs index b69b76c66..38626fd62 100644 --- a/packages/client/src/handle.rs +++ b/packages/client/src/handle.rs @@ -12,13 +12,14 @@ mod pipe; mod process; mod pty; mod remote; +mod sandbox; mod tag; mod user; mod watch; pub use self::{ ext::Ext, module::Module, object::Object, pipe::Pipe, process::Process, pty::Pty, - remote::Remote, tag::Tag, user::User, watch::Watch, + remote::Remote, sandbox::Sandbox, tag::Tag, user::User, watch::Watch, }; pub mod dynamic; @@ -31,6 +32,7 @@ pub trait Handle: + Pipe + Pty + Remote + + Sandbox + Tag + User + Watch diff --git a/packages/client/src/handle/dynamic.rs b/packages/client/src/handle/dynamic.rs index 9d59daf36..e5134b7ed 100644 --- a/packages/client/src/handle/dynamic.rs +++ b/packages/client/src/handle/dynamic.rs @@ -11,6 +11,7 @@ mod pipe; mod process; mod pty; mod remote; +mod sandbox; mod tag; mod user; mod watch; diff --git a/packages/client/src/handle/dynamic/sandbox.rs b/packages/client/src/handle/dynamic/sandbox.rs new file mode 100644 index 000000000..4e86939dc --- /dev/null +++ b/packages/client/src/handle/dynamic/sandbox.rs @@ -0,0 +1,60 @@ +use crate::prelude::*; + +impl tg::handle::Sandbox for super::Handle { + fn create_sandbox( + &self, + arg: tg::sandbox::create::Arg, + ) -> impl Future> { + self.0.create_sandbox(arg) + } + + fn delete_sandbox(&self, id: &tg::sandbox::Id) -> impl Future> { + unsafe { + std::mem::transmute::<_, futures::future::BoxFuture<'_, tg::Result<()>>>( + self.0.delete_sandbox(id), + ) + } + } + + fn sandbox_spawn( + &self, + id: &tg::sandbox::Id, + arg: tg::sandbox::spawn::Arg, + ) -> impl Future> { + unsafe { + std::mem::transmute::< + _, + futures::future::BoxFuture<'_, tg::Result>, + >(self.0.sandbox_spawn(id, arg)) + } + } + + fn try_sandbox_wait_future( + &self, + id: &tg::sandbox::Id, + arg: tg::sandbox::wait::Arg, + ) -> impl Future< + Output = tg::Result< + Option< + impl Future>> + Send + 'static, + >, + >, + > { + unsafe { + std::mem::transmute::< + _, + futures::future::BoxFuture< + '_, + tg::Result< + Option< + futures::future::BoxFuture< + 'static, + tg::Result>, + >, + >, + >, + >, + >(self.0.try_sandbox_wait_future(id, arg)) + } + } +} diff --git a/packages/client/src/handle/either.rs b/packages/client/src/handle/either.rs index f5b50977a..704d4b26c 100644 --- a/packages/client/src/handle/either.rs +++ b/packages/client/src/handle/either.rs @@ -10,6 +10,7 @@ mod pipe; mod process; mod pty; mod remote; +mod sandbox; mod tag; mod user; mod watch; diff --git a/packages/client/src/handle/either/sandbox.rs b/packages/client/src/handle/either/sandbox.rs new file mode 100644 index 000000000..caba2ba0d --- /dev/null +++ b/packages/client/src/handle/either/sandbox.rs @@ -0,0 +1,61 @@ +use { + crate::prelude::*, + futures::{FutureExt as _, TryFutureExt as _}, +}; + +impl tg::handle::Sandbox for tg::Either +where + L: tg::handle::Sandbox, + R: tg::handle::Sandbox, +{ + fn create_sandbox( + &self, + arg: tg::sandbox::create::Arg, + ) -> impl Future> { + match self { + tg::Either::Left(s) => s.create_sandbox(arg).left_future(), + tg::Either::Right(s) => s.create_sandbox(arg).right_future(), + } + } + + fn delete_sandbox(&self, id: &tg::sandbox::Id) -> impl Future> { + match self { + tg::Either::Left(s) => s.delete_sandbox(id).left_future(), + tg::Either::Right(s) => s.delete_sandbox(id).right_future(), + } + } + + fn sandbox_spawn( + &self, + id: &tg::sandbox::Id, + arg: tg::sandbox::spawn::Arg, + ) -> impl Future> { + match self { + tg::Either::Left(s) => s.sandbox_spawn(id, arg).left_future(), + tg::Either::Right(s) => s.sandbox_spawn(id, arg).right_future(), + } + } + + fn try_sandbox_wait_future( + &self, + id: &tg::sandbox::Id, + arg: tg::sandbox::wait::Arg, + ) -> impl Future< + Output = tg::Result< + Option< + impl Future>> + Send + 'static, + >, + >, + > { + match self { + tg::Either::Left(s) => s + .try_sandbox_wait_future(id, arg) + .map_ok(|option| option.map(futures::FutureExt::left_future)) + .left_future(), + tg::Either::Right(s) => s + .try_sandbox_wait_future(id, arg) + .map_ok(|option| option.map(futures::FutureExt::right_future)) + .right_future(), + } + } +} diff --git a/packages/client/src/handle/erased.rs b/packages/client/src/handle/erased.rs index 6b705fe09..3e9da968a 100644 --- a/packages/client/src/handle/erased.rs +++ b/packages/client/src/handle/erased.rs @@ -10,17 +10,30 @@ mod pipe; mod process; mod pty; mod remote; +mod sandbox; mod tag; mod user; mod watch; pub use self::{ module::Module, object::Object, pipe::Pipe, process::Process, pty::Pty, remote::Remote, - tag::Tag, user::User, watch::Watch, + sandbox::Sandbox, tag::Tag, user::User, watch::Watch, }; pub trait Handle: - Module + Object + Process + Pipe + Pty + Remote + Tag + User + Watch + Send + Sync + 'static + Module + + Object + + Process + + Pipe + + Pty + + Remote + + Sandbox + + Tag + + User + + Watch + + Send + + Sync + + 'static { fn cache( &self, diff --git a/packages/client/src/handle/erased/sandbox.rs b/packages/client/src/handle/erased/sandbox.rs new file mode 100644 index 000000000..f6c70310d --- /dev/null +++ b/packages/client/src/handle/erased/sandbox.rs @@ -0,0 +1,65 @@ +use { + crate::prelude::*, + futures::{future::BoxFuture, prelude::*}, +}; + +pub trait Sandbox: Send + Sync + 'static { + fn create_sandbox( + &self, + arg: tg::sandbox::create::Arg, + ) -> BoxFuture<'_, tg::Result>; + + fn delete_sandbox<'a>(&'a self, id: &'a tg::sandbox::Id) -> BoxFuture<'a, tg::Result<()>>; + + fn sandbox_spawn<'a>( + &'a self, + id: &'a tg::sandbox::Id, + arg: tg::sandbox::spawn::Arg, + ) -> BoxFuture<'a, tg::Result>; + + fn try_sandbox_wait_future<'a>( + &'a self, + id: &'a tg::sandbox::Id, + arg: tg::sandbox::wait::Arg, + ) -> BoxFuture< + 'a, + tg::Result>>>>, + >; +} + +impl Sandbox for T +where + T: tg::handle::Sandbox, +{ + fn create_sandbox( + &self, + arg: tg::sandbox::create::Arg, + ) -> BoxFuture<'_, tg::Result> { + self.create_sandbox(arg).boxed() + } + + fn delete_sandbox<'a>(&'a self, id: &'a tg::sandbox::Id) -> BoxFuture<'a, tg::Result<()>> { + self.delete_sandbox(id).boxed() + } + + fn sandbox_spawn<'a>( + &'a self, + id: &'a tg::sandbox::Id, + arg: tg::sandbox::spawn::Arg, + ) -> BoxFuture<'a, tg::Result> { + self.sandbox_spawn(id, arg).boxed() + } + + fn try_sandbox_wait_future<'a>( + &'a self, + id: &'a tg::sandbox::Id, + arg: tg::sandbox::wait::Arg, + ) -> BoxFuture< + 'a, + tg::Result>>>>, + > { + self.try_sandbox_wait_future(id, arg) + .map_ok(|option| option.map(futures::FutureExt::boxed)) + .boxed() + } +} diff --git a/packages/client/src/handle/sandbox.rs b/packages/client/src/handle/sandbox.rs new file mode 100644 index 000000000..6f73d280e --- /dev/null +++ b/packages/client/src/handle/sandbox.rs @@ -0,0 +1,63 @@ +use crate::prelude::*; + +pub trait Sandbox: Clone + Unpin + Send + Sync + 'static { + fn create_sandbox( + &self, + arg: tg::sandbox::create::Arg, + ) -> impl Future> + Send; + + fn delete_sandbox(&self, id: &tg::sandbox::Id) -> impl Future> + Send; + + fn sandbox_spawn( + &self, + id: &tg::sandbox::Id, + arg: tg::sandbox::spawn::Arg, + ) -> impl Future> + Send; + + fn try_sandbox_wait_future( + &self, + id: &tg::sandbox::Id, + arg: tg::sandbox::wait::Arg, + ) -> impl Future< + Output = tg::Result< + Option< + impl Future>> + Send + 'static, + >, + >, + > + Send; +} + +impl tg::handle::Sandbox for tg::Client { + fn create_sandbox( + &self, + arg: tg::sandbox::create::Arg, + ) -> impl Future> { + self.create_sandbox(arg) + } + + fn delete_sandbox(&self, id: &tg::sandbox::Id) -> impl Future> { + self.delete_sandbox(id) + } + + fn sandbox_spawn( + &self, + id: &tg::sandbox::Id, + arg: tg::sandbox::spawn::Arg, + ) -> impl Future> { + self.sandbox_spawn(id, arg) + } + + fn try_sandbox_wait_future( + &self, + id: &tg::sandbox::Id, + arg: tg::sandbox::wait::Arg, + ) -> impl Future< + Output = tg::Result< + Option< + impl Future>> + Send + 'static, + >, + >, + > { + self.try_sandbox_wait_future(id, arg) + } +} diff --git a/packages/client/src/http.rs b/packages/client/src/http.rs index 5c0d9dafe..5fbd8c976 100644 --- a/packages/client/src/http.rs +++ b/packages/client/src/http.rs @@ -82,7 +82,7 @@ impl tg::Client { tower::service_fn(move |request| { let future = service.clone().call(request); async move { - match tokio::time::timeout(Duration::from_secs(60), future).await { + match tokio::time::timeout(Duration::from_mins(1), future).await { Ok(result) => result, Err(_) => Err(Error::Other(tg::error!("request timed out"))), } diff --git a/packages/client/src/id.rs b/packages/client/src/id.rs index f14e42f2d..c0013e54c 100644 --- a/packages/client/src/id.rs +++ b/packages/client/src/id.rs @@ -34,6 +34,7 @@ pub enum Kind { User, Request, Error, + Sandbox, } #[derive(Clone, Eq, Hash, Ord, PartialEq, PartialOrd)] @@ -104,6 +105,7 @@ impl Id { Kind::User => 9, Kind::Request => 10, Kind::Error => 11, + Kind::Sandbox => 12, }; writer .write_u8(kind) @@ -297,6 +299,7 @@ impl std::fmt::Display for Kind { Self::User => "usr", Self::Request => "req", Self::Error => "err", + Self::Sandbox => "sbx", }; write!(f, "{kind}")?; Ok(()) @@ -320,6 +323,7 @@ impl std::str::FromStr for Kind { "usr" | "user" => Self::User, "req" | "request" => Self::Request, "err" | "error" => Self::Error, + "sbx" | "sandbox" => Self::Sandbox, _ => { return Err(tg::error!(%s, "invalid kind")); }, diff --git a/packages/client/src/lib.rs b/packages/client/src/lib.rs index c18d38ed0..77e7ba7c1 100644 --- a/packages/client/src/lib.rs +++ b/packages/client/src/lib.rs @@ -85,6 +85,7 @@ pub mod reference; pub mod referent; pub mod remote; pub mod run; +pub mod sandbox; pub mod symlink; pub mod sync; pub mod tag; diff --git a/packages/client/src/mutation.rs b/packages/client/src/mutation.rs index d99f9a83a..4bae7e8af 100644 --- a/packages/client/src/mutation.rs +++ b/packages/client/src/mutation.rs @@ -41,7 +41,10 @@ impl Mutation { values.iter().flat_map(tg::Value::objects).collect() }, Self::Prefix { template, .. } | Self::Suffix { template, .. } => template.objects(), - Self::Merge { value } => value.iter().flat_map(|(_key, val)| val.objects()).collect(), + Self::Merge { value } => value + .values() + .flat_map(super::value::Value::objects) + .collect(), } } diff --git a/packages/client/src/process.rs b/packages/client/src/process.rs index c036c71d4..3b54fc8bc 100644 --- a/packages/client/src/process.rs +++ b/packages/client/src/process.rs @@ -10,8 +10,8 @@ use { }; pub use self::{ - data::Data, id::Id, metadata::Metadata, mount::Mount, signal::Signal, state::State, - status::Status, stdio::Stdio, wait::Wait, + data::Data, id::Id, metadata::Metadata, signal::Signal, state::State, status::Status, + stdio::Stdio, wait::Wait, }; pub mod cancel; @@ -24,7 +24,6 @@ pub mod id; pub mod list; pub mod log; pub mod metadata; -pub mod mount; pub mod put; pub mod queue; pub mod signal; diff --git a/packages/client/src/process/data.rs b/packages/client/src/process/data.rs index 7b38b383f..bcc3b20af 100644 --- a/packages/client/src/process/data.rs +++ b/packages/client/src/process/data.rs @@ -1,9 +1,4 @@ -use { - crate::prelude::*, - serde::Deserialize as _, - std::path::PathBuf, - tangram_util::serde::{is_false, is_true, return_true}, -}; +use {crate::prelude::*, serde::Deserialize as _, tangram_util::serde::is_false}; #[derive( Clone, @@ -63,13 +58,13 @@ pub struct Data { #[tangram_serialize(id = 12, default, skip_serializing_if = "Option::is_none")] pub log: Option, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - #[tangram_serialize(id = 13, default, skip_serializing_if = "Vec::is_empty")] - pub mounts: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[tangram_serialize(id = 13, default, skip_serializing_if = "Option::is_none")] + pub sandbox: Option, - #[serde(default, skip_serializing_if = "is_false")] - #[tangram_serialize(id = 14, default, skip_serializing_if = "is_false")] - pub network: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[tangram_serialize(id = 14, default, skip_serializing_if = "Option::is_none")] + pub pid: Option, #[serde( default, @@ -103,26 +98,6 @@ pub struct Data { pub stdout: Option, } -#[derive( - Clone, - Debug, - serde::Deserialize, - serde::Serialize, - tangram_serialize::Deserialize, - tangram_serialize::Serialize, -)] -pub struct Mount { - #[tangram_serialize(id = 0)] - pub source: PathBuf, - - #[tangram_serialize(id = 1)] - pub target: PathBuf, - - #[serde(default = "return_true", skip_serializing_if = "is_true")] - #[tangram_serialize(id = 2, default = "return_true", skip_serializing_if = "is_true")] - pub readonly: bool, -} - fn deserialize_output<'de, D>(deserializer: D) -> Result, D::Error> where D: serde::Deserializer<'de>, diff --git a/packages/client/src/process/mount.rs b/packages/client/src/process/mount.rs deleted file mode 100644 index ad1654fe4..000000000 --- a/packages/client/src/process/mount.rs +++ /dev/null @@ -1,88 +0,0 @@ -use { - crate::prelude::*, - std::{path::PathBuf, str::FromStr}, -}; - -#[derive(Clone, Debug)] -pub struct Mount { - pub source: PathBuf, - pub target: PathBuf, - pub readonly: bool, -} - -impl Mount { - #[must_use] - pub fn children(&self) -> Vec { - Vec::new() - } - - #[must_use] - pub fn to_data(&self) -> tg::process::data::Mount { - let source = self.source.clone(); - let target = self.target.clone(); - let readonly = self.readonly; - tg::process::data::Mount { - source, - target, - readonly, - } - } -} - -impl From for Mount { - fn from(value: tg::process::data::Mount) -> Self { - Self { - source: value.source, - target: value.target, - readonly: value.readonly, - } - } -} - -impl FromStr for Mount { - type Err = tg::Error; - - fn from_str(s: &str) -> Result { - let (s, readonly) = if let Some((s, ro)) = s.split_once(',') { - if ro == "ro" { - (s, Some(true)) - } else if ro == "rw" { - (s, Some(false)) - } else { - return Err(tg::error!("unknown option: {ro:#?}")); - } - } else { - (s, None) - }; - let (source, target) = s - .split_once(':') - .ok_or_else(|| tg::error!("expected a target path"))?; - let target = PathBuf::from(target); - if !target.is_absolute() { - return Err(tg::error!(target = %target.display(), "expected an absolute path")); - } - let source = source.into(); - let readonly = readonly.unwrap_or(false); - Ok(Self { - source, - target, - readonly, - }) - } -} - -#[cfg(test)] -mod tests { - #[test] - fn parse() { - let _mount = "./source:/target,ro" - .parse::() - .expect("failed to parse"); - let _mount = "./source:/target,rw" - .parse::() - .expect("failed to parse"); - let _mount = "./source:/target" - .parse::() - .expect("failed to parse"); - } -} diff --git a/packages/client/src/process/spawn.rs b/packages/client/src/process/spawn.rs index 1afcfc527..19cc9147d 100644 --- a/packages/client/src/process/spawn.rs +++ b/packages/client/src/process/spawn.rs @@ -20,12 +20,6 @@ pub struct Arg { #[serde(default, skip_serializing_if = "Option::is_none")] pub local: Option, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub mounts: Vec, - - #[serde(default, skip_serializing_if = "is_false")] - pub network: bool, - #[serde(default, skip_serializing_if = "Option::is_none")] pub parent: Option, @@ -36,6 +30,9 @@ pub struct Arg { #[serde(default, skip_serializing_if = "is_false")] pub retry: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub sandbox: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] pub stderr: Option, @@ -118,11 +115,10 @@ impl Arg { checksum: None, command, local: None, - mounts: Vec::new(), - network: false, parent: None, remotes: None, retry: false, + sandbox: None, stderr: None, stdin: None, stdout: None, @@ -139,11 +135,10 @@ impl Arg { checksum, command, local: None, - mounts: Vec::new(), - network: false, parent: None, remotes: None, retry: false, + sandbox: None, stderr: None, stdin: None, stdout: None, diff --git a/packages/client/src/process/state.rs b/packages/client/src/process/state.rs index 51d5de4ee..42035dd02 100644 --- a/packages/client/src/process/state.rs +++ b/packages/client/src/process/state.rs @@ -14,10 +14,10 @@ pub struct State { pub expected_checksum: Option, pub finished_at: Option, pub log: Option, - pub mounts: Vec, - pub network: bool, pub output: Option, + pub pid: Option, pub retry: bool, + pub sandbox: Option, pub started_at: Option, pub status: tg::process::Status, pub stderr: Option, @@ -55,10 +55,10 @@ impl TryFrom for tg::process::State { let expected_checksum = value.expected_checksum; let finished_at = value.finished_at; let log = value.log.map(tg::Blob::with_id); - let mounts = value.mounts.into_iter().map(Into::into).collect(); - let network = value.network; let output = value.output.map(tg::Value::try_from).transpose()?; + let pid = value.pid; let retry = value.retry; + let sandbox = value.sandbox; let started_at = value.started_at; let status = value.status; let stderr = value.stderr; @@ -77,10 +77,10 @@ impl TryFrom for tg::process::State { expected_checksum, finished_at, log, - mounts, - network, output, + pid, retry, + sandbox, started_at, status, stderr, diff --git a/packages/client/src/progress.rs b/packages/client/src/progress.rs index bcc1eb613..0d2939495 100644 --- a/packages/client/src/progress.rs +++ b/packages/client/src/progress.rs @@ -85,11 +85,10 @@ impl std::fmt::Display for Indicator { const LENGTH: u64 = 20; if let (Some(current), Some(total)) = (self.current, self.total) { write!(f, "[")?; - let n = if total > 0 { - (current * LENGTH / total).min(LENGTH) - } else { - LENGTH - }; + let n = (current * LENGTH) + .checked_div(total) + .map_or(LENGTH, |n| n.min(LENGTH)); + for _ in 0..n { write!(f, "=")?; } diff --git a/packages/client/src/run.rs b/packages/client/src/run.rs index a80f60032..39aac100d 100644 --- a/packages/client/src/run.rs +++ b/packages/client/src/run.rs @@ -12,9 +12,7 @@ pub struct Arg { pub env: tg::value::Map, pub executable: Option, pub host: Option, - pub mounts: Option>>, pub name: Option, - pub network: Option, pub parent: Option, pub progress: bool, pub remote: Option, @@ -73,24 +71,6 @@ where env.remove("TANGRAM_PROCESS"); env.remove("TANGRAM_URL"); builder = builder.env(env); - let mut command_mounts = vec![]; - let mut process_mounts = vec![]; - if let Some(mounts) = arg.mounts { - for mount in mounts { - match mount { - tg::Either::Left(mount) => process_mounts.push(mount.to_data()), - tg::Either::Right(mount) => command_mounts.push(mount), - } - } - } else { - if let Some(mounts) = command.as_ref().map(|command| command.mounts.clone()) { - command_mounts = mounts; - } - if let Some(mounts) = state.as_ref().map(|state| state.mounts.clone()) { - process_mounts = mounts.iter().map(tg::process::Mount::to_data).collect(); - } - } - builder = builder.mounts(command_mounts); let stdin = if arg.stdin.is_none() { command .as_ref() @@ -112,10 +92,6 @@ where command.options.name.replace(name); } let checksum = arg.checksum; - let network = arg - .network - .or(state.as_ref().map(|state| state.network)) - .unwrap_or_default(); let stderr = arg .stderr .unwrap_or_else(|| state.as_ref().and_then(|state| state.stderr.clone())); @@ -134,22 +110,26 @@ where let stdout = arg .stdout .unwrap_or_else(|| state.as_ref().and_then(|state| state.stdout.clone())); - if network && checksum.is_none() { - return Err(tg::error!( - "a checksum is required to build with network enabled" - )); - } + // Read the current sandbox ID from the environment. + let sandbox = std::env::var("TANGRAM_SANDBOX") + .ok() + .map(|id| { + id.parse() + .map(tg::Either::Right) + .map_err(|source| tg::error!(!source, "failed to parse the sandbox id")) + }) + .transpose()?; + let progress = arg.progress; let arg = tg::process::spawn::Arg { cached: arg.cached, checksum, command, local: None, - mounts: process_mounts, - network, parent: arg.parent, remotes: arg.remote.map(|r| vec![r]), retry: arg.retry, + sandbox, stderr, stdin, stdout, diff --git a/packages/client/src/sandbox.rs b/packages/client/src/sandbox.rs new file mode 100644 index 000000000..845cbef3e --- /dev/null +++ b/packages/client/src/sandbox.rs @@ -0,0 +1,9 @@ +pub use self::id::Id; + +pub mod create; +pub mod data; +pub mod delete; +pub mod get; +pub mod id; +pub mod spawn; +pub mod wait; diff --git a/packages/client/src/sandbox/create.rs b/packages/client/src/sandbox/create.rs new file mode 100644 index 000000000..779fe4db3 --- /dev/null +++ b/packages/client/src/sandbox/create.rs @@ -0,0 +1,81 @@ +use { + crate::prelude::*, + std::path::PathBuf, + tangram_http::{request::builder::Ext as _, response::Ext as _}, + tangram_util::serde::{is_false, is_true, return_true}, +}; + +#[derive(Clone, Debug, Default, serde::Deserialize, serde::Serialize)] +pub struct Arg { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub hostname: Option, + + pub host: String, + + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub mounts: Vec, + + #[serde(default, skip_serializing_if = "is_false")] + pub network: bool, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub user: Option, +} + +#[derive( + Clone, + Debug, + serde::Deserialize, + serde::Serialize, + tangram_serialize::Deserialize, + tangram_serialize::Serialize, +)] +pub struct Mount { + #[tangram_serialize(id = 0)] + pub source: tg::Either, + + #[tangram_serialize(id = 1)] + pub target: PathBuf, + + #[serde(default = "return_true", skip_serializing_if = "is_true")] + #[tangram_serialize(id = 2, default = "return_true", skip_serializing_if = "is_true")] + pub readonly: bool, +} + +#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)] +pub struct Output { + pub id: tg::sandbox::Id, +} + +impl tg::Client { + pub async fn create_sandbox(&self, arg: Arg) -> tg::Result { + let method = http::Method::POST; + let uri = "/sandbox/create"; + let request = http::request::Builder::default() + .method(method) + .uri(uri) + .header(http::header::ACCEPT, mime::APPLICATION_JSON.to_string()) + .header( + http::header::CONTENT_TYPE, + mime::APPLICATION_JSON.to_string(), + ) + .json(arg) + .map_err(|source| tg::error!(!source, "failed to serialize the arg"))? + .unwrap(); + let response = self + .send_with_retry(request) + .await + .map_err(|source| tg::error!(!source, "failed to send the request"))?; + if !response.status().is_success() { + let error = response.json().await.map_err(|source| { + tg::error!(!source, "failed to deserialize the error response") + })?; + return Err(error); + } + let output = response + .json() + .await + .map_err(|source| tg::error!(!source, "failed to deserialize the response"))?; + Ok(output) + } +} diff --git a/packages/client/src/sandbox/data.rs b/packages/client/src/sandbox/data.rs new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/packages/client/src/sandbox/data.rs @@ -0,0 +1 @@ + diff --git a/packages/client/src/sandbox/delete.rs b/packages/client/src/sandbox/delete.rs new file mode 100644 index 000000000..c5d817eee --- /dev/null +++ b/packages/client/src/sandbox/delete.rs @@ -0,0 +1,27 @@ +use { + crate::prelude::*, + tangram_http::{request::builder::Ext as _, response::Ext as _}, +}; + +impl tg::Client { + pub async fn delete_sandbox(&self, id: &tg::sandbox::Id) -> tg::Result<()> { + let method = http::Method::DELETE; + let uri = format!("/sandbox/{id}"); + let request = http::request::Builder::default() + .method(method) + .uri(uri) + .empty() + .unwrap(); + let response = self + .send_with_retry(request) + .await + .map_err(|source| tg::error!(!source, "failed to send the request"))?; + if !response.status().is_success() { + let error = response.json().await.map_err(|source| { + tg::error!(!source, "failed to deserialize the error response") + })?; + return Err(error); + } + Ok(()) + } +} diff --git a/packages/client/src/sandbox/get.rs b/packages/client/src/sandbox/get.rs new file mode 100644 index 000000000..e30552847 --- /dev/null +++ b/packages/client/src/sandbox/get.rs @@ -0,0 +1,9 @@ +use {crate::prelude::*, std::path::PathBuf}; + +pub struct Arg { + pub id: tg::sandbox::Id, +} + +pub struct Output { + pub path: PathBuf, +} diff --git a/packages/client/src/sandbox/id.rs b/packages/client/src/sandbox/id.rs new file mode 100644 index 000000000..a1efdaac7 --- /dev/null +++ b/packages/client/src/sandbox/id.rs @@ -0,0 +1,81 @@ +use crate::prelude::*; + +#[derive( + Clone, + Eq, + Hash, + Ord, + PartialEq, + PartialOrd, + derive_more::Debug, + derive_more::Display, + serde::Deserialize, + serde::Serialize, + tangram_serialize::Deserialize, + tangram_serialize::Serialize, +)] +#[debug("tg::sandbox::Id(\"{_0}\")")] +#[serde(into = "tg::Id", try_from = "tg::Id")] +#[tangram_serialize(into = "tg::Id", try_from = "tg::Id")] +pub struct Id(tg::Id); + +impl tg::sandbox::Id { + #[expect(clippy::new_without_default)] + #[must_use] + pub fn new() -> Self { + Self(tg::Id::new_uuidv7(tg::id::Kind::Sandbox)) + } + + pub fn from_slice(bytes: &[u8]) -> tg::Result { + tg::Id::from_reader(bytes)?.try_into() + } +} + +impl std::ops::Deref for tg::sandbox::Id { + type Target = tg::Id; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl From for tg::Id { + fn from(value: tg::sandbox::Id) -> Self { + value.0 + } +} + +impl TryFrom for tg::sandbox::Id { + type Error = tg::Error; + + fn try_from(value: tg::Id) -> tg::Result { + if value.kind() != tg::id::Kind::Sandbox { + return Err(tg::error!(%value, "invalid kind")); + } + Ok(Self(value)) + } +} + +impl TryFrom> for tg::sandbox::Id { + type Error = tg::Error; + + fn try_from(value: Vec) -> tg::Result { + Self::from_slice(&value) + } +} + +impl std::str::FromStr for tg::sandbox::Id { + type Err = tg::Error; + + fn from_str(s: &str) -> tg::Result { + tg::Id::from_str(s)?.try_into() + } +} + +impl TryFrom for tg::sandbox::Id { + type Error = tg::Error; + + fn try_from(value: String) -> tg::Result { + value.parse() + } +} diff --git a/packages/client/src/sandbox/spawn.rs b/packages/client/src/sandbox/spawn.rs new file mode 100644 index 000000000..ae5ccbd94 --- /dev/null +++ b/packages/client/src/sandbox/spawn.rs @@ -0,0 +1,60 @@ +use { + crate::prelude::*, + std::{collections::BTreeMap, path::PathBuf}, + tangram_http::{request::builder::Ext as _, response::Ext as _}, +}; + +#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)] +pub struct Arg { + pub command: PathBuf, + + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub args: Vec, + + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub env: BTreeMap, + + pub stdin: tg::process::Stdio, + + pub stdout: tg::process::Stdio, + + pub stderr: tg::process::Stdio, +} + +#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)] +pub struct Output { + pub pid: i32, +} + +impl tg::Client { + pub async fn sandbox_spawn(&self, id: &tg::sandbox::Id, arg: Arg) -> tg::Result { + let method = http::Method::POST; + let uri = format!("/sandbox/{id}/spawn"); + let request = http::request::Builder::default() + .method(method) + .uri(uri) + .header(http::header::ACCEPT, mime::APPLICATION_JSON.to_string()) + .header( + http::header::CONTENT_TYPE, + mime::APPLICATION_JSON.to_string(), + ) + .json(arg) + .map_err(|source| tg::error!(!source, "failed to serialize the arg"))? + .unwrap(); + let response = self + .send_with_retry(request) + .await + .map_err(|source| tg::error!(!source, "failed to send the request"))?; + if !response.status().is_success() { + let error = response.json().await.map_err(|source| { + tg::error!(!source, "failed to deserialize the error response") + })?; + return Err(error); + } + let output = response + .json() + .await + .map_err(|source| tg::error!(!source, "failed to deserialize the response"))?; + Ok(output) + } +} diff --git a/packages/client/src/sandbox/wait.rs b/packages/client/src/sandbox/wait.rs new file mode 100644 index 000000000..3d0f31bb3 --- /dev/null +++ b/packages/client/src/sandbox/wait.rs @@ -0,0 +1,136 @@ +use { + crate::prelude::*, + futures::{StreamExt as _, TryFutureExt as _, TryStreamExt as _, future}, + tangram_futures::stream::TryExt as _, + tangram_http::{request::builder::Ext as _, response::Ext as _}, +}; + +#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)] +pub struct Arg { + pub pid: i32, +} + +#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)] +pub struct Output { + pub status: i32, +} + +#[derive(Clone, Debug)] +pub enum Event { + Output(Output), +} + +impl tg::Client { + pub async fn sandbox_wait(&self, id: &tg::sandbox::Id, arg: Arg) -> tg::Result { + let future = self + .try_sandbox_wait_future(id, arg) + .await? + .ok_or_else(|| tg::error!("the sandbox was not found"))?; + let output = future + .await? + .ok_or_else(|| tg::error!("expected a wait output"))?; + Ok(output) + } + + pub async fn try_sandbox_wait_future( + &self, + id: &tg::sandbox::Id, + arg: Arg, + ) -> tg::Result>> + Send + 'static>> { + let method = http::Method::POST; + let query = serde_urlencoded::to_string(&arg) + .map_err(|source| tg::error!(!source, "failed to serialize the arg"))?; + let uri = format!("/sandbox/{id}/wait?{query}"); + let request = http::request::Builder::default() + .method(method) + .uri(uri) + .header(http::header::ACCEPT, mime::TEXT_EVENT_STREAM.to_string()) + .empty() + .unwrap(); + let response = self + .send_with_retry(request) + .await + .map_err(|source| tg::error!(!source, "failed to send the request"))?; + if response.status() == http::StatusCode::NOT_FOUND { + return Ok(None); + } + if !response.status().is_success() { + let error = response.json().await.map_err(|source| { + tg::error!(!source, "failed to deserialize the error response") + })?; + return Err(error); + } + let content_type = response + .parse_header::(http::header::CONTENT_TYPE) + .transpose()?; + if !matches!( + content_type + .as_ref() + .map(|content_type| (content_type.type_(), content_type.subtype())), + Some((mime::TEXT, mime::EVENT_STREAM)), + ) { + return Err(tg::error!(?content_type, "invalid content type")); + } + let stream = response + .sse() + .map_err(|source| tg::error!(!source, "failed to read an event")) + .and_then(|event| { + future::ready( + if event.event.as_deref().is_some_and(|event| event == "error") { + match event.try_into() { + Ok(error) | Err(error) => Err(error), + } + } else { + event.try_into() + }, + ) + }) + .boxed(); + let future = stream.boxed().try_last().map_ok(|option| { + option.map(|output| { + let Event::Output(output) = output; + output + }) + }); + Ok(Some(future)) + } +} + +impl TryFrom for tangram_http::sse::Event { + type Error = tg::Error; + + fn try_from(value: Event) -> Result { + let event = match value { + Event::Output(output) => { + let data = serde_json::to_string(&output) + .map_err(|source| tg::error!(!source, "failed to serialize the event"))?; + tangram_http::sse::Event { + data, + event: Some("output".into()), + ..Default::default() + } + }, + }; + Ok(event) + } +} + +impl TryFrom for Event { + type Error = tg::Error; + + fn try_from(value: tangram_http::sse::Event) -> tg::Result { + match value.event.as_deref() { + Some("output") => { + let output = serde_json::from_str(&value.data) + .map_err(|source| tg::error!(!source, "failed to deserialize the event"))?; + Ok(Self::Output(output)) + }, + Some("error") => { + let error = serde_json::from_str(&value.data) + .map_err(|source| tg::error!(!source, "failed to deserialize the event"))?; + Err(error) + }, + value => Err(tg::error!(?value, "invalid event")), + } + } +} diff --git a/packages/client/src/value/parse.rs b/packages/client/src/value/parse.rs index e69486de2..4deb6bc81 100644 --- a/packages/client/src/value/parse.rs +++ b/packages/client/src/value/parse.rs @@ -668,7 +668,6 @@ fn command_inner(input: &mut Input) -> ModalResult { let mut env = BTreeMap::new(); let mut executable = None; let mut host = String::from("builtin"); - let mut mounts = Vec::new(); for (key, value) in entries { match key.as_str() { "args" => { @@ -698,18 +697,6 @@ fn command_inner(input: &mut Input) -> ModalResult { .map_err(|_| tg::error!("expected string for host"))?; host = value.clone(); }, - "mounts" => { - let value = value - .try_unwrap_array_ref() - .map_err(|_| tg::error!("expected array for mounts"))?; - for item in value { - let value = item - .try_unwrap_map_ref() - .map_err(|_| tg::error!("expected object for mount in mounts array"))?; - let mount = parse_mount(value)?; - mounts.push(mount); - } - }, _ => { return Err(tg::error!("unexpected field in command: {}", key)); }, @@ -722,7 +709,6 @@ fn command_inner(input: &mut Input) -> ModalResult { env, executable, host, - mounts, stdin: None, user: None, }; @@ -1353,35 +1339,6 @@ fn parse_module_referent(map: &tg::value::Map) -> tg::Result tg::Result { - let mut source = None; - let mut target = None; - for (key, value) in map { - match key.as_str() { - "source" => { - let value = value - .try_unwrap_object_ref() - .map_err(|_| tg::error!("expected object for source"))?; - let artifact = tg::Artifact::try_from(value.clone()) - .map_err(|error| tg::error!(!error, "expected artifact object for source"))?; - source = Some(artifact); - }, - "target" => { - let value = value - .try_unwrap_string_ref() - .map_err(|_| tg::error!("expected string for target"))?; - target = Some(PathBuf::from(value)); - }, - _ => { - return Err(tg::error!("unexpected field in mount: {}", key)); - }, - } - } - let source = source.ok_or_else(|| tg::error!("missing source field"))?; - let target = target.ok_or_else(|| tg::error!("missing target field"))?; - Ok(tg::command::Mount { source, target }) -} - fn whitespace(input: &mut Input) -> ModalResult<()> { take_while(0.., [' ', '\t', '\r', '\n']) .parse_next(input) diff --git a/packages/client/src/value/print.rs b/packages/client/src/value/print.rs index b1fcc5411..3b955d304 100644 --- a/packages/client/src/value/print.rs +++ b/packages/client/src/value/print.rs @@ -686,21 +686,6 @@ where }, })?; self.map_entry("host", |s| s.string(&object.host))?; - if !object.mounts.is_empty() { - self.map_entry("mounts", |s| { - s.start_array()?; - for mount in &object.mounts { - s.start_map()?; - s.map_entry("source", |s| s.artifact(&mount.source))?; - s.map_entry("target", |s| { - s.string(mount.target.to_string_lossy().as_ref()) - })?; - s.finish_map()?; - } - s.finish_array()?; - Ok(()) - })?; - } self.finish_map()?; write!(self.writer, ")")?; Ok(()) diff --git a/packages/clients/js/src/build.ts b/packages/clients/js/src/build.ts index 5544aabc6..ffa7921e0 100644 --- a/packages/clients/js/src/build.ts +++ b/packages/clients/js/src/build.ts @@ -79,10 +79,6 @@ async function inner(...args: tg.Args): Promise { } } - let commandMounts: Array | undefined; - if ("mounts" in arg && arg.mounts !== undefined) { - commandMounts = arg.mounts; - } let commandStdin: tg.Blob.Arg | undefined; if ("stdin" in arg && arg.stdin !== undefined) { commandStdin = arg.stdin; @@ -94,15 +90,15 @@ async function inner(...args: tg.Args): Promise { executable !== undefined ? { executable: executable } : undefined, "host" in arg ? { host: arg.host } : undefined, "user" in arg ? { user: arg.user } : undefined, - commandMounts !== undefined ? { mounts: commandMounts } : undefined, commandStdin !== undefined ? { stdin: commandStdin } : undefined, ); let checksum = arg.checksum; - let network = "network" in arg ? (arg.network ?? false) : false; - if (network === true && checksum === undefined) { - throw new Error("a checksum is required to build with network enabled"); - } + let host = arg.host ?? tg.process.env.TANGRAM_HOST; + tg.assert(host !== undefined, "expected the host to be set"); + tg.assert(typeof host === "string"); + let sandbox: tg.Process.Sandbox | undefined = + "sandbox" in arg ? arg.sandbox : { host }; let commandId = await command.store(); let commandReferent = { item: commandId, @@ -112,11 +108,10 @@ async function inner(...args: tg.Args): Promise { checksum, command: commandReferent, create: false, - mounts: [], - network, - parent: undefined, + parent: tg.Process.current?.id, remote: undefined, retry: false, + sandbox, stderr: undefined, stdin: undefined, stdout: undefined, @@ -216,7 +211,6 @@ async function arg_( env: object.env, executable: object.executable, host: object.host, - mounts: object.mounts, }; if (object.cwd !== undefined) { output.cwd = object.cwd; @@ -312,15 +306,8 @@ export class BuildBuilder< return this; } - mount(...mounts: Array>): this { - this.#args.push({ mounts }); - return this; - } - - mounts( - ...mounts: Array>>> - ): this { - this.#args.push(...mounts.map((mounts) => ({ mounts }))); + sandbox(sandbox: tg.Process.Sandbox | undefined): this { + this.#args.push({ sandbox }); return this; } @@ -329,11 +316,6 @@ export class BuildBuilder< return this; } - network(network: tg.Unresolved>): this { - this.#args.push({ network }); - return this; - } - then( onfulfilled?: | ((value: R) => TResult1 | PromiseLike) diff --git a/packages/clients/js/src/command.ts b/packages/clients/js/src/command.ts index 85254f6a9..2f85f0ba9 100644 --- a/packages/clients/js/src/command.ts +++ b/packages/clients/js/src/command.ts @@ -118,10 +118,6 @@ export class Command< }; } let host = arg.host ?? (tg.process.env.TANGRAM_HOST as string); - let mounts: Array = []; - if (arg.mounts && arg.mounts.length > 0) { - mounts = arg.mounts; - } if (executable === undefined) { throw new Error("cannot create a command without an executable"); } @@ -136,7 +132,6 @@ export class Command< env, executable, host, - mounts, stdin, user, }; @@ -258,12 +253,6 @@ export class Command< })(); } - get mounts(): Promise | undefined> { - return (async () => { - return (await this.object()).mounts; - })(); - } - build(...args: tg.UnresolvedArgs): tg.BuildBuilder<[], R> { return tg.build(this, { args }) as tg.BuildBuilder<[], R>; } @@ -291,7 +280,6 @@ export namespace Command { env?: tg.MaybeMutationMap | undefined; executable?: tg.Command.Arg.Executable | undefined; host?: string | undefined; - mounts?: Array | undefined; stdin?: tg.Blob.Arg | undefined; user?: string | undefined; }; @@ -348,7 +336,6 @@ export namespace Command { env: { [key: string]: tg.Value }; executable: tg.Command.Executable; host: string; - mounts: Array; stdin: tg.Blob | undefined; user: string | undefined; }; @@ -369,9 +356,6 @@ export namespace Command { if (object.cwd !== undefined) { output.cwd = object.cwd; } - if (object.mounts.length > 0) { - output.mounts = object.mounts.map(tg.Command.Mount.toData); - } if (object.stdin !== undefined) { output.stdin = object.stdin.id; } @@ -393,7 +377,6 @@ export namespace Command { ), executable: tg.Command.Executable.fromData(data.executable), host: data.host, - mounts: (data.mounts ?? []).map(tg.Command.Mount.fromData), stdin: data.stdin !== undefined ? tg.Blob.withId(data.stdin) : undefined, user: data.user, @@ -407,7 +390,6 @@ export namespace Command { tg.Value.objects(value), ), ...tg.Command.Executable.children(object.executable), - ...object.mounts.map(({ source }) => source), ...(object.stdin !== undefined ? [object.stdin] : []), ]; }; @@ -496,34 +478,12 @@ export namespace Command { }; } - export type Mount = { - source: tg.Artifact; - target: string; - }; - - export namespace Mount { - export let toData = (data: tg.Command.Mount): tg.Command.Data.Mount => { - return { - source: data.source.id, - target: data.target, - }; - }; - - export let fromData = (data: tg.Command.Data.Mount): tg.Command.Mount => { - return { - source: tg.Artifact.withId(data.source), - target: data.target, - }; - }; - } - export type Data = { args?: Array; cwd?: string; env?: { [key: string]: tg.Value.Data }; executable: tg.Command.Data.Executable; host: string; - mounts?: Array; stdin?: tg.Blob.Id; user?: string; }; @@ -549,11 +509,6 @@ export namespace Command { path: string; }; } - - export type Mount = { - source: tg.Artifact.Id; - target: string; - }; } } @@ -623,18 +578,6 @@ export class CommandBuilder< return this; } - mount(...mounts: Array>): this { - this.#args.push({ mounts }); - return this; - } - - mounts( - ...mounts: Array>>> - ): this { - this.#args.push(...mounts.map((mounts) => ({ mounts }))); - return this; - } - then, TResult2 = never>( onfulfilled?: | ((value: tg.Command) => TResult1 | PromiseLike) diff --git a/packages/clients/js/src/handle.ts b/packages/clients/js/src/handle.ts index db9c58114..05b5e4cb3 100644 --- a/packages/clients/js/src/handle.ts +++ b/packages/clients/js/src/handle.ts @@ -12,6 +12,7 @@ export type Handle = { write(bytes: string | Uint8Array): Promise; } & tg.Handle.Object & tg.Handle.Process & + tg.Handle.Sandbox & tg.Handle.System; export namespace Handle { @@ -29,11 +30,10 @@ export namespace Handle { checksum: tg.Checksum | undefined; command: tg.Referent; create: boolean; - mounts: Array; - network: boolean; parent: tg.Process.Id | undefined; remote: string | undefined; retry: boolean; + sandbox: tg.Process.Sandbox | undefined; stderr: string | undefined; stdin: string | undefined; stdout: string | undefined; @@ -78,6 +78,49 @@ export namespace Handle { ): Promise; }; + export type Sandbox = { + createSandbox(arg: SandboxCreateArg): Promise; + + deleteSandbox(id: string): Promise; + + sandboxSpawn(id: string, arg: SandboxSpawnArg): Promise; + + sandboxWait(id: string, arg: SandboxWaitArg): Promise; + }; + + export type SandboxCreateArg = { + hostname?: string | undefined; + host: string; + mounts?: Array | undefined; + network?: boolean | undefined; + user?: string | undefined; + }; + + export type SandboxCreateOutput = { + id: string; + }; + + export type SandboxSpawnArg = { + command: string; + args?: Array | undefined; + env?: Record | undefined; + stdin: string; + stdout: string; + stderr: string; + }; + + export type SandboxSpawnOutput = { + pid: number; + }; + + export type SandboxWaitArg = { + pid: number; + }; + + export type SandboxWaitOutput = { + status: number; + }; + export type LogStream = "stdout" | "stderr"; export type System = { diff --git a/packages/clients/js/src/process.ts b/packages/clients/js/src/process.ts index 0528c064d..21183b733 100644 --- a/packages/clients/js/src/process.ts +++ b/packages/clients/js/src/process.ts @@ -110,21 +110,6 @@ export class Process { })(); } - get mounts(): Promise> { - return (async () => { - let commandMounts = await (await this.command).mounts; - await this.load(); - return [...this.#state!.mounts, ...(commandMounts ?? [])]; - })(); - } - - get network(): Promise { - return (async () => { - await this.load(); - return this.#state!.network; - })(); - } - get user(): Promise { return (async () => { return await ( @@ -159,9 +144,8 @@ export namespace Process { env?: tg.MaybeMutationMap | undefined; executable?: tg.Command.Arg.Executable | undefined; host?: string | undefined; - mounts?: Array | undefined; name?: string | undefined; - network?: boolean | undefined; + sandbox?: tg.Process.Sandbox | undefined; stdin?: tg.Blob.Arg | undefined; user?: string | undefined; }; @@ -181,9 +165,8 @@ export namespace Process { env?: tg.MaybeMutationMap | undefined; executable?: tg.Command.Arg.Executable | undefined; host?: string | undefined; - mounts?: Array | undefined; name?: string | undefined; - network?: boolean | undefined; + sandbox?: tg.Process.Sandbox | undefined; stderr?: undefined; stdin?: tg.Blob.Arg | undefined; stdout?: undefined; @@ -194,9 +177,8 @@ export namespace Process { command: tg.Command; error: tg.Error | undefined; exit: number | undefined; - mounts: Array; - network: boolean; output?: tg.Value; + pid: number | undefined; status: tg.Process.Status; stderr: string | undefined; stdin: string | undefined; @@ -215,15 +197,12 @@ export namespace Process { if (value.exit !== undefined) { output.exit = value.exit; } - if (value.mounts.length > 0) { - output.mounts = value.mounts; - } - if (value.network) { - output.network = value.network; - } if ("output" in value) { output.output = tg.Value.toData(value.output); } + if (value.pid !== undefined) { + output.pid = value.pid; + } if (value.stderr !== undefined) { output.stderr = value.stderr; } @@ -246,8 +225,7 @@ export namespace Process { : tg.Error.fromData(data.error) : undefined, exit: data.exit, - mounts: data.mounts ?? [], - network: data.network ?? false, + pid: data.pid, status: data.status, stderr: data.stderr, stdin: data.stdin, @@ -260,11 +238,23 @@ export namespace Process { }; } - export type Mount = { - source: string; - target: string; - readonly?: boolean; - }; + export type Sandbox = tg.Process.Sandbox.Create | string; + + export namespace Sandbox { + export type Create = { + hostname?: string | undefined; + host: string; + mounts?: Array | undefined; + network?: boolean | undefined; + user?: string | undefined; + }; + + export type Mount = { + source: string; + target: string; + readonly?: boolean; + }; + } export type Status = | "created" @@ -277,9 +267,8 @@ export namespace Process { command: tg.Command.Id; error?: tg.Error.Data | tg.Error.Id; exit?: number; - mounts?: Array; - network?: boolean; output?: tg.Value.Data; + pid?: number; status: tg.Process.Status; stderr?: string; stdin?: string; diff --git a/packages/clients/js/src/run.ts b/packages/clients/js/src/run.ts index 82d4aea0b..8e67d6222 100644 --- a/packages/clients/js/src/run.ts +++ b/packages/clients/js/src/run.ts @@ -93,23 +93,13 @@ async function inner(...args: tg.Args): Promise { } let checksum = arg.checksum; - let processMounts: Array = []; - let commandMounts: Array | undefined; - if ("mounts" in arg && arg.mounts !== undefined) { - for (let mount of arg.mounts) { - if (tg.Artifact.is(mount.source)) { - if (commandMounts === undefined) { - commandMounts = []; - } - commandMounts.push(mount as tg.Command.Mount); - } else { - processMounts.push(mount as tg.Process.Mount); - } - } - } else { - commandMounts = await currentCommand?.mounts; - processMounts = tg.Process.current?.state?.mounts ?? []; - } + let currentSandbox = tg.process.env.TANGRAM_SANDBOX; + let sandbox: tg.Process.Sandbox | undefined = + "sandbox" in arg + ? arg.sandbox + : typeof currentSandbox === "string" + ? currentSandbox + : undefined; let processStdin = tg.Process.current?.state?.stdin; let commandStdin: tg.Blob.Arg | undefined; if ("stdin" in arg) { @@ -135,14 +125,9 @@ async function inner(...args: tg.Args): Promise { executable !== undefined ? { executable: executable } : undefined, "host" in arg ? { host: arg.host } : undefined, "user" in arg ? { user: arg.user } : undefined, - commandMounts !== undefined ? { mounts: commandMounts } : undefined, commandStdin !== undefined ? { stdin: commandStdin } : undefined, ); - let network = - "network" in arg - ? (arg.network ?? false) - : (tg.Process.current?.state?.network ?? false); let commandId = await command.store(); let commandReferent = { item: commandId, @@ -152,11 +137,10 @@ async function inner(...args: tg.Args): Promise { checksum, command: commandReferent, create: false, - mounts: processMounts, - network, - parent: undefined, + parent: tg.Process.current?.id, remote: undefined, retry: false, + sandbox, stderr, stdin: processStdin, stdout, @@ -256,7 +240,6 @@ async function arg_( env: object.env, executable: object.executable, host: object.host, - mounts: object.mounts, }; if (object.cwd !== undefined) { output.cwd = object.cwd; @@ -352,21 +335,8 @@ export class RunBuilder< return this; } - mount( - ...mounts: Array> - ): this { - this.#args.push({ mounts }); - return this; - } - - mounts( - ...mounts: Array< - tg.Unresolved< - tg.MaybeMutation> - > - > - ): this { - this.#args.push(...mounts.map((mounts) => ({ mounts }))); + sandbox(sandbox: tg.Process.Sandbox | undefined): this { + this.#args.push({ sandbox }); return this; } @@ -375,11 +345,6 @@ export class RunBuilder< return this; } - network(network: tg.Unresolved>): this { - this.#args.push({ network }); - return this; - } - then( onfulfilled?: | ((value: R) => TResult1 | PromiseLike) diff --git a/packages/js/src/handle.ts b/packages/js/src/handle.ts index 3e5433593..e5e010f72 100644 --- a/packages/js/src/handle.ts +++ b/packages/js/src/handle.ts @@ -35,6 +35,28 @@ export let handle: tg.Handle = { return syscall("process_wait", id, arg); }, + createSandbox(arg: tg.Handle.SandboxCreateArg): Promise { + return syscall("sandbox_create", arg); + }, + + deleteSandbox(id: string): Promise { + return syscall("sandbox_delete", id); + }, + + sandboxSpawn( + id: string, + arg: tg.Handle.SandboxSpawnArg, + ): Promise { + return syscall("sandbox_spawn", id, arg); + }, + + sandboxWait( + id: string, + arg: tg.Handle.SandboxWaitArg, + ): Promise { + return syscall("sandbox_wait", id, arg); + }, + checksum( input: string | Uint8Array, algorithm: tg.Checksum.Algorithm, diff --git a/packages/js/src/syscall.ts b/packages/js/src/syscall.ts index 908045107..d2328a088 100644 --- a/packages/js/src/syscall.ts +++ b/packages/js/src/syscall.ts @@ -82,6 +82,28 @@ declare global { arg: tg.Handle.ReadArg, ): Promise; + function syscall( + syscall: "sandbox_create", + arg: tg.Handle.SandboxCreateArg, + ): Promise; + + function syscall( + syscall: "sandbox_delete", + id: string, + ): Promise; + + function syscall( + syscall: "sandbox_spawn", + id: string, + arg: tg.Handle.SandboxSpawnArg, + ): Promise; + + function syscall( + syscall: "sandbox_wait", + id: string, + arg: tg.Handle.SandboxWaitArg, + ): Promise; + function syscall(syscall: "sleep", duration: number): Promise; function syscall( diff --git a/packages/js/src/tangram.d.ts b/packages/js/src/tangram.d.ts index 2626b188c..3a70a8986 100644 --- a/packages/js/src/tangram.d.ts +++ b/packages/js/src/tangram.d.ts @@ -645,9 +645,6 @@ declare namespace tg { /** Get this command's object. */ object(): Promise; - /** Get this command's mounts. */ - get mounts(): Promise | undefined>; - /** Get this command's user. */ get user(): Promise; @@ -686,9 +683,6 @@ declare namespace tg { /** The command's host. */ host?: string | undefined; - /** The command's mounts. */ - mounts?: Array | undefined; - /** The command's user. */ user?: string | undefined; @@ -726,7 +720,6 @@ declare namespace tg { env: { [key: string]: tg.Value }; executable: tg.Command.Executable; host: string; - mounts: Array | undefined; stdin: tg.Blob | undefined; user: string | undefined; }; @@ -751,12 +744,6 @@ declare namespace tg { path: string; }; } - - /** A mount. */ - export type Mount = { - source: tg.Artifact; - target: string; - }; } export namespace path { @@ -1346,12 +1333,6 @@ declare namespace tg { /** Get this process's command's executable. */ get executable(): Promise; - /** Get the mounts for this process and its command. */ - get mounts(): Promise>; - - /** Get whether this process has the network enabled. */ - get network(): Promise; - /** Get this process's command's user. */ get user(): Promise; } @@ -1386,11 +1367,8 @@ declare namespace tg { /** The command's host. */ host?: string | undefined; - /** The command's mounts. */ - mounts?: Array | undefined; - - /** Configure whether the process has access to the network. **/ - network?: boolean | undefined; + /** The sandbox configuration. */ + sandbox?: tg.Process.Sandbox | undefined; /** Ignore stdin, or set it to a blob. */ stdin?: tg.Blob.Arg | undefined; @@ -1426,11 +1404,8 @@ declare namespace tg { /** The command's host. */ host?: string | undefined; - /** The command's or process's mounts. */ - mounts?: Array | undefined; - - /** Configure whether the process has access to the network. **/ - network?: boolean | undefined; + /** The sandbox configuration. */ + sandbox?: tg.Process.Sandbox | undefined; /** Suppress stderr. */ stderr?: undefined; @@ -1445,12 +1420,24 @@ declare namespace tg { user?: string | undefined; }; - /** A mount. */ - export type Mount = { - source: string; - target: string; - readonly: boolean; - }; + /** A sandbox configuration, either an inline create arg or a sandbox ID. */ + export type Sandbox = tg.Process.Sandbox.Create | string; + + export namespace Sandbox { + export type Create = { + hostname?: string | undefined; + host: string; + mounts?: Array | undefined; + network?: boolean | undefined; + user?: string | undefined; + }; + + export type Mount = { + source: string; + target: string; + readonly?: boolean; + }; + } } export interface BuildBuilder< @@ -1486,16 +1473,10 @@ declare namespace tg { host(host: tg.Unresolved>): this; - mount(...mounts: Array>): this; - - mounts( - ...mounts: Array>>> - ): this; + sandbox(sandbox: tg.Process.Sandbox | undefined): this; named(name: tg.Unresolved>): this; - network(network: tg.Unresolved>): this; - then( this: tg.BuildBuilder<[], R>, onfulfilled?: @@ -1542,12 +1523,6 @@ declare namespace tg { host(host: tg.Unresolved>): this; - mount(...mounts: Array>): this; - - mounts( - ...mounts: Array>>> - ): this; - /** Build this command and return the process's output. */ build(...args: tg.UnresolvedArgs): tg.BuildBuilder<[], R>; @@ -1599,22 +1574,10 @@ declare namespace tg { host(host: tg.Unresolved>): this; - mount( - ...mounts: Array> - ): this; - - mounts( - ...mounts: Array< - tg.Unresolved< - tg.MaybeMutation> - > - > - ): this; + sandbox(sandbox: tg.Process.Sandbox | undefined): this; named(name: tg.Unresolved>): this; - network(network: tg.Unresolved>): this; - then( this: tg.RunBuilder<[], R>, onfulfilled?: diff --git a/packages/sandbox/Cargo.toml b/packages/sandbox/Cargo.toml index d40fd7ae2..01d08767e 100644 --- a/packages/sandbox/Cargo.toml +++ b/packages/sandbox/Cargo.toml @@ -17,3 +17,13 @@ bytes = { workspace = true } indoc = { workspace = true } libc = { workspace = true } num = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +serde-untagged = { workspace = true } +tangram_client = { workspace = true } +tangram_futures = { workspace = true } +time = { workspace = true } +tokio = { workspace = true } +tracing = { workspace = true } +rustix = { workspace = true } +futures = { workspace = true } \ No newline at end of file diff --git a/packages/sandbox/src/client.rs b/packages/sandbox/src/client.rs new file mode 100644 index 000000000..0fd4442d0 --- /dev/null +++ b/packages/sandbox/src/client.rs @@ -0,0 +1,285 @@ +use { + num::ToPrimitive as _, + std::{ + collections::BTreeMap, + os::{fd::RawFd, unix::io::AsRawFd}, + path::PathBuf, + sync::{Arc, Mutex, atomic::AtomicU32}, + }, + tangram_client::prelude::*, + tangram_futures::{read::Ext as _, write::Ext as _}, + tokio::{ + io::{AsyncReadExt, AsyncWriteExt as _}, + net::UnixStream, + sync::oneshot, + }, +}; + +type Requests = Arc>>>>; + +pub struct Client { + counter: AtomicU32, + read_task: tokio::task::JoinHandle<()>, + write_task: tokio::task::JoinHandle<()>, + sender: tokio::sync::mpsc::Sender<(Request, oneshot::Sender>)>, +} + +#[derive(Debug, serde::Serialize, serde::Deserialize)] +pub struct Request { + pub id: u32, + pub kind: RequestKind, +} + +#[derive(Debug, serde::Serialize, serde::Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum RequestKind { + Spawn(SpawnRequest), + Wait(WaitRequest), +} + +#[derive(Debug, serde::Serialize, serde::Deserialize)] +pub struct Response { + pub id: u32, + pub kind: ResponseKind, +} + +#[derive(Debug, serde::Serialize, serde::Deserialize)] +pub enum ResponseKind { + Spawn(SpawnResponse), + Wait(WaitResponse), +} + +#[derive(Debug, serde::Serialize, serde::Deserialize)] +pub struct SpawnRequest { + pub command: crate::Command, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub fds: Vec, +} + +#[derive(Debug, serde::Serialize, serde::Deserialize)] +pub struct SpawnResponse { + pub pid: Option, + pub error: Option, +} + +#[derive(Debug, serde::Serialize, serde::Deserialize)] +pub struct WaitRequest { + pub pid: i32, +} + +#[derive(Debug, serde::Serialize, serde::Deserialize)] +pub struct WaitResponse { + pub status: Option, +} + +impl Client { + pub fn new(socket: UnixStream) -> Self { + let (sender, mut receiver) = + tokio::sync::mpsc::channel::<(Request, oneshot::Sender>)>(128); + let requests: Requests = Arc::new(Mutex::new(BTreeMap::new())); + + // Get the raw fd for sendmsg before splitting the socket. + let fd = socket.as_raw_fd(); + let (mut read_half, mut write_half) = socket.into_split(); + + // Spawn the write task. This task reads requests from the channel and writes them to the socket. + let write_task = tokio::spawn({ + let requests = Arc::clone(&requests); + async move { + while let Some((request, sender)) = receiver.recv().await { + match Self::try_send(&mut write_half, fd, &request).await { + Ok(()) => { + requests.lock().unwrap().insert(request.id, sender); + }, + Err(error) + if matches!( + error.kind(), + std::io::ErrorKind::UnexpectedEof + | std::io::ErrorKind::ConnectionReset + ) => + { + break; + }, + Err(error) => { + sender + .send(Err(tg::error!(!error, "failed to send the request"))) + .ok(); + }, + } + } + } + }); + + // Spawn the read task. This task reads responses from the socket and resolves pending requests. + let read_task = tokio::spawn({ + let requests = Arc::clone(&requests); + async move { + loop { + match Self::try_receive(&mut read_half).await { + Ok(Some(response)) => { + let sender = requests.lock().unwrap().remove(&response.id); + if let Some(sender) = sender { + sender.send(Ok(response)).ok(); + } + }, + Ok(None) => (), + Err(error) + if matches!( + error.kind(), + std::io::ErrorKind::UnexpectedEof + | std::io::ErrorKind::ConnectionReset + ) => + { + break; + }, + Err(error) => { + tracing::error!(?error, "failed to receive the response"); + break; + }, + } + } + } + }); + + Self { + read_task, + write_task, + sender, + counter: AtomicU32::new(0), + } + } + + pub async fn connect(path: PathBuf) -> tg::Result { + let socket = tokio::net::UnixStream::connect(&path) + .await + .map_err(|source| tg::error!(!source, "failed to connect to the socket"))?; + Ok(Self::new(socket)) + } + + async fn try_send( + socket: &mut tokio::net::unix::OwnedWriteHalf, + fd: RawFd, + request: &Request, + ) -> std::io::Result<()> { + let bytes = serde_json::to_vec(&request).unwrap(); + let length = bytes.len().to_u64().unwrap(); + socket.write_uvarint(length).await?; + socket.write_all(&bytes).await?; + let RequestKind::Spawn(request) = &request.kind else { + return Ok(()); + }; + let fds = request.fds.clone(); + socket + .as_ref() + .async_io(tokio::io::Interest::WRITABLE, move || unsafe { + let buffer = [0u8; 1]; + let iov = libc::iovec { + iov_base: buffer.as_ptr() as *mut _, + iov_len: 1, + }; + let cmsg_space = + libc::CMSG_SPACE(std::mem::size_of_val(fds.as_slice()).to_u32().unwrap()); + let mut cmsg_buffer = vec![0u8; cmsg_space as _]; + let mut msg: libc::msghdr = std::mem::zeroed(); + msg.msg_iov = (&raw const iov).cast_mut(); + msg.msg_iovlen = 1; + msg.msg_control = cmsg_buffer.as_mut_ptr().cast(); + msg.msg_controllen = cmsg_space as _; + let cmsg = libc::CMSG_FIRSTHDR(&raw const msg); + if cmsg.is_null() { + let error = std::io::Error::other("failed to get the control message header"); + return Err(error); + } + (*cmsg).cmsg_level = libc::SOL_SOCKET; + (*cmsg).cmsg_type = libc::SCM_RIGHTS; + (*cmsg).cmsg_len = + libc::CMSG_LEN(std::mem::size_of_val(fds.as_slice()).to_u32().unwrap()) as _; + let data = libc::CMSG_DATA(cmsg); + std::ptr::copy_nonoverlapping(fds.as_ptr(), data.cast(), fds.len()); + let ret = libc::sendmsg(fd, &raw const msg, 0); + if ret < 0 { + return Err(std::io::Error::last_os_error()); + } + Ok(()) + }) + .await?; + Ok(()) + } + + async fn try_receive( + socket: &mut tokio::net::unix::OwnedReadHalf, + ) -> std::io::Result> { + let Some(length) = socket.try_read_uvarint().await? else { + return Err(std::io::ErrorKind::UnexpectedEof.into()); + }; + let mut buf = vec![0; length.to_usize().unwrap()]; + socket.read_exact(&mut buf).await?; + let response = serde_json::from_slice(&buf) + .inspect_err(|error| tracing::error!(?error, "failed to deserialize the response")) + .ok(); + Ok(response) + } + + pub async fn spawn(&self, mut command: crate::Command) -> tg::Result { + // Get the fds as indices. + let mut fds = Vec::new(); + command.stdin = command.stdin.map(|fd| { + let index = fds.len().to_i32().unwrap(); + fds.push(fd); + index + }); + command.stdout = command.stdout.map(|fd| { + let index = fds.len().to_i32().unwrap(); + fds.push(fd); + index + }); + command.stderr = command.stderr.map(|fd| { + let index = fds.len().to_i32().unwrap(); + fds.push(fd); + index + }); + let response = self + .send_request(RequestKind::Spawn(SpawnRequest { command, fds })) + .await?; + let ResponseKind::Spawn(response) = response.kind else { + return Err(tg::error!("expected a spawn response")); + }; + if let Some(error) = response.error { + let error = error.try_into()?; + return Err(error); + } + response.pid.ok_or_else(|| tg::error!("expected a PID")) + } + + pub async fn wait(&self, pid: i32) -> tg::Result { + let response = self + .send_request(RequestKind::Wait(WaitRequest { pid })) + .await?; + let ResponseKind::Wait(wait) = response.kind else { + return Err(tg::error!("expected a wait response")); + }; + wait.status.ok_or_else(|| tg::error!("expected a status")) + } + + async fn send_request(&self, kind: RequestKind) -> tg::Result { + let id = self + .counter + .fetch_add(1, std::sync::atomic::Ordering::AcqRel); + let (sender, receiver) = oneshot::channel(); + let request = Request { id, kind }; + self.sender + .send((request, sender)) + .await + .map_err(|source| tg::error!(!source, "the channel was closed"))?; + receiver + .await + .map_err(|_| tg::error!("expected a response"))? + } +} + +impl Drop for Client { + fn drop(&mut self) { + self.read_task.abort(); + self.write_task.abort(); + } +} diff --git a/packages/sandbox/src/common.rs b/packages/sandbox/src/common.rs index 57a6c9dec..ba7e17057 100644 --- a/packages/sandbox/src/common.rs +++ b/packages/sandbox/src/common.rs @@ -1,6 +1,7 @@ use std::{ ffi::{CString, OsStr}, os::unix::ffi::OsStrExt as _, + path::Path, }; pub struct CStringVec { @@ -20,6 +21,16 @@ pub fn cstring(s: impl AsRef) -> CString { CString::new(s.as_ref().as_bytes()).unwrap() } +#[allow(dead_code)] +pub fn envstring(k: impl AsRef, v: impl AsRef) -> CString { + let string = format!( + "{}={}", + k.as_ref().to_string_lossy(), + v.as_ref().to_string_lossy() + ); + CString::new(string).unwrap() +} + impl FromIterator for CStringVec { fn from_iter>(iter: T) -> Self { let mut strings = Vec::new(); @@ -36,6 +47,20 @@ impl FromIterator for CStringVec { } } +/// Resolve a non-absolute executable path by searching the given PATH value. +pub fn which(path: &Path, executable: &std::path::Path) -> Option { + if executable.is_absolute() { + return Some(executable.to_owned()); + } + for dir in std::env::split_paths(path) { + let candidate = dir.join(executable); + if candidate.is_file() { + return Some(candidate); + } + } + None +} + #[macro_export] macro_rules! abort { ($($t:tt)*) => {{ @@ -57,5 +82,3 @@ macro_rules! abort_errno { std::process::exit(std::io::Error::last_os_error().raw_os_error().unwrap_or(1)); }}; } - -pub use abort_errno; diff --git a/packages/sandbox/src/daemon.rs b/packages/sandbox/src/daemon.rs new file mode 100644 index 000000000..ed5d27efc --- /dev/null +++ b/packages/sandbox/src/daemon.rs @@ -0,0 +1,5 @@ +#[cfg(target_os = "macos")] +pub mod darwin; + +#[cfg(target_os = "linux")] +pub mod linux; diff --git a/packages/sandbox/src/darwin.rs b/packages/sandbox/src/daemon/darwin.rs similarity index 78% rename from packages/sandbox/src/darwin.rs rename to packages/sandbox/src/daemon/darwin.rs index e5dbaf57d..cf1f81df9 100644 --- a/packages/sandbox/src/darwin.rs +++ b/packages/sandbox/src/daemon/darwin.rs @@ -1,7 +1,7 @@ use { crate::{ - Command, - common::{CStringVec, abort_errno, cstring}, + Command, Options, abort_errno, + common::{CStringVec, cstring, which}, }, indoc::writedoc, num::ToPrimitive as _, @@ -13,30 +13,44 @@ use { }, }; -struct Context { - argv: CStringVec, - cwd: CString, - executable: CString, - profile: CString, +pub fn enter(options: &Options) -> std::io::Result<()> { + let profile = create_sandbox_profile(&options); + unsafe { + let mut error = std::ptr::null::(); + let ret = sandbox_init(profile.as_ptr(), 0, std::ptr::addr_of_mut!(error)); + if ret != 0 { + let error = error; + let message = CStr::from_ptr(error); + let result = std::io::Error::other(format!( + "failed to enter sandbox: {}", + message.to_string_lossy() + )); + sandbox_free_error(error); + return Err(result); + } + } + Ok(()) } -#[expect(clippy::needless_pass_by_value)] -pub fn spawn(command: Command) -> std::io::Result { +pub fn spawn(command: Command) -> std::io::Result { // Create the argv. let argv = std::iter::once(cstring(&command.executable)) .chain(command.trailing.iter().map(cstring)) .collect::(); // Create the cwd. - let cwd = command - .cwd - .clone() - .map_or_else(std::env::current_dir, Ok::<_, std::io::Error>) - .inspect_err(|_| eprintln!("failed to get cwd")) - .map(cstring)?; + let cwd = command.cwd.clone().map(cstring); // Create the executable. - let executable = cstring(&command.executable); + let executable = command + .env + .iter() + .find_map(|(key, value)| { + (key == "PATH") + .then_some(value) + .and_then(|path| which(path.as_ref(), &command.executable).map(cstring)) + }) + .unwrap_or_else(|| cstring(&command.executable)); if command.chroot.is_some() { return Err(std::io::Error::other("chroot is not allowed on darwin")); @@ -46,80 +60,52 @@ pub fn spawn(command: Command) -> std::io::Result { return Err(std::io::Error::other("uid/gid is not allowed on darwin")); } - // Set the environment. - unsafe { - for (key, _) in std::env::vars_os() { - std::env::remove_var(key); - } - for (key, value) in &command.env { - std::env::set_var(key, value); - } - } - - // Create the sandbox profile. - let profile = create_sandbox_profile(&command); - - // Create the context. - let context = Context { - argv, - cwd, - executable, - profile, - }; - // Fork. let pid = unsafe { libc::fork() }; if pid < 0 { return Err(std::io::Error::last_os_error()); } + + // Run the child process. if pid == 0 { - // Initialize the sandbox. unsafe { - let error = std::ptr::null_mut::<*const libc::c_char>(); - let ret = sandbox_init(context.profile.as_ptr(), 0, error); - - // Handle an error from `sandbox_init`. - if ret != 0 { - let error = *error; - let message = CStr::from_ptr(error); - sandbox_free_error(error); - abort_errno!("failed to setup the sandbox: {}", message.to_string_lossy()); + // Redirect stdio file descriptors. + if let Some(fd) = command.stdin { + libc::dup2(fd, libc::STDIN_FILENO); + } + if let Some(fd) = command.stdout { + libc::dup2(fd, libc::STDOUT_FILENO); + } + if let Some(fd) = command.stderr { + libc::dup2(fd, libc::STDERR_FILENO); } - } - // Change directories if necessary. - if unsafe { libc::chdir(context.cwd.as_ptr()) } != 0 { - abort_errno!("failed to change working directory"); - } + // Change the working directory. + if let Some(cwd) = &cwd { + let ret = libc::chdir(cwd.as_ptr()); + if ret == -1 { + abort_errno!("failed to set the working directory {:?}", command.cwd); + } + } - // Exec. - unsafe { - libc::execvp(context.executable.as_ptr(), context.argv.as_ptr()); - abort_errno!("failed to exec"); - } - } + // Set the environment. + for (key, _) in std::env::vars_os() { + std::env::remove_var(key); + } + for (key, value) in &command.env { + std::env::set_var(key, value); + } - // Wait for the child process to exit. - let mut status = 0; - unsafe { - libc::waitpid(pid, std::ptr::addr_of_mut!(status), 0); + // Exec. + libc::execvp(executable.as_ptr(), argv.as_ptr().cast()); + abort_errno!("execvp failed"); + } } - // Reap its children. - kill_process_tree(pid); - - let status = if libc::WIFEXITED(status) { - libc::WEXITSTATUS(status).to_u8().unwrap().into() - } else if libc::WIFSIGNALED(status) { - (128 + libc::WTERMSIG(status).to_u8().unwrap()).into() - } else { - return Err(std::io::Error::other("unknown process termination")); - }; - - Ok(status) + Ok(pid as _) } -fn create_sandbox_profile(command: &Command) -> CString { +fn create_sandbox_profile(options: &Options) -> CString { let mut profile = String::new(); writedoc!( profile, @@ -129,7 +115,7 @@ fn create_sandbox_profile(command: &Command) -> CString { ) .unwrap(); - let root_mount = command.mounts.iter().any(|mount| { + let root_mount = options.mounts.iter().any(|mount| { mount.source == mount.target && mount .target @@ -251,7 +237,7 @@ fn create_sandbox_profile(command: &Command) -> CString { } // Write the network profile. - if command.network { + if options.network { writedoc!( profile, r#" @@ -283,7 +269,7 @@ fn create_sandbox_profile(command: &Command) -> CString { .unwrap(); } - for mount in &command.mounts { + for mount in &options.mounts { if !root_mount { let path = mount.source.as_ref().unwrap(); if (mount.flags & libc::MNT_RDONLY.to_u64().unwrap()) != 0 { @@ -334,6 +320,7 @@ fn create_sandbox_profile(command: &Command) -> CString { CString::new(profile).unwrap() } +#[allow(dead_code)] fn kill_process_tree(pid: i32) { let mut pids = vec![pid]; let mut i = 0; diff --git a/packages/sandbox/src/daemon/linux.rs b/packages/sandbox/src/daemon/linux.rs new file mode 100644 index 000000000..a5edf94c4 --- /dev/null +++ b/packages/sandbox/src/daemon/linux.rs @@ -0,0 +1,312 @@ +use { + crate::{ + Command, Options, abort_errno, + common::{CStringVec, cstring, envstring, which}, + }, + num::ToPrimitive as _, + std::{ + ffi::{CString, OsStr}, + mem::MaybeUninit, + path::PathBuf, + }, +}; + +pub fn enter(options: &Options) -> std::io::Result<()> { + let (uid, gid) = get_user(options.user.as_ref()).expect("failed to get the uid/gid"); + unsafe { + // Update the uid map. + let proc_uid = libc::getuid(); + let proc_gid = libc::getgid(); + + let result = libc::unshare(libc::CLONE_NEWUSER); + if result < 0 { + return Err(std::io::Error::last_os_error()); + } + std::fs::write("/proc/self/uid_map", format!("{uid} {proc_uid} 1\n")) + .expect("failed to write the uid map"); + + // Deny setgroups. + std::fs::write("/proc/self/setgroups", "deny").expect("failed to deny setgroups"); + + // Update the gid map. + std::fs::write("/proc/self/gid_map", format!("{gid} {proc_gid} 1\n")) + .expect("failed to write the gid map"); + // Enter a new PID and mount namespace. The first child process will have pid 1. Mounts performed here will not be visible outside the sandbox. + let result = libc::unshare(libc::CLONE_NEWPID | libc::CLONE_NEWNS); + if result < 0 { + return Err(std::io::Error::last_os_error()); + } + + // If sandboxing in a network, enter a new network namespace. + if !options.network { + let result = libc::unshare(libc::CLONE_NEWNET); + if result < 0 { + return Err(std::io::Error::last_os_error()); + } + } + + // If a new hostname is requested, enter a new UTS namespace. + if let Some(hostname) = &options.hostname { + let result = libc::unshare(libc::CLONE_NEWUTS); + if result < 0 { + return Err(std::io::Error::last_os_error()); + } + let result = libc::sethostname(hostname.as_ptr().cast(), hostname.len()); + if result < 0 { + return Err(std::io::Error::last_os_error()); + } + } + } + + // Sort the mounts by target path component count so that parent mounts (like overlays at /) are mounted before child mounts (like bind mounts at /dev). + let mut mounts = options.mounts.clone(); + mounts.sort_unstable_by_key(|mount| { + mount + .target + .as_ref() + .map_or(0, |path| path.components().count()) + }); + for m in &mounts { + mount(m, options.chroot.as_ref())?; + } + + // chroot + if let Some(chroot) = &options.chroot { + unsafe { + let name = cstring(chroot); + if libc::chroot(name.as_ptr()) != 0 { + eprintln!("chroot failed"); + return Err(std::io::Error::last_os_error()); + } + } + } + + Ok(()) +} + +pub fn spawn(command: &Command) -> std::io::Result { + // Create argv, cwd, and envp strings. + let argv = std::iter::once(cstring(&command.executable)) + .chain(command.trailing.iter().map(cstring)) + .collect::(); + let cwd = command.cwd.clone().map(cstring); + let envp = command + .env + .iter() + .map(|(key, value)| envstring(key, value)) + .collect::(); + + let executable = command + .env + .iter() + .find_map(|(key, value)| { + (key == "PATH") + .then_some(value) + .and_then(|path| which(path.as_ref(), &command.executable).map(cstring)) + }) + .unwrap_or_else(|| cstring(&command.executable)); + + let mut clone_args: libc::clone_args = libc::clone_args { + flags: 0, + stack: 0, + stack_size: 0, + pidfd: 0, + child_tid: 0, + parent_tid: 0, + exit_signal: libc::SIGCHLD as u64, + tls: 0, + set_tid: 0, + set_tid_size: 0, + cgroup: 0, + }; + + let pid = unsafe { + libc::syscall( + libc::SYS_clone3, + std::ptr::addr_of_mut!(clone_args), + std::mem::size_of::(), + ) + }; + + // Check if clone3 failed. + if pid < 0 { + eprintln!("clone3 failed"); + return Err(std::io::Error::last_os_error()); + } + + // Run the process. + if pid == 0 { + unsafe { + if let Some(fd) = command.stdin { + libc::dup2(fd, libc::STDIN_FILENO); + } + if let Some(fd) = command.stdout { + libc::dup2(fd, libc::STDOUT_FILENO); + } + if let Some(fd) = command.stderr { + libc::dup2(fd, libc::STDERR_FILENO); + } + if let Some(cwd) = &cwd { + let ret = libc::chdir(cwd.as_ptr()); + if ret == -1 { + abort_errno!("failed to set the working directory {:?}", command.cwd); + } + } + libc::execvpe( + executable.as_ptr(), + argv.as_ptr().cast(), + envp.as_ptr().cast(), + ); + abort_errno!("execvpe failed {}", command.executable.display()); + } + } + + Ok(pid.to_i32().unwrap()) +} + +fn mount(mount: &crate::Mount, chroot: Option<&PathBuf>) -> std::io::Result<()> { + // Remap the target path. + let target = mount.target.as_ref().map(|target| { + if let Some(chroot) = &chroot { + chroot.join(target.strip_prefix("/").unwrap()) + } else { + target.clone() + } + }); + let source = mount.source.as_ref().map(cstring); + let flags = if let Some(source) = &source + && mount.fstype.is_none() + { + let existing = get_existing_mount_flags(source)?; + existing | mount.flags + } else { + mount.flags + }; + let mut target = target.map(cstring); + let fstype = mount.fstype.as_ref().map(cstring); + let data = mount.data.as_ref().map_or(std::ptr::null_mut(), |bytes| { + bytes.as_ptr().cast::().cast_mut() + }); + unsafe { + if let (Some(source), Some(target)) = (&source, &mut target) { + create_mountpoint_if_not_exists(source, target); + } + let source = source.as_ref().map_or(std::ptr::null(), |c| c.as_ptr()); + let target = target.as_ref().map_or(std::ptr::null(), |c| c.as_ptr()); + let fstype = fstype.as_ref().map_or(std::ptr::null(), |c| c.as_ptr()); + let result = libc::mount(source, target, fstype, flags, data); + if result < 0 { + eprintln!("failed to mount {source:?}:{target:?}"); + return Err(std::io::Error::last_os_error()); + } + if (flags & libc::MS_BIND != 0) && (flags & libc::MS_RDONLY != 0) { + let flags = flags | libc::MS_REMOUNT; + let result = libc::mount(source, target, fstype, flags, data); + if result < 0 { + eprintln!("failed to remount {target:?} as read-only"); + return Err(std::io::Error::last_os_error()); + } + } + } + Ok(()) +} + +fn create_mountpoint_if_not_exists(source: &CString, target: &mut CString) { + unsafe { + #[cfg_attr(all(target_arch = "x86_64"), expect(clippy::cast_possible_wrap))] + const BACKSLASH: libc::c_char = b'\\' as _; + #[cfg_attr(all(target_arch = "x86_64"), expect(clippy::cast_possible_wrap))] + const SLASH: libc::c_char = b'/' as _; + const NULL: libc::c_char = 0; + + // Determine if the target is a directory or not. + let is_dir = 'a: { + if source.as_bytes() == b"overlay" { + break 'a true; + } + let mut stat = MaybeUninit::::zeroed(); + if libc::stat(source.as_ptr(), stat.as_mut_ptr().cast()) < 0 { + abort_errno!("failed to stat source"); + } + let stat = stat.assume_init(); + if !(stat.st_mode & libc::S_IFDIR != 0 || stat.st_mode & libc::S_IFREG != 0) { + abort_errno!("mount source is not a directory or regular file"); + } + stat.st_mode & libc::S_IFDIR != 0 + }; + + let ptr = target.as_ptr().cast_mut(); + let len = target.as_bytes_with_nul().len(); + let mut esc = false; + for n in 1..len { + match (*ptr.add(n), esc) { + (SLASH, false) => { + *ptr.add(n) = 0; + libc::mkdir(target.as_ptr(), 0o755); + *ptr.add(n) = SLASH; + }, + (BACKSLASH, false) => { + esc = true; + }, + (NULL, _) => { + break; + }, + _ => { + esc = false; + }, + } + } + if is_dir { + libc::mkdir(target.as_ptr(), 0o755); + } else { + libc::creat(target.as_ptr(), 0o777); + } + } +} + +fn get_user(name: Option>) -> std::io::Result<(libc::uid_t, libc::gid_t)> { + let Some(name) = name else { + unsafe { + let uid = libc::getuid(); + let gid = libc::getgid(); + return Ok((uid, gid)); + } + }; + unsafe { + let passwd = libc::getpwnam(cstring(name.as_ref()).as_ptr()); + if passwd.is_null() { + return Err(std::io::Error::other("getpwname failed")); + } + let uid = (*passwd).pw_uid; + let gid = (*passwd).pw_gid; + Ok((uid, gid)) + } +} + +fn get_existing_mount_flags(path: &CString) -> std::io::Result { + const FLAGS: [(u64, u64); 7] = [ + (libc::MS_RDONLY, libc::ST_RDONLY), + (libc::MS_NODEV, libc::ST_NODEV), + (libc::MS_NOEXEC, libc::ST_NOEXEC), + (libc::MS_NOSUID, libc::ST_NOSUID), + (libc::MS_NOATIME, libc::ST_NOATIME), + (libc::MS_RELATIME, libc::ST_RELATIME), + (libc::MS_NODIRATIME, libc::ST_NODIRATIME), + ]; + let statfs = unsafe { + let mut statfs = std::mem::MaybeUninit::zeroed(); + let ret = libc::statfs64(path.as_ptr(), statfs.as_mut_ptr()); + if ret != 0 { + eprintln!("failed to statfs {}", path.to_string_lossy()); + return Err(std::io::Error::last_os_error()); + } + statfs.assume_init() + }; + let mut flags = 0; + for (mount_flag, stat_flag) in FLAGS { + if (statfs.f_flags.abs().to_u64().unwrap() & stat_flag) != 0 { + flags |= mount_flag; + } + } + Ok(flags) +} diff --git a/packages/sandbox/src/lib.rs b/packages/sandbox/src/lib.rs index 38ba44725..f024658b6 100644 --- a/packages/sandbox/src/lib.rs +++ b/packages/sandbox/src/lib.rs @@ -3,26 +3,24 @@ use { std::{ffi::OsString, path::PathBuf}, }; +pub mod client; +pub mod server; + mod common; -#[cfg(target_os = "macos")] -pub mod darwin; -#[cfg(target_os = "linux")] -pub mod linux; +mod daemon; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] pub struct Command { - pub chroot: Option, pub cwd: Option, pub env: Vec<(String, String)>, pub executable: PathBuf, - pub hostname: Option, - pub mounts: Vec, - pub network: bool, pub trailing: Vec, - pub user: Option, + pub stdin: Option, + pub stdout: Option, + pub stderr: Option, } -#[derive(Clone, Debug)] +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] pub struct Mount { pub source: Option, pub target: Option, @@ -30,3 +28,12 @@ pub struct Mount { pub flags: libc::c_ulong, pub data: Option, } + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct Options { + pub user: Option, + pub network: bool, + pub hostname: Option, + pub chroot: Option, + pub mounts: Vec, +} diff --git a/packages/sandbox/src/linux.rs b/packages/sandbox/src/linux.rs deleted file mode 100644 index 999c127ad..000000000 --- a/packages/sandbox/src/linux.rs +++ /dev/null @@ -1,295 +0,0 @@ -use { - crate::{ - Command, - common::{CStringVec, cstring}, - }, - bytes::Bytes, - num::ToPrimitive as _, - std::{ - ffi::{CString, OsStr}, - io::Write, - }, -}; - -mod guest; -mod root; - -#[derive(Debug)] -struct Mount { - source: Option, - target: Option, - fstype: Option, - flags: libc::c_ulong, - data: Option, -} - -struct Context { - argv: CStringVec, - cwd: CString, - envp: CStringVec, - executable: CString, - hostname: Option, - root: Option, - mounts: Vec, - network: bool, - socket: std::os::unix::net::UnixStream, -} - -pub fn spawn(mut command: Command) -> std::io::Result { - if !command.mounts.is_empty() && command.chroot.is_none() { - return Err(std::io::Error::other( - "cannot create mounts without a chroot directory", - )); - } - - // Sort the mounts. - command.mounts.sort_unstable_by_key(|mount| { - mount - .target - .as_ref() - .map_or(0, |path| path.components().count()) - }); - - // Create argv, cwd, and envp strings. - let argv = std::iter::once(cstring(&command.executable)) - .chain(command.trailing.iter().map(cstring)) - .collect::(); - let cwd = command - .cwd - .clone() - .map_or_else(std::env::current_dir, Ok::<_, std::io::Error>) - .inspect_err(|_| eprintln!("failed to get cwd")) - .map(cstring)?; - let envp = command - .env - .iter() - .map(|(key, value)| envstring(key, value)) - .collect::(); - let executable = cstring(&command.executable); - let hostname = command.hostname.as_ref().map(cstring); - - // Create the mounts. - let mut mounts = Vec::with_capacity(command.mounts.len()); - for mount in &command.mounts { - // Remap the target path. - let target = mount.target.as_ref().map(|target| { - if let Some(chroot) = &command.chroot { - chroot.join(target.strip_prefix("/").unwrap()) - } else { - target.clone() - } - }); - let source = mount.source.as_ref().map(cstring); - let flags = if let Some(source) = &source - && mount.fstype.is_none() - { - let existing = get_existing_mount_flags(source)?; - existing | mount.flags - } else { - mount.flags - }; - // Create the mount. - let mount = Mount { - source, - target: target.map(cstring), - fstype: mount.fstype.as_ref().map(cstring), - flags, - data: mount.data.clone(), - }; - mounts.push(mount); - } - - // Get the chroot path. - let root = command.chroot.as_ref().map(cstring); - - // Create the socket for guest control. This will be used to send the guest process its PID with respect to the parent's PID namespace and to indicate to the child when it may exec. - let (mut parent_socket, child_socket) = std::os::unix::net::UnixStream::pair() - .inspect_err(|_| eprintln!("failed to create socket"))?; - - // Create the context. - let context = Context { - argv, - cwd, - envp, - executable, - hostname, - root, - mounts, - network: command.network, - socket: child_socket, - }; - - // Set PATH. - unsafe { - let path = command - .env - .iter() - .find_map(|(key, value)| (key == "PATH").then_some(value)); - if let Some(path) = path { - std::env::set_var("PATH", path); - } else { - std::env::remove_var("PATH"); - } - } - - // Fork. - let mut clone_args: libc::clone_args = libc::clone_args { - flags: (libc::CLONE_NEWUSER | libc::CLONE_NEWPID) - .try_into() - .unwrap(), - stack: 0, - stack_size: 0, - pidfd: 0, - child_tid: 0, - parent_tid: 0, - exit_signal: 0, - tls: 0, - set_tid: 0, - set_tid_size: 0, - cgroup: 0, - }; - let root_pid = unsafe { - libc::syscall( - libc::SYS_clone3, - std::ptr::addr_of_mut!(clone_args), - std::mem::size_of::(), - ) - }; - let pid = root_pid.to_i32().unwrap(); - - // Check if clone3 failed. - if pid < 0 { - eprintln!("clone3 failed"); - return Err(std::io::Error::last_os_error()); - } - - // Run the root process. - if pid == 0 { - root::main(context); - } - - // Signal the root/guest process to start and get the guest PID. - let (uid, gid) = get_user(command.user.as_ref())?; - - // Start the process and get the pid of the guest process. - match try_start(command.chroot.is_some(), pid, gid, uid, &mut parent_socket) { - Ok(pid) => pid, - Err(error) => unsafe { - libc::kill(pid, libc::SIGKILL); - return Err(error); - }, - } - - // Wait for the root process to exit. - let mut status: libc::c_int = 0; - let ret = unsafe { libc::waitpid(pid, std::ptr::addr_of_mut!(status), libc::__WALL) }; - if ret == -1 { - eprintln!("wait failed"); - return Err(std::io::Error::last_os_error()); - } - - let status = if libc::WIFEXITED(status) { - libc::WEXITSTATUS(status).to_u8().unwrap().into() - } else if libc::WIFSIGNALED(status) { - (128 + libc::WTERMSIG(status).to_u8().unwrap()).into() - } else { - return Err(std::io::Error::other("unknown process termination")); - }; - - Ok(status) -} - -fn try_start( - chroot: bool, - pid: libc::pid_t, - child_gid: libc::gid_t, - child_uid: libc::gid_t, - socket: &mut std::os::unix::net::UnixStream, -) -> std::io::Result<()> { - // If the guest process is running in a chroot jail, it's current state is blocked waiting for the host process (the caller) to update its uid and gid maps. We need to wait for the root process to notify the host of the guest's PID after it is cloned. - if chroot { - // Write the guest process's UID map. - let uid = unsafe { libc::getuid() }; - std::fs::write( - format!("/proc/{pid}/uid_map"), - format!("{child_uid} {uid} 1\n"), - ) - .inspect_err(|_| eprintln!("failed to write uid map"))?; - - // Deny setgroups to the process. - std::fs::write(format!("/proc/{pid}/setgroups"), "deny") - .inspect_err(|_| eprintln!("failed to deny setgroups"))?; - - // Write the guest process's GID map. - let gid = unsafe { libc::getgid() }; - std::fs::write( - format!("/proc/{pid}/gid_map"), - format!("{child_gid} {gid} 1\n"), - ) - .inspect_err(|_| eprintln!("failed to write gid map"))?; - } - - // Notify the guest that it may continue. - socket - .write_all(&[1u8]) - .inspect_err(|_| eprintln!("failed to signal process"))?; - - // Return the child pid. - Ok(()) -} - -fn get_user(name: Option>) -> std::io::Result<(libc::uid_t, libc::gid_t)> { - let Some(name) = name else { - unsafe { - let uid = libc::getuid(); - let gid = libc::getgid(); - return Ok((uid, gid)); - } - }; - unsafe { - let passwd = libc::getpwnam(cstring(name.as_ref()).as_ptr()); - if passwd.is_null() { - return Err(std::io::Error::other("getpwname failed")); - } - let uid = (*passwd).pw_uid; - let gid = (*passwd).pw_gid; - Ok((uid, gid)) - } -} - -fn envstring(k: impl AsRef, v: impl AsRef) -> CString { - let string = format!( - "{}={}", - k.as_ref().to_string_lossy(), - v.as_ref().to_string_lossy() - ); - CString::new(string).unwrap() -} - -fn get_existing_mount_flags(path: &CString) -> std::io::Result { - const FLAGS: [(u64, u64); 7] = [ - (libc::MS_RDONLY, libc::ST_RDONLY), - (libc::MS_NODEV, libc::ST_NODEV), - (libc::MS_NOEXEC, libc::ST_NOEXEC), - (libc::MS_NOSUID, libc::ST_NOSUID), - (libc::MS_NOATIME, libc::ST_NOATIME), - (libc::MS_RELATIME, libc::ST_RELATIME), - (libc::MS_NODIRATIME, libc::ST_NODIRATIME), - ]; - let statfs = unsafe { - let mut statfs = std::mem::MaybeUninit::zeroed(); - let ret = libc::statfs64(path.as_ptr(), statfs.as_mut_ptr()); - if ret != 0 { - eprintln!("failed to statfs {}", path.to_string_lossy()); - return Err(std::io::Error::last_os_error()); - } - statfs.assume_init() - }; - let mut flags = 0; - for (mount_flag, stat_flag) in FLAGS { - if (statfs.f_flags.abs().to_u64().unwrap() & stat_flag) != 0 { - flags |= mount_flag; - } - } - Ok(flags) -} diff --git a/packages/sandbox/src/linux/guest.rs b/packages/sandbox/src/linux/guest.rs deleted file mode 100644 index 83c974c14..000000000 --- a/packages/sandbox/src/linux/guest.rs +++ /dev/null @@ -1,179 +0,0 @@ -use { - super::Context, - crate::abort_errno, - std::{ffi::CString, mem::MaybeUninit, os::fd::AsRawFd}, -}; - -pub fn main(mut context: Context) -> ! { - unsafe { - // Set hostname. - if let Some(hostname) = context.hostname.take() - && libc::sethostname(hostname.as_ptr(), hostname.as_bytes().len()) != 0 - { - abort_errno!("failed to set hostname"); - } - - // Wait for the notification from the host process to continue. - let mut notification = 0u8; - let ret = libc::recv( - context.socket.as_raw_fd(), - std::ptr::addr_of_mut!(notification).cast(), - std::mem::size_of_val(¬ification), - 0, - ); - if ret == -1 { - abort_errno!( - "the guest process failed to receive the notification from the host process to continue" - ); - } - assert_eq!(notification, 1); - - // If requested to spawn in a chroot, perform the mounts and chroot. - if context.root.is_some() { - mount_and_chroot(&mut context); - } - - // Set the working directory. - let ret = libc::chdir(context.cwd.as_ptr()); - if ret == -1 { - abort_errno!("failed to set the working directory"); - } - - // Finally, exec the process. - libc::execvpe( - context.executable.as_ptr(), - context.argv.as_ptr().cast(), - context.envp.as_ptr().cast(), - ); - - abort_errno!("execvpe failed") - } -} - -fn mount_and_chroot(context: &mut Context) { - unsafe { - let root = context.root.as_ref().unwrap(); - for mount in &mut context.mounts { - // Create the mount point. - if let (Some(source), Some(target)) = (&mount.source, &mut mount.target) { - create_mountpoint_if_not_exists(source, target); - } - let source = mount - .source - .as_ref() - .map_or_else(std::ptr::null, |s| s.as_ptr()); - let target = mount - .target - .as_ref() - .map_or_else(std::ptr::null, |s| s.as_ptr()); - let fstype = mount - .fstype - .as_ref() - .map_or_else(std::ptr::null, |value| value.as_ptr()); - let flags = mount.flags; - let data = mount - .data - .as_ref() - .map_or_else(std::ptr::null, |bytes| bytes.as_ptr()) - .cast(); - let ret = libc::mount(source, target, fstype, flags, data); - if ret == -1 { - abort_errno!("failed to mount {mount:#?}"); - } - - // The initial bind mount ignores flags other than MS_BIND and MS_REC. To support read-only bind mounts we must immediately remount the source/target pair with MS_REMOUNT to change the permissions of the mount point. - if (flags & libc::MS_BIND != 0) && (flags & libc::MS_RDONLY != 0) { - let flags = flags | libc::MS_REMOUNT; - let ret = libc::mount(source, target, fstype, flags, data); - if ret == -1 { - abort_errno!("failed to remount bind mount as read-only {mount:#?}"); - } - } - } - - // Mount the root, required by the pivot_root syscall. - let ret = libc::mount( - root.as_ptr(), - root.as_ptr(), - std::ptr::null(), - libc::MS_BIND | libc::MS_REC, - std::ptr::null(), - ); - if ret == -1 { - abort_errno!("failed to mount the root"); - } - - // Change the working directory to the pivoted root. - if let Some(root) = &context.root { - let ret = libc::chdir(root.as_ptr()); - if ret == -1 { - abort_errno!("failed to change directory to the root"); - } - } - - // Pivot the root. - let ret = libc::syscall(libc::SYS_pivot_root, c".".as_ptr(), c".".as_ptr()); - if ret == -1 { - abort_errno!("failed to pivot the root"); - } - - // Lazily unmount the root. - let ret = libc::umount2(c".".as_ptr().cast(), libc::MNT_DETACH); - if ret == -1 { - abort_errno!("failed to unmount the root"); - } - } -} - -fn create_mountpoint_if_not_exists(source: &CString, target: &mut CString) { - unsafe { - #[cfg_attr(all(target_arch = "x86_64"), expect(clippy::cast_possible_wrap))] - const BACKSLASH: libc::c_char = b'\\' as _; - #[cfg_attr(all(target_arch = "x86_64"), expect(clippy::cast_possible_wrap))] - const SLASH: libc::c_char = b'/' as _; - const NULL: libc::c_char = 0; - - // Determine if the target is a directory or not. - let is_dir = 'a: { - if source.as_bytes() == b"overlay" { - break 'a true; - } - let mut stat = MaybeUninit::::zeroed(); - if libc::stat(source.as_ptr(), stat.as_mut_ptr().cast()) < 0 { - abort_errno!("failed to stat source"); - } - let stat = stat.assume_init(); - if !(stat.st_mode & libc::S_IFDIR != 0 || stat.st_mode & libc::S_IFREG != 0) { - abort_errno!("mount source is not a directory or regular file"); - } - stat.st_mode & libc::S_IFDIR != 0 - }; - - let ptr = target.as_ptr().cast_mut(); - let len = target.as_bytes_with_nul().len(); - let mut esc = false; - for n in 1..len { - match (*ptr.add(n), esc) { - (SLASH, false) => { - *ptr.add(n) = 0; - libc::mkdir(target.as_ptr(), 0o755); - *ptr.add(n) = SLASH; - }, - (BACKSLASH, false) => { - esc = true; - }, - (NULL, _) => { - break; - }, - _ => { - esc = false; - }, - } - } - if is_dir { - libc::mkdir(target.as_ptr(), 0o755); - } else { - libc::creat(target.as_ptr(), 0o777); - } - } -} diff --git a/packages/sandbox/src/linux/root.rs b/packages/sandbox/src/linux/root.rs deleted file mode 100644 index 970a9ec22..000000000 --- a/packages/sandbox/src/linux/root.rs +++ /dev/null @@ -1,118 +0,0 @@ -use { - super::{Context, guest}, - crate::common::abort_errno, - num::ToPrimitive as _, - std::os::fd::AsRawFd as _, -}; - -// The "root" process takes over after the host spawns with CLONE_NEWUSER. -pub fn main(context: Context) -> ! { - unsafe { - // If the host process dies, kill this process. - let ret = libc::prctl(libc::PR_SET_PDEATHSIG, libc::SIGKILL); - if ret == -1 { - abort_errno!("prctl failed"); - } - - // Register signal handlers. - for signum in FORWARDED_SIGNALS { - libc::signal(signum, handler as *const () as _); - } - - // Get the clone flags. - let mut flags = 0; - if context.root.is_some() { - flags |= libc::CLONE_NEWNS; - flags |= libc::CLONE_NEWUTS; - } - if !context.network { - flags |= libc::CLONE_NEWNET; - } - - // Spawn the guest process. - let mut clone_args = libc::clone_args { - flags: flags.try_into().unwrap(), - stack: 0, - stack_size: 0, - pidfd: 0, - child_tid: 0, - parent_tid: 0, - exit_signal: 0, - tls: 0, - set_tid: 0, - set_tid_size: 0, - cgroup: 0, - }; - let pid = libc::syscall( - libc::SYS_clone3, - std::ptr::addr_of_mut!(clone_args), - std::mem::size_of::(), - ); - let pid = pid.to_i32().unwrap(); - if pid == -1 { - libc::close(context.socket.as_raw_fd()); - abort_errno!("clone3 failed"); - } - - // If this is the child process, run either init or guest. - if pid == 0 { - guest::main(context); - } - - // Wait for the child. - let mut status = 0; - let ret = libc::waitpid(pid, std::ptr::addr_of_mut!(status), libc::__WALL); - if ret == -1 { - abort_errno!("waitpid failed"); - } - if libc::WIFEXITED(status) { - let status = libc::WEXITSTATUS(status); - libc::exit(status); - } - - if libc::WIFSIGNALED(status) { - let signal = libc::WTERMSIG(status); - libc::exit(signal + 128); - } - - // Exit with the same status. - libc::exit(1); - } -} - -// We forward all signals except SIGCHILD. -const FORWARDED_SIGNALS: [libc::c_int; 29] = [ - libc::SIGINT, - libc::SIGQUIT, - libc::SIGILL, - libc::SIGTRAP, - libc::SIGABRT, - libc::SIGBUS, - libc::SIGFPE, - libc::SIGKILL, - libc::SIGUSR1, - libc::SIGSEGV, - libc::SIGUSR2, - libc::SIGPIPE, - libc::SIGALRM, - libc::SIGTERM, - libc::SIGSTKFLT, - libc::SIGCONT, - libc::SIGSTOP, - libc::SIGTSTP, - libc::SIGTTIN, - libc::SIGTTOU, - libc::SIGURG, - libc::SIGXCPU, - libc::SIGXFSZ, - libc::SIGVTALRM, - libc::SIGPROF, - libc::SIGWINCH, - libc::SIGPOLL, - libc::SIGPWR, - libc::SIGSYS, -]; - -unsafe extern "C" fn handler(signal: libc::c_int) { - unsafe { libc::kill(-1, signal) }; -} diff --git a/packages/sandbox/src/server.rs b/packages/sandbox/src/server.rs new file mode 100644 index 000000000..49300952f --- /dev/null +++ b/packages/sandbox/src/server.rs @@ -0,0 +1,353 @@ +use { + crate::{ + Options, + client::{Request, RequestKind, Response, ResponseKind, SpawnResponse, WaitResponse}, + }, + futures::future, + num::ToPrimitive as _, + std::{ + collections::BTreeMap, + os::fd::{AsRawFd, RawFd}, + path::Path, + pin::pin, + sync::{Arc, Mutex}, + }, + tangram_client::prelude::*, + tangram_futures::{read::Ext as _, write::Ext as _}, + tokio::{ + io::{AsyncReadExt as _, AsyncWriteExt as _}, + sync::{mpsc, oneshot}, + }, +}; + +#[derive(Clone)] +pub struct Server { + inner: Arc>, + sender: tokio::sync::mpsc::Sender<(Request, oneshot::Sender>)>, +} + +struct Inner { + task: Option>, + waits: Waits, +} + +type Waits = BTreeMap>)>>>; + +impl Drop for Inner { + fn drop(&mut self) { + if let Some(task) = self.task.take() { + task.abort(); + } + } +} + +impl Server { + pub fn new() -> tg::Result { + let (sender, mut receiver) = + mpsc::channel::<(Request, oneshot::Sender>)>(128); + let inner = Arc::new(Mutex::new(Inner { + task: None, + waits: BTreeMap::new(), + })); + let task = tokio::spawn({ + let inner = Arc::clone(&inner); + async move { + let Ok(mut signal) = + tokio::signal::unix::signal(tokio::signal::unix::SignalKind::child()) + else { + tracing::error!("failed to create a ready signal"); + return; + }; + loop { + let signal = signal.recv(); + let request = receiver.recv(); + match future::select(pin!(signal), pin!(request)).await { + future::Either::Left(_) => { + let mut inner = inner.lock().unwrap(); + Self::reap_children(&mut inner.waits); + }, + future::Either::Right((request, _)) => { + let Some((request, sender)) = request else { + break; + }; + Self::handle_request(&inner, request, sender); + }, + } + } + } + }); + inner.lock().unwrap().task.replace(task); + Ok(Self { inner, sender }) + } + + fn handle_request( + inner: &Arc>, + request: Request, + sender: oneshot::Sender>, + ) { + match request.kind { + RequestKind::Spawn(spawn) => { + let result; + #[cfg(target_os = "linux")] + { + result = crate::daemon::linux::spawn(&spawn.command); + } + #[cfg(target_os = "macos")] + { + result = crate::daemon::darwin::spawn(spawn.command); + } + let kind = result.map_or_else( + |error| { + ResponseKind::Spawn(SpawnResponse { + pid: None, + error: Some(tg::error::Data { + message: Some(error.to_string()), + ..tg::error::Data::default() + }), + }) + }, + |pid| { + ResponseKind::Spawn(SpawnResponse { + pid: Some(pid), + error: None, + }) + }, + ); + let response = Response { + id: request.id, + kind, + }; + sender.send(Ok(response)).ok(); + }, + RequestKind::Wait(wait) => { + let mut inner = inner.lock().unwrap(); + if let Some(tg::Either::Left(status)) = inner.waits.get(&wait.pid) { + let kind = ResponseKind::Wait(WaitResponse { + status: Some(*status), + }); + let response = Response { + id: request.id, + kind, + }; + sender.send(Ok(response)).ok(); + return; + } + inner + .waits + .entry(wait.pid) + .or_insert(tg::Either::Right(Vec::new())) + .as_mut() + .unwrap_right() + .push((request.id, sender)); + }, + } + } + + fn reap_children(waits: &mut Waits) { + unsafe { + loop { + // Wait for any child processes without blocking. + let mut status = 0; + let pid = libc::waitpid(-1, std::ptr::addr_of_mut!(status), libc::WNOHANG); + let status = if libc::WIFEXITED(status) { + libc::WEXITSTATUS(status) + } else if libc::WIFSIGNALED(status) { + 128 + libc::WTERMSIG(status) + } else { + 1 + }; + if pid == 0 { + break; + } + if pid < 0 { + let error = std::io::Error::last_os_error(); + tracing::error!(?error, "error waiting for children"); + break; + } + + // Deliver the status to any pending waiters. + if let Some(tg::Either::Right(waiters)) = waits.remove(&pid) { + for (id, sender) in waiters { + let kind = ResponseKind::Wait(WaitResponse { + status: Some(status), + }); + let response = Response { id, kind }; + sender.send(Ok(response)).ok(); + } + } + + // Update the shared state. + waits.insert(pid, tg::Either::Left(status)); + } + } + } + + // Enter the sandbox. This is irreversible for the current process. + pub unsafe fn enter(options: &Options) -> tg::Result<()> { + #[cfg(target_os = "linux")] + crate::daemon::linux::enter(options) + .map_err(|source| tg::error!(!source, "failed to enter the sandbox"))?; + + #[cfg(target_os = "macos")] + crate::daemon::darwin::enter(&options) + .map_err(|source| tg::error!(!source, "failed to enter the sandbox"))?; + + Ok(()) + } + + pub fn bind(path: &Path) -> tg::Result { + // Bind the Unix listener to the specified path. + let listener = std::os::unix::net::UnixListener::bind(path) + .map_err(|source| tg::error!(!source, "failed to bind the listener"))?; + listener + .set_nonblocking(true) + .map_err(|source| tg::error!(!source, "failed to set the listener as non blocking"))?; + Ok(listener) + } + + pub async fn serve(&self, listener: tokio::net::UnixListener) -> tg::Result<()> { + // Acc t connections in a loop. + loop { + let (stream, _addr) = listener + .accept() + .await + .map_err(|source| tg::error!(!source, "failed to accept the connection"))?; + let server = self.clone(); + tokio::spawn(async move { + let result = server.handle_connection(stream).await; + if let Err(error) = result { + tracing::error!(?error, "connection handler failed"); + } + }); + } + } + + async fn handle_connection(&self, mut stream: tokio::net::UnixStream) -> tg::Result<()> { + loop { + // Read the message length. + let length = stream + .read_uvarint() + .await + .map_err(|source| tg::error!(!source, "failed to read the message length"))?; + + // Read the request. + let mut bytes = vec![0u8; length.to_usize().unwrap()]; + stream + .read_exact(&mut bytes) + .await + .map_err(|source| tg::error!(!source, "failed to read the message"))?; + + // Deserialize the request + let mut request = serde_json::from_slice::(&bytes) + .map_err(|source| tg::error!(!source, "failed to deserialize the message"))?; + + // If this is a spawn request, receive the FDs. + if let RequestKind::Spawn(request) = &mut request.kind { + let fd = stream.as_raw_fd(); + let fds = stream + .async_io(tokio::io::Interest::READABLE, move || unsafe { + // Receive the message. + let mut buffer = [0u8; 1]; + let iov = libc::iovec { + iov_base: buffer.as_mut_ptr().cast(), + iov_len: 1, + }; + let length = + libc::CMSG_SPACE((3 * std::mem::size_of::()).to_u32().unwrap()); + let mut cmsg_buffer = vec![0u8; length as _]; + let mut msg: libc::msghdr = std::mem::zeroed(); + msg.msg_iov = (&raw const iov).cast_mut(); + msg.msg_iovlen = 1; + msg.msg_control = cmsg_buffer.as_mut_ptr().cast(); + msg.msg_controllen = cmsg_buffer.len() as _; + let ret = libc::recvmsg(fd, &raw mut msg, 0); + if ret < 0 { + let error = std::io::Error::last_os_error(); + return Err(error); + } + + // Read the fds. + let mut fds = Vec::new(); + let cmsg = libc::CMSG_FIRSTHDR(&raw const msg); + if !cmsg.is_null() + && (*cmsg).cmsg_level == libc::SOL_SOCKET + && (*cmsg).cmsg_type == libc::SCM_RIGHTS + { + let data = libc::CMSG_DATA(cmsg); + let n = ((*cmsg).cmsg_len.to_usize().unwrap() + - libc::CMSG_LEN(0).to_usize().unwrap()) + / std::mem::size_of::(); + for i in 0..n { + fds.push( + data.add(i * std::mem::size_of::()) + .cast::() + .read_unaligned(), + ); + } + } + + Ok(fds) + }) + .await + .map_err(|source| tg::error!(!source, "failed to receive the fds"))?; + request.fds = fds; + request.command.stdin = request + .command + .stdin + .and_then(|i| request.fds.get(i.to_usize().unwrap()).copied()); + request.command.stdout = request + .command + .stdout + .and_then(|i| request.fds.get(i.to_usize().unwrap()).copied()); + request.command.stderr = request + .command + .stderr + .and_then(|i| request.fds.get(i.to_usize().unwrap()).copied()); + } + + // Collect the FDs to close after the spawn request is processed. + let fds_to_close = if let RequestKind::Spawn(spawn) = &request.kind { + spawn.fds.clone() + } else { + Vec::new() + }; + + // Send the request to the shared task. + let (sender, receiver) = oneshot::channel(); + self.sender + .send((request, sender)) + .await + .map_err(|source| tg::error!(!source, "failed to send the request"))?; + let response = receiver + .await + .map_err(|source| tg::error!(!source, "failed to receive the request"))??; + + // Close the received FDs so the pipes can get EOF when the child exits. + for fd in fds_to_close { + unsafe { + libc::close(fd); + } + } + + // Write the response. + let response = serde_json::to_vec(&response).unwrap(); + let length = response.len().to_u64().unwrap(); + stream + .write_uvarint(length) + .await + .map_err(|source| tg::error!(!source, "failed to send the response"))?; + stream + .write_all(&response) + .await + .map_err(|source| tg::error!(!source, "failed to send the response"))?; + } + } + + pub fn shutdown(&self) { + let mut inner = self.inner.lock().unwrap(); + let Some(task) = inner.task.take() else { + return; + }; + task.abort(); + } +} diff --git a/packages/server/Cargo.toml b/packages/server/Cargo.toml index 29bdc177f..bdc7ac714 100644 --- a/packages/server/Cargo.toml +++ b/packages/server/Cargo.toml @@ -81,6 +81,7 @@ tangram_ignore = { workspace = true } tangram_index = { workspace = true } tangram_js = { workspace = true, optional = true } tangram_messenger = { workspace = true } +tangram_sandbox = { workspace = true } tangram_serialize = { workspace = true } tangram_session = { workspace = true } tangram_store = { workspace = true } diff --git a/packages/server/src/check.rs b/packages/server/src/check.rs index 2d9250b4f..766c1bda4 100644 --- a/packages/server/src/check.rs +++ b/packages/server/src/check.rs @@ -40,7 +40,7 @@ impl Server { return Ok(output); } - if context.process.is_some() { + if context.sandbox.is_some() { return Err(tg::error!("forbidden")); } diff --git a/packages/server/src/checkin.rs b/packages/server/src/checkin.rs index 4e4860893..1c8a82f6b 100644 --- a/packages/server/src/checkin.rs +++ b/packages/server/src/checkin.rs @@ -63,8 +63,8 @@ impl Server { impl Stream>> + Send + use<>, > { // Handle host path conversion. - if let Some(process) = &context.process { - arg.path = process.host_path_for_guest_path(arg.path.clone()); + if let Some(sandbox) = &context.sandbox { + arg.path = sandbox.host_path_for_guest_path(arg.path.clone()); } // Guard against concurrent cleans. diff --git a/packages/server/src/checkin/input.rs b/packages/server/src/checkin/input.rs index 45e9decaa..17c70e65b 100644 --- a/packages/server/src/checkin/input.rs +++ b/packages/server/src/checkin/input.rs @@ -571,9 +571,9 @@ impl Server { // If the target is absolute, then get the host path if necessary. if target.is_absolute() - && let Some(process) = &state.context.process + && let Some(sandbox) = &state.context.sandbox { - target = process.host_path_for_guest_path(target); + target = sandbox.host_path_for_guest_path(target); } // Canonicalize the target. diff --git a/packages/server/src/checkout.rs b/packages/server/src/checkout.rs index 3f7e04574..8cbec17f1 100644 --- a/packages/server/src/checkout.rs +++ b/packages/server/src/checkout.rs @@ -46,11 +46,11 @@ impl Server { ) -> tg::Result< impl Stream>> + Send + use<>, > { - // If there is a process in the context, then replace the path with the host path. - if let Some(process) = &context.process + // If there is a sandbox in the context, then replace the path with the host path. + if let Some(sandbox) = &context.sandbox && let Some(path) = &mut arg.path { - *path = process.host_path_for_guest_path(path.clone()); + *path = sandbox.host_path_for_guest_path(path.clone()); } // If the path is not provided, then cache. @@ -88,8 +88,8 @@ impl Server { }; // Map the path if necessary. - let path = if let Some(process) = &context.process { - process.guest_path_for_host_path(path)? + let path = if let Some(sandbox) = &context.sandbox { + sandbox.guest_path_for_host_path(path)? } else { path }; @@ -114,8 +114,8 @@ impl Server { path }; - let path = if let Some(process) = &context.process { - process.guest_path_for_host_path(path.clone())? + let path = if let Some(sandbox) = &context.sandbox { + sandbox.guest_path_for_host_path(path.clone())? } else { path }; @@ -195,8 +195,8 @@ impl Server { .stream() .map_ok(move |event| { if let tg::progress::Event::Output(mut output) = event { - if let Some(process) = &context.process { - output.path = process.host_path_for_guest_path(output.path.clone()); + if let Some(sandbox) = &context.sandbox { + output.path = sandbox.host_path_for_guest_path(output.path.clone()); } tg::progress::Event::Output(output) } else { diff --git a/packages/server/src/clean.rs b/packages/server/src/clean.rs index 476363444..859efe973 100644 --- a/packages/server/src/clean.rs +++ b/packages/server/src/clean.rs @@ -22,7 +22,7 @@ impl Server { ) -> tg::Result< impl Stream>> + Send + use<>, > { - if context.process.is_some() { + if context.sandbox.is_some() { return Err(tg::error!("forbidden")); } diff --git a/packages/server/src/config.rs b/packages/server/src/config.rs index 52caa6ce4..d4b4844dd 100644 --- a/packages/server/src/config.rs +++ b/packages/server/src/config.rs @@ -646,7 +646,7 @@ impl Default for Cleaner { concurrency: 1, partition_count: 256, partition_start: 0, - ttl: Duration::from_secs(86400), + ttl: Duration::from_hours(24), } } } @@ -853,7 +853,7 @@ impl Default for SyncPutStore { impl Default for Tag { fn default() -> Self { Self { - cache_ttl: Duration::from_secs(600), + cache_ttl: Duration::from_mins(10), } } } @@ -862,7 +862,7 @@ impl Default for Vfs { fn default() -> Self { Self { cache_size: 4096, - cache_ttl: Duration::from_secs(3600), + cache_ttl: Duration::from_hours(1), database_connections: 4, io: VfsIo::Auto, passthrough: VfsPassthrough::Auto, @@ -873,7 +873,7 @@ impl Default for Vfs { impl Default for Watch { fn default() -> Self { Self { - ttl: Duration::from_secs(3600), + ttl: Duration::from_hours(1), } } } @@ -884,7 +884,7 @@ impl Default for Watchdog { batch_size: 100, interval: Duration::from_secs(1), max_depth: 1024, - ttl: Duration::from_secs(60), + ttl: Duration::from_mins(1), } } } diff --git a/packages/server/src/context.rs b/packages/server/src/context.rs index 53413a778..6f62dc6e1 100644 --- a/packages/server/src/context.rs +++ b/packages/server/src/context.rs @@ -2,14 +2,14 @@ use {std::path::PathBuf, std::sync::Arc, tangram_client::prelude::*}; #[derive(Clone, Debug, Default)] pub struct Context { - pub process: Option>, + pub sandbox: Option>, pub token: Option, pub untrusted: bool, } #[derive(Clone, Debug)] -pub struct Process { - pub id: tg::process::Id, +pub struct Sandbox { + pub id: tg::sandbox::Id, pub paths: Option, pub remote: Option, pub retry: bool, @@ -23,7 +23,7 @@ pub struct Paths { pub root_host: PathBuf, } -impl Process { +impl Sandbox { pub fn host_path_for_guest_path(&self, path: PathBuf) -> PathBuf { let Some(path_map) = &self.paths else { return path; diff --git a/packages/server/src/database/postgres.sql b/packages/server/src/database/postgres.sql index b8160957d..1a7d35496 100644 --- a/packages/server/src/database/postgres.sql +++ b/packages/server/src/database/postgres.sql @@ -15,9 +15,9 @@ create table processes ( host text not null, id text primary key, log text, - mounts text, - network boolean not null, output text, + pid int4, + sandbox text, retry boolean not null, started_at int8, status text not null, diff --git a/packages/server/src/database/sqlite.sql b/packages/server/src/database/sqlite.sql index e0d44dd22..5bab92467 100644 --- a/packages/server/src/database/sqlite.sql +++ b/packages/server/src/database/sqlite.sql @@ -15,9 +15,9 @@ create table processes ( host text not null, id text primary key, log text, - mounts text, - network integer not null, output text, + pid integer, + sandbox text, retry integer not null, started_at integer, status text not null, diff --git a/packages/server/src/document.rs b/packages/server/src/document.rs index 59654e549..06fcf6ce9 100644 --- a/packages/server/src/document.rs +++ b/packages/server/src/document.rs @@ -40,7 +40,7 @@ impl Server { return Ok(output); } - if context.process.is_some() { + if context.sandbox.is_some() { return Err(tg::error!("forbidden")); } diff --git a/packages/server/src/format.rs b/packages/server/src/format.rs index f6a215576..09a84a1ea 100644 --- a/packages/server/src/format.rs +++ b/packages/server/src/format.rs @@ -12,7 +12,7 @@ impl Server { context: &Context, arg: tg::format::Arg, ) -> tg::Result<()> { - if context.process.is_some() { + if context.sandbox.is_some() { return Err(tg::error!("forbidden")); } diff --git a/packages/server/src/get.rs b/packages/server/src/get.rs index 92417b145..90d29e005 100644 --- a/packages/server/src/get.rs +++ b/packages/server/src/get.rs @@ -15,7 +15,7 @@ impl Server { ) -> tg::Result< impl Stream>>> + Send + use<>, > { - if context.process.is_some() { + if context.sandbox.is_some() { return Err(tg::error!("forbidden")); } let stream = match reference.item() { diff --git a/packages/server/src/handle.rs b/packages/server/src/handle.rs index c537a4dd1..b40005659 100644 --- a/packages/server/src/handle.rs +++ b/packages/server/src/handle.rs @@ -11,6 +11,7 @@ mod pipe; mod process; mod pty; mod remote; +mod sandbox; mod tag; mod user; mod watch; diff --git a/packages/server/src/handle/sandbox.rs b/packages/server/src/handle/sandbox.rs new file mode 100644 index 000000000..21a86765b --- /dev/null +++ b/packages/server/src/handle/sandbox.rs @@ -0,0 +1,116 @@ +use { + super::ServerWithContext, + crate::{Context, Server, Shared}, + tangram_client::prelude::*, +}; + +impl tg::handle::Sandbox for Shared { + async fn create_sandbox( + &self, + arg: tg::sandbox::create::Arg, + ) -> tg::Result { + self.0 + .create_sandbox_with_context(&Context::default(), arg) + .await + } + + async fn delete_sandbox(&self, id: &tg::sandbox::Id) -> tg::Result<()> { + self.0 + .delete_sandbox_with_context(&Context::default(), id) + .await + } + + async fn sandbox_spawn( + &self, + id: &tg::sandbox::Id, + arg: tg::sandbox::spawn::Arg, + ) -> tg::Result { + self.0 + .sandbox_spawn_with_context(&Context::default(), id, arg) + .await + } + + async fn try_sandbox_wait_future( + &self, + id: &tg::sandbox::Id, + arg: tg::sandbox::wait::Arg, + ) -> tg::Result< + Option< + impl Future>> + Send + 'static, + >, + > { + self.0 + .sandbox_wait_with_context(&Context::default(), id, arg) + .await + } +} + +impl tg::handle::Sandbox for Server { + async fn create_sandbox( + &self, + arg: tg::sandbox::create::Arg, + ) -> tg::Result { + self.create_sandbox_with_context(&Context::default(), arg) + .await + } + + async fn delete_sandbox(&self, id: &tg::sandbox::Id) -> tg::Result<()> { + self.delete_sandbox_with_context(&Context::default(), id) + .await + } + + async fn sandbox_spawn( + &self, + id: &tg::sandbox::Id, + arg: tg::sandbox::spawn::Arg, + ) -> tg::Result { + self.sandbox_spawn_with_context(&Context::default(), id, arg) + .await + } + + async fn try_sandbox_wait_future( + &self, + id: &tg::sandbox::Id, + arg: tg::sandbox::wait::Arg, + ) -> tg::Result< + Option< + impl Future>> + Send + 'static, + >, + > { + self.sandbox_wait_with_context(&Context::default(), id, arg) + .await + } +} + +impl tg::handle::Sandbox for ServerWithContext { + async fn create_sandbox( + &self, + arg: tg::sandbox::create::Arg, + ) -> tg::Result { + self.0.create_sandbox_with_context(&self.1, arg).await + } + + async fn delete_sandbox(&self, id: &tg::sandbox::Id) -> tg::Result<()> { + self.0.delete_sandbox_with_context(&self.1, id).await + } + + async fn sandbox_spawn( + &self, + id: &tg::sandbox::Id, + arg: tg::sandbox::spawn::Arg, + ) -> tg::Result { + self.0.sandbox_spawn_with_context(&self.1, id, arg).await + } + + async fn try_sandbox_wait_future( + &self, + id: &tg::sandbox::Id, + arg: tg::sandbox::wait::Arg, + ) -> tg::Result< + Option< + impl Future>> + Send + 'static, + >, + > { + self.0.sandbox_wait_with_context(&self.1, id, arg).await + } +} diff --git a/packages/server/src/health.rs b/packages/server/src/health.rs index 4a72af1d5..a63aa2ae2 100644 --- a/packages/server/src/health.rs +++ b/packages/server/src/health.rs @@ -13,7 +13,7 @@ impl Server { context: &Context, arg: tg::health::Arg, ) -> tg::Result { - if context.process.is_some() { + if context.sandbox.is_some() { return Err(tg::error!("forbidden")); } @@ -146,7 +146,7 @@ impl Server { } } *self.diagnostics.lock().unwrap() = diagnostics; - tokio::time::sleep(Duration::from_secs(3600)).await; + tokio::time::sleep(Duration::from_hours(1)).await; } } diff --git a/packages/server/src/http.rs b/packages/server/src/http.rs index 974104310..67e3f6d78 100644 --- a/packages/server/src/http.rs +++ b/packages/server/src/http.rs @@ -87,7 +87,7 @@ impl Server { .layer(tangram_http::layer::tracing::TracingLayer::new()) .layer(tower_http::timeout::TimeoutLayer::with_status_code( http::StatusCode::REQUEST_TIMEOUT, - Duration::from_secs(60), + Duration::from_mins(1), )) .add_extension(stop.clone()) .layer(tangram_http::layer::compression::RequestDecompressionLayer) @@ -507,6 +507,20 @@ impl Server { .handle_write_pty_request(request, &context, pty) .boxed(), + // Sandboxes. + (http::Method::POST, ["sandbox", "create"]) => server + .handle_create_sandbox_request(request, &context) + .boxed(), + (http::Method::DELETE, ["sandbox", sandbox]) => server + .handle_delete_sandbox_request(request, &context, sandbox) + .boxed(), + (http::Method::POST, ["sandbox", sandbox, "spawn"]) => server + .handle_sandbox_spawn_request(request, &context, sandbox) + .boxed(), + (http::Method::POST, ["sandbox", sandbox, "wait"]) => server + .handle_sandbox_wait_request(request, &context, sandbox) + .boxed(), + // Remotes. (http::Method::GET, ["remotes"]) => server .handle_list_remotes_request(request, &context) diff --git a/packages/server/src/index.rs b/packages/server/src/index.rs index 7609f546e..a28f573eb 100644 --- a/packages/server/src/index.rs +++ b/packages/server/src/index.rs @@ -214,7 +214,7 @@ impl Server { &self, context: &Context, ) -> tg::Result>> + Send + use<>> { - if context.process.is_some() { + if context.sandbox.is_some() { return Err(tg::error!("forbidden")); } if !self.config.advanced.single_process { diff --git a/packages/server/src/lib.rs b/packages/server/src/lib.rs index 54837f6ef..cf3f64d6b 100644 --- a/packages/server/src/lib.rs +++ b/packages/server/src/lib.rs @@ -51,6 +51,7 @@ mod push; mod read; mod remote; mod run; +mod sandbox; mod store; mod sync; mod tag; @@ -104,6 +105,7 @@ pub struct State { remotes: DashMap, remote_get_object_tasks: RemoteGetObjectTasks, remote_list_tags_tasks: RemoteListTagsTasks, + sandboxes: Sandboxes, store: Store, temps: DashSet, version: String, @@ -163,6 +165,8 @@ type RemoteListTagsTasks = tangram_futures::task::Map< fnv::FnvBuildHasher, >; +type Sandboxes = DashMap; + impl Owned { pub fn stop(&self) { self.task.stop(); @@ -268,9 +272,8 @@ impl Server { .map_err(|source| tg::error!(!source, "failed to create the tags directory"))?; // Get the available parallelism. - let parallelism = std::thread::available_parallelism() - .map(std::num::NonZeroUsize::get) - .unwrap_or(1); + let parallelism = + std::thread::available_parallelism().map_or(1, std::num::NonZeroUsize::get); // Remove an existing socket file. let socket_path = path.join("socket"); @@ -541,6 +544,9 @@ impl Server { }, }; + // Create the sandboxes. + let sandboxes = DashMap::default(); + // Create the temp paths. let temps = DashSet::default(); @@ -581,6 +587,7 @@ impl Server { remotes, remote_get_object_tasks, remote_list_tags_tasks, + sandboxes, store, temps, version, @@ -1002,6 +1009,20 @@ impl Server { tracing::trace!("indexer task"); } + // Kill all sandbox processes. + let sandbox_ids = server + .sandboxes + .iter() + .map(|r| r.key().clone()) + .collect::>(); + for id in sandbox_ids { + if let Some((_, mut sandbox)) = server.sandboxes.remove(&id) { + sandbox.serve_task.abort(); + sandbox.process.kill().await.ok(); + } + } + tracing::trace!("sandboxes"); + // Remove the temp paths. server .temps diff --git a/packages/server/src/lsp.rs b/packages/server/src/lsp.rs index 62820e020..817a6b052 100644 --- a/packages/server/src/lsp.rs +++ b/packages/server/src/lsp.rs @@ -15,7 +15,7 @@ impl Server { input: impl AsyncBufRead + Send + Unpin + 'static, output: impl AsyncWrite + Send + Unpin + 'static, ) -> tg::Result<()> { - if context.process.is_some() { + if context.sandbox.is_some() { return Err(tg::error!("forbidden")); } let compiler = self.create_compiler(); diff --git a/packages/server/src/pipe/close.rs b/packages/server/src/pipe/close.rs index b208bbf38..67dc59d01 100644 --- a/packages/server/src/pipe/close.rs +++ b/packages/server/src/pipe/close.rs @@ -28,7 +28,7 @@ impl Server { return Ok(()); } - if context.process.is_some() { + if context.sandbox.is_some() { return Err(tg::error!("forbidden")); } diff --git a/packages/server/src/pipe/create.rs b/packages/server/src/pipe/create.rs index 537ead285..24ee9f6a2 100644 --- a/packages/server/src/pipe/create.rs +++ b/packages/server/src/pipe/create.rs @@ -26,7 +26,7 @@ impl Server { .map_err(|source| tg::error!(!source, "failed to create the pipe on the remote")); } - if context.process.is_some() { + if context.sandbox.is_some() { return Err(tg::error!("forbidden")); } diff --git a/packages/server/src/pipe/delete.rs b/packages/server/src/pipe/delete.rs index 84b0d8578..10c4fadca 100644 --- a/packages/server/src/pipe/delete.rs +++ b/packages/server/src/pipe/delete.rs @@ -26,7 +26,7 @@ impl Server { return Ok(()); } - if context.process.is_some() { + if context.sandbox.is_some() { return Err(tg::error!("forbidden")); } diff --git a/packages/server/src/process/children/get.rs b/packages/server/src/process/children/get.rs index 4315a0deb..1baf3e8b1 100644 --- a/packages/server/src/process/children/get.rs +++ b/packages/server/src/process/children/get.rs @@ -134,7 +134,7 @@ impl Server { .boxed(); // Create the interval. - let interval = IntervalStream::new(tokio::time::interval(Duration::from_secs(60))) + let interval = IntervalStream::new(tokio::time::interval(Duration::from_mins(1))) .map(|_| ()) .boxed(); diff --git a/packages/server/src/process/finish.rs b/packages/server/src/process/finish.rs index c7162cb18..4281e8658 100644 --- a/packages/server/src/process/finish.rs +++ b/packages/server/src/process/finish.rs @@ -2,7 +2,7 @@ use { crate::{Context, Server}, futures::{StreamExt as _, stream::FuturesUnordered}, indoc::formatdoc, - tangram_client::prelude::*, + tangram_client::{handle::Sandbox, prelude::*}, tangram_database::{self as db, prelude::*}, tangram_http::{body::Boxed as BoxBody, request::Ext as _}, tangram_messenger::prelude::*, @@ -36,7 +36,7 @@ impl Server { return Ok(()); } - if context.process.is_some() { + if context.sandbox.is_some() { return Err(tg::error!("forbidden")); } @@ -65,6 +65,25 @@ impl Server { return Err(tg::error!("failed to find the process")); }; + // Kill by pid if it exists. + if let Some(pid) = data.pid { + unsafe { libc::kill(pid, libc::SIGKILL) }; + } + + // Decrement the sandbox refcount and kill it if necessary. + if let Some(id) = &data.sandbox + && let Some(mut sandbox) = self.sandboxes.get_mut(id) + { + sandbox.refcount = sandbox.refcount.saturating_sub(1); + if sandbox.refcount == 0 { + drop(sandbox); + self.delete_sandbox(id) + .await + .inspect_err(|error| tracing::warn!(?error, "failed to delete the sandbox")) + .ok(); + } + } + // Get the process's children. let connection = self .database @@ -165,6 +184,7 @@ impl Server { finished_at = {p}4, heartbeat_at = null, output = {p}5, + pid = null, exit = {p}6, status = {p}7, touched_at = {p}8 diff --git a/packages/server/src/process/get/postgres.rs b/packages/server/src/process/get/postgres.rs index 284128cda..4c8d44bd0 100644 --- a/packages/server/src/process/get/postgres.rs +++ b/packages/server/src/process/get/postgres.rs @@ -42,12 +42,12 @@ impl Server { host: Option, #[tangram_database(as = "Option")] log: Option, - #[tangram_database(as = "Option>>")] - mounts: Option>, - network: Option, #[tangram_database(as = "Option>")] output: Option, + pid: Option, retry: Option, + #[tangram_database(as = "Option")] + sandbox: Option, started_at: Option, #[tangram_database(as = "Option")] status: Option, @@ -76,9 +76,9 @@ impl Server { host, log, output, + pid, retry, - mounts, - network, + sandbox, started_at, status, stderr, @@ -123,9 +123,6 @@ impl Server { let host = row .host .ok_or_else(|| tg::error!(%id, "missing host field"))?; - let network = row - .network - .ok_or_else(|| tg::error!(%id, "missing network field"))?; let retry = row .retry .ok_or_else(|| tg::error!(%id, "missing retry field"))?; @@ -163,9 +160,9 @@ impl Server { host, log: row.log, output: row.output, + pid: row.pid, retry, - mounts: row.mounts.unwrap_or_default(), - network, + sandbox: row.sandbox, started_at: row.started_at, status, stderr: row.stderr, diff --git a/packages/server/src/process/get/sqlite.rs b/packages/server/src/process/get/sqlite.rs index 06d5658a3..c046df930 100644 --- a/packages/server/src/process/get/sqlite.rs +++ b/packages/server/src/process/get/sqlite.rs @@ -63,11 +63,10 @@ impl Server { host: String, log: Option, output: Option, + pid: Option, #[tangram_database(as = "db::sqlite::value::TryFrom")] retry: u64, - mounts: Option, - #[tangram_database(as = "db::sqlite::value::TryFrom")] - network: u64, + sandbox: Option, started_at: Option, status: String, stderr: Option, @@ -90,9 +89,9 @@ impl Server { host, log, output, + pid, retry, - mounts, - network, + sandbox, started_at, status, stderr, @@ -157,13 +156,11 @@ impl Server { .transpose() .map_err(|source| tg::error!(!source, "failed to deserialize"))?; let retry = row.retry != 0; - let mounts = row - .mounts - .map(|s| serde_json::from_str(&s)) + let sandbox = row + .sandbox + .map(|s| s.parse()) .transpose() - .map_err(|source| tg::error!(!source, "failed to deserialize"))? - .unwrap_or_default(); - let network = row.network != 0; + .map_err(|source| tg::error!(!source, %id, "failed to parse the sandbox id"))?; let status = row .status .parse() @@ -234,9 +231,9 @@ impl Server { host: row.host, log, output, + pid: row.pid, retry, - mounts, - network, + sandbox, started_at: row.started_at, status, stderr, diff --git a/packages/server/src/process/heartbeat.rs b/packages/server/src/process/heartbeat.rs index c13077e3f..58b541b95 100644 --- a/packages/server/src/process/heartbeat.rs +++ b/packages/server/src/process/heartbeat.rs @@ -27,7 +27,7 @@ impl Server { return Ok(output); } - if context.process.is_some() { + if context.sandbox.is_some() { return Err(tg::error!("forbidden")); } diff --git a/packages/server/src/process/list.rs b/packages/server/src/process/list.rs index ac1098d23..a47d0d344 100644 --- a/packages/server/src/process/list.rs +++ b/packages/server/src/process/list.rs @@ -16,7 +16,7 @@ impl Server { context: &Context, arg: tg::process::list::Arg, ) -> tg::Result { - if context.process.is_some() { + if context.sandbox.is_some() { return Err(tg::error!("forbidden")); } diff --git a/packages/server/src/process/list/postgres.rs b/packages/server/src/process/list/postgres.rs index d85e19570..1617c4410 100644 --- a/packages/server/src/process/list/postgres.rs +++ b/packages/server/src/process/list/postgres.rs @@ -40,10 +40,10 @@ impl Server { log: Option, #[tangram_database(as = "Option>")] output: Option, + pid: Option, retry: bool, - #[tangram_database(as = "Option>>")] - mounts: Option>, - network: bool, + #[tangram_database(as = "Option")] + sandbox: Option, started_at: Option, #[tangram_database(as = "db::postgres::value::FromStr")] status: tg::process::Status, @@ -72,9 +72,9 @@ impl Server { host, log, output, + pid, retry, - mounts, - network, + sandbox, started_at, status, stderr, @@ -127,9 +127,9 @@ impl Server { host: row.host, log: row.log, output: row.output, + pid: row.pid, retry: row.retry, - mounts: row.mounts.unwrap_or_default(), - network: row.network, + sandbox: row.sandbox, started_at: row.started_at, status: row.status, stderr: row.stderr, diff --git a/packages/server/src/process/list/sqlite.rs b/packages/server/src/process/list/sqlite.rs index a1c16b15b..fe556024e 100644 --- a/packages/server/src/process/list/sqlite.rs +++ b/packages/server/src/process/list/sqlite.rs @@ -49,10 +49,10 @@ impl Server { log: Option, #[tangram_database(as = "Option>")] output: Option, + pid: Option, retry: bool, - #[tangram_database(as = "Option>>")] - mounts: Option>, - network: bool, + #[tangram_database(as = "Option")] + sandbox: Option, started_at: Option, #[tangram_database(as = "db::sqlite::value::FromStr")] status: tg::process::Status, @@ -80,9 +80,9 @@ impl Server { host, log, output, + pid, retry, - mounts, - network, + sandbox, started_at, status, stderr, @@ -172,9 +172,9 @@ impl Server { host: row.host, log: row.log, output: row.output, + pid: row.pid, retry: row.retry, - mounts: row.mounts.unwrap_or_default(), - network: row.network, + sandbox: row.sandbox, started_at: row.started_at, status: row.status, stderr: row.stderr, diff --git a/packages/server/src/process/log/get.rs b/packages/server/src/process/log/get.rs index b55eb1181..bf36b7c6e 100644 --- a/packages/server/src/process/log/get.rs +++ b/packages/server/src/process/log/get.rs @@ -117,7 +117,7 @@ impl Server { .boxed(); // Create the interval. - let interval = IntervalStream::new(tokio::time::interval(Duration::from_secs(60))) + let interval = IntervalStream::new(tokio::time::interval(Duration::from_mins(1))) .map(|_| ()) .boxed(); diff --git a/packages/server/src/process/log/post.rs b/packages/server/src/process/log/post.rs index 8a697bd9d..8ad091a53 100644 --- a/packages/server/src/process/log/post.rs +++ b/packages/server/src/process/log/post.rs @@ -29,7 +29,7 @@ impl Server { return Ok(()); } - if context.process.is_some() { + if context.sandbox.is_some() { return Err(tg::error!("forbidden")); } diff --git a/packages/server/src/process/put.rs b/packages/server/src/process/put.rs index e3db41813..a6475fb3f 100644 --- a/packages/server/src/process/put.rs +++ b/packages/server/src/process/put.rs @@ -36,7 +36,7 @@ impl Server { return Ok(()); } - if context.process.is_some() { + if context.sandbox.is_some() { return Err(tg::error!("forbidden")); } let now = time::OffsetDateTime::now_utc().unix_timestamp(); diff --git a/packages/server/src/process/put/postgres.rs b/packages/server/src/process/put/postgres.rs index a208f409a..e4361f132 100644 --- a/packages/server/src/process/put/postgres.rs +++ b/packages/server/src/process/put/postgres.rs @@ -53,10 +53,9 @@ impl Server { let mut finished_ats: Vec> = Vec::with_capacity(items.len()); let mut hosts: Vec = Vec::with_capacity(items.len()); let mut logs: Vec> = Vec::with_capacity(items.len()); - let mut mounts: Vec> = Vec::with_capacity(items.len()); - let mut networks: Vec = Vec::with_capacity(items.len()); let mut outputs: Vec> = Vec::with_capacity(items.len()); let mut retries: Vec = Vec::with_capacity(items.len()); + let mut sandboxes: Vec> = Vec::with_capacity(items.len()); let mut started_ats: Vec> = Vec::with_capacity(items.len()); let mut statuses = Vec::with_capacity(items.len()); let mut stderrs: Vec> = Vec::with_capacity(items.len()); @@ -92,16 +91,13 @@ impl Server { finished_ats.push(data.finished_at); hosts.push(data.host.clone()); logs.push(data.log.as_ref().map(ToString::to_string)); - mounts.push( - (!data.mounts.is_empty()).then(|| serde_json::to_string(&data.mounts).unwrap()), - ); - networks.push(data.network); outputs.push( data.output .as_ref() .map(|output| serde_json::to_string(output).unwrap()), ); retries.push(data.retry); + sandboxes.push(data.sandbox.as_ref().map(ToString::to_string)); started_ats.push(data.started_at); statuses.push(data.status.to_string()); stderrs.push(data.stderr.as_ref().map(ToString::to_string)); @@ -129,10 +125,9 @@ impl Server { finished_at, host, log, - mounts, - network, output, retry, + sandbox, started_at, status, stderr, @@ -159,14 +154,13 @@ impl Server { unnest($15::text[]), unnest($16::bool[]), unnest($17::text[]), - unnest($18::bool[]), - unnest($19::int8[]), + unnest($18::int8[]), + unnest($19::text[]), unnest($20::text[]), unnest($21::text[]), unnest($22::text[]), - unnest($23::text[]), - unnest($24::int8[]), - unnest($25::int8[]) + unnest($23::int8[]), + unnest($24::int8[]) on conflict (id) do update set actual_checksum = excluded.actual_checksum, cacheable = excluded.cacheable, @@ -181,10 +175,9 @@ impl Server { finished_at = excluded.finished_at, host = excluded.host, log = excluded.log, - mounts = excluded.mounts, - network = excluded.network, output = excluded.output, retry = excluded.retry, + sandbox = excluded.sandbox, started_at = excluded.started_at, status = excluded.status, stderr = excluded.stderr, @@ -212,10 +205,9 @@ impl Server { &finished_ats, &hosts, &logs, - &mounts, - &networks, &outputs, &retries, + &sandboxes, &started_ats, &statuses, &stderrs, diff --git a/packages/server/src/process/put/sqlite.rs b/packages/server/src/process/put/sqlite.rs index ec34ac44b..7eb3867dc 100644 --- a/packages/server/src/process/put/sqlite.rs +++ b/packages/server/src/process/put/sqlite.rs @@ -76,10 +76,9 @@ impl Server { finished_at, host, log, - mounts, - network, output, retry, + sandbox, started_at, status, stderr, @@ -112,8 +111,7 @@ impl Server { ?21, ?22, ?23, - ?24, - ?25 + ?24 ) on conflict (id) do update set actual_checksum = ?2, @@ -129,17 +127,16 @@ impl Server { finished_at = ?12, host = ?13, log = ?14, - mounts = ?15, - network = ?16, - output = ?17, - retry = ?18, - started_at = ?19, - status = ?20, - stderr = ?21, - stdin = ?22, - stdout = ?23, - token_count = ?24, - touched_at = ?25 + output = ?15, + retry = ?16, + sandbox = ?17, + started_at = ?18, + status = ?19, + stderr = ?20, + stdin = ?21, + stdout = ?22, + token_count = ?23, + touched_at = ?24 " ); let mut process_stmt = cache @@ -168,8 +165,6 @@ impl Server { tg::Either::Left(data) => data.code.map(|code| code.to_string()), tg::Either::Right(_) => None, }); - let mounts_json = - (!data.mounts.is_empty()).then(|| serde_json::to_string(&data.mounts).unwrap()); let output_json = data .output .as_ref() @@ -190,10 +185,9 @@ impl Server { data.finished_at, data.host, data.log.as_ref().map(ToString::to_string), - mounts_json, - data.network, output_json, data.retry, + data.sandbox.as_ref().map(ToString::to_string), data.started_at, data.status.to_string(), data.stderr.as_ref().map(ToString::to_string), diff --git a/packages/server/src/process/queue.rs b/packages/server/src/process/queue.rs index d91582c36..3251bd1c3 100644 --- a/packages/server/src/process/queue.rs +++ b/packages/server/src/process/queue.rs @@ -30,7 +30,7 @@ impl Server { context: &Context, _arg: tg::process::queue::Arg, ) -> tg::Result> { - if context.process.is_some() { + if context.sandbox.is_some() { return Err(tg::error!("forbidden")); } diff --git a/packages/server/src/process/spawn.rs b/packages/server/src/process/spawn.rs index 88bd749fd..556f1c9a1 100644 --- a/packages/server/src/process/spawn.rs +++ b/packages/server/src/process/spawn.rs @@ -6,7 +6,7 @@ use { }, indoc::{formatdoc, indoc}, std::{fmt::Write, pin::pin}, - tangram_client::prelude::*, + tangram_client::{handle::Sandbox, prelude::*}, tangram_database::{self as db, prelude::*}, tangram_futures::{stream::Ext as _, task::Task}, tangram_http::{body::Boxed as BoxBody, request::Ext as _}, @@ -31,13 +31,12 @@ impl Server { ) -> tg::Result< BoxStream<'static, tg::Result>>>, > { - // If the process context is set, update the parent, remotes, and retry. - if let Some(process) = &context.process { - arg.parent = Some(process.id.clone()); - if let Some(remote) = &process.remote { + // If the sandbox context is set, update the remotes and retry. + if let Some(sandbox) = &context.sandbox { + if let Some(remote) = &sandbox.remote { arg.remotes = Some(vec![remote.clone()]); } - arg.retry = process.retry; + arg.retry = sandbox.retry; } // Create the progress. @@ -45,9 +44,12 @@ impl Server { // Spawn the task. let task = Task::spawn({ + let context = context.clone(); let server = self.clone(); let progress = progress.clone(); - async move |_| match Box::pin(server.try_spawn_process_task(arg, &progress)).await { + async move |_| match Box::pin(server.try_spawn_process_task(arg, &context, &progress)) + .await + { Ok(output) => { progress.output(output); }, @@ -64,6 +66,7 @@ impl Server { async fn try_spawn_process_task( &self, arg: tg::process::spawn::Arg, + context: &Context, progress: &crate::progress::Handle>, ) -> tg::Result> { // Forward to remote if requested. @@ -133,12 +136,26 @@ impl Server { .map_err(|source| tg::error!(!source, "failed to begin a transaction"))?; // Determine if the process is cacheable. + let reproducible_sandbox = arg.sandbox.as_ref().is_some_and(|sbx| { + sbx.as_ref().left().is_some_and(|sandbox| { + !sandbox.network && sandbox.mounts.iter().all(|mount| mount.source.is_right()) + }) + }); + let reproducible_stdin = arg.stdin.is_none(); + let reproducible_stdout = arg + .stdout + .as_ref() + .is_none_or(|stdio| matches!(stdio, tg::process::Stdio::Pipe(_))); + let reproducible_stderr = arg + .stderr + .as_ref() + .is_none_or(|stdio| matches!(stdio, tg::process::Stdio::Pipe(_))); + let cacheable = arg.checksum.is_some() - || (arg.mounts.is_empty() - && !arg.network - && arg.stdin.is_none() - && arg.stdout.is_none() - && arg.stderr.is_none()); + || (reproducible_sandbox + && reproducible_stdin + && reproducible_stdout + && reproducible_stderr); // Get or create a local process. let mut output = if cacheable @@ -167,7 +184,7 @@ impl Server { } else if matches!(arg.cached, None | Some(false)) { let host = host.ok_or_else(|| tg::error!("expected the host to be set"))?; let output = self - .create_local_process(&transaction, &arg, cacheable, &host) + .create_local_process(&transaction, &arg, cacheable, &host, context) .await .map_err(|source| tg::error!(!source, "failed to create a local process"))?; tracing::trace!(?output, "created local process"); @@ -604,10 +621,9 @@ impl Server { expected_checksum, finished_at, host, - mounts, - network, output, retry, + sandbox, status, token_count, touched_at @@ -629,8 +645,7 @@ impl Server { {p}14, {p}15, {p}16, - {p}17, - {p}18 + {p}17 ) on conflict (id) do update set actual_checksum = {p}2, @@ -643,13 +658,12 @@ impl Server { expected_checksum = {p}9, finished_at = {p}10, host = {p}11, - mounts = {p}12, - network = {p}13, - output = {p}14, - retry = {p}15, - status = {p}16, - token_count = {p}17, - touched_at = {p}18; + output = {p}12, + retry = {p}13, + sandbox = {p}14, + status = {p}15, + token_count = {p}16, + touched_at = {p}17; " ); let now: i64 = time::OffsetDateTime::now_utc().unix_timestamp(); @@ -675,10 +689,12 @@ impl Server { expected_checksum.to_string(), now, host, - (!arg.mounts.is_empty()).then(|| db::value::Json(arg.mounts.clone())), - arg.network, output.clone().map(db::value::Json), arg.retry, + arg.sandbox.as_ref().and_then(|s| match s { + tg::Either::Left(_) => None, + tg::Either::Right(id) => Some(id.to_string()), + }), status.to_string(), 0, now, @@ -707,9 +723,27 @@ impl Server { arg: &tg::process::spawn::Arg, cacheable: bool, host: &str, + context: &crate::Context, ) -> tg::Result { let p = transaction.p(); + // Get or create a sandbox. + let sandbox = match &arg.sandbox { + Some(tg::Either::Left(arg)) => { + let id = self.create_sandbox(arg.clone()).await?.id; + Some(id) + }, + Some(tg::Either::Right(id)) => Some(id.clone()), + None => context.sandbox.as_ref().map(|sbx| sbx.id.clone()), + }; + + // Increment the sandbox refcount. + if let Some(sandbox_id) = &sandbox + && let Some(mut sandbox) = self.sandboxes.get_mut(sandbox_id) + { + sandbox.refcount += 1; + } + // Create an ID. let id = tg::process::Id::new(); @@ -752,9 +786,8 @@ impl Server { expected_checksum, heartbeat_at, host, - mounts, - network, retry, + sandbox, started_at, status, stderr, @@ -781,8 +814,7 @@ impl Server { {p}15, {p}16, {p}17, - {p}18, - {p}19 + {p}18 ) on conflict (id) do update set cacheable = {p}2, @@ -793,16 +825,15 @@ impl Server { expected_checksum = {p}7, heartbeat_at = {p}8, host = {p}9, - mounts = {p}10, - network = {p}11, - retry = {p}12, - started_at = {p}13, - status = {p}14, - stderr = {p}15, - stdin = {p}16, - stdout = {p}17, - token_count = {p}18, - touched_at = {p}19; + retry = {p}10, + sandbox = {p}11, + started_at = {p}12, + status = {p}13, + stderr = {p}14, + stdin = {p}15, + stdout = {p}16, + token_count = {p}17, + touched_at = {p}18; " ); let now = time::OffsetDateTime::now_utc().unix_timestamp(); @@ -818,9 +849,8 @@ impl Server { arg.checksum.as_ref().map(ToString::to_string), heartbeat_at, host, - (!arg.mounts.is_empty()).then(|| db::value::Json(arg.mounts.clone())), - arg.network, arg.retry, + sandbox.as_ref().map(ToString::to_string), started_at, status.to_string(), arg.stderr.as_ref().map(ToString::to_string), diff --git a/packages/server/src/process/start.rs b/packages/server/src/process/start.rs index adf56dc18..e7ad15b87 100644 --- a/packages/server/src/process/start.rs +++ b/packages/server/src/process/start.rs @@ -29,7 +29,7 @@ impl Server { return Ok(()); } - if context.process.is_some() { + if context.sandbox.is_some() { return Err(tg::error!("forbidden")); } diff --git a/packages/server/src/process/status.rs b/packages/server/src/process/status.rs index ae7a47d48..225baa3cf 100644 --- a/packages/server/src/process/status.rs +++ b/packages/server/src/process/status.rs @@ -73,7 +73,7 @@ impl Server { // Create the interval. let interval = - IntervalStream::new(tokio::time::interval(Duration::from_secs(60))).map(|_| ()); + IntervalStream::new(tokio::time::interval(Duration::from_mins(1))).map(|_| ()); // Create the stream. let server = self.clone(); diff --git a/packages/server/src/process/touch.rs b/packages/server/src/process/touch.rs index c3008aa0f..f9374fd34 100644 --- a/packages/server/src/process/touch.rs +++ b/packages/server/src/process/touch.rs @@ -28,7 +28,7 @@ impl Server { return Ok(()); } - if context.process.is_some() { + if context.sandbox.is_some() { return Err(tg::error!("forbidden")); } diff --git a/packages/server/src/pty.rs b/packages/server/src/pty.rs index 318e57014..aef212e6e 100644 --- a/packages/server/src/pty.rs +++ b/packages/server/src/pty.rs @@ -21,11 +21,11 @@ mod size; mod write; pub(crate) struct Pty { - master: Option, - slave: Option, + pub(crate) master: Option, + pub(crate) slave: Option, #[expect(dead_code)] - name: CString, - session: Option, + pub(crate) name: CString, + pub(crate) session: Option, pub(crate) temp: Temp, } diff --git a/packages/server/src/pty/close.rs b/packages/server/src/pty/close.rs index c18263753..fd82783d5 100644 --- a/packages/server/src/pty/close.rs +++ b/packages/server/src/pty/close.rs @@ -28,7 +28,7 @@ impl Server { return Ok(()); } - if context.process.is_some() { + if context.sandbox.is_some() { return Err(tg::error!("forbidden")); } diff --git a/packages/server/src/pty/create.rs b/packages/server/src/pty/create.rs index 274169825..b1523f566 100644 --- a/packages/server/src/pty/create.rs +++ b/packages/server/src/pty/create.rs @@ -27,7 +27,7 @@ impl Server { .map_err(|source| tg::error!(!source, "failed to create the pty on the remote")); } - if context.process.is_some() { + if context.sandbox.is_some() { return Err(tg::error!("forbidden")); } diff --git a/packages/server/src/pty/delete.rs b/packages/server/src/pty/delete.rs index f1aa5a31c..a9d86560c 100644 --- a/packages/server/src/pty/delete.rs +++ b/packages/server/src/pty/delete.rs @@ -27,7 +27,7 @@ impl Server { return Ok(()); } - if context.process.is_some() { + if context.sandbox.is_some() { return Err(tg::error!("forbidden")); } diff --git a/packages/server/src/remote/delete.rs b/packages/server/src/remote/delete.rs index 51a94026c..48dedc9c3 100644 --- a/packages/server/src/remote/delete.rs +++ b/packages/server/src/remote/delete.rs @@ -12,7 +12,7 @@ impl Server { context: &Context, name: &str, ) -> tg::Result<()> { - if context.process.is_some() { + if context.sandbox.is_some() { return Err(tg::error!("forbidden")); } diff --git a/packages/server/src/remote/get.rs b/packages/server/src/remote/get.rs index 8fcb2010e..0400007a7 100644 --- a/packages/server/src/remote/get.rs +++ b/packages/server/src/remote/get.rs @@ -13,7 +13,7 @@ impl Server { context: &Context, name: &str, ) -> tg::Result> { - if context.process.is_some() { + if context.sandbox.is_some() { return Err(tg::error!("forbidden")); } diff --git a/packages/server/src/remote/list.rs b/packages/server/src/remote/list.rs index d7c0e6873..622880a04 100644 --- a/packages/server/src/remote/list.rs +++ b/packages/server/src/remote/list.rs @@ -13,7 +13,7 @@ impl Server { context: &Context, _arg: tg::remote::list::Arg, ) -> tg::Result { - if context.process.is_some() { + if context.sandbox.is_some() { return Err(tg::error!("forbidden")); } diff --git a/packages/server/src/remote/put.rs b/packages/server/src/remote/put.rs index c72b6288e..c52aad557 100644 --- a/packages/server/src/remote/put.rs +++ b/packages/server/src/remote/put.rs @@ -13,7 +13,7 @@ impl Server { name: &str, arg: tg::remote::put::Arg, ) -> tg::Result<()> { - if context.process.is_some() { + if context.sandbox.is_some() { return Err(tg::error!("forbidden")); } diff --git a/packages/server/src/run.rs b/packages/server/src/run.rs index b0dcbe79c..8c21aa73d 100644 --- a/packages/server/src/run.rs +++ b/packages/server/src/run.rs @@ -1,8 +1,9 @@ use { crate::{ProcessPermit, Server}, futures::{FutureExt as _, TryFutureExt as _, future}, - std::{collections::BTreeSet, path::Path, sync::Arc, time::Duration}, + std::{collections::BTreeSet, sync::Arc, time::Duration}, tangram_client::prelude::*, + tangram_database::{self as db, prelude::*}, }; mod common; @@ -242,30 +243,8 @@ impl Server { .await .map_err(|source| tg::error!(!source, "failed to get the host"))?; - // Determine if the root is mounted. - let root_mounted = state - .mounts - .iter() - .any(|mount| mount.source == mount.target && mount.target == Path::new("/")); - - // Get the server's user. - let whoami = - util::whoami().map_err(|error| tg::error!(!error, "failed to get username"))?; - - // Determine if the process is unsandboxed. - let mounts = command - .mounts(self) - .await - .map_err(|source| tg::error!(!source, "failed to get the mounts"))?; - let user = command - .user(self) - .await - .map_err(|source| tg::error!(!source, "failed to get the user"))?; - let unsandboxed = root_mounted - && (mounts.is_empty() && state.mounts.len() == 1) - && state.network - && user.as_ref().is_none_or(|user| user == &whoami); - let sandboxed = !unsandboxed; + // Determine if the process is sandboxed. + let sandboxed = state.sandbox.is_some(); let result = { match host.as_str() { @@ -327,6 +306,33 @@ impl Server { Ok(output) } + pub(crate) async fn update_process_pid( + &self, + id: &tg::process::Id, + pid: i32, + ) -> tg::Result<()> { + let connection = self + .database + .write_connection() + .await + .map_err(|source| tg::error!(!source, "failed to get a database connection"))?; + let p = connection.p(); + let statement = format!( + " + update processes + set pid = {p}1 + where id = {p}2; + " + ); + let params = db::params![pid, id.to_string()]; + connection + .execute(statement.into(), params) + .await + .map_err(|source| tg::error!(!source, "failed to update the process pid"))?; + drop(connection); + Ok(()) + } + async fn compute_checksum( &self, value: &tg::Value, diff --git a/packages/server/src/run/common.rs b/packages/server/src/run/common.rs index a8f5305cd..a8e76186d 100644 --- a/packages/server/src/run/common.rs +++ b/packages/server/src/run/common.rs @@ -86,10 +86,10 @@ pub async fn run(mut arg: Arg<'_>) -> tg::Result { }; // Get the output path. - let path = temp.path().join("output/output"); + let path = temp.path().join("output"); let exists = tokio::fs::try_exists(&path) .await - .map_err(|source| tg::error!(!source, "failed to determine if the output path exists"))?; + .map_err(|source| tg::error!(!source, path = %path.display(), "failed to determine if the output path exists"))?; // Try to read the user.tangram.output xattr. if let Ok(Some(bytes)) = xattr::get(&path, "user.tangram.output") { @@ -112,8 +112,8 @@ pub async fn run(mut arg: Arg<'_>) -> tg::Result { // Check in the output. if output.output.is_none() && exists { - let path = if let Some(process) = &context.process { - process + let path = if let Some(sandbox) = &context.sandbox { + sandbox .guest_path_for_host_path(path.clone()) .map_err(|source| tg::error!(!source, "failed to map the output path"))? } else { @@ -319,6 +319,12 @@ async fn run_session(arg: Arg<'_>, pty: &tg::pty::Id) -> tg::Result { .to_i32() .unwrap(); + // Update the process pid in the database. + server + .update_process_pid(id, pid) + .await + .map_err(|source| tg::error!(!source, %id, "failed to update the process pid"))?; + // Drop the FDs. drop(fds); @@ -515,6 +521,12 @@ async fn run_inner(arg: Arg<'_>) -> tg::Result { drop(cmd); let pid = child.id().unwrap().to_i32().unwrap(); + // Update the process pid in the database. + server + .update_process_pid(id, pid) + .await + .map_err(|source| tg::error!(!source, %id, "failed to update the process pid"))?; + // Spawn the stdio task. let stdio_task = tokio::spawn({ let server = server.clone(); @@ -571,7 +583,7 @@ async fn run_inner(arg: Arg<'_>) -> tg::Result { Ok(exit) } -async fn stdio_task( +pub(super) async fn stdio_task( server: &Server, id: &tg::process::Id, remote: Option<&String>, @@ -663,7 +675,7 @@ where Ok::<_, tg::Error>(()) } -async fn stdio_task_inner( +pub(super) async fn stdio_task_inner( server: &Server, id: &tg::process::Id, remote: Option<&String>, diff --git a/packages/server/src/run/darwin.rs b/packages/server/src/run/darwin.rs index acc63c0ea..1ce5ee037 100644 --- a/packages/server/src/run/darwin.rs +++ b/packages/server/src/run/darwin.rs @@ -1,13 +1,14 @@ use { - super::util::{cache_children, render_args_dash_a, render_args_string, render_env, whoami}, + super::util::{cache_children, render_args_dash_a, render_args_string, render_env}, crate::{Context, Server, temp::Temp}, std::{ - collections::BTreeMap, - path::{Path, PathBuf}, + os::fd::{AsFd as _, AsRawFd as _}, + path::PathBuf, sync::Arc, }, tangram_client::prelude::*, - tangram_futures::task::Task, + tangram_futures::{read::Ext as _, stream::TryExt as _, task::Task, write::Ext as _}, + tangram_sandbox as sandbox, tangram_uri::Uri, }; @@ -36,28 +37,28 @@ impl Server { .await .map_err(|source| tg::error!(!source, "failed to cache the children"))?; - // Determine if the root is mounted. - let root_mounted = state - .mounts - .iter() - .any(|mount| mount.source == mount.target && mount.target == Path::new("/")); - // Get the artifacts path. let artifacts_path = self.artifacts_path(); - // Create the temp. - let temp = Temp::new(self); - tokio::fs::create_dir_all(&temp) - .await - .map_err(|source| tg::error!(!source, "failed to create the temp directory"))?; - - // Create the output directory. - tokio::fs::create_dir_all(temp.path().join("output")) - .await - .map_err(|source| tg::error!(!source, "failed to create the output directory"))?; - // Get the output path. - let output_path = temp.path().join("output/output"); + let temp = Temp::new(self); + let output_path = if let Some(id)=&state.sandbox { + let sandbox = self + .sandboxes + .get(id) + .ok_or_else(|| tg::error!("failed to find the sandbox"))?; + let path = sandbox.temp.path().join("output/output"); + drop(sandbox); + path + } else { + tokio::fs::create_dir_all(&temp) + .await + .map_err(|source| tg::error!(!source, "failed to create the temp directory"))?; + tokio::fs::create_dir_all(temp.path().join("output")) + .await + .map_err(|source| tg::error!(!source, "failed to create the output directory"))?; + temp.path().join("output/output") + }; // Render the args. let mut args = match command.host.as_str() { @@ -66,10 +67,13 @@ impl Server { }; // Create the working directory. - let cwd = command - .cwd - .clone() - .unwrap_or_else(|| temp.path().join("work")); + let cwd = if let Some(cwd) = &command.cwd { + cwd.clone() + } else if state.sandbox.is_none() { + temp.path().join("work") + } else { + "/".into() + }; tokio::fs::create_dir_all(&cwd) .await .map_err(|source| tg::error!(!source, "failed to create the working directory"))?; @@ -112,57 +116,6 @@ impl Server { }, }; - let path = temp.path().join(".tangram"); - tokio::fs::create_dir_all(&path).await.map_err( - |source| tg::error!(!source, path = %path.display(), "failed to create the directory"), - )?; - - // Listen. - let socket_path = path.join("socket").display().to_string(); - let mut url = if socket_path.len() <= MAX_URL_LEN { - tangram_uri::Uri::builder() - .scheme("http+unix") - .authority(&socket_path) - .path("") - .build() - .unwrap() - } else { - "http://localhost:0".to_owned().parse::().unwrap() - }; - let listener = Server::listen(&url) - .await - .map_err(|source| tg::error!(!source, "failed to listen"))?; - let listener_addr = listener - .local_addr() - .map_err(|source| tg::error!(!source, "failed to get listener address"))?; - if let tokio_util::either::Either::Right(listener) = listener_addr { - let port = listener.port(); - url = format!("http://localhost:{port}").parse::().unwrap(); - } - - // Serve. - let server = self.clone(); - let context = Context { - process: Some(Arc::new(crate::context::Process { - id: process.id().clone(), - paths: None, - remote: remote.cloned(), - retry: *process - .retry(self) - .await - .map_err(|source| tg::error!(!source, "failed to get the process retry"))?, - })), - ..Default::default() - }; - let task = Task::spawn({ - let context = context.clone(); - |stop| async move { - server.serve(listener, context, stop).await; - } - }); - - let serve_task = Some((task, url)); - // Set `$TANGRAM_OUTPUT`. env.insert( "TANGRAM_OUTPUT".to_owned(), @@ -172,81 +125,407 @@ impl Server { // Set `$TANGRAM_PROCESS`. env.insert("TANGRAM_PROCESS".to_owned(), id.to_string()); - // Set `$TANGRAM_URL`. - let url = serve_task.as_ref().map(|(_, url)| url.to_string()).unwrap(); - env.insert("TANGRAM_URL".to_owned(), url); - - // Get the server's user. - let whoami = whoami().map_err(|error| tg::error!(!error, "failed to get username"))?; + // Set `$TANGRAM_SANDBOX`. + if let Some(sandbox_id) = &state.sandbox { + env.insert("TANGRAM_SANDBOX".to_owned(), sandbox_id.to_string()); + } - // Determine if the process is sandboxed. - let unsandboxed = root_mounted - && (command.mounts.is_empty() && state.mounts.len() == 1) - && state.network - && command.user.as_ref().is_none_or(|user| user == &whoami); + // Run the process. + let output = if let Some(sandbox_id) = &state.sandbox { + // Set `$TANGRAM_URL` from the sandbox's socket. + let sandbox = self + .sandboxes + .get(sandbox_id) + .ok_or_else(|| tg::error!("failed to find the sandbox"))?; + let url = if let Some(url) = &sandbox.proxy_url { + url.clone() + } else { + let socket_path = sandbox.temp.path().join(".tangram/socket"); + tangram_uri::Uri::builder() + .scheme("http+unix") + .authority(socket_path.to_str().unwrap()) + .path("") + .build() + .unwrap() + }; + drop(sandbox); + env.insert("TANGRAM_URL".to_owned(), url.to_string()); - // Sandbox if necessary. - let (args, cwd, env, executable) = if unsandboxed { - (args, cwd, env, executable) + drop(temp); + self.run_darwin_sandboxed( + state, command, id, remote, sandbox_id, executable, args, env, cwd, + ) + .await? } else { - let args = { - let mut args_ = vec!["sandbox".to_owned()]; - args_.push("-C".to_owned()); - args_.push(cwd.display().to_string()); - for (name, value) in &env { - args_.push("-e".to_owned()); - args_.push(format!("{name}={value}")); - } - if !root_mounted { - args_.push("--mount".to_owned()); - args_.push(format!("source={}", temp.path().display())); - args_.push("--mount".to_owned()); - args_.push(format!("source={},ro", artifacts_path.display())); - args_.push("--mount".to_owned()); - args_.push(format!("source={}", cwd.display())); - } - for mount in &state.mounts { - let mount = if mount.readonly { - format!("source={},ro", mount.source.display()) - } else { - format!("source={}", mount.source.display()) - }; - args_.push("--mount".to_owned()); - args_.push(mount); - } - if state.network { - args_.push("--network".to_owned()); + // Set up the serve task. + let path = temp.path().join(".tangram"); + tokio::fs::create_dir_all(&path).await.map_err( + |source| tg::error!(!source, path = %path.display(), "failed to create the directory"), + )?; + + // Listen. + let socket_path = path.join("socket").display().to_string(); + let mut url = if socket_path.len() <= MAX_URL_LEN { + tangram_uri::Uri::builder() + .scheme("http+unix") + .authority(&socket_path) + .path("") + .build() + .unwrap() + } else { + "http://localhost:0".to_owned().parse::().unwrap() + }; + let listener = Server::listen(&url) + .await + .map_err(|source| tg::error!(!source, "failed to listen"))?; + let listener_addr = listener + .local_addr() + .map_err(|source| tg::error!(!source, "failed to get listener address"))?; + if let tokio_util::either::Either::Right(listener) = listener_addr { + let port = listener.port(); + url = format!("http://localhost:{port}").parse::().unwrap(); + } + + // Serve. + let server = self.clone(); + let context = Context::default(); + let task = Task::spawn({ + let context = context.clone(); + |stop| async move { + server.serve(listener, context, stop).await; } - args_.push(executable.display().to_string()); - args_.push("--".to_owned()); - args_.extend(args); - args_ + }); + + let serve_task = Some((task, url.clone())); + + // Set `$TANGRAM_URL`. + env.insert("TANGRAM_URL".to_owned(), url.to_string()); + + let arg = crate::run::common::Arg { + args, + command, + context: &context, + cwd, + env, + executable, + id, + remote, + serve_task, + server: self, + state, + temp: &temp, }; - let cwd = PathBuf::from("/"); - let env = BTreeMap::new(); - let executable = tangram_util::env::current_exe() - .map_err(|source| tg::error!(!source, "failed to get the current executable"))?; - (args, cwd, env, executable) + crate::run::common::run(arg) + .await + .map_err(|source| tg::error!(!source, "failed to run the process"))? }; - // Run the process. - let arg = crate::run::common::Arg { - args, - command, - context: &context, - cwd, - env, + Ok(output) + } + + #[allow(clippy::too_many_arguments)] + async fn run_darwin_sandboxed( + &self, + state: &tg::process::State, + command: &tg::command::Data, + id: &tg::process::Id, + remote: Option<&String>, + sandbox_id: &tg::sandbox::Id, + executable: PathBuf, + args: Vec, + env: std::collections::BTreeMap, + cwd: PathBuf, + ) -> tg::Result { + // Get the sandbox client and the host output path. + let sandbox = self + .sandboxes + .get(sandbox_id) + .ok_or_else(|| tg::error!("failed to find the sandbox"))?; + let client = Arc::clone(&sandbox.client); + let output_path = sandbox.temp.path().join("output/output"); + drop(sandbox); + + // Collect FDs that need to be kept alive until after the spawn call. + let mut fds = Vec::new(); + + // Handle stdin. + let (stdin, stdin_writer) = if command.stdin.is_some() { + let (sender, receiver) = tokio::net::unix::pipe::pipe() + .map_err(|source| tg::error!(!source, "failed to create a pipe for stdin"))?; + let sender = sender.boxed(); + let fd = receiver + .into_blocking_fd() + .map_err(|source| tg::error!(!source, "failed to get the fd from the pipe"))?; + let raw_fd = fd.as_raw_fd(); + fds.push(fd); + (Some(raw_fd), Some(sender)) + } else { + match state.stdin.as_ref() { + Some(tg::process::Stdio::Pipe(pipe_id)) => { + let pipe = self + .pipes + .get(pipe_id) + .ok_or_else(|| tg::error!("failed to find the pipe"))?; + let fd = pipe + .receiver + .as_fd() + .try_clone_to_owned() + .map_err(|source| tg::error!(!source, "failed to clone the receiver"))?; + let receiver = tokio::net::unix::pipe::Receiver::from_owned_fd_unchecked(fd) + .map_err(|source| tg::error!(!source, "io error"))?; + let fd = receiver.into_blocking_fd().map_err(|source| { + tg::error!(!source, "failed to get the fd from the pipe") + })?; + let raw_fd = fd.as_raw_fd(); + fds.push(fd); + (Some(raw_fd), None) + }, + Some(tg::process::Stdio::Pty(pty_id)) => { + let pty = self + .ptys + .get(pty_id) + .ok_or_else(|| tg::error!("failed to find the pty"))?; + let slave = pty + .slave + .as_ref() + .ok_or_else(|| tg::error!("the pty slave is closed"))?; + let fd = slave + .try_clone() + .map_err(|source| tg::error!(!source, "failed to clone the pty slave"))?; + let raw_fd = fd.as_raw_fd(); + fds.push(fd); + (Some(raw_fd), None) + }, + None => (None, None), + } + }; + + // Handle stdout. + let (stdout, stdout_reader) = match state.stdout.as_ref() { + Some(tg::process::Stdio::Pipe(pipe_id)) => { + let pipe = self + .pipes + .get(pipe_id) + .ok_or_else(|| tg::error!("failed to find the pipe"))?; + let fd = pipe + .sender + .as_ref() + .ok_or_else(|| tg::error!("the pipe is closed"))? + .as_fd() + .try_clone_to_owned() + .map_err(|source| tg::error!(!source, "failed to clone the sender"))?; + let sender = tokio::net::unix::pipe::Sender::from_owned_fd_unchecked(fd) + .map_err(|source| tg::error!(!source, "io error"))?; + let fd = sender + .into_blocking_fd() + .map_err(|source| tg::error!(!source, "failed to get the fd from the pipe"))?; + let raw_fd = fd.as_raw_fd(); + fds.push(fd); + (Some(raw_fd), None) + }, + Some(tg::process::Stdio::Pty(pty_id)) => { + let pty = self + .ptys + .get(pty_id) + .ok_or_else(|| tg::error!("failed to find the pty"))?; + let slave = pty + .slave + .as_ref() + .ok_or_else(|| tg::error!("the pty slave is closed"))?; + let fd = slave + .try_clone() + .map_err(|source| tg::error!(!source, "failed to clone the pty slave"))?; + let raw_fd = fd.as_raw_fd(); + fds.push(fd); + (Some(raw_fd), None) + }, + None => { + let (sender, receiver) = tokio::net::unix::pipe::pipe() + .map_err(|source| tg::error!(!source, "failed to create a pipe for stdout"))?; + let fd = sender + .into_blocking_fd() + .map_err(|source| tg::error!(!source, "failed to get the fd from the pipe"))?; + let raw_fd = fd.as_raw_fd(); + fds.push(fd); + let receiver = receiver.boxed(); + (Some(raw_fd), Some(receiver)) + }, + }; + + // Handle stderr. + let (stderr, stderr_reader) = match state.stderr.as_ref() { + Some(tg::process::Stdio::Pipe(pipe_id)) => { + let pipe = self + .pipes + .get(pipe_id) + .ok_or_else(|| tg::error!("failed to find the pipe"))?; + let fd = pipe + .sender + .as_ref() + .ok_or_else(|| tg::error!("the pipe is closed"))? + .as_fd() + .try_clone_to_owned() + .map_err(|source| tg::error!(!source, "failed to clone the sender"))?; + let sender = tokio::net::unix::pipe::Sender::from_owned_fd_unchecked(fd) + .map_err(|source| tg::error!(!source, "io error"))?; + let fd = sender + .into_blocking_fd() + .map_err(|source| tg::error!(!source, "failed to get the fd from the pipe"))?; + let raw_fd = fd.as_raw_fd(); + fds.push(fd); + (Some(raw_fd), None) + }, + Some(tg::process::Stdio::Pty(pty_id)) => { + let pty = self + .ptys + .get(pty_id) + .ok_or_else(|| tg::error!("failed to find the pty"))?; + let slave = pty + .slave + .as_ref() + .ok_or_else(|| tg::error!("the pty slave is closed"))?; + let fd = slave + .try_clone() + .map_err(|source| tg::error!(!source, "failed to clone the pty slave"))?; + let raw_fd = fd.as_raw_fd(); + fds.push(fd); + (Some(raw_fd), None) + }, + None => { + let (sender, receiver) = tokio::net::unix::pipe::pipe() + .map_err(|source| tg::error!(!source, "failed to create a pipe for stderr"))?; + let fd = sender + .into_blocking_fd() + .map_err(|source| tg::error!(!source, "failed to get the fd from the pipe"))?; + let raw_fd = fd.as_raw_fd(); + fds.push(fd); + let receiver = receiver.boxed(); + (Some(raw_fd), Some(receiver)) + }, + }; + + // Create the sandbox command. + let sandbox_command = sandbox::Command { + cwd: Some(cwd), + env: env.into_iter().collect(), executable, - id, - remote, - serve_task, - server: self, - state, - temp: &temp, + stdin, + stdout, + stderr, + trailing: args, }; - let output = crate::run::common::run(arg) + + // Spawn the command in the sandbox. + let pid = client + .spawn(sandbox_command) + .await + .map_err(|source| tg::error!(!source, "failed to spawn the process in the sandbox"))?; + + // Update the process pid in the database. + self.update_process_pid(id, pid) .await - .map_err(|source| tg::error!(!source, "failed to run the process"))?; + .map_err(|source| tg::error!(!source, %id, "failed to update the process pid"))?; + + // Drop the FDs now that the spawn has completed. + drop(fds); + + // Spawn the stdio task. + let stdio_task = tokio::spawn({ + let server = self.clone(); + let id = id.clone(); + let remote = remote.cloned(); + let stdin_blob = command.stdin.clone().map(tg::Blob::with_id); + async move { + super::common::stdio_task( + &server, + &id, + remote.as_ref(), + stdin_blob, + stdin_writer, + stdout_reader, + stderr_reader, + ) + .await?; + Ok::<_, tg::Error>(()) + } + }); + + // Wait for the process in the sandbox. + let status = client.wait(pid).await.map_err(|source| { + tg::error!(!source, "failed to wait for the process in the sandbox") + })?; + + // Await the stdio task. + stdio_task + .await + .map_err(|source| tg::error!(!source, "the stdio task panicked"))??; + + // Create the output. + let exit = u8::try_from(status).unwrap_or(1); + let mut output = super::Output { + checksum: None, + error: None, + exit, + output: None, + }; + + // Get the output path on the host. + let exists = tokio::fs::try_exists(&output_path) + .await + .map_err(|source| { + tg::error!(!source, "failed to determine if the output path exists") + })?; + + // Try to read the user.tangram.output xattr. + if let Ok(Some(bytes)) = xattr::get(&output_path, "user.tangram.output") { + let tgon = String::from_utf8(bytes) + .map_err(|source| tg::error!(!source, "failed to decode the output xattr"))?; + output.output = Some( + tgon.parse::() + .map_err(|source| tg::error!(!source, "failed to parse the output xattr"))?, + ); + } + + // Try to read the user.tangram.error xattr. + if let Ok(Some(bytes)) = xattr::get(&output_path, "user.tangram.error") { + let error = serde_json::from_slice::(&bytes) + .map_err(|source| tg::error!(!source, "failed to deserialize the error xattr"))?; + let error = tg::Error::try_from(error) + .map_err(|source| tg::error!(!source, "failed to convert the error data"))?; + output.error = Some(error); + } + + // Check in the output. + if output.output.is_none() && exists { + let context = self + .sandboxes + .get(sandbox_id) + .map(|sandbox| sandbox.context.clone()) + .unwrap_or_default(); + let arg = tg::checkin::Arg { + options: tg::checkin::Options { + destructive: true, + deterministic: true, + ignore: false, + lock: None, + locked: true, + root: true, + ..Default::default() + }, + path: output_path.clone(), + updates: Vec::new(), + }; + let checkin_output = self + .checkin_with_context(&context, arg) + .await + .map_err(|source| tg::error!(!source, "failed to check in the output"))? + .try_last() + .await? + .and_then(|event| event.try_unwrap_output().ok()) + .ok_or_else(|| tg::error!("stream ended without output"))?; + let value = tg::Artifact::with_id(checkin_output.artifact.item).into(); + output.output = Some(value); + } Ok(output) } diff --git a/packages/server/src/run/js.rs b/packages/server/src/run/js.rs index ae6c9ed6c..195ce2219 100644 --- a/packages/server/src/run/js.rs +++ b/packages/server/src/run/js.rs @@ -49,9 +49,8 @@ impl Server { // Spawn the task. let local_pool_handle = self.local_pool_handle.get_or_init(|| { - let parallelism = std::thread::available_parallelism() - .map(std::num::NonZeroUsize::get) - .unwrap_or(1); + let parallelism = + std::thread::available_parallelism().map_or(1, std::num::NonZeroUsize::get); let concurrency = self.config.runner.as_ref().map_or(parallelism, |config| { config.concurrency.unwrap_or(parallelism) }); @@ -61,14 +60,17 @@ impl Server { let server = self.clone(); let process = process.clone(); move || async move { - let process = crate::context::Process { - id: process.id().clone(), + let sandbox = crate::context::Sandbox { + id: state + .sandbox + .clone() + .ok_or_else(|| tg::error!("expected a sandbox"))?, paths: None, remote: process.remote().cloned(), retry: *process.retry(&server).await?, }; let context = Context { - process: Some(Arc::new(process)), + sandbox: Some(Arc::new(sandbox)), ..Default::default() }; let handle = ServerWithContext(server, context); diff --git a/packages/server/src/run/linux.rs b/packages/server/src/run/linux.rs index 37eeda465..708664765 100644 --- a/packages/server/src/run/linux.rs +++ b/packages/server/src/run/linux.rs @@ -1,39 +1,16 @@ use { - super::util::{cache_children, render_args_dash_a, render_args_string, render_env, whoami}, + super::util::{cache_children, render_args_dash_a, render_args_string, render_env}, crate::{Context, Server, temp::Temp}, - indoc::formatdoc, std::{ - collections::{BTreeMap, HashMap}, - os::unix::ffi::OsStrExt as _, - path::{Path, PathBuf}, + os::fd::{AsFd as _, AsRawFd as _}, + path::Path, sync::Arc, }, tangram_client::prelude::*, - tangram_futures::task::Task, + tangram_futures::{read::Ext as _, stream::TryExt as _, write::Ext as _}, + tangram_sandbox as sandbox, }; -struct SandboxArg<'a> { - args: &'a [String], - command: &'a tg::command::Data, - cwd: &'a Path, - env: &'a BTreeMap, - executable: &'a Path, - id: &'a tg::process::Id, - mounts: &'a [tg::Either<&'a tg::process::Mount, &'a tg::command::data::Mount>], - root_mounted: bool, - server: &'a Server, - state: &'a tg::process::State, - temp: &'a Temp, -} - -struct SandboxOutput { - args: Vec, - cwd: PathBuf, - env: BTreeMap, - executable: PathBuf, - root: PathBuf, -} - impl Server { pub(crate) async fn run_linux(&self, process: &tg::Process) -> tg::Result { let id = process.id(); @@ -52,38 +29,25 @@ impl Server { .await .map_err(|source| tg::error!(!source, "failed to get the command data"))?; - // Cache the process's chidlren. + // Cache the process's children. cache_children(self, process) .await .map_err(|source| tg::error!(!source, "failed to cache the children"))?; - // Create the temp. - let temp = Temp::new(self); - tokio::fs::create_dir_all(temp.path()) - .await - .map_err(|source| tg::error!(!source, "failed to create the temp directory"))?; - - // Create the output directory. - tokio::fs::create_dir_all(temp.path().join("output")) - .await - .map_err(|source| tg::error!(!source, "failed to create the output directory"))?; - - // Determine if the root is mounted. - let root_mounted = state - .mounts - .iter() - .any(|mount| mount.source == mount.target && mount.target == Path::new("/")); - // Get the artifacts path. - let artifacts_path = if root_mounted { + let artifacts_path = if state.sandbox.is_none() { self.artifacts_path() } else { "/.tangram/artifacts".into() }; // Get the output path. - let output_path = if root_mounted { - temp.path().join("output/output") + let temp = Temp::new(self); + let output_path = if state.sandbox.is_none() { + tokio::fs::create_dir_all(temp.path()) + .await + .map_err(|source| tg::error!(!source, "failed to create output directory"))?; + temp.path().join("output") } else { Path::new("/output/output").to_owned() }; @@ -139,12 +103,6 @@ impl Server { }, }; - // Create mounts. - let mounts = std::iter::empty() - .chain(state.mounts.iter().map(tg::Either::Left)) - .chain(command.mounts.iter().map(tg::Either::Right)) - .collect::>(); - // Set `$TANGRAM_OUTPUT`. env.insert( "TANGRAM_OUTPUT".to_owned(), @@ -154,374 +112,370 @@ impl Server { // Set `$TANGRAM_PROCESS`. env.insert("TANGRAM_PROCESS".to_owned(), id.to_string()); - // Create the guest uri. - let guest_socket = if root_mounted { - temp.path().join(".tangram/socket") + // Set `$TANGRAM_SANDBOX`. + if let Some(sandbox_id) = &state.sandbox { + env.insert("TANGRAM_SANDBOX".to_owned(), sandbox_id.to_string()); + } + + // Set `$TANGRAM_URL`. + let socket = if state.sandbox.is_none() { + self.path.join("socket") } else { Path::new("/.tangram/socket").to_owned() }; - let guest_socket = guest_socket.to_str().unwrap(); - let guest_uri = tangram_uri::Uri::builder() + let url = tangram_uri::Uri::builder() .scheme("http+unix") - .authority(guest_socket) + .authority(socket.to_str().unwrap()) .path("") .build() .unwrap(); + env.insert("TANGRAM_URL".to_owned(), url.to_string()); - // Set `$TANGRAM_URL`. - env.insert("TANGRAM_URL".to_owned(), guest_uri.to_string()); - - // Get the server's user. - let whoami = whoami().map_err(|error| tg::error!(!error, "failed to get username"))?; - - // Determine if the process is sandboxed. - let unsandboxed = root_mounted - && (command.mounts.is_empty() && state.mounts.len() == 1) - && state.network - && command.user.as_ref().is_none_or(|user| user == &whoami); - - // Sandbox if necessary. - let (args, cwd, env, executable, root) = if unsandboxed { - let root = temp.path().join("root"); - (args, cwd, env, executable, root) + // Run the process. + let output = if let Some(sandbox_id) = &state.sandbox { + drop(temp); + self.run_linux_sandboxed( + state, command, id, remote, sandbox_id, executable, args, env, cwd, + ) + .await? } else { - let arg = SandboxArg { - args: &args, + tokio::fs::create_dir_all(temp.path()) + .await + .map_err(|source| tg::error!(!source, "failed to create output directory"))?; + let context = Context::default(); + let arg = crate::run::common::Arg { + args, command, - cwd: &cwd, - env: &env, - executable: &executable, + context: &context, + cwd, + env, + executable, id, - mounts: &mounts, - root_mounted, + remote, + serve_task: None, server: self, state, temp: &temp, }; - let output = sandbox(arg) + crate::run::common::run(arg) .await - .map_err(|source| tg::error!(!source, "failed to create the sandbox"))?; - let SandboxOutput { - args, - cwd, - env, - executable, - root, - } = output; - (args, cwd, env, executable, root) + .map_err(|source| tg::error!(!source, "failed to run the process"))? }; - // Create the paths for sandboxed processes. - let paths = if root_mounted { - None + Ok(output) + } + + #[allow(clippy::too_many_arguments)] + async fn run_linux_sandboxed( + &self, + state: &tg::process::State, + command: &tg::command::Data, + id: &tg::process::Id, + remote: Option<&String>, + sandbox_id: &tg::sandbox::Id, + executable: std::path::PathBuf, + args: Vec, + env: std::collections::BTreeMap, + cwd: std::path::PathBuf, + ) -> tg::Result { + // Get the sandbox client. + let sandbox = self + .sandboxes + .get(sandbox_id) + .ok_or_else(|| tg::error!("failed to find the sandbox"))?; + let client = Arc::clone(&sandbox.client); + let output_path = sandbox.temp.path().join("output/output"); + drop(sandbox); + + // Collect FDs that need to be kept alive until after the spawn call. + let mut fds = Vec::new(); + + // Handle stdin. + let (stdin, stdin_writer) = if command.stdin.is_some() { + let (sender, receiver) = tokio::net::unix::pipe::pipe() + .map_err(|source| tg::error!(!source, "failed to create a pipe for stdin"))?; + let sender = sender.boxed(); + let fd = receiver + .into_blocking_fd() + .map_err(|source| tg::error!(!source, "failed to get the fd from the pipe"))?; + let raw_fd = fd.as_raw_fd(); + fds.push(fd); + (Some(raw_fd), Some(sender)) } else { - Some(crate::context::Paths { - server_host: self.path.clone(), - output_host: temp.path().join("output"), - output_guest: "/output".into(), - root_host: root, - }) + match state.stdin.as_ref() { + Some(tg::process::Stdio::Pipe(pipe_id)) => { + let pipe = self + .pipes + .get(pipe_id) + .ok_or_else(|| tg::error!("failed to find the pipe"))?; + let fd = pipe + .receiver + .as_fd() + .try_clone_to_owned() + .map_err(|source| tg::error!(!source, "failed to clone the receiver"))?; + let receiver = tokio::net::unix::pipe::Receiver::from_owned_fd_unchecked(fd) + .map_err(|source| tg::error!(!source, "io error"))?; + let fd = receiver.into_blocking_fd().map_err(|source| { + tg::error!(!source, "failed to get the fd from the pipe") + })?; + let raw_fd = fd.as_raw_fd(); + fds.push(fd); + (Some(raw_fd), None) + }, + Some(tg::process::Stdio::Pty(pty_id)) => { + let pty = self + .ptys + .get(pty_id) + .ok_or_else(|| tg::error!("failed to find the pty"))?; + let slave = pty + .slave + .as_ref() + .ok_or_else(|| tg::error!("the pty slave is closed"))?; + let fd = slave + .try_clone() + .map_err(|source| tg::error!(!source, "failed to clone the pty slave"))?; + let raw_fd = fd.as_raw_fd(); + fds.push(fd); + (Some(raw_fd), None) + }, + None => (None, None), + } }; - // Create the host uri. - let host_socket = temp.path().join(".tangram/socket"); - tokio::fs::create_dir_all(host_socket.parent().unwrap()) - .await - .map_err(|source| tg::error!(!source, "failed to create the host path"))?; - let host_socket = host_socket - .to_str() - .ok_or_else(|| tg::error!(path = %host_socket.display(), "invalid path"))?; - let host_uri = tangram_uri::Uri::builder() - .scheme("http+unix") - .authority(host_socket) - .path("") - .build() - .unwrap(); - - // Listen. - let listener = Server::listen(&host_uri) - .await - .map_err(|source| tg::error!(!source, "failed to listen"))?; - - // Serve. - let server = self.clone(); - let context = Context { - process: Some(Arc::new(crate::context::Process { - id: process.id().clone(), - paths, - remote: remote.cloned(), - retry: *process - .retry(self) - .await - .map_err(|source| tg::error!(!source, "failed to get the process retry"))?, - })), - ..Default::default() + // Handle stdout. + let (stdout, stdout_reader) = match state.stdout.as_ref() { + Some(tg::process::Stdio::Pipe(pipe_id)) => { + let pipe = self + .pipes + .get(pipe_id) + .ok_or_else(|| tg::error!("failed to find the pipe"))?; + let fd = pipe + .sender + .as_ref() + .ok_or_else(|| tg::error!("the pipe is closed"))? + .as_fd() + .try_clone_to_owned() + .map_err(|source| tg::error!(!source, "failed to clone the sender"))?; + let sender = tokio::net::unix::pipe::Sender::from_owned_fd_unchecked(fd) + .map_err(|source| tg::error!(!source, "io error"))?; + let fd = sender + .into_blocking_fd() + .map_err(|source| tg::error!(!source, "failed to get the fd from the pipe"))?; + let raw_fd = fd.as_raw_fd(); + fds.push(fd); + (Some(raw_fd), None) + }, + Some(tg::process::Stdio::Pty(pty_id)) => { + let pty = self + .ptys + .get(pty_id) + .ok_or_else(|| tg::error!("failed to find the pty"))?; + let slave = pty + .slave + .as_ref() + .ok_or_else(|| tg::error!("the pty slave is closed"))?; + let fd = slave + .try_clone() + .map_err(|source| tg::error!(!source, "failed to clone the pty slave"))?; + let raw_fd = fd.as_raw_fd(); + fds.push(fd); + (Some(raw_fd), None) + }, + None => { + let (sender, receiver) = tokio::net::unix::pipe::pipe() + .map_err(|source| tg::error!(!source, "failed to create a pipe for stdout"))?; + let fd = sender + .into_blocking_fd() + .map_err(|source| tg::error!(!source, "failed to get the fd from the pipe"))?; + let raw_fd = fd.as_raw_fd(); + fds.push(fd); + let receiver = receiver.boxed(); + (Some(raw_fd), Some(receiver)) + }, }; - let task = Task::spawn({ - let context = context.clone(); - |stop| async move { - server.serve(listener, context, stop).await; - } - }); - let serve_task = Some((task, guest_uri)); + // Handle stderr. + let (stderr, stderr_reader) = match state.stderr.as_ref() { + Some(tg::process::Stdio::Pipe(pipe_id)) => { + let pipe = self + .pipes + .get(pipe_id) + .ok_or_else(|| tg::error!("failed to find the pipe"))?; + let fd = pipe + .sender + .as_ref() + .ok_or_else(|| tg::error!("the pipe is closed"))? + .as_fd() + .try_clone_to_owned() + .map_err(|source| tg::error!(!source, "failed to clone the sender"))?; + let sender = tokio::net::unix::pipe::Sender::from_owned_fd_unchecked(fd) + .map_err(|source| tg::error!(!source, "io error"))?; + let fd = sender + .into_blocking_fd() + .map_err(|source| tg::error!(!source, "failed to get the fd from the pipe"))?; + let raw_fd = fd.as_raw_fd(); + fds.push(fd); + (Some(raw_fd), None) + }, + Some(tg::process::Stdio::Pty(pty_id)) => { + let pty = self + .ptys + .get(pty_id) + .ok_or_else(|| tg::error!("failed to find the pty"))?; + let slave = pty + .slave + .as_ref() + .ok_or_else(|| tg::error!("the pty slave is closed"))?; + let fd = slave + .try_clone() + .map_err(|source| tg::error!(!source, "failed to clone the pty slave"))?; + let raw_fd = fd.as_raw_fd(); + fds.push(fd); + (Some(raw_fd), None) + }, + None => { + let (sender, receiver) = tokio::net::unix::pipe::pipe() + .map_err(|source| tg::error!(!source, "failed to create a pipe for stderr"))?; + let fd = sender + .into_blocking_fd() + .map_err(|source| tg::error!(!source, "failed to get the fd from the pipe"))?; + let raw_fd = fd.as_raw_fd(); + fds.push(fd); + let receiver = receiver.boxed(); + (Some(raw_fd), Some(receiver)) + }, + }; - // Run the process. - let arg = crate::run::common::Arg { - args, - command, - context: &context, - cwd, - env, + // Create the sandbox command. + let sandbox_command = sandbox::Command { + cwd: Some(cwd), + env: env.into_iter().collect(), executable, - id, - remote, - serve_task, - server: self, - state, - temp: &temp, + stdin, + stdout, + stderr, + trailing: args, }; - let output = crate::run::common::run(arg) - .await - .map_err(|source| tg::error!(!source, "failed to run the process"))?; - Ok(output) - } -} - -async fn sandbox(arg: SandboxArg<'_>) -> tg::Result { - let SandboxArg { - args, - cwd, - env, - executable, - id, - mounts, - root_mounted, - server, - temp, - state, - command, - } = arg; - - // Create the output path. - let output_path = temp.path().join("output"); - - // Initialize the output with all fields. - let mut output = SandboxOutput { - root: temp.path().join("root"), - args: vec!["sandbox".to_owned()], - cwd: PathBuf::from("/"), - env: BTreeMap::new(), - executable: tangram_util::env::current_exe() - .map_err(|source| tg::error!(!source, "failed to get the current executable"))?, - }; - - let mut overlays = HashMap::new(); - for mount in mounts { - match mount { - tg::Either::Left(mount) => { - output.args.push("--mount".to_owned()); - output - .args - .push(bind(&mount.source, &mount.target, mount.readonly)); - }, - tg::Either::Right(mount) => { - // Create the overlay state if it does not exist. Since we use async here, we can't use the .entry() api. - if !overlays.contains_key(&mount.target) { - let lowerdirs = Vec::new(); - let upperdir = temp.path().join("upper").join(overlays.len().to_string()); - let workdir = temp.path().join("work").join(overlays.len().to_string()); - tokio::fs::create_dir_all(&upperdir).await.ok(); - tokio::fs::create_dir_all(&workdir).await.ok(); - if mount.target == Path::new("/") { - output.root = upperdir.clone(); - } - overlays.insert(mount.target.clone(), (lowerdirs, upperdir, workdir)); - } - - // Compute the path. - let path = server.artifacts_path().join(mount.source.to_string()); - - // Get the lower dirs. - let (lowerdirs, _, _) = overlays.get_mut(&mount.target).unwrap(); + // Spawn the command in the sandbox. + let pid = client + .spawn(sandbox_command) + .await + .map_err(|source| tg::error!(!source, "failed to spawn the process in the sandbox"))?; - // Add this path to the lowerdirs. - lowerdirs.push(path); - }, - } - } + // Update the process pid in the database. + self.update_process_pid(id, pid) + .await + .map_err(|source| tg::error!(!source, %id, "failed to update the process pid"))?; + + // Drop the FDs now that the spawn has completed. + drop(fds); + + // Spawn the stdio task. + let stdio_task = tokio::spawn({ + let server = self.clone(); + let id = id.clone(); + let remote = remote.cloned(); + let stdin_blob = command.stdin.clone().map(tg::Blob::with_id); + async move { + super::common::stdio_task( + &server, + &id, + remote.as_ref(), + stdin_blob, + stdin_writer, + stdout_reader, + stderr_reader, + ) + .await?; + Ok::<_, tg::Error>(()) + } + }); - // Add additional mounts. - if !root_mounted { - let path = temp.path().join(".tangram"); - tokio::fs::create_dir_all(&path).await.map_err( - |source| tg::error!(!source, path = %path.display(), "failed to create the data directory"), - )?; + // Wait for the process in the sandbox. + let status = client.wait(pid).await.map_err(|source| { + tg::error!(!source, "failed to wait for the process in the sandbox") + })?; - // Create /etc. - tokio::fs::create_dir_all(temp.path().join("lower/etc")) + // Await the stdio task. + stdio_task .await - .ok(); + .map_err(|source| tg::error!(!source, "the stdio task panicked"))??; + + // Create the output. + let exit = u8::try_from(status).unwrap_or(1); + let mut output = super::Output { + checksum: None, + error: None, + exit, + output: None, + }; - // Create /tmp. - tokio::fs::create_dir_all(temp.path().join("lower/tmp")) - .await - .ok(); - - // Create nsswitch.conf. - tokio::fs::write( - temp.path().join("lower/etc/nsswitch.conf"), - formatdoc!( - " - passwd: files compat - shadow: files compat - hosts: files dns compat - " - ), - ) - .await - .map_err(|source| tg::error!(!source, "failed to create /etc/nsswitch.conf"))?; - - // Create /etc/passwd. - tokio::fs::write( - temp.path().join("lower/etc/passwd"), - formatdoc!( - " - root:!:0:0:root:/nonexistent:/bin/false - nobody:!:65534:65534:nobody:/nonexistent:/bin/false - " - ), - ) - .await - .map_err(|source| tg::error!(!source, "failed to create /etc/passwd"))?; - - // Copy resolv.conf. - if state.network { - tokio::fs::copy( - "/etc/resolv.conf", - temp.path().join("lower/etc/resolv.conf"), - ) + // Get the output path on the host. + let exists = tokio::fs::try_exists(&output_path) .await .map_err(|source| { - tg::error!(!source, "failed to copy /etc/resolv.conf to the sandbox") + tg::error!(!source, "failed to determine if the output path exists") })?; - output.args.push("--network".to_owned()); - } - // Get or create the root overlay. - if !overlays.contains_key(Path::new("/")) { - let lowerdirs = Vec::new(); - let upperdir = temp.path().join("upper").join(overlays.len().to_string()); - let workdir = temp.path().join("work").join(overlays.len().to_string()); - tokio::fs::create_dir_all(&upperdir).await.ok(); - tokio::fs::create_dir_all(&workdir).await.ok(); - output.root = upperdir.clone(); - overlays.insert("/".into(), (lowerdirs, upperdir, workdir)); + // Try to read the user.tangram.output xattr. + if let Ok(Some(bytes)) = xattr::get(&output_path, "user.tangram.output") { + let tgon = String::from_utf8(bytes) + .map_err(|source| tg::error!(!source, "failed to decode the output xattr"))?; + output.output = Some( + tgon.parse::() + .map_err(|source| tg::error!(!source, "failed to parse the output xattr"))?, + ); } - let (lowerdirs, _, _) = overlays.get_mut(Path::new("/")).unwrap(); - lowerdirs.push(temp.path().join("lower")); - - // Add mounts for /dev, /proc, /tmp, /.tangram, /.tangram/artifacts, and /output. - output.args.push("--mount".to_owned()); - output.args.push(bind("/dev", "/dev", false)); - output.args.push("--mount".to_owned()); - output.args.push(bind("/proc", "/proc", false)); - output.args.push("--mount".to_owned()); - output - .args - .push(bind(temp.path().join(".tangram"), "/.tangram", false)); - output.args.push("--mount".to_owned()); - output - .args - .push(bind(server.artifacts_path(), "/.tangram/artifacts", true)); - output.args.push("--mount".to_owned()); - output.args.push(bind(&output_path, "/output", false)); - } - - // Add the overlay mounts. - for (merged, (lowerdirs, upperdir, workdir)) in &overlays { - output.args.push("--mount".to_owned()); - output - .args - .push(overlay(lowerdirs, upperdir, workdir, merged)); - } - output.args.push("-C".to_owned()); - output.args.push(cwd.display().to_string()); - - // Add env vars to the command. - for (name, value) in env { - output.args.push("-e".to_owned()); - output.args.push(format!("{name}={value}")); - } - - // Set the user. - let user = command.user.as_deref().unwrap_or("root"); - output.args.push("--user".to_owned()); - output.args.push(user.to_owned()); - - // Set the hostname - output.args.push("--hostname".to_owned()); - output.args.push(id.to_string()); - - // Set the chroot. - output.args.push("--chroot".to_owned()); - output.args.push(output.root.display().to_string()); - - // Add the executable and original args. - output.args.push(executable.display().to_string()); - output.args.push("--".to_owned()); - output.args.extend(args.iter().cloned()); - - Ok(output) -} - -fn bind(source: impl AsRef, target: impl AsRef, readonly: bool) -> String { - let mut string = format!( - "type=bind,source={},target={}", - source.as_ref().display(), - target.as_ref().display() - ); - if readonly { - string.push_str(",ro"); - } - string -} - -fn overlay(lowerdirs: &[PathBuf], upperdir: &Path, workdir: &Path, merged: &Path) -> String { - fn escape(out: &mut Vec, path: &[u8]) { - for byte in path.iter().copied() { - if byte == 0 { - break; - } - if byte == b':' { - out.push(b'\\'); - } - out.push(byte); + // Try to read the user.tangram.error xattr. + if let Ok(Some(bytes)) = xattr::get(&output_path, "user.tangram.error") { + let error = serde_json::from_slice::(&bytes) + .map_err(|source| tg::error!(!source, "failed to deserialize the error xattr"))?; + let error = tg::Error::try_from(error) + .map_err(|source| tg::error!(!source, "failed to convert the error data"))?; + output.error = Some(error); } - } - - // Create the mount options. - let mut data = b"type=overlay,source=overlay,target=".to_vec(); - data.extend_from_slice(merged.as_os_str().as_bytes()); - // Add the lower directories. - data.extend_from_slice(b",userxattr,lowerdir="); - for (n, dir) in lowerdirs.iter().enumerate() { - escape(&mut data, dir.as_os_str().as_bytes()); - if n != lowerdirs.len() - 1 { - data.push(b':'); + // Check in the output. + if output.output.is_none() && exists { + let context = self + .sandboxes + .get(sandbox_id) + .map(|sandbox| sandbox.context.clone()) + .unwrap_or_default(); + let checkin_path = if let Some(sandbox) = &context.sandbox { + sandbox + .guest_path_for_host_path(output_path.clone()) + .map_err(|source| tg::error!(!source, "failed to map the output path"))? + } else { + output_path.clone() + }; + let arg = tg::checkin::Arg { + options: tg::checkin::Options { + destructive: true, + deterministic: true, + ignore: false, + lock: None, + locked: true, + root: true, + ..Default::default() + }, + path: checkin_path, + updates: Vec::new(), + }; + let checkin_output = self + .checkin_with_context(&context, arg) + .await + .map_err(|source| tg::error!(!source, "failed to check in the output"))? + .try_last() + .await? + .and_then(|event| event.try_unwrap_output().ok()) + .ok_or_else(|| tg::error!("stream ended without output"))?; + let value = tg::Artifact::with_id(checkin_output.artifact.item).into(); + output.output = Some(value); } - } - - // Add the upper directory. - data.extend_from_slice(b",upperdir="); - data.extend_from_slice(upperdir.as_os_str().as_bytes()); - // Add the working directory. - data.extend_from_slice(b",workdir="); - data.extend_from_slice(workdir.as_os_str().as_bytes()); - - String::from_utf8(data).unwrap() + Ok(output) + } } diff --git a/packages/server/src/run/util.rs b/packages/server/src/run/util.rs index bbf49d352..238707b3d 100644 --- a/packages/server/src/run/util.rs +++ b/packages/server/src/run/util.rs @@ -201,19 +201,3 @@ pub fn render_value_string( _ => Ok(tg::Value::try_from_data(value.clone()).unwrap().to_string()), } } - -pub fn whoami() -> tg::Result { - unsafe { - let uid = libc::getuid(); - let pwd = libc::getpwuid(uid); - if pwd.is_null() { - let source = std::io::Error::last_os_error(); - return Err(tg::error!(!source, "failed to get username")); - } - let username = std::ffi::CStr::from_ptr((*pwd).pw_name) - .to_str() - .map_err(|source| tg::error!(!source, "non-utf8 username"))? - .to_owned(); - Ok(username) - } -} diff --git a/packages/server/src/sandbox.rs b/packages/server/src/sandbox.rs new file mode 100644 index 000000000..5ebc98eb1 --- /dev/null +++ b/packages/server/src/sandbox.rs @@ -0,0 +1,30 @@ +use { + crate::temp::Temp, + std::{path::PathBuf, sync::Arc}, + tangram_futures::task::Task, + tangram_sandbox as sandbox, + tangram_uri::Uri, +}; + +pub mod create; +#[cfg(target_os = "macos")] +mod darwin; +pub mod delete; +#[cfg(target_os = "linux")] +mod linux; +pub mod spawn; +pub mod wait; + +pub struct Sandbox { + pub process: tokio::process::Child, + pub client: Arc, + pub context: crate::Context, + pub refcount: usize, + #[allow(dead_code, reason = "required by darwin")] + pub root: PathBuf, + pub serve_task: Task<()>, + #[allow(dead_code, reason = "owns the piped stderr fd for the sandbox daemon")] + pub stderr: Option, + pub temp: Temp, + pub proxy_url: Option, +} diff --git a/packages/server/src/sandbox/create.rs b/packages/server/src/sandbox/create.rs new file mode 100644 index 000000000..370ea6441 --- /dev/null +++ b/packages/server/src/sandbox/create.rs @@ -0,0 +1,276 @@ +use { + crate::{Context, Server, temp::Temp}, + byteorder::ReadBytesExt as _, + std::{ + future::Future, + os::{fd::AsRawFd as _, unix::process::ExitStatusExt as _}, + sync::Arc, + time::Duration, + }, + tangram_client::prelude::*, + tangram_futures::task::Task, + tangram_http::{body::Boxed as BoxBody, request::Ext as _}, + tangram_sandbox as sandbox, + tangram_uri::Uri, +}; + +const MAX_URL_LEN: usize = 100; + +impl Server { + pub(crate) fn create_sandbox_with_context<'a>( + &'a self, + context: &'a Context, + arg: tg::sandbox::create::Arg, + ) -> impl Future> + Send + 'a { + self.create_sandbox_with_context_inner(context, arg) + } + + async fn create_sandbox_with_context_inner<'a>( + &'a self, + context: &'a Context, + arg: tg::sandbox::create::Arg, + ) -> tg::Result { + if context.sandbox.is_some() { + return Err(tg::error!("forbidden")); + } + + // Create the sandbox ID and temp directory. + let id = tg::sandbox::Id::new(); + let temp = Temp::new(self); + tokio::fs::create_dir_all(temp.path()) + .await + .map_err(|source| tg::error!(!source, "failed to create the temp directory"))?; + + // Create the output directory. + tokio::fs::create_dir_all(temp.path().join("output")) + .await + .map_err(|source| tg::error!(!source, "failed to create the output directory"))?; + + let path; + #[cfg(target_os = "linux")] + { + path = temp.path().join("socket"); + } + #[cfg(target_os = "macos")] + { + path = std::path::Path::new("/tmp").join(format!("{id}.socket")); + } + let (root, args) = self.get_sandbox_args(&id, &arg, &temp).await?; + + // Set up a pipe to notify the server once the child process has bound its server. + let (mut reader, writer) = std::io::pipe() + .map_err(|source| tg::error!(!source, "failed to create the ready pipe"))?; + let flags = unsafe { libc::fcntl(writer.as_raw_fd(), libc::F_GETFD) }; + if flags < 0 { + return Err(tg::error!( + source = std::io::Error::last_os_error(), + "failed to get the ready pipe flags" + )); + } + if unsafe { libc::fcntl(writer.as_raw_fd(), libc::F_SETFD, flags & !libc::FD_CLOEXEC) } < 0 + { + return Err(tg::error!( + source = std::io::Error::last_os_error(), + "failed to set the ready pipe flags" + )); + } + let ready_fd = writer.as_raw_fd(); + + // Spawn the sandbox process. + let executable = tangram_util::env::current_exe() + .map_err(|source| tg::error!(!source, "failed to get the current executable"))?; + let mut process = tokio::process::Command::new(executable) + .kill_on_drop(true) + .process_group(0) + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::inherit()) + .arg("sandbox") + .arg("serve") + .arg("--socket") + .arg(&path) + .arg("--ready-fd") + .arg(ready_fd.to_string()) + .args(&args) + .spawn() + .map_err(|source| tg::error!(!source, "failed to spawn the sandbox process"))?; + drop(writer); + + // Wait for the sandbox process to signal readiness. + let task = tokio::task::spawn_blocking(move || reader.read_u8()); + let result = tokio::time::timeout(Duration::from_secs(5), task) + .await + .map_err(|source| tg::error!(!source, "timed out waiting for the sandbox ready signal")) + .and_then(|output| { + output.map_err(|source| tg::error!(!source, "the sandbox ready task panicked")) + }) + .and_then(|output| { + output.map_err(|source| { + tg::error!(!source, "failed to read the sandbox ready signal") + }) + }) + .and_then(|byte| { + if byte != 0x00 { + return Err(tg::error!("received an invalid ready byte {byte}")); + } + Ok(()) + }); + if let Err(source) = result { + process.start_kill().ok(); + let output = tokio::time::timeout(Duration::from_secs(1), process.wait_with_output()) + .await + .ok() + .and_then(Result::ok); + let stderr = output + .as_ref() + .map(|output| String::from_utf8_lossy(&output.stderr)); + let exit_code = output.as_ref().and_then(|output| output.status.code()); + let signal = output + .as_ref() + .and_then(|output| output.status.stopped_signal()); + let error = if let Some(stderr) = stderr { + tg::error!( + !source, + stderr = %stderr, + exit_code = ?exit_code, + signal = ?signal, + "failed to start the sandbox" + ) + } else { + tg::error!( + !source, + exit_code = ?exit_code, + signal = ?signal, + "failed to start the sandbox" + ) + }; + return Err(error); + } + + // Connect to the sandbox. + let client = sandbox::client::Client::connect(path) + .await + .map_err(|source| tg::error!(!source, "failed to connect to the sandbox"))?; + + // Create the proxy server for this sandbox. + let socket_path = temp.path().join(".tangram/socket"); + tokio::fs::create_dir_all(socket_path.parent().unwrap()) + .await + .map_err(|source| tg::error!(!source, "failed to create the proxy socket directory"))?; + let host_socket = socket_path + .to_str() + .ok_or_else(|| tg::error!("the proxy socket path is not valid UTF-8"))?; + let url = if cfg!(target_os = "linux") || host_socket.len() <= MAX_URL_LEN { + tangram_uri::Uri::builder() + .scheme("http+unix") + .authority(host_socket) + .path("") + .build() + .unwrap() + } else { + "http://localhost:0".to_owned().parse::().unwrap() + }; + let listener = Server::listen(&url) + .await + .map_err(|source| tg::error!(!source, "failed to listen on the proxy socket"))?; + let listener_addr = listener + .local_addr() + .map_err(|source| tg::error!(!source, "failed to get listener address"))?; + + let proxy_url = if let tokio_util::either::Either::Right(listener) = listener_addr { + let port = listener.port(); + Some(format!("http://localhost:{port}").parse::().unwrap()) + } else { + None + }; + let paths = if cfg!(target_os = "linux") { + Some(crate::context::Paths { + server_host: self.path.clone(), + output_host: temp.path().join("output"), + output_guest: "/output".into(), + root_host: root.clone(), + }) + } else { + None + }; + let context = Context { + sandbox: Some(Arc::new(crate::context::Sandbox { + id: id.clone(), + paths, + remote: None, + retry: false, + })), + ..Default::default() + }; + let serve_task = { + let server = self.clone(); + let context = context.clone(); + Task::spawn(|stop| async move { + server.serve(listener, context, stop).await; + }) + }; + + // Take the stderr handle from the child process. + let stderr = process.stderr.take(); + + // Store the sandbox. + self.sandboxes.insert( + id.clone(), + super::Sandbox { + process, + client: Arc::new(client), + context, + refcount: 0, + root, + serve_task, + stderr, + temp, + proxy_url, + }, + ); + + Ok(tg::sandbox::create::Output { id }) + } + + pub(crate) async fn handle_create_sandbox_request( + &self, + request: http::Request, + context: &Context, + ) -> tg::Result> { + let accept = request + .parse_header::(http::header::ACCEPT) + .transpose() + .map_err(|source| tg::error!(!source, "failed to parse the accept header"))?; + + let arg = request + .json_or_default() + .await + .map_err(|source| tg::error!(!source, "failed to deserialize the request body"))?; + + let output = self + .create_sandbox_with_context(context, arg) + .await + .map_err(|source| tg::error!(!source, "failed to create the sandbox"))?; + + let (content_type, body) = match accept + .as_ref() + .map(|accept| (accept.type_(), accept.subtype())) + { + None | Some((mime::STAR, mime::STAR) | (mime::APPLICATION, mime::JSON)) => { + let content_type = mime::APPLICATION_JSON; + let body = serde_json::to_vec(&output).unwrap(); + (Some(content_type), BoxBody::with_bytes(body)) + }, + Some((type_, subtype)) => { + return Err(tg::error!(%type_, %subtype, "invalid accept type")); + }, + }; + + let mut response = http::Response::builder(); + if let Some(content_type) = content_type { + response = response.header(http::header::CONTENT_TYPE, content_type.to_string()); + } + let response = response.body(body).unwrap(); + Ok(response) + } +} diff --git a/packages/server/src/sandbox/darwin.rs b/packages/server/src/sandbox/darwin.rs new file mode 100644 index 000000000..403458899 --- /dev/null +++ b/packages/server/src/sandbox/darwin.rs @@ -0,0 +1,65 @@ +use { + crate::{Server, temp::Temp}, + std::path::{Path, PathBuf}, + tangram_client::prelude::*, +}; + +impl Server { + pub(super) async fn get_sandbox_args( + &self, + _id: &tg::sandbox::Id, + arg: &tg::sandbox::create::Arg, + temp: &Temp, + ) -> tg::Result<(PathBuf, Vec)> { + // Determine if the root is mounted. + let root_mounted = arg.mounts.iter().any(|mount| { + mount + .source + .as_ref() + .left() + .is_some_and(|source| source == &mount.target && mount.target == Path::new("/")) + }); + + let mut args = Vec::new(); + let root = temp.path().to_path_buf(); + + // Add bind mounts. + for mount in &arg.mounts { + match &mount.source { + tg::Either::Left(path) => { + let mount_arg = if mount.readonly { + format!("source={},ro", path.display()) + } else { + format!("source={}", path.display()) + }; + args.push("--mount".to_owned()); + args.push(mount_arg); + }, + tg::Either::Right(_) => { + return Err(tg::error!("overlay mounts are not supported on darwin")); + }, + } + } + + if !root_mounted { + // Create the .tangram directory. + let path = temp.path().join(".tangram"); + tokio::fs::create_dir_all(&path).await.map_err( + |source| tg::error!(!source, path = %path.display(), "failed to create the data directory"), + )?; + + // Add mounts for the temp directory, artifacts, and the working directory. + args.push("--mount".to_owned()); + args.push(format!("source={}", temp.path().display())); + args.push("--mount".to_owned()); + args.push(format!("source={},ro", self.artifacts_path().display())); + } + + // Add the network flag. + if arg.network { + args.push("--network".to_owned()); + } + + Ok((root, args)) + } +} diff --git a/packages/server/src/sandbox/delete.rs b/packages/server/src/sandbox/delete.rs new file mode 100644 index 000000000..0fc487bb3 --- /dev/null +++ b/packages/server/src/sandbox/delete.rs @@ -0,0 +1,73 @@ +use { + crate::{Context, Server}, + tangram_client::prelude::*, + tangram_http::{body::Boxed as BoxBody, request::Ext as _}, +}; + +impl Server { + pub(crate) async fn delete_sandbox_with_context( + &self, + context: &Context, + id: &tg::sandbox::Id, + ) -> tg::Result<()> { + if context.sandbox.is_some() { + return Err(tg::error!("forbidden")); + } + + // Remove the sandbox from the map. + let (_, mut sandbox) = self + .sandboxes + .remove(id) + .ok_or_else(|| tg::error!("the sandbox was not found"))?; + + // Stop the proxy serve task. + sandbox.serve_task.stop(); + sandbox.serve_task.wait().await.ok(); + + // Kill and wait for the sandbox process. + sandbox + .process + .kill() + .await + .map_err(|source| tg::error!(!source, "failed to kill the sandbox process"))?; + + Ok(()) + } + + pub(crate) async fn handle_delete_sandbox_request( + &self, + request: http::Request, + context: &Context, + id: &str, + ) -> tg::Result> { + // Get the accept header. + let accept = request + .parse_header::(http::header::ACCEPT) + .transpose() + .map_err(|source| tg::error!(!source, "failed to parse the accept header"))?; + + // Parse the sandbox id. + let id = id + .parse() + .map_err(|source| tg::error!(!source, "failed to parse the sandbox id"))?; + + // Delete the sandbox. + self.delete_sandbox_with_context(context, &id) + .await + .map_err(|source| tg::error!(!source, %id, "failed to delete the sandbox"))?; + + // Create the response. + match accept + .as_ref() + .map(|accept| (accept.type_(), accept.subtype())) + { + None | Some((mime::STAR, mime::STAR)) => (), + Some((type_, subtype)) => { + return Err(tg::error!(%type_, %subtype, "invalid accept type")); + }, + } + + let response = http::Response::builder().body(BoxBody::empty()).unwrap(); + Ok(response) + } +} diff --git a/packages/server/src/sandbox/linux.rs b/packages/server/src/sandbox/linux.rs new file mode 100644 index 000000000..b7c3f92f2 --- /dev/null +++ b/packages/server/src/sandbox/linux.rs @@ -0,0 +1,215 @@ +use { + crate::{Server, temp::Temp}, + indoc::formatdoc, + std::{ + collections::HashMap, + os::unix::ffi::OsStrExt as _, + path::{Path, PathBuf}, + }, + tangram_client::prelude::*, +}; + +impl Server { + pub(super) async fn get_sandbox_args( + &self, + id: &tg::sandbox::Id, + arg: &tg::sandbox::create::Arg, + temp: &Temp, + ) -> tg::Result<(PathBuf, Vec)> { + // Determine if the root is mounted. + let root_mounted = arg.mounts.iter().any(|mount| { + mount + .source + .as_ref() + .left() + .is_some_and(|source| source == &mount.target && mount.target == Path::new("/")) + }); + + let mut args = Vec::new(); + let root = temp.path().join("root"); + tokio::fs::create_dir_all(&root) + .await + .map_err(|source| tg::error!(!source, "failed to create the root directory"))?; + let mut overlays = HashMap::new(); + for mount in &arg.mounts { + match &mount.source { + tg::Either::Left(source) => { + args.push("--mount".to_owned()); + args.push(bind(source, &mount.target, mount.readonly)); + }, + tg::Either::Right(id) => { + // Create the overlay state if it does not exist. Since we use async here, we can't use the .entry() api. + if !overlays.contains_key(&mount.target) { + let lowerdirs = Vec::new(); + let upperdir = temp.path().join("upper").join(overlays.len().to_string()); + let workdir = temp.path().join("work").join(overlays.len().to_string()); + tokio::fs::create_dir_all(&upperdir).await.ok(); + tokio::fs::create_dir_all(&workdir).await.ok(); + overlays.insert(mount.target.clone(), (lowerdirs, upperdir, workdir)); + } + + // Compute the path. + let path = self.artifacts_path().join(id.to_string()); + + // Get the lower dirs. + let (lowerdirs, _, _) = overlays.get_mut(&mount.target).unwrap(); + + // Add this path to the lowerdirs. + lowerdirs.push(path); + }, + } + } + + if !root_mounted { + // Create the .tangram directory. + let path = temp.path().join(".tangram"); + tokio::fs::create_dir_all(&path).await.map_err( + |source| tg::error!(!source, path = %path.display(), "failed to create the data directory"), + )?; + + // Create /etc. + tokio::fs::create_dir_all(temp.path().join("lower/etc")) + .await + .ok(); + + // Create /tmp. + tokio::fs::create_dir_all(temp.path().join("lower/tmp")) + .await + .ok(); + + // Create nsswitch.conf. + tokio::fs::write( + temp.path().join("lower/etc/nsswitch.conf"), + formatdoc!( + " + passwd: files compat + shadow: files compat + hosts: files dns compat + " + ), + ) + .await + .map_err(|source| tg::error!(!source, "failed to create /etc/nsswitch.conf"))?; + + // Create /etc/passwd. + tokio::fs::write( + temp.path().join("lower/etc/passwd"), + formatdoc!( + " + root:!:0:0:root:/nonexistent:/bin/false + nobody:!:65534:65534:nobody:/nonexistent:/bin/false + " + ), + ) + .await + .map_err(|source| tg::error!(!source, "failed to create /etc/passwd"))?; + + // Copy resolv.conf. + if arg.network { + tokio::fs::copy( + "/etc/resolv.conf", + temp.path().join("lower/etc/resolv.conf"), + ) + .await + .map_err(|source| { + tg::error!(!source, "failed to copy /etc/resolv.conf to the sandbox") + })?; + args.push("--network".to_owned()); + } + + // Get or create the root overlay. + if !overlays.contains_key(Path::new("/")) { + let lowerdirs = Vec::new(); + let upperdir = temp.path().join("upper").join(overlays.len().to_string()); + let workdir = temp.path().join("work").join(overlays.len().to_string()); + tokio::fs::create_dir_all(&upperdir).await.ok(); + tokio::fs::create_dir_all(&workdir).await.ok(); + overlays.insert("/".into(), (lowerdirs, upperdir, workdir)); + } + let (lowerdirs, _, _) = overlays.get_mut(Path::new("/")).unwrap(); + lowerdirs.push(temp.path().join("lower")); + + // Add mounts for /dev, /proc, /tmp, /.tangram, /.tangram/artifacts, and /output. + args.push("--mount".to_owned()); + args.push(bind("/dev", "/dev", false)); + args.push("--mount".to_owned()); + args.push(bind("/proc", "/proc", false)); + args.push("--mount".to_owned()); + args.push(bind(temp.path().join(".tangram"), "/.tangram", false)); + args.push("--mount".to_owned()); + args.push(bind(self.artifacts_path(), "/.tangram/artifacts", true)); + args.push("--mount".to_owned()); + args.push(bind(temp.path().join("output"), "/output", false)); + } + + // Add the overlay mounts. + for (merged, (lowerdirs, upperdir, workdir)) in &overlays { + args.push("--mount".to_owned()); + args.push(overlay(lowerdirs, upperdir, workdir, merged)); + } + + // Set the user. + let user = arg.user.as_deref().unwrap_or("root"); + args.push("--user".to_owned()); + args.push(user.to_owned()); + + // Set the hostname + args.push("--hostname".to_owned()); + args.push(id.to_string()); + + // Set the chroot. + args.push("--root".to_owned()); + args.push(root.display().to_string()); + + Ok((root, args)) + } +} + +fn bind(source: impl AsRef, target: impl AsRef, readonly: bool) -> String { + let mut string = format!( + "type=bind,source={},target={}", + source.as_ref().display(), + target.as_ref().display() + ); + if readonly { + string.push_str(",ro"); + } + string +} + +fn overlay(lowerdirs: &[PathBuf], upperdir: &Path, workdir: &Path, merged: &Path) -> String { + fn escape(out: &mut Vec, path: &[u8]) { + for byte in path.iter().copied() { + if byte == 0 { + break; + } + if byte == b':' { + out.push(b'\\'); + } + out.push(byte); + } + } + + // Create the mount options. + let mut data = b"type=overlay,source=overlay,target=".to_vec(); + data.extend_from_slice(merged.as_os_str().as_bytes()); + + // Add the lower directories. + data.extend_from_slice(b",userxattr,lowerdir="); + for (n, dir) in lowerdirs.iter().enumerate() { + escape(&mut data, dir.as_os_str().as_bytes()); + if n != lowerdirs.len() - 1 { + data.push(b':'); + } + } + + // Add the upper directory. + data.extend_from_slice(b",upperdir="); + data.extend_from_slice(upperdir.as_os_str().as_bytes()); + + // Add the working directory. + data.extend_from_slice(b",workdir="); + data.extend_from_slice(workdir.as_os_str().as_bytes()); + + String::from_utf8(data).unwrap() +} diff --git a/packages/server/src/sandbox/spawn.rs b/packages/server/src/sandbox/spawn.rs new file mode 100644 index 000000000..ad222ca8c --- /dev/null +++ b/packages/server/src/sandbox/spawn.rs @@ -0,0 +1,233 @@ +use { + crate::{Context, Server}, + std::os::fd::{AsFd as _, AsRawFd as _}, + tangram_client::prelude::*, + tangram_http::{body::Boxed as BoxBody, request::Ext as _}, + tangram_sandbox as sandbox, +}; + +impl Server { + pub(crate) async fn sandbox_spawn_with_context( + &self, + context: &Context, + id: &tg::sandbox::Id, + arg: tg::sandbox::spawn::Arg, + ) -> tg::Result { + if context.sandbox.is_some() { + return Err(tg::error!("forbidden")); + } + let sandbox = self + .sandboxes + .get(id) + .ok_or_else(|| tg::error!("sandbox not found"))?; + let client = sandbox.client.clone(); + let cwd; + #[cfg(target_os = "linux")] + { + cwd = Some("/".into()); + } + #[cfg(target_os = "macos")] + { + cwd = Some(sandbox.root.clone()); + } + drop(sandbox); + + // Collect FDs that need to be kept alive until after the spawn call. + let mut fds = Vec::new(); + + // Handle stdin. + let stdin = match &arg.stdin { + tg::process::Stdio::Pipe(pipe_id) => { + let pipe = self + .pipes + .get(pipe_id) + .ok_or_else(|| tg::error!("failed to find the pipe"))?; + let fd = pipe + .receiver + .as_fd() + .try_clone_to_owned() + .map_err(|source| tg::error!(!source, "failed to clone the receiver"))?; + let receiver = tokio::net::unix::pipe::Receiver::from_owned_fd_unchecked(fd) + .map_err(|source| tg::error!(!source, "io error"))?; + let fd = receiver + .into_blocking_fd() + .map_err(|source| tg::error!(!source, "failed to get the fd from the pipe"))?; + let raw_fd = fd.as_raw_fd(); + fds.push(fd); + raw_fd + }, + tg::process::Stdio::Pty(pty_id) => { + let pty = self + .ptys + .get(pty_id) + .ok_or_else(|| tg::error!("failed to find the pty"))?; + let slave = pty + .slave + .as_ref() + .ok_or_else(|| tg::error!("the pty slave is closed"))?; + let fd = slave + .try_clone() + .map_err(|source| tg::error!(!source, "failed to clone the pty slave"))?; + let raw_fd = fd.as_raw_fd(); + fds.push(fd); + raw_fd + }, + }; + + // Handle stdout. + let stdout = match &arg.stdout { + tg::process::Stdio::Pipe(pipe_id) => { + let pipe = self + .pipes + .get(pipe_id) + .ok_or_else(|| tg::error!("failed to find the pipe"))?; + let fd = pipe + .sender + .as_ref() + .ok_or_else(|| tg::error!("the pipe is closed"))? + .as_fd() + .try_clone_to_owned() + .map_err(|source| tg::error!(!source, "failed to clone the sender"))?; + let sender = tokio::net::unix::pipe::Sender::from_owned_fd_unchecked(fd) + .map_err(|source| tg::error!(!source, "io error"))?; + let fd = sender + .into_blocking_fd() + .map_err(|source| tg::error!(!source, "failed to get the fd from the pipe"))?; + let raw_fd = fd.as_raw_fd(); + fds.push(fd); + raw_fd + }, + tg::process::Stdio::Pty(pty_id) => { + let pty = self + .ptys + .get(pty_id) + .ok_or_else(|| tg::error!("failed to find the pty"))?; + let slave = pty + .slave + .as_ref() + .ok_or_else(|| tg::error!("the pty slave is closed"))?; + let fd = slave + .try_clone() + .map_err(|source| tg::error!(!source, "failed to clone the pty slave"))?; + let raw_fd = fd.as_raw_fd(); + fds.push(fd); + raw_fd + }, + }; + + // Handle stderr. + let stderr = match &arg.stderr { + tg::process::Stdio::Pipe(pipe_id) => { + let pipe = self + .pipes + .get(pipe_id) + .ok_or_else(|| tg::error!("failed to find the pipe"))?; + let fd = pipe + .sender + .as_ref() + .ok_or_else(|| tg::error!("the pipe is closed"))? + .as_fd() + .try_clone_to_owned() + .map_err(|source| tg::error!(!source, "failed to clone the sender"))?; + let sender = tokio::net::unix::pipe::Sender::from_owned_fd_unchecked(fd) + .map_err(|source| tg::error!(!source, "io error"))?; + let fd = sender + .into_blocking_fd() + .map_err(|source| tg::error!(!source, "failed to get the fd from the pipe"))?; + let raw_fd = fd.as_raw_fd(); + fds.push(fd); + raw_fd + }, + tg::process::Stdio::Pty(pty_id) => { + let pty = self + .ptys + .get(pty_id) + .ok_or_else(|| tg::error!("failed to find the pty"))?; + let slave = pty + .slave + .as_ref() + .ok_or_else(|| tg::error!("the pty slave is closed"))?; + let fd = slave + .try_clone() + .map_err(|source| tg::error!(!source, "failed to clone the pty slave"))?; + let raw_fd = fd.as_raw_fd(); + fds.push(fd); + raw_fd + }, + }; + + // Create the command. + let command = sandbox::Command { + cwd, + env: arg.env.into_iter().collect(), + executable: arg.command, + stdin: Some(stdin), + stdout: Some(stdout), + stderr: Some(stderr), + trailing: arg.args, + }; + + // Spawn the command via the sandbox client. + let pid = client + .spawn(command) + .await + .map_err(|source| tg::error!(!source, "failed to spawn the process"))?; + + // Drop the FDs now that the spawn has completed. + drop(fds); + + Ok(tg::sandbox::spawn::Output { pid }) + } + + pub(crate) async fn handle_sandbox_spawn_request( + &self, + request: http::Request, + context: &Context, + id: &str, + ) -> tg::Result> { + // Get the accept header. + let accept = request + .parse_header::(http::header::ACCEPT) + .transpose() + .map_err(|source| tg::error!(!source, "failed to parse the accept header"))?; + + // Parse the sandbox ID. + let id = id + .parse() + .map_err(|source| tg::error!(!source, "failed to parse the sandbox id"))?; + + // Get the arg. + let arg = request + .json() + .await + .map_err(|source| tg::error!(!source, "failed to deserialize the request body"))?; + + // Spawn the command. + let output = self + .sandbox_spawn_with_context(context, &id, arg) + .await + .map_err(|source| tg::error!(!source, "failed to spawn the command in the sandbox"))?; + + // Create the response. + let (content_type, body) = match accept + .as_ref() + .map(|accept| (accept.type_(), accept.subtype())) + { + None | Some((mime::STAR, mime::STAR) | (mime::APPLICATION, mime::JSON)) => { + let content_type = mime::APPLICATION_JSON; + let body = serde_json::to_vec(&output).unwrap(); + (Some(content_type), BoxBody::with_bytes(body)) + }, + Some((type_, subtype)) => { + return Err(tg::error!(%type_, %subtype, "invalid accept type")); + }, + }; + + let mut response = http::Response::builder(); + if let Some(content_type) = content_type { + response = response.header(http::header::CONTENT_TYPE, content_type.to_string()); + } + let response = response.body(body).unwrap(); + Ok(response) + } +} diff --git a/packages/server/src/sandbox/wait.rs b/packages/server/src/sandbox/wait.rs new file mode 100644 index 000000000..136fdb1ab --- /dev/null +++ b/packages/server/src/sandbox/wait.rs @@ -0,0 +1,117 @@ +use { + crate::{Context, Server}, + futures::{StreamExt as _, stream}, + std::sync::Arc, + tangram_client::prelude::*, + tangram_futures::task::Stop, + tangram_http::{ + body::Boxed as BoxBody, + request::Ext as _, + response::{Ext as _, builder::Ext as _}, + }, +}; + +impl Server { + pub(crate) async fn sandbox_wait_with_context( + &self, + context: &Context, + id: &tg::sandbox::Id, + arg: tg::sandbox::wait::Arg, + ) -> tg::Result< + Option< + impl Future>> + Send + 'static + use<>, + >, + > { + if context.sandbox.is_some() { + return Err(tg::error!("forbidden")); + } + let Some(sandbox) = self.sandboxes.get(id) else { + return Ok(None); + }; + let client = Arc::clone(&sandbox.client); + drop(sandbox); + let future = async move { + let status = client + .wait(arg.pid) + .await + .map_err(|source| tg::error!(!source, "failed to wait for the process"))?; + Ok(Some(tg::sandbox::wait::Output { status })) + }; + Ok(Some(future)) + } + + pub(crate) async fn handle_sandbox_wait_request( + &self, + request: http::Request, + context: &Context, + id: &str, + ) -> tg::Result> { + // Parse the ID. + let id = id + .parse::() + .map_err(|source| tg::error!(!source, "failed to parse the sandbox id"))?; + + // Parse the arg. + let arg = request + .query_params() + .transpose() + .map_err(|source| tg::error!(!source, "failed to parse the query params"))? + .ok_or_else(|| tg::error!("expected a process"))?; + + // Get the accept header. + let accept: Option = request + .parse_header(http::header::ACCEPT) + .transpose() + .map_err(|source| tg::error!(!source, "failed to parse the accept header"))?; + + // Get the future. + let Some(future) = self.sandbox_wait_with_context(context, &id, arg).await? else { + return Ok(http::Response::builder() + .not_found() + .empty() + .unwrap() + .boxed_body()); + }; + + // Create the stream. + let stream = stream::once(future).filter_map(|result| async move { + match result { + Ok(Some(value)) => Some(Ok(tg::sandbox::wait::Event::Output(value))), + Ok(None) => None, + Err(error) => Some(Err(error)), + } + }); + + // Stop the stream when the server stops. + let stop = request.extensions().get::().cloned().unwrap(); + let stop = async move { stop.wait().await }; + let stream = stream.take_until(stop); + + // Create the body. + let (content_type, body) = match accept + .as_ref() + .map(|accept| (accept.type_(), accept.subtype())) + { + None | Some((mime::STAR, mime::STAR) | (mime::TEXT, mime::EVENT_STREAM)) => { + let content_type = mime::TEXT_EVENT_STREAM; + let stream = stream.map(|result| match result { + Ok(event) => event.try_into(), + Err(error) => error.try_into(), + }); + (Some(content_type), BoxBody::with_sse_stream(stream)) + }, + Some((type_, subtype)) => { + return Err(tg::error!(%type_, %subtype, "invalid accept type")); + }, + }; + + // Create the response. + let mut response = http::Response::builder(); + if let Some(content_type) = content_type { + response = response.header(http::header::CONTENT_TYPE, content_type.to_string()); + } + let response = response.body(body).unwrap(); + + Ok(response) + } +} diff --git a/packages/server/src/tag/batch.rs b/packages/server/src/tag/batch.rs index a0f58608f..66fa0051f 100644 --- a/packages/server/src/tag/batch.rs +++ b/packages/server/src/tag/batch.rs @@ -35,7 +35,7 @@ impl Server { return Ok(()); } - if context.process.is_some() { + if context.sandbox.is_some() { return Err(tg::error!("forbidden")); } diff --git a/packages/server/src/tag/delete.rs b/packages/server/src/tag/delete.rs index 6af2d5059..0bc9060b9 100644 --- a/packages/server/src/tag/delete.rs +++ b/packages/server/src/tag/delete.rs @@ -35,7 +35,7 @@ impl Server { return Ok(output); } - if context.process.is_some() { + if context.sandbox.is_some() { return Err(tg::error!("forbidden")); } diff --git a/packages/server/src/tag/list.rs b/packages/server/src/tag/list.rs index 28ca513bd..1a210f503 100644 --- a/packages/server/src/tag/list.rs +++ b/packages/server/src/tag/list.rs @@ -25,7 +25,7 @@ impl Server { context: &Context, arg: tg::tag::list::Arg, ) -> tg::Result { - if context.process.is_some() { + if context.sandbox.is_some() { return Err(tg::error!("forbidden")); } diff --git a/packages/server/src/tag/put.rs b/packages/server/src/tag/put.rs index ede5b743d..edb003784 100644 --- a/packages/server/src/tag/put.rs +++ b/packages/server/src/tag/put.rs @@ -36,7 +36,7 @@ impl Server { return Ok(()); } - if context.process.is_some() { + if context.sandbox.is_some() { return Err(tg::error!("forbidden")); } diff --git a/packages/server/src/user.rs b/packages/server/src/user.rs index 8ed0ef4de..3a9ee20e6 100644 --- a/packages/server/src/user.rs +++ b/packages/server/src/user.rs @@ -12,7 +12,7 @@ impl Server { context: &Context, token: &str, ) -> tg::Result> { - if context.process.is_some() { + if context.sandbox.is_some() { return Err(tg::error!("forbidden")); } diff --git a/packages/server/src/watch/delete.rs b/packages/server/src/watch/delete.rs index 912642adf..f61d0bae0 100644 --- a/packages/server/src/watch/delete.rs +++ b/packages/server/src/watch/delete.rs @@ -10,7 +10,7 @@ impl Server { context: &Context, mut arg: tg::watch::delete::Arg, ) -> tg::Result<()> { - if context.process.is_some() { + if context.sandbox.is_some() { return Err(tg::error!("forbidden")); } diff --git a/packages/server/src/watch/list.rs b/packages/server/src/watch/list.rs index 7f1c56204..83616fd36 100644 --- a/packages/server/src/watch/list.rs +++ b/packages/server/src/watch/list.rs @@ -10,7 +10,7 @@ impl Server { context: &Context, _arg: tg::watch::list::Arg, ) -> tg::Result { - if context.process.is_some() { + if context.sandbox.is_some() { return Err(tg::error!("forbidden")); } let data = self diff --git a/packages/server/src/watch/touch.rs b/packages/server/src/watch/touch.rs index ac49fe781..08aad12c7 100644 --- a/packages/server/src/watch/touch.rs +++ b/packages/server/src/watch/touch.rs @@ -10,7 +10,7 @@ impl Server { context: &Context, arg: tg::watch::touch::Arg, ) -> tg::Result<()> { - if context.process.is_some() { + if context.sandbox.is_some() { return Err(tg::error!("forbidden")); } diff --git a/packages/vfs/src/fuse.rs b/packages/vfs/src/fuse.rs index 87da1b3bd..6c46d0ff6 100644 --- a/packages/vfs/src/fuse.rs +++ b/packages/vfs/src/fuse.rs @@ -273,9 +273,8 @@ where sqpoll_ring = Some(ring); // Create threads. - let preferred_thread_count = std::thread::available_parallelism() - .map(std::num::NonZero::get) - .unwrap_or(1); + let preferred_thread_count = + std::thread::available_parallelism().map_or(1, std::num::NonZero::get); let mut thread_count = preferred_thread_count; let mut first_cloned_fd = None; if preferred_thread_count > 1 {