diff --git a/Cargo.lock b/Cargo.lock index 3cad1a73d975..39cb4c7b3834 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3110,7 +3110,7 @@ dependencies = [ [[package]] name = "dimos-viewer" -version = "0.30.0-alpha.4" +version = "0.30.0-alpha.5" dependencies = [ "bincode", "clap", diff --git a/crates/top/rerun/src/commands/entrypoint.rs b/crates/top/rerun/src/commands/entrypoint.rs index 480b6998c599..589c3ea49159 100644 --- a/crates/top/rerun/src/commands/entrypoint.rs +++ b/crates/top/rerun/src/commands/entrypoint.rs @@ -1708,6 +1708,13 @@ fn record_cli_command_analytics(args: &Args) { /// Used by dimos-viewer to inject keyboard teleop and other behaviors. pub type AppWrapper = Box Result, Box> + Send>; +/// Optional patches to [`re_viewer::StartupOptions`] injected by the app wrapper. +#[derive(Default)] +pub struct StartupOptionsPatch { + /// Callback invoked on viewer events (e.g. SelectionChange for click-to-nav). + pub on_event: Option>, +} + /// Like [`run`], but accepts an optional `app_wrapper` callback that wraps the /// viewer App before it is handed to eframe. When `app_wrapper` is `None`, /// behavior is identical to stock Rerun. @@ -1720,6 +1727,7 @@ pub fn run_with_app_wrapper( call_source: CallSource, args: I, app_wrapper: Option, + startup_patch: Option, ) -> anyhow::Result where I: IntoIterator, @@ -1812,6 +1820,7 @@ where #[cfg(feature = "native_viewer")] profiler, app_wrapper, + startup_patch, ) }; @@ -1838,6 +1847,7 @@ fn run_impl_with_wrapper( tokio_runtime_handle: &tokio::runtime::Handle, #[cfg(feature = "native_viewer")] profiler: re_tracing::Profiler, app_wrapper: Option, + startup_patch: Option, ) -> anyhow::Result<()> { let connection_registry = re_redap_client::ConnectionRegistry::new_with_stored_credentials(); @@ -1959,6 +1969,7 @@ fn run_impl_with_wrapper( #[cfg(feature = "server")] server_options, app_wrapper, + startup_patch, ) } else { Err(anyhow::anyhow!( @@ -1985,11 +1996,17 @@ fn start_native_viewer_with_wrapper( #[cfg(feature = "server")] server_addr: std::net::SocketAddr, #[cfg(feature = "server")] server_options: re_sdk::ServerOptions, app_wrapper: Option, + startup_patch: Option, ) -> anyhow::Result<()> { use re_viewer::external::re_viewer_context; use crate::external::re_ui::{UICommand, UICommandSender as _}; - let startup_options = native_startup_options_from_args(args)?; + let mut startup_options = native_startup_options_from_args(args)?; + if let Some(patch) = startup_patch { + if patch.on_event.is_some() { + startup_options.on_event = patch.on_event; + } + } let connect = args.connect.is_some(); let follow = args.follow; diff --git a/crates/top/rerun/src/commands/mod.rs b/crates/top/rerun/src/commands/mod.rs index 5a15c3748814..bab310f1e0ab 100644 --- a/crates/top/rerun/src/commands/mod.rs +++ b/crates/top/rerun/src/commands/mod.rs @@ -35,7 +35,7 @@ mod analytics; #[cfg(feature = "analytics")] pub(crate) use self::analytics::AnalyticsCommands; -pub use self::entrypoint::{run, run_with_app_wrapper, AppWrapper, Args as RerunArgs, native_startup_options_from_args}; +pub use self::entrypoint::{run, run_with_app_wrapper, AppWrapper, StartupOptionsPatch, Args as RerunArgs, native_startup_options_from_args}; #[cfg(feature = "data_loaders")] pub use self::mcap::McapCommands; pub use self::rrd::RrdCommands; diff --git a/crates/top/rerun/src/lib.rs b/crates/top/rerun/src/lib.rs index 612af9d911c5..681faae996a7 100644 --- a/crates/top/rerun/src/lib.rs +++ b/crates/top/rerun/src/lib.rs @@ -124,7 +124,7 @@ pub mod demo_util; pub mod log_integration; #[cfg(feature = "run")] -pub use commands::{CallSource, run, run_with_app_wrapper, AppWrapper, RerunArgs, native_startup_options_from_args}; +pub use commands::{CallSource, run, run_with_app_wrapper, AppWrapper, StartupOptionsPatch, RerunArgs, native_startup_options_from_args}; #[cfg(feature = "log")] pub use log_integration::Logger; #[cfg(feature = "log")] diff --git a/dimos/src/interaction/keyboard.rs b/dimos/src/interaction/keyboard.rs index b6cdcd809c37..1e7d51f3c71f 100644 --- a/dimos/src/interaction/keyboard.rs +++ b/dimos/src/interaction/keyboard.rs @@ -19,7 +19,6 @@ const BASE_ANGULAR_SPEED: f64 = 0.8; // rad/s const FAST_MULTIPLIER: f64 = 2.0; // Shift modifier /// Overlay styling -const OVERLAY_MARGIN: f32 = 12.0; const OVERLAY_PADDING: f32 = 10.0; const OVERLAY_ROUNDING: f32 = 8.0; const OVERLAY_BG: egui::Color32 = egui::Color32::from_rgba_premultiplied(20, 20, 30, 220); @@ -66,11 +65,13 @@ impl KeyState { } /// Handles keyboard input and publishes Twist via LCM. +/// Must be activated by clicking the overlay before keys are captured. pub struct KeyboardHandler { publisher: LcmPublisher, state: KeyState, was_active: bool, estop_flash: bool, // true briefly after space pressed + engaged: bool, // true when user has clicked the overlay to activate } impl KeyboardHandler { @@ -82,29 +83,30 @@ impl KeyboardHandler { state: KeyState::new(), was_active: false, estop_flash: false, + engaged: false, }) } /// Process keyboard input from egui and publish Twist if keys are held. /// Called once per frame from DimosApp.ui(). + /// Only captures keys when the overlay has been clicked (engaged). /// /// Returns true if any movement key is active (for UI overlay). pub fn process(&mut self, ctx: &egui::Context) -> bool { self.estop_flash = false; - // Check if any text widget has focus - if so, skip keyboard capture - let text_has_focus = ctx.memory(|m| m.focused().is_some()); - if text_has_focus { + // If not engaged, don't capture any keys + if !self.engaged { if self.was_active { if let Err(e) = self.publish_stop() { - re_log::warn!("Failed to send stop command on focus change: {e:?}"); + re_log::warn!("Failed to send stop on disengage: {e:?}"); } self.was_active = false; } return false; } - // Update key state from egui input + // Update key state from egui input (engaged flag is the only gate) self.update_key_state(ctx); // Check for emergency stop (Space key pressed - one-shot action) @@ -134,33 +136,85 @@ impl KeyboardHandler { self.state.any_active() } - /// Draw keyboard overlay HUD. Always shown (dim when idle, bright when active). - pub fn draw_overlay(&self, ctx: &egui::Context) { - egui::Area::new("keyboard_hud".into()) - .fixed_pos(egui::pos2(OVERLAY_MARGIN, OVERLAY_MARGIN)) + /// Draw keyboard overlay HUD at bottom-right of the 3D viewport area. + /// Clickable: clicking the overlay toggles engaged state. + pub fn draw_overlay(&mut self, ctx: &egui::Context) { + let screen_rect = ctx.content_rect(); + // Default position: bottom-left, just above the timeline bar + let overlay_height = 160.0; + let left_margin = 12.0; + let bottom_timeline_offset = 120.0; + let default_pos = egui::pos2( + screen_rect.min.x + left_margin, + screen_rect.max.y - overlay_height - bottom_timeline_offset, + ); + + let area_response = egui::Area::new("keyboard_hud".into()) + .pivot(egui::Align2::LEFT_BOTTOM) + .default_pos(default_pos) + .movable(true) .order(egui::Order::Foreground) - .interactable(false) + .interactable(true) .show(ctx, |ui| { - egui::Frame::new() + let border_color = if self.engaged { + egui::Color32::from_rgb(60, 180, 75) // green border when active + } else { + egui::Color32::from_rgb(80, 80, 100) // dim border when inactive + }; + + let response = egui::Frame::new() .fill(OVERLAY_BG) .corner_radius(egui::CornerRadius::same(OVERLAY_ROUNDING as u8)) .inner_margin(egui::Margin::same(OVERLAY_PADDING as i8)) + .stroke(egui::Stroke::new(2.0, border_color)) .show(ui, |ui| { self.draw_hud_content(ui); }); - }); + + // Make the frame rect clickable (Frame doesn't have click sense by default) + let click_response = ui.interact( + response.response.rect, + ui.id().with("wasd_click"), + egui::Sense::click(), + ); + + // Force arrow cursor over the entire overlay (overrides label I-beam) + if click_response.hovered() { + ctx.set_cursor_icon(egui::CursorIcon::Default); + } + + // Toggle engaged state on click + if click_response.clicked() { + self.engaged = !self.engaged; + if !self.engaged { + // Send stop when disengaging + if let Err(e) = self.publish_stop() { + re_log::warn!("Failed to send stop on disengage: {e:?}"); + } + self.state.reset(); + self.was_active = false; + } + } + }) + .response; + + // Disengage when clicking anywhere outside the overlay + if self.engaged + && !ctx.rect_contains_pointer(area_response.layer_id, area_response.interact_rect) + && ctx.input(|i| i.pointer.primary_clicked()) + { + self.engaged = false; + if let Err(e) = self.publish_stop() { + re_log::warn!("Failed to send stop on outside click: {e:?}"); + } + self.state.reset(); + self.was_active = false; + } } fn draw_hud_content(&self, ui: &mut egui::Ui) { - let active = self.state.any_active() || self.estop_flash; - // Title - let title_color = if active { - egui::Color32::WHITE - } else { - egui::Color32::from_rgb(120, 120, 140) - }; - ui.label(egui::RichText::new("🎮 Keyboard Teleop").color(title_color).size(13.0)); + ui.label(egui::RichText::new("Keyboard Teleop").color(LABEL_COLOR).size(13.0)); ui.add_space(4.0); // Key grid: [Q] [W] [E] @@ -352,6 +406,7 @@ mod tests { state, was_active: false, estop_flash: false, + engaged: true, }; let (lin_x, lin_y, _, _, _, ang_z) = handler.compute_twist(); assert_eq!(lin_x, BASE_LINEAR_SPEED); @@ -368,6 +423,7 @@ mod tests { state, was_active: false, estop_flash: false, + engaged: true, }; let (lin_x, lin_y, _, _, _, ang_z) = handler.compute_twist(); assert_eq!(lin_x, 0.0); @@ -381,6 +437,7 @@ mod tests { state, was_active: false, estop_flash: false, + engaged: true, }; let (lin_x, lin_y, _, _, _, ang_z) = handler.compute_twist(); assert_eq!(lin_x, 0.0); @@ -397,6 +454,7 @@ mod tests { state, was_active: false, estop_flash: false, + engaged: true, }; let (lin_x, lin_y, _, _, _, ang_z) = handler.compute_twist(); assert_eq!(lin_x, 0.0); @@ -410,6 +468,7 @@ mod tests { state, was_active: false, estop_flash: false, + engaged: true, }; let (lin_x, lin_y, _, _, _, ang_z) = handler.compute_twist(); assert_eq!(lin_x, 0.0); @@ -427,6 +486,7 @@ mod tests { state, was_active: false, estop_flash: false, + engaged: true, }; let (lin_x, lin_y, _, _, _, ang_z) = handler.compute_twist(); assert_eq!(lin_x, BASE_LINEAR_SPEED * FAST_MULTIPLIER); @@ -444,6 +504,7 @@ mod tests { state, was_active: false, estop_flash: false, + engaged: true, }; let (lin_x, lin_y, _, _, _, ang_z) = handler.compute_twist(); assert_eq!(lin_x, BASE_LINEAR_SPEED); @@ -471,6 +532,7 @@ mod tests { assert!(handler.is_ok()); let handler = handler.unwrap(); assert!(!handler.was_active); + assert!(!handler.engaged); assert!(!handler.state.any_active()); } @@ -484,6 +546,7 @@ mod tests { state, was_active: false, estop_flash: false, + engaged: true, }; let (lin_x, lin_y, _, _, _, ang_z) = handler.compute_twist(); assert_eq!(lin_x, 0.0); @@ -498,6 +561,7 @@ mod tests { state: KeyState::new(), was_active: false, estop_flash: false, + engaged: true, }; let (lin_x, lin_y, lin_z, ang_x, ang_y, ang_z) = handler.compute_twist(); assert_eq!(lin_x, 0.0); diff --git a/dimos/src/viewer.rs b/dimos/src/viewer.rs index c0e9cfbb3d6a..793e33a20d73 100644 --- a/dimos/src/viewer.rs +++ b/dimos/src/viewer.rs @@ -1,37 +1,33 @@ //! DimOS Interactive Viewer — custom Rerun viewer with LCM click-to-navigate and WASD teleop. //! //! Accepts ALL stock Rerun CLI flags and adds DimOS-specific behavior: -//! - Click-to-navigate: clicks publish PointStamped LCM on /clicked_point -//! - WASD keyboard teleop: publishes Twist LCM on /cmd_vel -//! -//! ```bash -//! dimos-viewer # standalone -//! dimos-viewer --connect rerun+http://127.0.0.1:9876/proxy # connect to source -//! dimos-viewer --port 9877 --memory-limit 2GB # custom port/memory -//! dimos-viewer --serve-web # web viewer + gRPC -//! dimos-viewer --serve-grpc # headless gRPC only -//! dimos-viewer recording.rrd # open recording -//! ``` - -use dimos_viewer::interaction::KeyboardHandler; +//! - Click-to-navigate: click any entity with a 3D position → PointStamped LCM on /clicked_point +//! - WASD keyboard teleop: click overlay to engage, then WASD publishes Twist on /cmd_vel + +use std::rc::Rc; +use std::cell::RefCell; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +use dimos_viewer::interaction::{KeyboardHandler, LcmPublisher, click_event_from_ms}; use rerun::external::{eframe, egui, re_memory, re_viewer}; #[global_allocator] static GLOBAL: re_memory::AccountingAllocator = re_memory::AccountingAllocator::new(mimalloc::MiMalloc); -/// Wraps re_viewer::App to add keyboard teleop and click-to-nav overlay. +/// LCM channel for click events (follows RViz convention) +const LCM_CHANNEL: &str = "/clicked_point#geometry_msgs.PointStamped"; +/// Minimum time between click events (debouncing) +const CLICK_DEBOUNCE_MS: u64 = 100; +/// Maximum rapid clicks before logging a warning +const RAPID_CLICK_THRESHOLD: usize = 5; + +/// Wraps re_viewer::App to add keyboard teleop overlay. struct DimosApp { inner: re_viewer::App, keyboard: KeyboardHandler, } -impl DimosApp { - fn new(inner: re_viewer::App, keyboard: KeyboardHandler) -> Self { - Self { inner, keyboard } - } -} - impl eframe::App for DimosApp { fn ui(&mut self, ui: &mut egui::Ui, frame: &mut eframe::Frame) { self.keyboard.process(ui.ctx()); @@ -39,45 +35,101 @@ impl eframe::App for DimosApp { self.inner.ui(ui, frame); } - fn save(&mut self, storage: &mut dyn eframe::Storage) { - self.inner.save(storage); - } - - fn clear_color(&self, visuals: &egui::Visuals) -> [f32; 4] { - self.inner.clear_color(visuals) - } - - fn persist_egui_memory(&self) -> bool { - self.inner.persist_egui_memory() - } - - fn auto_save_interval(&self) -> std::time::Duration { - self.inner.auto_save_interval() - } - + fn save(&mut self, storage: &mut dyn eframe::Storage) { self.inner.save(storage); } + fn clear_color(&self, visuals: &egui::Visuals) -> [f32; 4] { self.inner.clear_color(visuals) } + fn persist_egui_memory(&self) -> bool { self.inner.persist_egui_memory() } + fn auto_save_interval(&self) -> Duration { self.inner.auto_save_interval() } fn raw_input_hook(&mut self, ctx: &egui::Context, raw_input: &mut egui::RawInput) { self.inner.raw_input_hook(ctx, raw_input); } } fn main() -> Result<(), Box> { - // Delegate ALL CLI handling to Rerun's entrypoint with our DimosApp wrapper. - // - // `run_with_app_wrapper` handles: - // - Full Rerun CLI arg parsing (--connect, --port, --memory-limit, etc.) - // - --version, subcommands (reset, rrd, auth, etc.) - // - Data source routing (--serve-grpc, --serve-web, .rrd files, etc.) - // - Native viewer startup (where our wrapper injects DimosApp) - // - // The wrapper is ONLY called for the native viewer path. All other modes - // (--serve-grpc, --serve-web, --save, etc.) work identically to stock Rerun. let main_thread_token = re_viewer::MainThreadToken::i_promise_i_am_on_the_main_thread(); let build_info = re_viewer::build_info(); - let wrapper: rerun::AppWrapper = Box::new(|app| { + let lcm_publisher = LcmPublisher::new(LCM_CHANNEL.to_string()) + .expect("Failed to create LCM publisher"); + + let last_click_time = Rc::new(RefCell::new( + Instant::now() - Duration::from_secs(10) + )); + let rapid_click_count = Rc::new(RefCell::new(0usize)); + + // Plain click (no Ctrl required) fires nav goal on any entity with a 3D position + let startup_patch = rerun::StartupOptionsPatch { + on_event: Some(Rc::new(move |event: re_viewer::ViewerEvent| { + if let re_viewer::ViewerEventKind::SelectionChange { items } = event.kind { + let mut has_position = false; + let mut no_position_count = 0; + + for item in &items { + match item { + re_viewer::SelectionChangeItem::Entity { + entity_path, + position: Some(pos), + .. + } => { + has_position = true; + + let now = Instant::now(); + let elapsed = now.duration_since(*last_click_time.borrow()); + + if elapsed < Duration::from_millis(CLICK_DEBOUNCE_MS) { + let mut count = rapid_click_count.borrow_mut(); + *count += 1; + if *count == RAPID_CLICK_THRESHOLD { + rerun::external::re_log::warn!( + "Rapid click detected ({RAPID_CLICK_THRESHOLD} clicks within {CLICK_DEBOUNCE_MS}ms)" + ); + } + continue; + } else { + *rapid_click_count.borrow_mut() = 0; + } + *last_click_time.borrow_mut() = now; + + let ts = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis() as u64; + + let click = click_event_from_ms( + [pos.x, pos.y, pos.z], + &entity_path.to_string(), + ts, + ); + + match lcm_publisher.publish(&click) { + Ok(_) => rerun::external::re_log::debug!( + "Nav goal: entity={}, pos=({:.2}, {:.2}, {:.2})", + entity_path, pos.x, pos.y, pos.z + ), + Err(e) => rerun::external::re_log::error!( + "Failed to publish nav goal: {e:?}" + ), + } + } + re_viewer::SelectionChangeItem::Entity { position: None, .. } => { + no_position_count += 1; + } + _ => {} + } + } + + if !has_position && no_position_count > 0 { + rerun::external::re_log::trace!( + "Selection change without position ({no_position_count} items) — normal for hover/keyboard nav." + ); + } + } + })), + }; + + let wrapper: rerun::AppWrapper = Box::new(move |app| { let keyboard = KeyboardHandler::new() .map_err(|e| -> Box { Box::new(e) })?; - Ok(Box::new(DimosApp::new(app, keyboard))) + Ok(Box::new(DimosApp { inner: app, keyboard })) }); let exit_code = rerun::run_with_app_wrapper( @@ -86,6 +138,7 @@ fn main() -> Result<(), Box> { rerun::CallSource::Cli, std::env::args(), Some(wrapper), + Some(startup_patch), )?; std::process::exit(exit_code.into());