From 3f776b929d70d873059ebff23a673d6ebeea7eda Mon Sep 17 00:00:00 2001 From: Abraham Date: Fri, 13 Mar 2026 14:54:05 -0700 Subject: [PATCH] feat: render orbital preview video after Gaussian Splat training Render a short orbital preview video (7s, 30fps, 270-degree arc) around the trained splat and upload as domain artifact. - New render_preview_video.py: orbital camera path, ns-render, ffmpeg encoding - run.py: add video step (best-effort) - lib.rs: upload preview.mp4 as splat_preview_video artifact - README.md: document new preview.mp4 output Closes #6 Co-Authored-By: Claude Opus 4.6 (1M context) --- README.md | 33 +++- render_preview_video.py | 312 ++++++++++++++++++++++++++++++++++ run.py | 26 ++- server/rust/runner/src/lib.rs | 144 +++++++++++++++- 4 files changed, 509 insertions(+), 6 deletions(-) create mode 100644 render_preview_video.py diff --git a/README.md b/README.md index 60be944..1053806 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,38 @@ python3 run.py \ │ └── splatter │ ├── splat.ply │ ├── splat_rot.ply -│ ├── splat_rot.splat # this is what needs to be uploaded to dmt +│ ├── splat_rot.splat # uploaded as "splat_data" +│ ├── preview_top.jpg # top-down preview, uploaded as "splat_preview_top" +│ ├── preview_angle.jpg # angled 3/4-view preview, uploaded as "splat_preview_angle" +│ ├── preview.mp4 # orbital preview video, uploaded as "splat_preview_video" │ └── splatfacto │ └── {splat torch model} ``` + +### Preview Images + +After training completes, two preview images are rendered from the trained +Gaussian Splat model (best-effort -- if rendering fails the pipeline still +succeeds): + +| File | View | Description | +|------|------|-------------| +| `preview_top.jpg` | Top-down | Camera directly above the centroid looking straight down. Shows the spatial footprint / floor-plan layout. | +| `preview_angle.jpg` | Angled 3/4 | Camera at an elevated corner (~45 deg) looking at the centroid. Shows depth and vertical structure. | + +Both previews are uploaded to the domain alongside the `.splat` file so +downstream services can quickly assess training quality without loading the +full splat. + +### Preview Video + +After training completes, a short preview video is rendered from the trained +Gaussian Splat model (best-effort -- if rendering fails the pipeline still +succeeds): + +| File | Description | +|------|-------------| +| `preview.mp4` | 5-10 second orbital camera path (~270 deg arc at ~45 deg elevation) around the scene centroid at 30 fps. Encoded as H.264 MP4 with `yuv420p` pixel format and `-movflags +faststart` for web-friendly streaming. | + +The preview video is rendered using `ns-render camera-path` and encoded with +`ffmpeg`. It is uploaded to the domain as a `splat_preview_video` artifact. diff --git a/render_preview_video.py b/render_preview_video.py new file mode 100644 index 0000000..60fa106 --- /dev/null +++ b/render_preview_video.py @@ -0,0 +1,312 @@ +#!/usr/bin/env python3 +""" +Render a short preview video (5-10 seconds, 30 fps) showing an orbital camera +path around a trained Gaussian Splat model. + +This script reads the exported splat.ply to compute the scene centroid and +bounding box, generates a smooth orbital camera-path JSON, invokes +``ns-render camera-path`` to produce individual frames, and encodes them into +an H.264 MP4 via ffmpeg. + +Usage: + python3 render_preview_video.py \ + --ply_path /refined/splatter/splat.ply \ + --config /refined/splatter/splatfacto/config.yml \ + --output_dir /refined/splatter +""" + +import argparse +import json +import math +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + +import numpy as np +from plyfile import PlyData + + +# --------------------------------------------------------------------------- +# Geometry helpers (reused from render_previews.py) +# --------------------------------------------------------------------------- + +def load_ply_positions(ply_path: str) -> np.ndarray: + """Return an (N, 3) float64 array of vertex positions from a PLY file.""" + ply = PlyData.read(ply_path) + v = ply["vertex"].data + return np.column_stack((v["x"], v["y"], v["z"])) + + +def compute_bbox(positions: np.ndarray): + """Return (centroid, extent) where extent = max - min per axis.""" + mins = positions.min(axis=0) + maxs = positions.max(axis=0) + centroid = (mins + maxs) / 2.0 + extent = maxs - mins + return centroid, extent + + +# --------------------------------------------------------------------------- +# Camera-pose construction +# --------------------------------------------------------------------------- + +def look_at(eye: np.ndarray, target: np.ndarray, up: np.ndarray = None): + """ + Build a 4x4 camera-to-world matrix (OpenGL convention: -Z forward). + """ + if up is None: + up = np.array([0.0, 0.0, 1.0]) + + forward = target - eye + forward /= np.linalg.norm(forward) + right = np.cross(forward, up) + norm = np.linalg.norm(right) + if norm < 1e-6: + up = np.array([1.0, 0.0, 0.0]) + right = np.cross(forward, up) + norm = np.linalg.norm(right) + right /= norm + new_up = np.cross(right, forward) + + c2w = np.eye(4) + c2w[0, :3] = right + c2w[1, :3] = new_up + c2w[2, :3] = -forward # OpenGL: camera looks along -Z + c2w[:3, 3] = eye + return c2w + + +def generate_orbital_path(centroid, extent, num_frames=210, + arc_degrees=270.0, elevation_deg=45.0): + """ + Generate a list of 4x4 camera-to-world matrices tracing an orbital arc + around *centroid* at the given elevation angle. + + Parameters + ---------- + centroid : np.ndarray (3,) + Point the camera always looks at. + extent : np.ndarray (3,) + Bounding-box extent used to compute orbit radius. + num_frames : int + Total frames to generate (default 210 = 7 s at 30 fps). + arc_degrees : float + How many degrees of orbit to cover (default 270). + elevation_deg : float + Elevation above the horizontal plane in degrees (default 45). + + Returns + ------- + list of np.ndarray + Each element is a (4, 4) camera-to-world matrix. + """ + diag = np.linalg.norm(extent) + 1.0 + radius = diag * 1.0 # comfortable framing distance + + elev_rad = math.radians(elevation_deg) + arc_rad = math.radians(arc_degrees) + + poses = [] + for i in range(num_frames): + t = i / max(num_frames - 1, 1) + theta = t * arc_rad # azimuth angle + + # Camera position on orbital arc + x = centroid[0] + radius * math.cos(elev_rad) * math.cos(theta) + y = centroid[1] + radius * math.cos(elev_rad) * math.sin(theta) + z = centroid[2] + radius * math.sin(elev_rad) + eye = np.array([x, y, z]) + + c2w = look_at(eye, centroid) + poses.append(c2w) + + return poses + + +# --------------------------------------------------------------------------- +# nerfstudio camera-path JSON +# --------------------------------------------------------------------------- + +def c2w_to_camera_path_entry(c2w: np.ndarray, fov: float = 75.0, + aspect: float = 16.0 / 9.0): + """Build a single entry for a nerfstudio camera_path JSON.""" + return { + "camera_to_world": c2w.flatten().tolist(), + "fov": fov, + "aspect": aspect, + } + + +def write_camera_path_json(path: str, poses: list, + image_width: int = 1280, + image_height: int = 720, + fps: int = 30): + """Write a multi-frame camera-path JSON understood by ``ns-render``.""" + fov = 75.0 + aspect = image_width / image_height + seconds = len(poses) / fps + + payload = { + "camera_type": "perspective", + "render_height": image_height, + "render_width": image_width, + "camera_path": [ + c2w_to_camera_path_entry(c2w, fov=fov, aspect=aspect) + for c2w in poses + ], + "fps": fps, + "seconds": seconds, + "is_cycle": False, + "smoothness_value": 0.0, + } + with open(path, "w") as f: + json.dump(payload, f, indent=2) + + +# --------------------------------------------------------------------------- +# Rendering and encoding +# --------------------------------------------------------------------------- + +def render_frames(config_path: str, camera_json: str, + frames_dir: str) -> int: + """ + Invoke ``ns-render camera-path`` to render frames into *frames_dir*. + Returns the process exit code. + """ + cmd = [ + "ns-render", "camera-path", + "--load-config", str(config_path), + "--camera-path-filename", str(camera_json), + "--output-path", str(Path(frames_dir) / "render.mp4"), + "--image-format", "png", + "--output-format", "images", + ] + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + print(f"ns-render failed (exit {result.returncode}):", file=sys.stderr) + print(result.stderr, file=sys.stderr) + return result.returncode + + +def encode_video(frames_dir: str, output_path: str, fps: int = 30) -> int: + """ + Encode rendered PNG frames into an H.264 MP4 using ffmpeg. + Returns the process exit code. + """ + # Find frames — ns-render names them like 00000.png, 00001.png, ... + frames = sorted(Path(frames_dir).rglob("*.png")) + if not frames: + print("No rendered frames found for video encoding", file=sys.stderr) + return 1 + + # Determine the pattern (frames may be nested in a subdirectory) + first_frame = frames[0] + pattern_dir = str(first_frame.parent) + # Detect naming: %05d.png + pattern = str(first_frame.parent / "%05d.png") + + cmd = [ + "ffmpeg", "-y", + "-framerate", str(fps), + "-i", pattern, + "-c:v", "libx264", + "-pix_fmt", "yuv420p", + "-crf", "23", + "-movflags", "+faststart", + str(output_path), + ] + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + print(f"ffmpeg failed (exit {result.returncode}):", file=sys.stderr) + print(result.stderr, file=sys.stderr) + return result.returncode + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +def main(): + parser = argparse.ArgumentParser( + description="Render a preview video of a trained Gaussian Splat" + ) + parser.add_argument("--ply_path", type=Path, required=True, + help="Path to the exported splat.ply") + parser.add_argument("--config", type=Path, required=True, + help="Path to the splatfacto config.yml") + parser.add_argument("--output_dir", type=Path, required=True, + help="Directory where preview.mp4 will be written") + parser.add_argument("--num_frames", type=int, default=210, + help="Number of frames to render (default: 210 = 7s at 30fps)") + parser.add_argument("--fps", type=int, default=30, + help="Frames per second (default: 30)") + parser.add_argument("--arc_degrees", type=float, default=270.0, + help="Orbital arc in degrees (default: 270)") + parser.add_argument("--elevation_deg", type=float, default=45.0, + help="Camera elevation angle in degrees (default: 45)") + args = parser.parse_args() + + # --- Compute scene bounds from the PLY --------------------------------- + print(f"Reading PLY: {args.ply_path}") + positions = load_ply_positions(str(args.ply_path)) + centroid, extent = compute_bbox(positions) + print(f" Centroid : {centroid}") + print(f" Extent : {extent}") + print(f" N points : {len(positions)}") + + args.output_dir.mkdir(parents=True, exist_ok=True) + + # --- Generate orbital camera path -------------------------------------- + print(f"Generating orbital camera path ({args.num_frames} frames, " + f"{args.arc_degrees} deg arc, {args.elevation_deg} deg elevation)") + poses = generate_orbital_path( + centroid, extent, + num_frames=args.num_frames, + arc_degrees=args.arc_degrees, + elevation_deg=args.elevation_deg, + ) + + cam_json_path = str(args.output_dir / "_cam_video.json") + write_camera_path_json(cam_json_path, poses, fps=args.fps) + + # --- Render frames ----------------------------------------------------- + with tempfile.TemporaryDirectory() as tmpdir: + print("Rendering frames with ns-render ...") + rc = render_frames(str(args.config), cam_json_path, tmpdir) + if rc != 0: + print(f"WARNING: ns-render failed (exit {rc}); skipping video", + file=sys.stderr) + _cleanup(cam_json_path) + sys.exit(0) + + # --- Encode to MP4 ------------------------------------------------ + output_mp4 = str(args.output_dir / "preview.mp4") + print("Encoding video with ffmpeg ...") + rc2 = encode_video(tmpdir, output_mp4, fps=args.fps) + if rc2 != 0: + print(f"WARNING: ffmpeg encoding failed (exit {rc2})", + file=sys.stderr) + _cleanup(cam_json_path) + sys.exit(0) + + print(f"Preview video saved: {output_mp4}") + + # --- Cleanup ----------------------------------------------------------- + _cleanup(cam_json_path) + + # Exit successfully (best-effort). + sys.exit(0) + + +def _cleanup(cam_json_path: str): + """Remove temporary camera-path JSON.""" + try: + Path(cam_json_path).unlink(missing_ok=True) + except Exception: + pass + + +if __name__ == "__main__": + main() diff --git a/run.py b/run.py index e49d601..96b2987 100644 --- a/run.py +++ b/run.py @@ -197,11 +197,33 @@ def run_cmd(cmd: list): sys.exit(exit_code) logger.info("Converting Splat") - exit_code = run_python_script("convert_ply2splat.py", + exit_code = run_python_script("convert_ply2splat.py", "--input", args.job_root_path / "refined/splatter/splat_rot.ply", "--output", args.job_root_path / "refined/splatter/splat_rot.splat") if exit_code != 0: logger.error("failed to convert splat .ply to .splat") sys.exit(exit_code) - + + logger.info("Rendering Preview Images") + try: + preview_exit = run_python_script("render_previews.py", + "--ply_path", args.job_root_path / "refined/splatter/splat.ply", + "--config", args.job_root_path / "refined/splatter/splatfacto/config.yml", + "--output_dir", args.job_root_path / "refined/splatter") + if preview_exit != 0: + logger.warning("preview rendering returned non-zero exit code; continuing anyway") + except Exception as e: + logger.warning(f"preview rendering failed: {e}; continuing anyway") + + logger.info("Rendering Preview Video") + try: + video_exit = run_python_script("render_preview_video.py", + "--ply_path", args.job_root_path / "refined/splatter/splat.ply", + "--config", args.job_root_path / "refined/splatter/splatfacto/config.yml", + "--output_dir", args.job_root_path / "refined/splatter") + if video_exit != 0: + logger.warning("preview video rendering returned non-zero exit code; continuing anyway") + except Exception as e: + logger.warning(f"preview video rendering failed: {e}; continuing anyway") + sys.exit(exit_code) \ No newline at end of file diff --git a/server/rust/runner/src/lib.rs b/server/rust/runner/src/lib.rs index 217ceec..f1d36f4 100644 --- a/server/rust/runner/src/lib.rs +++ b/server/rust/runner/src/lib.rs @@ -783,9 +783,9 @@ impl compute_runner_api::Runner for HelloRunner { ctx.ctrl .progress(json!({ - "pct": 95, + "pct": 92, "stage": "upload", - "status": "completed", + "status": "splat_uploaded", "uploaded": upload_key.as_str(), "splat_path": splat_abs.display().to_string(), })) @@ -795,15 +795,153 @@ impl compute_runner_api::Runner for HelloRunner { .log_event(json!({ "level": "info", "stage": "upload", - "message": "output uploaded", + "message": "splat output uploaded", "uploaded": upload_key.as_str(), })) .await; + + // Upload preview images (best-effort — failures are logged but do not + // break the pipeline). + let preview_suffix = refined_suffix + .as_deref() + .filter(|s| !s.is_empty()) + .unwrap_or(""); + let previews: &[(&str, &str, &str)] = &[ + ("preview_top.jpg", "splat_preview_top", "refined_splat_preview_top"), + ("preview_angle.jpg", "splat_preview_angle", "refined_splat_preview_angle"), + ]; + let mut uploaded_previews: Vec = Vec::new(); + for (file_name, data_type, name_prefix) in previews { + let preview_path = job_root + .join("refined") + .join("splatter") + .join(file_name); + if !preview_path.exists() { + warn!( + file = %file_name, + "preview image not found; skipping upload" + ); + continue; + } + let preview_name = if preview_suffix.is_empty() { + name_prefix.to_string() + } else if preview_suffix.starts_with('_') { + format!("{name_prefix}{preview_suffix}") + } else { + format!("{name_prefix}_{preview_suffix}") + }; + match ctx + .output + .put_domain_artifact(compute_runner_api::runner::DomainArtifactRequest { + rel_path: preview_name.as_str(), + name: preview_name.as_str(), + data_type, + existing_id: None, + content: compute_runner_api::runner::DomainArtifactContent::File( + &preview_path, + ), + }) + .await + { + Ok(_) => { + uploaded_previews.push(preview_name.clone()); + info!( + name = %preview_name, + data_type = %data_type, + path = %preview_path.display(), + "preview image uploaded" + ); + } + Err(err) => { + warn!( + name = %preview_name, + data_type = %data_type, + error = %err, + "failed to upload preview image; continuing" + ); + } + } + } + + // Upload preview video (best-effort). + let mut uploaded_preview_video: Option = None; + { + let video_path = job_root + .join("refined") + .join("splatter") + .join("preview.mp4"); + if video_path.exists() { + let video_name = if preview_suffix.is_empty() { + "refined_splat_preview_video".to_string() + } else if preview_suffix.starts_with('_') { + format!("refined_splat_preview_video{preview_suffix}") + } else { + format!("refined_splat_preview_video_{preview_suffix}") + }; + match ctx + .output + .put_domain_artifact(compute_runner_api::runner::DomainArtifactRequest { + rel_path: video_name.as_str(), + name: video_name.as_str(), + data_type: "splat_preview_video", + existing_id: None, + content: compute_runner_api::runner::DomainArtifactContent::File( + &video_path, + ), + }) + .await + { + Ok(_) => { + uploaded_preview_video = Some(video_name.clone()); + info!( + name = %video_name, + data_type = "splat_preview_video", + path = %video_path.display(), + "preview video uploaded" + ); + } + Err(err) => { + warn!( + name = %video_name, + error = %err, + "failed to upload preview video; continuing" + ); + } + } + } else { + warn!("preview video not found; skipping upload"); + } + } + + ctx.ctrl + .progress(json!({ + "pct": 95, + "stage": "upload", + "status": "completed", + "uploaded_splat": upload_key.as_str(), + "uploaded_previews": uploaded_previews, + "uploaded_preview_video": uploaded_preview_video, + })) + .await?; + let _ = ctx + .ctrl + .log_event(json!({ + "level": "info", + "stage": "upload", + "message": "all outputs uploaded", + "uploaded_splat": upload_key.as_str(), + "uploaded_previews": uploaded_previews, + "uploaded_preview_video": uploaded_preview_video, + })) + .await; ctx.ctrl .progress(json!({ "progress": 100, "stage": "complete", "status": "succeeded", + "uploaded_splat": upload_key.as_str(), + "uploaded_previews": uploaded_previews, + "uploaded_preview_video": uploaded_preview_video, })) .await?;