ns-render crashes with "need at least one array to stack" #3749
Description
Describe the bug
After training using splatfacto, a video is rendered using the command but an error shows.
To Reproduce
Steps to reproduce the behavior:
- Use (600) 5048x3340 processed by colmap
- Train dataset on splatfacto w/ bilateral grid for 30k
['ns-train',
'splatfacto',
'--viewer.quit-on-train-completion=True',
'--logging.local-writer.enable', 'False',
'--logging.profiler', 'none',
'--timestamp', 'train-stage-1',
'--pipeline.model.use-scale-regularization', 'True',
'--max-num-iterations', '30000',
'--pipeline.model.use-bilateral-grid', 'True',
'--pipeline.datamanager.dataloader-num-workers', '0',
'--pipeline.datamanager.cache-images', 'disk',
'colmap',
'--data', '/mnt/data/workflow-input',
'--downscale-factor', '1',
'--auto-scale-poses', 'False'] - Export trajectory video
[
'ns-render',
'interpolate',
'--load-config', 'outputs/unnamed/splatfacto/train-stage-1/config.yml',
'--output-path', '/mnt/data/workflow-input/exports/render.mp4',
'--frame-rate', '10']
Expected behavior
No error, output the video.
Screenshots
If applicable, add screenshots to help explain your problem.
Additional context
INFO 03/20/2026 12:56:19 AM Component command: ['ns-train', 'splatfacto', '--viewer.quit-on-train-completion=True', '--logging.local-writer.enable', 'False', '--logging.profiler', 'none',
'--timestamp', 'train-stage-1', '--pipeline.model.use-scale-regularization', 'True', '--max-num-iterations', '30000', '--pipeline.model.use-bilateral-grid', 'True',
'--pipeline.datamanager.dataloader-num-workers', '0', '--pipeline.datamanager.cache-images', 'disk', 'colmap', '--data', '/mnt/data/workflow-input', '--downscale-factor', '1',
'--auto-scale-poses', 'False']
/opt/ml/code/nerfstudio/nerfstudio/process_data/process_data_utils.py:205: SyntaxWarning: invalid escape sequence '\,'
select_cmd = "select='" + "+".join([f"eq(n\,{idx})" for idx in frame_indices]) + "',setpts=N/TB,"
/opt/ml/code/nerfstudio/nerfstudio/field_components/activations.py:32: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
@custom_fwd(cast_inputs=torch.float32)
/opt/ml/code/nerfstudio/nerfstudio/field_components/activations.py:38: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
@custom_bwd
──────────────────────────────────────────────────────── Config ────────────────────────────────────────────────────────
TrainerConfig(
_target=<class 'nerfstudio.engine.trainer.Trainer'>,
output_dir=PosixPath('outputs'),
method_name='splatfacto',
experiment_name=None,
project_name='nerfstudio-project',
timestamp='train-stage-1',
machine=MachineConfig(seed=42, num_devices=1, num_machines=1, machine_rank=0, dist_url='auto', device_type='cuda'),
logging=LoggingConfig(
relative_log_dir=PosixPath('.'),
steps_per_log=10,
max_buffer_size=20,
local_writer=LocalWriterConfig(
_target=<class 'nerfstudio.utils.writer.LocalWriter'>,
enable=False,
stats_to_track=(
<EventName.ITER_TRAIN_TIME: 'Train Iter (time)'>,
<EventName.TRAIN_RAYS_PER_SEC: 'Train Rays / Sec'>,
<EventName.CURR_TEST_PSNR: 'Test PSNR'>,
<EventName.VIS_RAYS_PER_SEC: 'Vis Rays / Sec'>,
<EventName.TEST_RAYS_PER_SEC: 'Test Rays / Sec'>,
<EventName.ETA: 'ETA (time)'>
),
max_log_size=10
),
profiler='none'
),
viewer=ViewerConfig(
relative_log_filename='viewer_log_filename.txt',
websocket_port=None,
websocket_port_default=7007,
websocket_host='0.0.0.0',
num_rays_per_chunk=32768,
max_num_display_images=512,
quit_on_train_completion=True,
image_format='jpeg',
jpeg_quality=75,
make_share_url=False,
camera_frustum_scale=0.1,
default_composite_depth=True
),
pipeline=VanillaPipelineConfig(
_target=<class 'nerfstudio.pipelines.base_pipeline.VanillaPipeline'>,
datamanager=FullImageDatamanagerConfig(
_target=<class 'nerfstudio.data.datamanagers.full_images_datamanager.FullImageDatamanager'>,
data=None,
masks_on_gpu=False,
images_on_gpu=False,
dataparser=ColmapDataParserConfig(
_target=<class 'nerfstudio.data.dataparsers.colmap_dataparser.ColmapDataParser'>,
data=PosixPath('/mnt/data/workflow-input'),
scale_factor=1.0,
downscale_factor=1,
downscale_rounding_mode='floor',
tiling_factor=1,
scene_scale=1.0,
orientation_method='up',
center_method='poses',
auto_scale_poses=False,
assume_colmap_world_coordinate_convention=True,
eval_mode='interval',
train_split_fraction=0.9,
eval_interval=8,
depth_unit_scale_factor=0.001,
images_path=PosixPath('images'),
masks_path=None,
depths_path=None,
colmap_path=PosixPath('colmap/sparse/0'),
load_3D_points=True,
max_2D_matches_per_3D_point=0
),
camera_res_scale_factor=1.0,
eval_num_images_to_sample_from=-1,
eval_num_times_to_repeat_images=-1,
cache_images='disk',
cache_images_type='uint8',
max_thread_workers=None,
train_cameras_sampling_strategy='random',
train_cameras_sampling_seed=42,
fps_reset_every=100,
dataloader_num_workers=0,
prefetch_factor=4,
cache_compressed_images=False
),
model=SplatfactoModelConfig(
_target=<class 'nerfstudio.models.splatfacto.SplatfactoModel'>,
enable_collider=True,
collider_params={'near_plane': 2.0, 'far_plane': 6.0},
loss_coefficients={'rgb_loss_coarse': 1.0, 'rgb_loss_fine': 1.0},
eval_num_rays_per_chunk=4096,
prompt=None,
warmup_length=500,
refine_every=100,
resolution_schedule=3000,
background_color='random',
num_downscales=2,
cull_alpha_thresh=0.1,
cull_scale_thresh=0.5,
reset_alpha_every=30,
densify_grad_thresh=0.0008,
use_absgrad=True,
densify_size_thresh=0.01,
n_split_samples=2,
sh_degree_interval=1000,
cull_screen_size=0.15,
split_screen_size=0.05,
stop_screen_size_at=4000,
random_init=False,
num_random=50000,
random_scale=10.0,
ssim_lambda=0.2,
stop_split_at=15000,
sh_degree=3,
use_scale_regularization=True,
max_gauss_ratio=10.0,
output_depth_during_training=False,
rasterize_mode='classic',
camera_optimizer=CameraOptimizerConfig(
_target=<class 'nerfstudio.cameras.camera_optimizers.CameraOptimizer'>,
mode='off',
trans_l2_penalty=0.01,
rot_l2_penalty=0.001,
optimizer=None,
scheduler=None
),
use_bilateral_grid=True,
grid_shape=(16, 16, 8),
color_corrected_metrics=False,
strategy='default',
max_gs_num=1000000,
noise_lr=500000.0,
mcmc_opacity_reg=0.01,
mcmc_scale_reg=0.01
)
),
optimizers={
'means': {
'optimizer': AdamOptimizerConfig(
_target=<class 'torch.optim.adam.Adam'>,
lr=0.00016,
eps=1e-15,
max_norm=None,
weight_decay=0
),
'scheduler': ExponentialDecaySchedulerConfig(
_target=<class 'nerfstudio.engine.schedulers.ExponentialDecayScheduler'>,
lr_pre_warmup=1e-08,
lr_final=1.6e-06,
warmup_steps=0,
max_steps=30000,
ramp='cosine'
)
},
'features_dc': {
'optimizer': AdamOptimizerConfig(
_target=<class 'torch.optim.adam.Adam'>,
lr=0.0025,
eps=1e-15,
max_norm=None,
weight_decay=0
),
'scheduler': None
},
'features_rest': {
'optimizer': AdamOptimizerConfig(
_target=<class 'torch.optim.adam.Adam'>,
lr=0.000125,
eps=1e-15,
max_norm=None,
weight_decay=0
),
'scheduler': None
},
'opacities': {
'optimizer': AdamOptimizerConfig(
_target=<class 'torch.optim.adam.Adam'>,
lr=0.05,
eps=1e-15,
max_norm=None,
weight_decay=0
),
'scheduler': None
},
'scales': {
'optimizer': AdamOptimizerConfig(
_target=<class 'torch.optim.adam.Adam'>,
lr=0.005,
eps=1e-15,
max_norm=None,
weight_decay=0
),
'scheduler': None
},
'quats': {
'optimizer': AdamOptimizerConfig(
_target=<class 'torch.optim.adam.Adam'>,
lr=0.001,
eps=1e-15,
max_norm=None,
weight_decay=0
),
'scheduler': None
},
'camera_opt': {
'optimizer': AdamOptimizerConfig(
_target=<class 'torch.optim.adam.Adam'>,
lr=0.0001,
eps=1e-15,
max_norm=None,
weight_decay=0
),
'scheduler': ExponentialDecaySchedulerConfig(
_target=<class 'nerfstudio.engine.schedulers.ExponentialDecayScheduler'>,
lr_pre_warmup=0,
lr_final=5e-07,
warmup_steps=1000,
max_steps=30000,
ramp='cosine'
)
},
'bilateral_grid': {
'optimizer': AdamOptimizerConfig(
_target=<class 'torch.optim.adam.Adam'>,
lr=0.002,
eps=1e-15,
max_norm=None,
weight_decay=0
),
'scheduler': ExponentialDecaySchedulerConfig(
_target=<class 'nerfstudio.engine.schedulers.ExponentialDecayScheduler'>,
lr_pre_warmup=0,
lr_final=0.0001,
warmup_steps=1000,
max_steps=30000,
ramp='cosine'
)
}
},
vis='viewer',
data=None,
prompt=None,
relative_model_dir=PosixPath('nerfstudio_models'),
load_scheduler=True,
steps_per_save=2000,
steps_per_eval_batch=0,
steps_per_eval_image=100,
steps_per_eval_all_images=1000,
max_num_iterations=30000,
mixed_precision=False,
use_grad_scaler=False,
save_only_latest_checkpoint=True,
load_dir=None,
load_step=None,
load_config=None,
load_checkpoint=None,
log_gradients=False,
gradient_accumulation_steps={},
start_paused=False
)
────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
[00:56:28] Saving config to: outputs/unnamed/splatfacto/train-stage-1/config.yml experiment_config.py:136
/opt/ml/code/nerfstudio/nerfstudio/engine/trainer.py:137: FutureWarning: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead.
self.grad_scaler = GradScaler(enabled=self.use_grad_scaler)
Saving checkpoints to: outputs/unnamed/splatfacto/train-stage-1/nerfstudio_models trainer.py:142
╭──────────────── viser ────────────────╮
│ ╷ │
│ HTTP │ http://localhost:7007 │
│ Websocket │ ws://localhost:7007 │
│ ╵ │
╰───────────────────────────────────────╯
(viser) Passing ['initial_value'] as positional arguments to add_dropdown is deprecated. Please use keyword arguments instead: initial_value=not set
(viser) Passing ['initial_value'] as positional arguments to add_dropdown is deprecated. Please use keyword arguments instead: initial_value=default
(viser) Passing ['initial_value'] as positional arguments to add_dropdown is deprecated. Please use keyword arguments instead: initial_value=not set
(viser) Passing ['initial_value'] as positional arguments to add_dropdown is deprecated. Please use keyword arguments instead: initial_value=default
[NOTE] Not running eval iterations since only viewer is enabled.
Use --vis {wandb, tensorboard, viewer+wandb, viewer+tensorboard} to run with eval.
No Nerfstudio checkpoint to load, so training from scratch.
Disabled comet/tensorboard/wandb event writers
[00:56:37] disabled local writer writer.py:186
Step 600: 0 GSs duplicated, 3463 GSs split. Now having 7140 GSs.
Step 600: 822 GSs pruned. Now having 6318 GSs.
Step 700: 0 GSs duplicated, 5817 GSs split. Now having 12135 GSs.
Step 700: 1212 GSs pruned. Now having 10923 GSs.
Step 800: 1 GSs duplicated, 9891 GSs split. Now having 20815 GSs.
Step 800: 1568 GSs pruned. Now having 19247 GSs.
Step 900: 14 GSs duplicated, 16857 GSs split. Now having 36118 GSs.
Step 900: 2069 GSs pruned. Now having 34049 GSs.
Step 1000: 66 GSs duplicated, 26954 GSs split. Now having 61069 GSs.
Step 1000: 3118 GSs pruned. Now having 57951 GSs.
Step 1100: 248 GSs duplicated, 40002 GSs split. Now having 98201 GSs.
Step 1100: 4380 GSs pruned. Now having 93821 GSs.
Step 1200: 702 GSs duplicated, 49860 GSs split. Now having 144383 GSs.
Step 1200: 6053 GSs pruned. Now having 138330 GSs.
Step 1300: 807 GSs duplicated, 43964 GSs split. Now having 183101 GSs.
Step 1300: 8759 GSs pruned. Now having 174342 GSs.
Step 1400: 1583 GSs duplicated, 46594 GSs split. Now having 222519 GSs.
Step 1400: 5643 GSs pruned. Now having 216876 GSs.
Step 1500: 1923 GSs duplicated, 40003 GSs split. Now having 258802 GSs.
Step 1500: 6354 GSs pruned. Now having 252448 GSs.
Step 1600: 1994 GSs duplicated, 33571 GSs split. Now having 288013 GSs.
Step 1600: 5994 GSs pruned. Now having 282019 GSs.
Step 1700: 1806 GSs duplicated, 28402 GSs split. Now having 312227 GSs.
Step 1700: 5384 GSs pruned. Now having 306843 GSs.
Step 1800: 1589 GSs duplicated, 26324 GSs split. Now having 334756 GSs.
Step 1800: 4472 GSs pruned. Now having 330284 GSs.
Step 1900: 1413 GSs duplicated, 26031 GSs split. Now having 357728 GSs.
Step 1900: 4083 GSs pruned. Now having 353645 GSs.
Step 2000: 1369 GSs duplicated, 26592 GSs split. Now having 381606 GSs.
Step 2000: 3546 GSs pruned. Now having 378060 GSs.
Step 2100: 1231 GSs duplicated, 26130 GSs split. Now having 405421 GSs.
Step 2100: 3207 GSs pruned. Now having 402214 GSs.
Step 2200: 1185 GSs duplicated, 23102 GSs split. Now having 426501 GSs.
Step 2200: 2817 GSs pruned. Now having 423684 GSs.
Step 2300: 1157 GSs duplicated, 18645 GSs split. Now having 443486 GSs.
Step 2300: 2439 GSs pruned. Now having 441047 GSs.
Step 2400: 960 GSs duplicated, 12655 GSs split. Now having 454662 GSs.
Step 2400: 2161 GSs pruned. Now having 452501 GSs.
Step 2500: 864 GSs duplicated, 8545 GSs split. Now having 461910 GSs.
Step 2500: 1622 GSs pruned. Now having 460288 GSs.
Step 2600: 767 GSs duplicated, 5726 GSs split. Now having 466781 GSs.
Step 2600: 1242 GSs pruned. Now having 465539 GSs.
Step 2700: 673 GSs duplicated, 4731 GSs split. Now having 470943 GSs.
Step 2700: 934 GSs pruned. Now having 470009 GSs.
Step 2800: 668 GSs duplicated, 3818 GSs split. Now having 474495 GSs.
Step 2800: 740 GSs pruned. Now having 473755 GSs.
Step 2900: 577 GSs duplicated, 3301 GSs split. Now having 477633 GSs.
Step 2900: 626 GSs pruned. Now having 477007 GSs.
Step 3200: 3141 GSs duplicated, 124927 GSs split. Now having 605075 GSs.
Step 3200: 42380 GSs pruned. Now having 562695 GSs.
Step 3300: 9204 GSs duplicated, 151910 GSs split. Now having 723809 GSs.
Step 3300: 19792 GSs pruned. Now having 704017 GSs.
Step 3400: 5178 GSs duplicated, 77751 GSs split. Now having 786946 GSs.
Step 3400: 23468 GSs pruned. Now having 763478 GSs.
Step 3500: 3265 GSs duplicated, 46568 GSs split. Now having 813311 GSs.
Step 3500: 14770 GSs pruned. Now having 798541 GSs.
Step 3600: 2192 GSs duplicated, 30233 GSs split. Now having 830966 GSs.
Step 3600: 9871 GSs pruned. Now having 821095 GSs.
Step 3700: 1534 GSs duplicated, 20831 GSs split. Now having 843460 GSs.
Step 3700: 7091 GSs pruned. Now having 836369 GSs.
Step 3800: 1226 GSs duplicated, 16311 GSs split. Now having 853906 GSs.
Step 3800: 5270 GSs pruned. Now having 848636 GSs.
Step 3900: 959 GSs duplicated, 12883 GSs split. Now having 862478 GSs.
Step 3900: 4202 GSs pruned. Now having 858276 GSs.
Step 4000: 885 GSs duplicated, 9956 GSs split. Now having 869117 GSs.
Step 4000: 3913 GSs pruned. Now having 865204 GSs.
Step 4100: 676 GSs duplicated, 8263 GSs split. Now having 874143 GSs.
Step 4100: 3300 GSs pruned. Now having 870843 GSs.
Step 4200: 631 GSs duplicated, 8000 GSs split. Now having 879474 GSs.
Step 4200: 2763 GSs pruned. Now having 876711 GSs.
Step 4300: 533 GSs duplicated, 6206 GSs split. Now having 883450 GSs.
Step 4300: 2573 GSs pruned. Now having 880877 GSs.
Step 4400: 437 GSs duplicated, 5311 GSs split. Now having 886625 GSs.
Step 4400: 2321 GSs pruned. Now having 884304 GSs.
Step 4500: 415 GSs duplicated, 5190 GSs split. Now having 889909 GSs.
Step 4500: 2009 GSs pruned. Now having 887900 GSs.
Step 4600: 365 GSs duplicated, 4723 GSs split. Now having 892988 GSs.
Step 4600: 1930 GSs pruned. Now having 891058 GSs.
Step 4700: 367 GSs duplicated, 5130 GSs split. Now having 896555 GSs.
Step 4700: 1813 GSs pruned. Now having 894742 GSs.
Step 4800: 400 GSs duplicated, 5208 GSs split. Now having 900350 GSs.
Step 4800: 1763 GSs pruned. Now having 898587 GSs.
Step 4900: 307 GSs duplicated, 4159 GSs split. Now having 903053 GSs.
Step 4900: 1777 GSs pruned. Now having 901276 GSs.
Step 5000: 345 GSs duplicated, 5027 GSs split. Now having 906648 GSs.
Step 5000: 1579 GSs pruned. Now having 905069 GSs.
Step 5100: 276 GSs duplicated, 3672 GSs split. Now having 909017 GSs.
Step 5100: 1746 GSs pruned. Now having 907271 GSs.
Step 5200: 317 GSs duplicated, 4128 GSs split. Now having 911716 GSs.
Step 5200: 1427 GSs pruned. Now having 910289 GSs.
Step 5300: 270 GSs duplicated, 3578 GSs split. Now having 914137 GSs.
Step 5300: 1522 GSs pruned. Now having 912615 GSs.
Step 5400: 264 GSs duplicated, 3466 GSs split. Now having 916345 GSs.
Step 5400: 1422 GSs pruned. Now having 914923 GSs.
Step 5500: 245 GSs duplicated, 3152 GSs split. Now having 918320 GSs.
Step 5500: 1453 GSs pruned. Now having 916867 GSs.
Step 5600: 284 GSs duplicated, 3665 GSs split. Now having 920816 GSs.
Step 5600: 1370 GSs pruned. Now having 919446 GSs.
Step 5700: 291 GSs duplicated, 3639 GSs split. Now having 923376 GSs.
Step 5700: 1384 GSs pruned. Now having 921992 GSs.
Step 5800: 195 GSs duplicated, 2246 GSs split. Now having 924433 GSs.
Step 5800: 1488 GSs pruned. Now having 922945 GSs.
Step 5900: 189 GSs duplicated, 2660 GSs split. Now having 925794 GSs.
Step 5900: 1175 GSs pruned. Now having 924619 GSs.
Step 6200: 5262 GSs duplicated, 167218 GSs split. Now having 1097099 GSs.
Step 6200: 17149 GSs pruned. Now having 1079950 GSs.
Step 6300: 11312 GSs duplicated, 182879 GSs split. Now having 1274141 GSs.
Step 6300: 36508 GSs pruned. Now having 1237633 GSs.
Step 6400: 4624 GSs duplicated, 83917 GSs split. Now having 1326174 GSs.
Step 6400: 40524 GSs pruned. Now having 1285650 GSs.
Step 6500: 2518 GSs duplicated, 50259 GSs split. Now having 1338427 GSs.
Step 6500: 22576 GSs pruned. Now having 1315851 GSs.
Step 6600: 1674 GSs duplicated, 34122 GSs split. Now having 1351647 GSs.
Step 6600: 15321 GSs pruned. Now having 1336326 GSs.
Step 6700: 1190 GSs duplicated, 25590 GSs split. Now having 1363106 GSs.
Step 6700: 11511 GSs pruned. Now having 1351595 GSs.
Step 6800: 879 GSs duplicated, 20835 GSs split. Now having 1373309 GSs.
Step 6800: 9086 GSs pruned. Now having 1364223 GSs.
Step 6900: 781 GSs duplicated, 17902 GSs split. Now having 1382906 GSs.
Step 6900: 7820 GSs pruned. Now having 1375086 GSs.
Step 7000: 680 GSs duplicated, 15900 GSs split. Now having 1391666 GSs.
Step 7000: 6695 GSs pruned. Now having 1384971 GSs.
Step 7100: 551 GSs duplicated, 14392 GSs split. Now having 1399914 GSs.
Step 7100: 6206 GSs pruned. Now having 1393708 GSs.
Step 7200: 529 GSs duplicated, 13285 GSs split. Now having 1407522 GSs.
Step 7200: 5752 GSs pruned. Now having 1401770 GSs.
Step 7300: 473 GSs duplicated, 12234 GSs split. Now having 1414477 GSs.
Step 7300: 5316 GSs pruned. Now having 1409161 GSs.
Step 7400: 421 GSs duplicated, 11507 GSs split. Now having 1421089 GSs.
Step 7400: 5065 GSs pruned. Now having 1416024 GSs.
Step 7500: 413 GSs duplicated, 10903 GSs split. Now having 1427340 GSs.
Step 7500: 4719 GSs pruned. Now having 1422621 GSs.
Step 7600: 367 GSs duplicated, 10111 GSs split. Now having 1433099 GSs.
Step 7600: 4471 GSs pruned. Now having 1428628 GSs.
Step 7700: 329 GSs duplicated, 9426 GSs split. Now having 1438383 GSs.
Step 7700: 4418 GSs pruned. Now having 1433965 GSs.
Step 7800: 320 GSs duplicated, 9544 GSs split. Now having 1443829 GSs.
Step 7800: 4231 GSs pruned. Now having 1439598 GSs.
Step 7900: 344 GSs duplicated, 9157 GSs split. Now having 1449099 GSs.
Step 7900: 4201 GSs pruned. Now having 1444898 GSs.
Step 8000: 288 GSs duplicated, 8691 GSs split. Now having 1453877 GSs.
Step 8000: 3974 GSs pruned. Now having 1449903 GSs.
Step 8100: 311 GSs duplicated, 8682 GSs split. Now having 1458896 GSs.
Step 8100: 3862 GSs pruned. Now having 1455034 GSs.
Step 8200: 261 GSs duplicated, 7910 GSs split. Now having 1463205 GSs.
Step 8200: 3890 GSs pruned. Now having 1459315 GSs.
Step 8300: 287 GSs duplicated, 8388 GSs split. Now having 1467990 GSs.
Step 8300: 3618 GSs pruned. Now having 1464372 GSs.
Step 8400: 274 GSs duplicated, 7909 GSs split. Now having 1472555 GSs.
Step 8400: 3653 GSs pruned. Now having 1468902 GSs.
Step 8500: 237 GSs duplicated, 7369 GSs split. Now having 1476508 GSs.
Step 8500: 3550 GSs pruned. Now having 1472958 GSs.
Step 8600: 292 GSs duplicated, 7839 GSs split. Now having 1481089 GSs.
Step 8600: 3468 GSs pruned. Now having 1477621 GSs.
Step 8700: 238 GSs duplicated, 6935 GSs split. Now having 1484794 GSs.
Step 8700: 3565 GSs pruned. Now having 1481229 GSs.
Step 8800: 239 GSs duplicated, 6896 GSs split. Now having 1488364 GSs.
Step 8800: 3385 GSs pruned. Now having 1484979 GSs.
Step 8900: 223 GSs duplicated, 6717 GSs split. Now having 1491919 GSs.
Step 8900: 3255 GSs pruned. Now having 1488664 GSs.
Step 9200: 101 GSs duplicated, 6044 GSs split. Now having 1494809 GSs.
Step 9200: 12278 GSs pruned. Now having 1482531 GSs.
Step 9300: 152 GSs duplicated, 6798 GSs split. Now having 1489481 GSs.
Step 9300: 2896 GSs pruned. Now having 1486585 GSs.
Step 9400: 131 GSs duplicated, 5708 GSs split. Now having 1492424 GSs.
Step 9400: 3139 GSs pruned. Now having 1489285 GSs.
Step 9500: 119 GSs duplicated, 5572 GSs split. Now having 1494976 GSs.
Step 9500: 2837 GSs pruned. Now having 1492139 GSs.
Step 9600: 136 GSs duplicated, 5570 GSs split. Now having 1497845 GSs.
Step 9600: 2823 GSs pruned. Now having 1495022 GSs.
Step 9700: 135 GSs duplicated, 5516 GSs split. Now having 1500673 GSs.
Step 9700: 2842 GSs pruned. Now having 1497831 GSs.
Step 9800: 153 GSs duplicated, 5746 GSs split. Now having 1503730 GSs.
Step 9800: 2796 GSs pruned. Now having 1500934 GSs.
Step 9900: 138 GSs duplicated, 5273 GSs split. Now having 1506345 GSs.
Step 9900: 2905 GSs pruned. Now having 1503440 GSs.
Step 10000: 137 GSs duplicated, 4801 GSs split. Now having 1508378 GSs.
Step 10000: 2710 GSs pruned. Now having 1505668 GSs.
Step 10100: 167 GSs duplicated, 5398 GSs split. Now having 1511233 GSs.
Step 10100: 2693 GSs pruned. Now having 1508540 GSs.
Step 10200: 132 GSs duplicated, 5407 GSs split. Now having 1514079 GSs.
Step 10200: 2786 GSs pruned. Now having 1511293 GSs.
Step 10300: 143 GSs duplicated, 5023 GSs split. Now having 1516459 GSs.
Step 10300: 2718 GSs pruned. Now having 1513741 GSs.
Step 10400: 156 GSs duplicated, 5213 GSs split. Now having 1519110 GSs.
Step 10400: 2634 GSs pruned. Now having 1516476 GSs.
Step 10500: 149 GSs duplicated, 5331 GSs split. Now having 1521956 GSs.
Step 10500: 2695 GSs pruned. Now having 1519261 GSs.
Step 10600: 144 GSs duplicated, 5230 GSs split. Now having 1524635 GSs.
Step 10600: 2793 GSs pruned. Now having 1521842 GSs.
Step 10700: 151 GSs duplicated, 5499 GSs split. Now having 1527492 GSs.
Step 10700: 2727 GSs pruned. Now having 1524765 GSs.
Step 10800: 190 GSs duplicated, 5571 GSs split. Now having 1530526 GSs.
Step 10800: 2853 GSs pruned. Now having 1527673 GSs.
Step 10900: 141 GSs duplicated, 4926 GSs split. Now having 1532740 GSs.
Step 10900: 2933 GSs pruned. Now having 1529807 GSs.
Step 11000: 146 GSs duplicated, 4917 GSs split. Now having 1534870 GSs.
Step 11000: 2719 GSs pruned. Now having 1532151 GSs.
Step 11100: 194 GSs duplicated, 5136 GSs split. Now having 1537481 GSs.
Step 11100: 2638 GSs pruned. Now having 1534843 GSs.
Step 11200: 137 GSs duplicated, 4323 GSs split. Now having 1539303 GSs.
Step 11200: 2654 GSs pruned. Now having 1536649 GSs.
Step 11300: 126 GSs duplicated, 4298 GSs split. Now having 1541073 GSs.
Step 11300: 2468 GSs pruned. Now having 1538605 GSs.
Step 11400: 105 GSs duplicated, 4376 GSs split. Now having 1543086 GSs.
Step 11400: 2400 GSs pruned. Now having 1540686 GSs.
Step 11500: 114 GSs duplicated, 4195 GSs split. Now having 1544995 GSs.
Step 11500: 2500 GSs pruned. Now having 1542495 GSs.
Step 11600: 135 GSs duplicated, 4432 GSs split. Now having 1547062 GSs.
Step 11600: 2413 GSs pruned. Now having 1544649 GSs.
Step 11700: 113 GSs duplicated, 4585 GSs split. Now having 1549347 GSs.
Step 11700: 2516 GSs pruned. Now having 1546831 GSs.
Step 11800: 155 GSs duplicated, 4768 GSs split. Now having 1551754 GSs.
Step 11800: 2580 GSs pruned. Now having 1549174 GSs.
Step 11900: 143 GSs duplicated, 4653 GSs split. Now having 1553970 GSs.
Step 11900: 2619 GSs pruned. Now having 1551351 GSs.
Step 12200: 73 GSs duplicated, 4271 GSs split. Now having 1555695 GSs.
Step 12200: 10794 GSs pruned. Now having 1544901 GSs.
Step 12300: 73 GSs duplicated, 3854 GSs split. Now having 1548828 GSs.
Step 12300: 2353 GSs pruned. Now having 1546475 GSs.
Step 12400: 63 GSs duplicated, 3329 GSs split. Now having 1549867 GSs.
Step 12400: 2199 GSs pruned. Now having 1547668 GSs.
Step 12500: 76 GSs duplicated, 3437 GSs split. Now having 1551181 GSs.
Step 12500: 2107 GSs pruned. Now having 1549074 GSs.
Step 12600: 97 GSs duplicated, 3672 GSs split. Now having 1552843 GSs.
Step 12600: 2110 GSs pruned. Now having 1550733 GSs.
Step 12700: 99 GSs duplicated, 3599 GSs split. Now having 1554431 GSs.
Step 12700: 2173 GSs pruned. Now having 1552258 GSs.
Step 12800: 99 GSs duplicated, 3591 GSs split. Now having 1555948 GSs.
Step 12800: 2108 GSs pruned. Now having 1553840 GSs.
Step 12900: 110 GSs duplicated, 3870 GSs split. Now having 1557820 GSs.
Step 12900: 2153 GSs pruned. Now having 1555667 GSs.
Step 13000: 103 GSs duplicated, 3653 GSs split. Now having 1559423 GSs.
Step 13000: 2337 GSs pruned. Now having 1557086 GSs.
Step 13100: 98 GSs duplicated, 3731 GSs split. Now having 1560915 GSs.
Step 13100: 2280 GSs pruned. Now having 1558635 GSs.
Step 13200: 100 GSs duplicated, 4104 GSs split. Now having 1562839 GSs.
Step 13200: 2176 GSs pruned. Now having 1560663 GSs.
Step 13300: 110 GSs duplicated, 3853 GSs split. Now having 1564626 GSs.
Step 13300: 2357 GSs pruned. Now having 1562269 GSs.
Step 13400: 113 GSs duplicated, 3958 GSs split. Now having 1566340 GSs.
Step 13400: 2327 GSs pruned. Now having 1564013 GSs.
Step 13500: 133 GSs duplicated, 4087 GSs split. Now having 1568233 GSs.
Step 13500: 2416 GSs pruned. Now having 1565817 GSs.
Step 13600: 103 GSs duplicated, 3524 GSs split. Now having 1569444 GSs.
Step 13600: 2351 GSs pruned. Now having 1567093 GSs.
Step 13700: 136 GSs duplicated, 3916 GSs split. Now having 1571145 GSs.
Step 13700: 2238 GSs pruned. Now having 1568907 GSs.
Step 13800: 102 GSs duplicated, 3502 GSs split. Now having 1572511 GSs.
Step 13800: 2282 GSs pruned. Now having 1570229 GSs.
Step 13900: 100 GSs duplicated, 3667 GSs split. Now having 1573996 GSs.
Step 13900: 2222 GSs pruned. Now having 1571774 GSs.
Step 14000: 108 GSs duplicated, 3845 GSs split. Now having 1575727 GSs.
Step 14000: 2308 GSs pruned. Now having 1573419 GSs.
Step 14100: 118 GSs duplicated, 3710 GSs split. Now having 1577247 GSs.
Step 14100: 2289 GSs pruned. Now having 1574958 GSs.
Step 14200: 127 GSs duplicated, 4128 GSs split. Now having 1579213 GSs.
Step 14200: 2284 GSs pruned. Now having 1576929 GSs.
Step 14300: 110 GSs duplicated, 3723 GSs split. Now having 1580762 GSs.
Step 14300: 2487 GSs pruned. Now having 1578275 GSs.
Step 14400: 113 GSs duplicated, 3655 GSs split. Now having 1582043 GSs.
Step 14400: 2286 GSs pruned. Now having 1579757 GSs.
Step 14500: 114 GSs duplicated, 3732 GSs split. Now having 1583603 GSs.
Step 14500: 2360 GSs pruned. Now having 1581243 GSs.
Step 14600: 97 GSs duplicated, 3643 GSs split. Now having 1584983 GSs.
Step 14600: 2292 GSs pruned. Now having 1582691 GSs.
Step 14700: 125 GSs duplicated, 3857 GSs split. Now having 1586673 GSs.
Step 14700: 2348 GSs pruned. Now having 1584325 GSs.
Step 14800: 118 GSs duplicated, 3584 GSs split. Now having 1588027 GSs.
Step 14800: 2430 GSs pruned. Now having 1585597 GSs.
Step 14900: 118 GSs duplicated, 3594 GSs split. Now having 1589309 GSs.
Step 14900: 2233 GSs pruned. Now having 1587076 GSs.
╭─────────────────────────────── 🎉 Training Finished 🎉 ───────────────────────────────╮
│ ╷ │
│ Config File │ outputs/unnamed/splatfacto/train-stage-1/config.yml │
│ Checkpoint Directory │ outputs/unnamed/splatfacto/train-stage-1/nerfstudio_models │
│ ╵ │
╰───────────────────────────────────────────────────────────────────────────────────────╯
[W320 06:58:18.682105649 AllocatorConfig.cpp:28] Warning: PYTORCH_CUDA_ALLOC_CONF is deprecated, use PYTORCH_ALLOC_CONF instead (function operator())
03/20/2026 06:58:18 AM INFO 03/20/2026 06:58:18 AM Training complete. Brief pause before export... main.py:2872
03/20/2026 06:58:23 AM INFO 03/20/2026 06:58:23 AM Copied checkpoint to output for crash recovery: /mnt/data/workflow-input/exports/nerfstudio_models main.py:2889
INFO 03/20/2026 06:58:23 AM Copied config to output for crash recovery: /mnt/data/workflow-input/exports/config.yml main.py:2892
INFO 03/20/2026 06:58:23 AM Running component: Nerfstudio-Export main.py:2608
INFO 03/20/2026 06:58:23 AM Reset CUDA_VISIBLE_DEVICES for export: 0 main.py:2785
INFO 03/20/2026 06:58:23 AM Running component Nerfstudio-Export pipeline.py:180
INFO 03/20/2026 06:58:23 AM Component command: ['ns-export', 'gaussian-splat', '--load-config', 'outputs/unnamed/splatfacto/train-stage-1/config.yml', '--output-dir', '/mnt/data/workflow-input/exports'] pipeline.py:214
/opt/ml/code/nerfstudio/nerfstudio/process_data/process_data_utils.py:205: SyntaxWarning: invalid escape sequence '\,'
select_cmd = "select='" + "+".join([f"eq(n\,{idx})" for idx in frame_indices]) + "',setpts=N/TB,"
/opt/ml/code/nerfstudio/nerfstudio/field_components/activations.py:32: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
@custom_fwd(cast_inputs=torch.float32)
/opt/ml/code/nerfstudio/nerfstudio/field_components/activations.py:38: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
@custom_bwd
Loading latest checkpoint from load_dir
✅ Done loading checkpoint from outputs/unnamed/splatfacto/train-stage-1/nerfstudio_models/step-000029999.ckpt
0 Gaussians have NaN/Inf and 38 have low opacity, only export 1587038/1587076
[W320 06:59:38.961171707 AllocatorConfig.cpp:28] Warning: PYTORCH_CUDA_ALLOC_CONF is deprecated, use PYTORCH_ALLOC_CONF instead (function operator())
03/20/2026 06:59:39 AM INFO 03/20/2026 06:59:39 AM Running component: Nerfstudio-Metrics main.py:2608
INFO 03/20/2026 06:59:39 AM Running component Nerfstudio-Metrics pipeline.py:180
INFO 03/20/2026 06:59:39 AM Component command: ['ns-eval', '--load-config', 'outputs/unnamed/splatfacto/train-stage-1/config.yml', '--output-path', '/opt/ml/model/dataset/eval/metrics.json'] pipeline.py:214
/opt/ml/code/nerfstudio/nerfstudio/process_data/process_data_utils.py:205: SyntaxWarning: invalid escape sequence '\,'
select_cmd = "select='" + "+".join([f"eq(n\,{idx})" for idx in frame_indices]) + "',setpts=N/TB,"
/opt/ml/code/nerfstudio/nerfstudio/field_components/activations.py:32: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
@custom_fwd(cast_inputs=torch.float32)
/opt/ml/code/nerfstudio/nerfstudio/field_components/activations.py:38: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
@custom_bwd
Loading latest checkpoint from load_dir
✅ Done loading checkpoint from outputs/unnamed/splatfacto/train-stage-1/nerfstudio_models/step-000029999.ckpt
/opt/ml/code/nerfstudio/nerfstudio/pipelines/base_pipeline.py:406: UserWarning: std_mean(): degrees of freedom is <= 0. Correction should be strictly less than the reduction factor (input numel divided by output numel). (Triggered internally at /pytorch/aten/src/ATen/native/ReduceOps.cpp:1857.)
key_std, key_mean = torch.std_mean(
Saved results to: /opt/ml/model/dataset/eval/metrics.json
[W320 06:59:51.168663145 AllocatorConfig.cpp:28] Warning: PYTORCH_CUDA_ALLOC_CONF is deprecated, use PYTORCH_ALLOC_CONF instead (function operator())
03/20/2026 06:59:51 AM INFO 03/20/2026 06:59:51 AM Evaluation Metrics - PSNR: 12.1217, SSIM: 0.1234, LPIPS: 0.4417 main.py:3099
INFO 03/20/2026 06:59:51 AM Running component: Ply-to-Video main.py:2608
INFO 03/20/2026 06:59:51 AM POST_PROCESSING started main.py:2631
INFO 03/20/2026 06:59:51 AM Running component Ply-to-Video pipeline.py:180
INFO 03/20/2026 06:59:51 AM Component command: ['ns-render', 'interpolate', '--load-config', 'outputs/unnamed/splatfacto/train-stage-1/config.yml', '--output-path', pipeline.py:214
'/mnt/data/workflow-input/exports/render.mp4', '--frame-rate', '10']
/opt/ml/code/nerfstudio/nerfstudio/process_data/process_data_utils.py:205: SyntaxWarning: invalid escape sequence '\,'
select_cmd = "select='" + "+".join([f"eq(n\,{idx})" for idx in frame_indices]) + "',setpts=N/TB,"
/opt/ml/code/nerfstudio/nerfstudio/field_components/activations.py:32: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
@custom_fwd(cast_inputs=torch.float32)
/opt/ml/code/nerfstudio/nerfstudio/field_components/activations.py:38: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
@custom_bwd
Loading latest checkpoint from load_dir
✅ Done loading checkpoint from outputs/unnamed/splatfacto/train-stage-1/nerfstudio_models/step-000029999.ckpt
Traceback (most recent call last):
File "/usr/local/bin/ns-render", line 8, in <module>
sys.exit(entrypoint())
^^^^^^^^^^^^
File "/opt/ml/code/nerfstudio/nerfstudio/scripts/render.py", line 917, in entrypoint
tyro.cli(Commands).main()
File "/opt/ml/code/nerfstudio/nerfstudio/scripts/render.py", line 624, in main
camera_path = get_interpolated_camera_path(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/ml/code/nerfstudio/nerfstudio/cameras/camera_paths.py", line 43, in get_interpolated_camera_path
poses, Ks, times = get_interpolated_poses_many(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/ml/code/nerfstudio/nerfstudio/cameras/camera_utils.py", line 303, in get_interpolated_poses_many
traj = np.stack(traj, axis=0)
^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/site-packages/numpy/core/shape_base.py", line 445, in stack
raise ValueError('need at least one array to stack')
ValueError: need at least one array to stack
[W320 07:00:01.951549783 AllocatorConfig.cpp:28] Warning: PYTORCH_CUDA_ALLOC_CONF is deprecated, use PYTORCH_ALLOC_CONF instead (function operator())
Command 'ns-render interpolate --load-config outputs/unnamed/splatfacto/train-stage-1/config.yml --output-path /mnt/data/workflow-input/exports/render.mp4 --frame-rate 10' failed with return code 1
[W320 07:00:02.751128137 AllocatorConfig.cpp:28] Warning: PYTORCH_CUDA_ALLOC_CONF is deprecated, use PYTORCH_ALLOC_CONF instead (function operator())
(base) ubuntu@ip-172-31-3-20:/mnt/efs/guidance-for-open-source-3d-reconstruction-toolbox-for-gaussian-splats-on-aws/source/container$