From 36780db507808d87c2f86984f528151c7bee8d5c Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Mon, 13 Apr 2026 10:42:33 -0400 Subject: [PATCH 1/3] err_out_screen: send traceback to LOG and to stderr --- .../NextGen_Forcings_Engine/core/err_handler.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/err_handler.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/err_handler.py index fcbfc9db..7f492b1a 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/err_handler.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/err_handler.py @@ -12,6 +12,12 @@ LOG = ewts.get_logger(ewts.FORCING_ID) +def in_exception_context() -> bool: + if sys.exc_info()[0] is not None: + return True + return False + + def err_out_screen(err_msg: str, exc: BaseException | None = None): """Print an error message to the screen and exit the program gracefully. @@ -27,8 +33,15 @@ def err_out_screen(err_msg: str, exc: BaseException | None = None): if exc is not None: err_msg += f" - {exc}" err_msg_out = "ERROR: " + err_msg + print(err_msg_out, flush=True) - traceback.print_exc() # Only prints if an exception is currently being handled + LOG.critical(err_msg_out) + + if in_exception_context(): + tb = traceback.format_exc() + LOG.critical(f"TRACEBACK: {tb}") + print(tb, flush=True, file=sys.stderr) + sys.exit(1) From be73f71c30457de01ff29c30faaa8748f878025f Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Mon, 13 Apr 2026 10:44:05 -0400 Subject: [PATCH 2/3] MR lagged ensemble: fix forcing config files ForecastInputHorizons --- .../config_templates/medium_range_mem2_config.yml | 2 +- .../config_templates/medium_range_mem3_config.yml | 2 +- .../config_templates/medium_range_mem4_config.yml | 2 +- .../config_templates/medium_range_mem5_config.yml | 2 +- .../config_templates/medium_range_mem6_config.yml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem2_config.yml b/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem2_config.yml index 4410bd60..0465be73 100644 --- a/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem2_config.yml +++ b/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem2_config.yml @@ -18,7 +18,7 @@ LookBack : -9999 RefcstBDateProc: "202412020000" ForecastFrequency: 60 ForecastShift: 0 -ForecastInputHorizons: [12246] +ForecastInputHorizons: [12600] ForecastInputOffsets: [0] Geopackage: "{root_dir}/gpkg/conus/{gage}.gpkg" GeogridIn: "{root_dir}/esmf_mesh/{gage}_ESMF_Mesh.nc" diff --git a/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem3_config.yml b/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem3_config.yml index d93e2982..d62983b2 100644 --- a/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem3_config.yml +++ b/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem3_config.yml @@ -18,7 +18,7 @@ LookBack : -9999 RefcstBDateProc: "202412020000" ForecastFrequency: 60 ForecastShift: 0 -ForecastInputHorizons: [12252] +ForecastInputHorizons: [12960] ForecastInputOffsets: [0] Geopackage: "{root_dir}/gpkg/conus/{gage}.gpkg" GeogridIn: "{root_dir}/esmf_mesh/{gage}_ESMF_Mesh.nc" diff --git a/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem4_config.yml b/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem4_config.yml index 733a5a93..a3681e6b 100644 --- a/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem4_config.yml +++ b/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem4_config.yml @@ -18,7 +18,7 @@ LookBack : -9999 RefcstBDateProc: "202412020000" ForecastFrequency: 60 ForecastShift: 0 -ForecastInputHorizons: [12258] +ForecastInputHorizons: [13320] ForecastInputOffsets: [0] Geopackage: "{root_dir}/gpkg/conus/{gage}.gpkg" GeogridIn: "{root_dir}/esmf_mesh/{gage}_ESMF_Mesh.nc" diff --git a/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem5_config.yml b/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem5_config.yml index 1e79c788..2d590ee3 100644 --- a/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem5_config.yml +++ b/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem5_config.yml @@ -18,7 +18,7 @@ LookBack : -9999 RefcstBDateProc: "202412020000" ForecastFrequency: 60 ForecastShift: 0 -ForecastInputHorizons: [12264] +ForecastInputHorizons: [13680] ForecastInputOffsets: [0] Geopackage: "{root_dir}/gpkg/conus/{gage}.gpkg" GeogridIn: "{root_dir}/esmf_mesh/{gage}_ESMF_Mesh.nc" diff --git a/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem6_config.yml b/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem6_config.yml index 0a63c730..bbaa72ed 100644 --- a/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem6_config.yml +++ b/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem6_config.yml @@ -18,7 +18,7 @@ LookBack : -9999 RefcstBDateProc: "202412020000" ForecastFrequency: 60 ForecastShift: 0 -ForecastInputHorizons: [12270] +ForecastInputHorizons: [14040] ForecastInputOffsets: [0] Geopackage: "{root_dir}/gpkg/conus/{gage}.gpkg" GeogridIn: "{root_dir}/esmf_mesh/{gage}_ESMF_Mesh.nc" From 534ebdff0c52b812674e592e23820e817b5d9db3 Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Tue, 14 Apr 2026 09:06:43 -0400 Subject: [PATCH 3/3] Improve messages in function err_out_screen --- .../NextGen_Forcings_Engine/core/err_handler.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/err_handler.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/err_handler.py index 7f492b1a..d2af210c 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/err_handler.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/err_handler.py @@ -1,3 +1,4 @@ +import inspect import logging import os import sys @@ -39,8 +40,13 @@ def err_out_screen(err_msg: str, exc: BaseException | None = None): if in_exception_context(): tb = traceback.format_exc() - LOG.critical(f"TRACEBACK: {tb}") - print(tb, flush=True, file=sys.stderr) + tb_msg = f"TRACEBACK: {tb}" + print(tb_msg, flush=True, file=sys.stderr) + LOG.critical(tb_msg) + + final_msg = f"Calling sys.exit(1) from object {repr(inspect.currentframe().f_code.co_name)}" + print(final_msg, flush=True, file=sys.stderr) + LOG.critical(final_msg) sys.exit(1)