From bd789917ce41f254b3838bd0eb0240ecb445e319 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Sat, 15 Nov 2025 08:17:38 -0500 Subject: [PATCH 01/31] Add scheduled obsid --- kadi/commands/commands_v2.py | 11 ++++++++++- kadi/commands/states.py | 9 +++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/kadi/commands/commands_v2.py b/kadi/commands/commands_v2.py index 6c06f249..37ce887a 100644 --- a/kadi/commands/commands_v2.py +++ b/kadi/commands/commands_v2.py @@ -969,7 +969,7 @@ def get_cmds_obs_final( ) else: starcat_idx = cmd["idx"] - elif tlmsid == "COAOSQID": + elif tlmsid == "OBSID": obsid = cmd["params"]["id"] # Look for obsid change within obs, likely an undercover # (target="cold blank ECS"). First stop the initial obs at the time @@ -1452,6 +1452,15 @@ def parse_backstop(load_name: str, backstop_text: str): idx = cmds.colnames.index("timeline_id") cmds.add_column(load_name, index=idx, name="source") del cmds["timeline_id"] + + # Add OBSID load event commands to track the scheduled obsid in the event of an + # SCS-107 where the original COAOSQID commands in the observing loads are dropped. + cmds_obsid = cmds[cmds["tlmsid"] == "COAOSQID"] + cmds_obsid["type"] = "LOAD_EVENT" + cmds_obsid["tlmsid"] = "OBSID" + cmds_obsid["scs"] -= 3 # Move these load event cmds to vehicle loads + cmds = cmds.add_cmds(cmds_obsid) + return cmds diff --git a/kadi/commands/states.py b/kadi/commands/states.py index 08e3ee98..648c8ac4 100644 --- a/kadi/commands/states.py +++ b/kadi/commands/states.py @@ -880,6 +880,15 @@ class ObsidTransition(ParamTransition): cmd_param_key = "id" +class ObsidSchedTransition(ParamTransition): + """Scheduled Obsid update""" + + command_attributes = {"tlmsid": "OBSID"} + state_keys = ["obsid_sched"] + transition_key = "obsid_sched" + cmd_param_key = "id" + + class EclipseEntryTimerTransition(ParamTransition): """Eclipse entry timer update""" From 569f6caf1b00190c6f7a1fbf69bfed2113cbe3ae Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Mon, 17 Nov 2025 10:28:39 -0500 Subject: [PATCH 02/31] Support not matching command blocks in update script --- kadi/scripts/update_cmds_v2.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/kadi/scripts/update_cmds_v2.py b/kadi/scripts/update_cmds_v2.py index 9282c223..a3b2b14b 100644 --- a/kadi/scripts/update_cmds_v2.py +++ b/kadi/scripts/update_cmds_v2.py @@ -30,6 +30,12 @@ def get_opt(args=None): action="version", version="%(prog)s {version}".format(version=__version__), ) + parser.add_argument( + "--no-match-prev-cmds", + action="store_true", + help="Do not enforce matching previous command block when updating cmds v2 " + "(experts only, this can produce an invalid commands table)", + ) args = parser.parse_args(args) return args @@ -48,6 +54,7 @@ def main(args=None): log_level=opt.log_level, scenario=opt.scenario, data_root=opt.data_root, + match_prev_cmds=not opt.no_match_prev_cmds, ) From 948f34c66b8895274b702a6541a600e583ac9276 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Mon, 17 Nov 2025 10:46:42 -0500 Subject: [PATCH 03/31] Add matching-block-size as conf and option --- kadi/commands/commands_v2.py | 8 ++------ kadi/config.py | 1 + kadi/scripts/update_cmds_v2.py | 9 +++++++++ 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/kadi/commands/commands_v2.py b/kadi/commands/commands_v2.py index 37ce887a..3f79518b 100644 --- a/kadi/commands/commands_v2.py +++ b/kadi/commands/commands_v2.py @@ -44,10 +44,6 @@ __all__ = ["clear_caches", "get_cmds"] -# TODO configuration options, but use DEFAULT_* in the mean time - -MATCHING_BLOCK_SIZE = 500 - # TODO: cache translation from cmd_events to CommandTable's [Probably not] # Formally approved load products @@ -1619,11 +1615,11 @@ def get_matching_block_idx(cmds_arch, cmds_recent): logger.info(" {}".format(opcode)) # Find the first matching block that is sufficiently long for block in matching_blocks: - if block.size > MATCHING_BLOCK_SIZE: + if block.size > conf.matching_block_size: break else: raise ValueError( - f"No matching blocks at least {MATCHING_BLOCK_SIZE} long. This most likely " + f"No matching blocks at least {conf.matching_block_size} long. This most likely " "means that you have not recently synced your local Ska data using `ska_sync`." ) diff --git a/kadi/config.py b/kadi/config.py index 9bfeba03..d9128b7e 100644 --- a/kadi/config.py +++ b/kadi/config.py @@ -101,6 +101,7 @@ class Conf(ConfigNamespace): "step size may be smaller." ), ) + matching_block_size = ConfigItem(500, "Matching block size for command blocks.") # Create a configuration instance for the user diff --git a/kadi/scripts/update_cmds_v2.py b/kadi/scripts/update_cmds_v2.py index a3b2b14b..4660dd77 100644 --- a/kadi/scripts/update_cmds_v2.py +++ b/kadi/scripts/update_cmds_v2.py @@ -5,6 +5,7 @@ from kadi import __version__ from kadi.commands.commands_v2 import update_cmds_archive +from kadi.config import conf def get_opt(args=None): @@ -36,6 +37,11 @@ def get_opt(args=None): help="Do not enforce matching previous command block when updating cmds v2 " "(experts only, this can produce an invalid commands table)", ) + parser.add_argument( + "--matching-block-size", + type=int, + help=f"Matching block size (default={conf.matching_block_size})", + ) args = parser.parse_args(args) return args @@ -48,6 +54,9 @@ def main(args=None): opt = get_opt(args) log_run_info(log_func=print, opt=opt) + if opt.matching_block_size is not None: + conf.matching_block_size = opt.matching_block_size + update_cmds_archive( lookback=opt.lookback, stop=opt.stop, From 8e726b7ae31efa0c6fdb300997bc2d8b7c74e3d4 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Mon, 17 Nov 2025 15:41:05 -0500 Subject: [PATCH 04/31] Show matching block logging info only for exceptions --- kadi/commands/commands_v2.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/kadi/commands/commands_v2.py b/kadi/commands/commands_v2.py index 3f79518b..324c204b 100644 --- a/kadi/commands/commands_v2.py +++ b/kadi/commands/commands_v2.py @@ -1606,18 +1606,19 @@ def get_matching_block_idx(cmds_arch, cmds_recent): diff = difflib.SequenceMatcher(a=arch_vals, b=recent_vals, autojunk=False) matching_blocks = diff.get_matching_blocks() - logger.info("Matching blocks for (a) recent commands and (b) existing HDF5") - for block in matching_blocks: - logger.info(" {}".format(block)) opcodes = diff.get_opcodes() - logger.info("Diffs between (a) recent commands and (b) existing HDF5") - for opcode in opcodes: - logger.info(" {}".format(opcode)) # Find the first matching block that is sufficiently long for block in matching_blocks: if block.size > conf.matching_block_size: break else: + logger.info("Matching blocks for (a) recent commands and (b) existing HDF5") + for block in matching_blocks: + logger.info(" {}".format(block)) + logger.info("Diffs between (a) recent commands and (b) existing HDF5") + for opcode in opcodes: + logger.info(" {}".format(opcode)) + raise ValueError( f"No matching blocks at least {conf.matching_block_size} long. This most likely " "means that you have not recently synced your local Ska data using `ska_sync`." From 416b8be9ec8cd467be8ac780a535a434bcd1e8e1 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Mon, 17 Nov 2025 15:42:00 -0500 Subject: [PATCH 05/31] Add `obsid_sched` to observation --- kadi/commands/commands_v2.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/kadi/commands/commands_v2.py b/kadi/commands/commands_v2.py index 324c204b..6f40a207 100644 --- a/kadi/commands/commands_v2.py +++ b/kadi/commands/commands_v2.py @@ -715,6 +715,7 @@ def get_state_cmds(cmds): "AOUPTARQ", "AONM2NPE", "AONM2NPD", + "OBSID", ] if cmds["tlmsid"].dtype.kind == "S": @@ -946,6 +947,7 @@ def get_cmds_obs_final( # use values that are not None to avoid errors. For `sim_pos`, if the SIM # has not been commanded in a long time then it will be at -99616. obsid = -1 if prev_obsid is None else prev_obsid + obsid_sched = -1 if prev_obsid is None else prev_obsid starcat_idx = None starcat_date = None sim_pos = -99616 if prev_simpos is None else prev_simpos @@ -965,7 +967,11 @@ def get_cmds_obs_final( ) else: starcat_idx = cmd["idx"] + elif tlmsid == "OBSID": + obsid_sched = cmd["params"]["id"] + + elif tlmsid == "COAOSQID": obsid = cmd["params"]["id"] # Look for obsid change within obs, likely an undercover # (target="cold blank ECS"). First stop the initial obs at the time @@ -993,6 +999,7 @@ def get_cmds_obs_final( elif tlmsid == "OBS": obs_params = cmd["params"] obs_params["obsid"] = obsid + obs_params["obsid_sched"] = obsid_sched obs_params["simpos"] = sim_pos # matches states 'simpos' obs_params["obs_start"] = cmd["date"] if obs_params["npnt_enab"]: @@ -1454,7 +1461,9 @@ def parse_backstop(load_name: str, backstop_text: str): cmds_obsid = cmds[cmds["tlmsid"] == "COAOSQID"] cmds_obsid["type"] = "LOAD_EVENT" cmds_obsid["tlmsid"] = "OBSID" - cmds_obsid["scs"] -= 3 # Move these load event cmds to vehicle loads + # Move these load event cmds to vehicle loads. They will be stopped if vehicle loads + # are stopped. + cmds_obsid["scs"] -= 3 cmds = cmds.add_cmds(cmds_obsid) return cmds From acf55fe7c7d4b0b1300cc485a655c67ffdc7a823 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Wed, 19 Nov 2025 07:14:25 -0500 Subject: [PATCH 06/31] Remove support for use_ska_dir (no longer needed) --- kadi/commands/commands_v2.py | 40 ++++++++++++------------------------ 1 file changed, 13 insertions(+), 27 deletions(-) diff --git a/kadi/commands/commands_v2.py b/kadi/commands/commands_v2.py index 6f40a207..d868a0f5 100644 --- a/kadi/commands/commands_v2.py +++ b/kadi/commands/commands_v2.py @@ -1367,7 +1367,6 @@ def get_load_cmds_from_occweb_or_local( dir_year_month=None, load_name=None, *, - use_ska_dir=False, archive=True, ) -> CommandTable: """Get the load cmds (backstop) for ``load_name`` within ``dir_year_month`` @@ -1408,36 +1407,23 @@ def get_load_cmds_from_occweb_or_local( cmds = pickle.load(fh) return cmds - if use_ska_dir: - ska_dir = load_dir_from_load_name(load_name) - for filename in ska_dir.glob("CR????????.backstop"): - backstop_text = filename.read_text() - logger.info(f"Got backstop from {filename}") + load_dir_contents = occweb.get_occweb_dir(dir_year_month / load_name, timeout=5) + for filename in load_dir_contents["Name"]: + if re.match(r"CR\d{3}.\d{4}\.backstop", filename): + # Download the backstop file from OCCweb + backstop_text = occweb.get_occweb_page( + dir_year_month / load_name / filename, + cache=conf.cache_loads_in_astropy_cache, + timeout=10, + ) cmds = parse_backstop(load_name, backstop_text) if archive: write_backstop(cmds, cmds_filename) break - else: - raise ValueError(f"No backstop file found in {ska_dir}") - - else: # use OCCweb - load_dir_contents = occweb.get_occweb_dir(dir_year_month / load_name, timeout=5) - for filename in load_dir_contents["Name"]: - if re.match(r"CR\d{3}.\d{4}\.backstop", filename): - # Download the backstop file from OCCweb - backstop_text = occweb.get_occweb_page( - dir_year_month / load_name / filename, - cache=conf.cache_loads_in_astropy_cache, - timeout=10, - ) - cmds = parse_backstop(load_name, backstop_text) - if archive: - write_backstop(cmds, cmds_filename) - break - else: - raise ValueError( - f"Could not find backstop file in {dir_year_month / load_name}" - ) + else: + raise ValueError( + f"Could not find backstop file in {dir_year_month / load_name}" + ) return cmds From 57d08058f3637a2408998dca22e4fc60aa51cc0b Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Wed, 19 Nov 2025 08:46:01 -0500 Subject: [PATCH 07/31] Change `archive` option to `in_work` --- kadi/commands/commands_v2.py | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/kadi/commands/commands_v2.py b/kadi/commands/commands_v2.py index d868a0f5..3cd720b6 100644 --- a/kadi/commands/commands_v2.py +++ b/kadi/commands/commands_v2.py @@ -409,7 +409,7 @@ def update_cmds_list_for_loads_in_backstop( cmds = get_load_cmds_from_occweb_or_local( load_dir, load_name, - archive=False, + in_work=True, ) cmd_rltt = cmds.get_rltt_cmd() cmd_rltt["params"]["load_path"] = load_path @@ -1367,13 +1367,13 @@ def get_load_cmds_from_occweb_or_local( dir_year_month=None, load_name=None, *, - archive=True, + in_work=False, ) -> CommandTable: """Get the load cmds (backstop) for ``load_name`` within ``dir_year_month`` - If the backstop file is already available locally, use that. Otherwise, the - file is downloaded from OCCweb and is then parsed and saved as a gzipped - pickle file of the corresponding CommandTable object. + If the backstop file is already available locally, use that. Otherwise, the file is + downloaded from OCCweb and is then parsed and saved as a gzipped pickle file of the + corresponding CommandTable object. Parameters ---------- @@ -1381,12 +1381,9 @@ def get_load_cmds_from_occweb_or_local( Path to the directory containing the ``load_name`` directory. load_name : str Load name in the usual format e.g. JAN0521A. - use_ska_dir : bool - If True, get the backstop from the SKA directory structure instead of - OCCweb. - archive : bool - If True, save the backstop commands as a gzipped pickle file in the - loads archive directory. + in_work : bool + If True then the backstop file is in-work and should not be saved in the command + loads archive (~/.kadi/loads by default). Returns ------- @@ -1395,7 +1392,7 @@ def get_load_cmds_from_occweb_or_local( """ # Determine output file name and make directory if necessary. cmds_filename = None - if archive: + if not in_work: loads_dir = paths.LOADS_ARCHIVE_DIR() loads_dir.mkdir(parents=True, exist_ok=True) cmds_filename = loads_dir / f"{load_name}.pkl.gz" @@ -1417,7 +1414,7 @@ def get_load_cmds_from_occweb_or_local( timeout=10, ) cmds = parse_backstop(load_name, backstop_text) - if archive: + if not in_work: write_backstop(cmds, cmds_filename) break else: From fcc893c79f092a4100dad45e497f097d5f829ee8 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Wed, 19 Nov 2025 11:49:59 -0500 Subject: [PATCH 08/31] Do not include OBSID commands prior to writing to loads archive --- kadi/commands/commands_v2.py | 60 ++++++++++++++++++++---------------- 1 file changed, 34 insertions(+), 26 deletions(-) diff --git a/kadi/commands/commands_v2.py b/kadi/commands/commands_v2.py index 3cd720b6..39daddc3 100644 --- a/kadi/commands/commands_v2.py +++ b/kadi/commands/commands_v2.py @@ -1390,37 +1390,36 @@ def get_load_cmds_from_occweb_or_local( CommandTable Backstop commands for the load. """ - # Determine output file name and make directory if necessary. - cmds_filename = None - if not in_work: - loads_dir = paths.LOADS_ARCHIVE_DIR() - loads_dir.mkdir(parents=True, exist_ok=True) - cmds_filename = loads_dir / f"{load_name}.pkl.gz" + # Determine archived local gzip file name + cmds_filename = paths.LOADS_ARCHIVE_DIR() / f"{load_name}.pkl.gz" + if not in_work and cmds_filename.exists(): # If the output file already exists, read the commands and return them. - if cmds_filename.exists(): - logger.info(f"Already have {cmds_filename}") - with gzip.open(cmds_filename, "rb") as fh: - cmds = pickle.load(fh) - return cmds + logger.info(f"Already have {cmds_filename}") + with gzip.open(cmds_filename, "rb") as fh: + cmds = pickle.load(fh) + cmds = add_load_event_obsid_cmds(cmds) + return cmds + # Find the backstop file on OCCweb load_dir_contents = occweb.get_occweb_dir(dir_year_month / load_name, timeout=5) for filename in load_dir_contents["Name"]: if re.match(r"CR\d{3}.\d{4}\.backstop", filename): - # Download the backstop file from OCCweb - backstop_text = occweb.get_occweb_page( - dir_year_month / load_name / filename, - cache=conf.cache_loads_in_astropy_cache, - timeout=10, - ) - cmds = parse_backstop(load_name, backstop_text) - if not in_work: - write_backstop(cmds, cmds_filename) break else: raise ValueError( f"Could not find backstop file in {dir_year_month / load_name}" ) + # Download the backstop file from OCCweb + backstop_text = occweb.get_occweb_page( + dir_year_month / load_name / filename, + cache=conf.cache_loads_in_astropy_cache, + timeout=10, + ) + cmds = parse_backstop(load_name, backstop_text) + if not in_work: + write_backstop(cmds, cmds_filename) + cmds = add_load_event_obsid_cmds(cmds) return cmds @@ -1439,17 +1438,23 @@ def parse_backstop(load_name: str, backstop_text: str): cmds.add_column(load_name, index=idx, name="source") del cmds["timeline_id"] - # Add OBSID load event commands to track the scheduled obsid in the event of an - # SCS-107 where the original COAOSQID commands in the observing loads are dropped. + return cmds + + +def add_load_event_obsid_cmds(cmds: CommandTable) -> CommandTable: + """Add OBSID commands in vehicle loads corresponding to COAOSQID commands. + + This allows tracking of the scheduled OBSID in the event of an SCS-107 where the + original COAOSQID commands in the observing loads are dropped. The load event cmds + are placed in the vehicle loads so they get stopped if vehicle loads are stopped + by NSM etc. + """ cmds_obsid = cmds[cmds["tlmsid"] == "COAOSQID"] cmds_obsid["type"] = "LOAD_EVENT" cmds_obsid["tlmsid"] = "OBSID" - # Move these load event cmds to vehicle loads. They will be stopped if vehicle loads - # are stopped. cmds_obsid["scs"] -= 3 - cmds = cmds.add_cmds(cmds_obsid) - return cmds + return cmds.add_cmds(cmds_obsid) def write_backstop(cmds: CommandTable, cmds_filename: str | Path): @@ -1458,6 +1463,8 @@ def write_backstop(cmds: CommandTable, cmds_filename: str | Path): This function saves a CommandTable object to disk as a compressed pickle file. ``cmds_filename`` is normally the kadi loads archive directory. + It also creates the parent directories as needed. + Parameters ---------- cmds : CommandTable @@ -1468,6 +1475,7 @@ def write_backstop(cmds: CommandTable, cmds_filename: str | Path): """ logger.info(f"Saving {cmds_filename}") + Path(cmds_filename).parent.mkdir(parents=True, exist_ok=True) with gzip.open(cmds_filename, "wb") as fh: pickle.dump(cmds, fh) From eaa8bad69bf84b056de5f0d12e9ac75dd58b750f Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Fri, 30 Jan 2026 06:59:42 -0500 Subject: [PATCH 09/31] Fix unused import from rebase --- kadi/commands/commands_v2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kadi/commands/commands_v2.py b/kadi/commands/commands_v2.py index 39daddc3..999b6798 100644 --- a/kadi/commands/commands_v2.py +++ b/kadi/commands/commands_v2.py @@ -19,7 +19,7 @@ import requests from astropy.table import Table from cxotime import CxoTime -from parse_cm.paths import ParseLoadNameError, load_dir_from_load_name, parse_load_name +from parse_cm.paths import ParseLoadNameError, parse_load_name from ska_helpers.retry import retry_func from ska_sun import get_nsm_attitude from testr.test_helper import has_internet From 95a61fa4bd3494afafc3fe99345517f90aee237f Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Fri, 30 Jan 2026 10:27:50 -0500 Subject: [PATCH 10/31] Ignore scheduled obsid in command matching --- kadi/commands/commands_v2.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/kadi/commands/commands_v2.py b/kadi/commands/commands_v2.py index 999b6798..02b6c9d4 100644 --- a/kadi/commands/commands_v2.py +++ b/kadi/commands/commands_v2.py @@ -1637,6 +1637,11 @@ def get_list_for_matching(cmds: CommandTable) -> list[tuple]: keys = ("date", "type", "tlmsid", "scs", "step", "source", "vcdu") rows = [] for cmd in cmds: + if cmd["type"] == "LOAD_EVENT" and cmd["tlmsid"] == "OBSID": + # Ignore OBSID LOAD_EVENT commands (aka scheduled obsid) for matching + # because the commands archive may or may not include these commands. + continue + row = tuple( cmd[key].decode("ascii") if isinstance(cmd[key], bytes) else str(cmd[key]) for key in keys @@ -1650,7 +1655,9 @@ def get_list_for_matching(cmds: CommandTable) -> list[tuple]: # commands are not mutable in this way we just apply this for OBS commands. if cmd["tlmsid"] == "OBS": row_params = tuple( - (key, cmd["params"][key]) for key in sorted(cmd["params"]) + (key, cmd["params"][key]) + for key in sorted(cmd["params"]) + if key != "obsid_sched" ) row += row_params rows.append(row) From 6b16b68399bccd4edcc214c81b2cea38747a171c Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Sat, 31 Jan 2026 12:43:37 -0500 Subject: [PATCH 11/31] Many changes to hopefully get this working --- kadi/commands/commands_v2.py | 153 +++++++++++++++++++++++++-------- kadi/commands/core.py | 4 +- kadi/config.py | 28 +++++- kadi/scripts/update_cmds_v2.py | 46 ++++++++-- 4 files changed, 184 insertions(+), 47 deletions(-) diff --git a/kadi/commands/commands_v2.py b/kadi/commands/commands_v2.py index 02b6c9d4..efc1aba7 100644 --- a/kadi/commands/commands_v2.py +++ b/kadi/commands/commands_v2.py @@ -21,6 +21,7 @@ from cxotime import CxoTime from parse_cm.paths import ParseLoadNameError, parse_load_name from ska_helpers.retry import retry_func +from ska_helpers.utils import temp_env_var from ska_sun import get_nsm_attitude from testr.test_helper import has_internet @@ -71,6 +72,7 @@ # APR1420B was the first load set to have RLTT (backstop 6.9) RLTT_ERA_START = CxoTime("2020-04-14") +RLTT_ERA_START_LOAD = "APR1420B" HAS_INTERNET = has_internet() @@ -489,6 +491,8 @@ def update_cmd_events_and_loads_and_get_cmds_recent( loads_backstop_path = paths.LOADS_BACKSTOP_PATH(load_name) with gzip.open(loads_backstop_path, "rb") as fh: cmds: CommandTable = pickle.load(fh) + if check_add_scheduled_obsid_cmds(): + cmds = add_scheduled_obsid_cmds(cmds) # Filter commands if loads (vehicle and/or observing) were approved but # never uplinked @@ -1398,28 +1402,28 @@ def get_load_cmds_from_occweb_or_local( logger.info(f"Already have {cmds_filename}") with gzip.open(cmds_filename, "rb") as fh: cmds = pickle.load(fh) - cmds = add_load_event_obsid_cmds(cmds) - return cmds - - # Find the backstop file on OCCweb - load_dir_contents = occweb.get_occweb_dir(dir_year_month / load_name, timeout=5) - for filename in load_dir_contents["Name"]: - if re.match(r"CR\d{3}.\d{4}\.backstop", filename): - break else: - raise ValueError( - f"Could not find backstop file in {dir_year_month / load_name}" + # Find the backstop file on OCCweb + load_dir_contents = occweb.get_occweb_dir(dir_year_month / load_name, timeout=5) + for filename in load_dir_contents["Name"]: + if re.match(r"CR\d{3}.\d{4}\.backstop", filename): + break + else: + raise ValueError( + f"Could not find backstop file in {dir_year_month / load_name}" + ) + # Download the backstop file from OCCweb + backstop_text = occweb.get_occweb_page( + dir_year_month / load_name / filename, + cache=conf.cache_loads_in_astropy_cache, + timeout=10, ) - # Download the backstop file from OCCweb - backstop_text = occweb.get_occweb_page( - dir_year_month / load_name / filename, - cache=conf.cache_loads_in_astropy_cache, - timeout=10, - ) - cmds = parse_backstop(load_name, backstop_text) - if not in_work: - write_backstop(cmds, cmds_filename) - cmds = add_load_event_obsid_cmds(cmds) + cmds = parse_backstop(load_name, backstop_text) + if not in_work: + write_backstop(cmds, cmds_filename) + + if check_add_scheduled_obsid_cmds(): + cmds = add_scheduled_obsid_cmds(cmds) return cmds @@ -1441,18 +1445,74 @@ def parse_backstop(load_name: str, backstop_text: str): return cmds -def add_load_event_obsid_cmds(cmds: CommandTable) -> CommandTable: +@functools.cache +def check_add_scheduled_obsid_cmds() -> bool: + """Determine whether to add scheduled OBSID commands to the commands archive. + + This function evaluates the configuration setting + `conf.add_scheduled_obsid_commands` to determine if scheduled OBSID commands should + be added. The logic is: + + - If the config is explicitly set to True or False, return that value + - If the config is None (default), auto-detect based on whether existing commands + archive already contains OBSID commands + + Returns + ------- + bool + True if scheduled OBSID commands should be added, False otherwise. + + Raises + ------ + ValueError + If conf.add_scheduled_obsid_commands is not a bool or None. + + Notes + ----- + This function is cached to avoid repeated evaluation during processing. Scheduled + OBSID commands allow tracking of the scheduled OBSID in the event of an SCS-107 + where original COAOSQID commands in observing loads are dropped. + """ + if isinstance(conf.add_scheduled_obsid_commands, bool): + logger.info( + f"add_scheduled_obsid_commands set to {conf.add_scheduled_obsid_commands}" + ) + out = conf.add_scheduled_obsid_commands + elif conf.add_scheduled_obsid_commands is None: + out = (has_obsid := np.any(IDX_CMDS["tlmsid"] == "OBSID")) + logger.info( + "add_scheduled_obsid_commands auto-detected as " + f"{out} based on existing OBSID commands={has_obsid})" + ) + else: + raise ValueError("conf.add_scheduled_obsid_commands must be a bool or None") + + return out + + +def add_scheduled_obsid_cmds(cmds: CommandTable) -> CommandTable: """Add OBSID commands in vehicle loads corresponding to COAOSQID commands. This allows tracking of the scheduled OBSID in the event of an SCS-107 where the original COAOSQID commands in the observing loads are dropped. The load event cmds are placed in the vehicle loads so they get stopped if vehicle loads are stopped by NSM etc. + + Parameters + ---------- + cmds : CommandTable + CommandTable of load commands + + Returns + ------- + CommandTable + New CommandTable with original table and added OBSID load event commands. """ cmds_obsid = cmds[cmds["tlmsid"] == "COAOSQID"] cmds_obsid["type"] = "LOAD_EVENT" cmds_obsid["tlmsid"] = "OBSID" cmds_obsid["scs"] -= 3 + logger.info(f"Adding {len(cmds_obsid)} OBSID load event commands") return cmds.add_cmds(cmds_obsid) @@ -1487,7 +1547,6 @@ def update_cmds_archive( log_level=logging.INFO, scenario=None, data_root=".", - match_prev_cmds=True, ): """Update cmds2.h5 and cmds2.pkl archive files. @@ -1520,16 +1579,19 @@ def update_cmds_archive( # Local context manager for log_level and data_root kadi_logger = logging.getLogger("kadi") log_level_orig = kadi_logger.level - with conf.set_temp("commands_dir", data_root): + with conf.set_temp("commands_dir", data_root), temp_env_var("KADI", data_root): try: kadi_logger.setLevel(log_level) - _update_cmds_archive(lookback, stop, match_prev_cmds, scenario, data_root) + _update_cmds_archive(lookback, stop, scenario, data_root) finally: kadi_logger.setLevel(log_level_orig) -def _update_cmds_archive(lookback, stop_loads, match_prev_cmds, scenario, data_root): +def _update_cmds_archive(lookback, stop_loads, scenario, data_root): """Do the real work of updating the cmds archive""" + # Either no-match-prev-cmds or matching RLTT start disables matching previous cmds + match_prev_cmds = not (conf.no_match_prev_cmds or conf.match_from_rltt_start) + idx_cmds_path = Path(data_root) / "cmds2.h5" pars_dict_path = Path(data_root) / "cmds2.pkl" @@ -1544,7 +1606,7 @@ def _update_cmds_archive(lookback, stop_loads, match_prev_cmds, scenario, data_r ) del cmds_arch["timeline_id"] pars_dict = {} - match_prev_cmds = False # No matching of previous commands + match_prev_cmds = False # No match of previous commands since there are none. cmds_recent = update_cmd_events_and_loads_and_get_cmds_recent( scenario=scenario, @@ -1556,10 +1618,23 @@ def _update_cmds_archive(lookback, stop_loads, match_prev_cmds, scenario, data_r if match_prev_cmds: idx0_arch, idx0_recent = get_matching_block_idx(cmds_arch, cmds_recent) else: - idx0_arch = len(cmds_arch) idx0_recent = 0 + if conf.match_from_rltt_start: + # Special case for reprocessing the commands archive starting at the RLTT + # era from load RLTT_ERA_START_LOAD (APR1420B). Find the index of the first + # command in these loads. Note np.argmax() returns the first matching index + # in the case of multiple matches. + idx0_arch = np.argmax(cmds_arch["source"] == RLTT_ERA_START_LOAD) + logger.info( + f"Matching from RLTT start load {RLTT_ERA_START_LOAD}, " + f"idx0_arch={idx0_arch}, idx0_recent={idx0_recent}" + ) + else: + # Append to end of existing cmds archive file + idx0_arch = len(cmds_arch) # Convert from `params` col of dicts to index into same params in pars_dict. + cmds_recent[-100:].pprint_like_backstop() for cmd in cmds_recent: cmd["idx"] = get_par_idx_update_pars_dict(pars_dict, cmd) @@ -1567,7 +1642,9 @@ def _update_cmds_archive(lookback, stop_loads, match_prev_cmds, scenario, data_r # For the command below the no-op logic should be clear: # cmds_arch = vstack([cmds_arch[:idx0_arch], cmds_recent[idx0_recent:]]) if idx0_arch == len(cmds_arch) and idx0_recent == len(cmds_recent): - logger.info(f"No new commands found, skipping writing {idx_cmds_path}") + logger.info( + f"No new commands found, skipping writing {idx_cmds_path.absolute()}" + ) return # Merge the recent commands with the existing archive. @@ -1586,19 +1663,21 @@ def _update_cmds_archive(lookback, stop_loads, match_prev_cmds, scenario, data_r # Save the updated archive and pars_dict. cmds_arch_new = vstack_exact([cmds_arch[:idx0_arch], cmds_recent[idx0_recent:]]) - logger.info(f"Writing {len(cmds_arch_new)} commands to {idx_cmds_path}") - cmds_arch_new.write(str(idx_cmds_path), path="data", format="hdf5", overwrite=True) + logger.info(f"Writing {len(cmds_arch_new)} commands to {idx_cmds_path.absolute()}") + cmds_arch_new.write( + str(idx_cmds_path.absolute()), path="data", format="hdf5", overwrite=True + ) - logger.info(f"Writing updated pars_dict to {pars_dict_path}") - pickle.dump(pars_dict, open(pars_dict_path, "wb")) + logger.info(f"Writing updated pars_dict to {pars_dict_path.absolute()}") + pickle.dump(pars_dict, open(pars_dict_path.absolute(), "wb")) def get_matching_block_idx(cmds_arch, cmds_recent): # Find place in archive where the recent commands start. idx_arch_recent = cmds_arch.find_date(cmds_recent["date"][0]) - logger.info("Selecting commands from cmds_arch[{}:]".format(idx_arch_recent)) cmds_arch_recent = cmds_arch[idx_arch_recent:] cmds_arch_recent.rev_pars_dict = weakref.ref(REV_PARS_DICT) + logger.info("Selecting commands from cmds_arch[{}:]".format(idx_arch_recent)) arch_vals = get_list_for_matching(cmds_arch_recent) recent_vals = get_list_for_matching(cmds_recent) @@ -1608,13 +1687,15 @@ def get_matching_block_idx(cmds_arch, cmds_recent): matching_blocks = diff.get_matching_blocks() opcodes = diff.get_opcodes() # Find the first matching block that is sufficiently long + logger.info("Matching blocks for (a) recent commands and (b) existing HDF5") for block in matching_blocks: + logger.info(" {}".format(block)) if block.size > conf.matching_block_size: + logger.info( + f"Found matching block of size {block.size} > {conf.matching_block_size}" + ) break else: - logger.info("Matching blocks for (a) recent commands and (b) existing HDF5") - for block in matching_blocks: - logger.info(" {}".format(block)) logger.info("Diffs between (a) recent commands and (b) existing HDF5") for opcode in opcodes: logger.info(" {}".format(opcode)) diff --git a/kadi/commands/core.py b/kadi/commands/core.py index 2c4d7023..9ebd6369 100644 --- a/kadi/commands/core.py +++ b/kadi/commands/core.py @@ -86,7 +86,7 @@ def load_idx_cmds(version=None, file=None): file = IDX_CMDS_PATH(version) with tables.open_file(file, mode="r") as h5: idx_cmds = CommandTable(h5.root.data[:]) - logger.info(f"Loaded {file} with {len(idx_cmds)} commands") + logger.info(f"Loaded {Path(file).absolute()} with {len(idx_cmds)} commands") # For V2 add the params column here to make IDX_CMDS be same as regular cmds if version == 2: @@ -101,7 +101,7 @@ def load_pars_dict(version=None, file=None): file = PARS_DICT_PATH(version) with open(file, "rb") as fh: pars_dict = pickle.load(fh, encoding="ascii") - logger.info(f"Loaded {file} with {len(pars_dict)} pars") + logger.info(f"Loaded {Path(file).absolute()} with {len(pars_dict)} pars") return pars_dict diff --git a/kadi/config.py b/kadi/config.py index d9128b7e..21db1976 100644 --- a/kadi/config.py +++ b/kadi/config.py @@ -101,7 +101,33 @@ class Conf(ConfigNamespace): "step size may be smaller." ), ) - matching_block_size = ConfigItem(500, "Matching block size for command blocks.") + + no_match_prev_cmds = ConfigItem( + False, + "Do not match previous command block when updating cmds v2. " + "Setting to True can produce an invalid commands table (experts only).", + ) + + matching_block_size = ConfigItem( + 500, + "Matching block size for command blocks.", + ) + + add_scheduled_obsid_commands = ConfigItem( + defaultvalue=None, + cfgtype="boolean", + description=( + "Add scheduled OBSID commands to commands archive. ", + "If None then check for OBSID commands in the existing archive and " + "behave accordingly. Developers only.", + ), + ) + + match_from_rltt_start = ConfigItem( + False, + "Match previous commands exactly from the start of the RLTT era (APR1420B). " + "Developers only.", + ) # Create a configuration instance for the user diff --git a/kadi/scripts/update_cmds_v2.py b/kadi/scripts/update_cmds_v2.py index 4660dd77..764cd08c 100644 --- a/kadi/scripts/update_cmds_v2.py +++ b/kadi/scripts/update_cmds_v2.py @@ -12,9 +12,18 @@ def get_opt(args=None): """ Get options for command line interface to update. """ - parser = argparse.ArgumentParser(description="Update HDF5 cmds v2 table") - parser.add_argument("--lookback", type=int, help="Lookback (default=30 days)") - parser.add_argument("--stop", help="Stop date for update (default=Now+21 days)") + parser = argparse.ArgumentParser( + description="Update HDF5 cmds v2 table", + ) + parser.add_argument( + "--lookback", + type=int, + help="Lookback (default=30 days)", + ) + parser.add_argument( + "--stop", + help="Stop date for update (default=Now+21 days)", + ) parser.add_argument( "--log-level", type=int, @@ -31,17 +40,31 @@ def get_opt(args=None): action="version", version="%(prog)s {version}".format(version=__version__), ) - parser.add_argument( + + # Developer-only options + dev_group = parser.add_argument_group("Developer-only options") + dev_group.add_argument( "--no-match-prev-cmds", action="store_true", help="Do not enforce matching previous command block when updating cmds v2 " "(experts only, this can produce an invalid commands table)", ) - parser.add_argument( + dev_group.add_argument( "--matching-block-size", type=int, help=f"Matching block size (default={conf.matching_block_size})", ) + dev_group.add_argument( + "--add-scheduled-obsid-commands", + action="store_true", + help="Add scheduled OBSID commands to commands archive", + ) + dev_group.add_argument( + "--match-from-rltt-start", + action="store_true", + help="Match previous commands exactly from the start of the RLTT era " + "(APR1420B). This implies --no-match-prev-cmds.", + ) args = parser.parse_args(args) return args @@ -54,8 +77,16 @@ def main(args=None): opt = get_opt(args) log_run_info(log_func=print, opt=opt) - if opt.matching_block_size is not None: - conf.matching_block_size = opt.matching_block_size + # Transfer these developer-only options to conf + for attr in ( + "no_match_prev_cmds", + "matching_block_size", + "add_scheduled_obsid_commands", + "match_from_rltt_start", + ): + if (value := getattr(opt, attr)) is not None: + setattr(conf, attr, value) + print(f"Set conf.{attr} = {value}") update_cmds_archive( lookback=opt.lookback, @@ -63,7 +94,6 @@ def main(args=None): log_level=opt.log_level, scenario=opt.scenario, data_root=opt.data_root, - match_prev_cmds=not opt.no_match_prev_cmds, ) From 5e5a4439f7d9c6774f60a8f3d97433fea347aa8e Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Sat, 31 Jan 2026 14:45:56 -0500 Subject: [PATCH 12/31] This seems to mostly work so far --- kadi/commands/commands_v2.py | 19 ++++++++----------- kadi/config.py | 10 ---------- kadi/scripts/update_cmds_v2.py | 6 ------ 3 files changed, 8 insertions(+), 27 deletions(-) diff --git a/kadi/commands/commands_v2.py b/kadi/commands/commands_v2.py index efc1aba7..cf0a16f8 100644 --- a/kadi/commands/commands_v2.py +++ b/kadi/commands/commands_v2.py @@ -1473,19 +1473,17 @@ def check_add_scheduled_obsid_cmds() -> bool: OBSID commands allow tracking of the scheduled OBSID in the event of an SCS-107 where original COAOSQID commands in observing loads are dropped. """ - if isinstance(conf.add_scheduled_obsid_commands, bool): + if conf.match_from_rltt_start: + logger.info('Adding "OBSID" commands due to conf.match_from_rltt_start=True') + out = True + elif np.any(IDX_CMDS["tlmsid"] == "OBSID"): logger.info( - f"add_scheduled_obsid_commands set to {conf.add_scheduled_obsid_commands}" - ) - out = conf.add_scheduled_obsid_commands - elif conf.add_scheduled_obsid_commands is None: - out = (has_obsid := np.any(IDX_CMDS["tlmsid"] == "OBSID")) - logger.info( - "add_scheduled_obsid_commands auto-detected as " - f"{out} based on existing OBSID commands={has_obsid})" + 'Adding "OBSID" commands due to existing "OBSID" commands in archive' ) + out = True else: - raise ValueError("conf.add_scheduled_obsid_commands must be a bool or None") + logger.info('Not adding "OBSID" commands') + out = False return out @@ -1634,7 +1632,6 @@ def _update_cmds_archive(lookback, stop_loads, scenario, data_root): idx0_arch = len(cmds_arch) # Convert from `params` col of dicts to index into same params in pars_dict. - cmds_recent[-100:].pprint_like_backstop() for cmd in cmds_recent: cmd["idx"] = get_par_idx_update_pars_dict(pars_dict, cmd) diff --git a/kadi/config.py b/kadi/config.py index 21db1976..0134bb69 100644 --- a/kadi/config.py +++ b/kadi/config.py @@ -113,16 +113,6 @@ class Conf(ConfigNamespace): "Matching block size for command blocks.", ) - add_scheduled_obsid_commands = ConfigItem( - defaultvalue=None, - cfgtype="boolean", - description=( - "Add scheduled OBSID commands to commands archive. ", - "If None then check for OBSID commands in the existing archive and " - "behave accordingly. Developers only.", - ), - ) - match_from_rltt_start = ConfigItem( False, "Match previous commands exactly from the start of the RLTT era (APR1420B). " diff --git a/kadi/scripts/update_cmds_v2.py b/kadi/scripts/update_cmds_v2.py index 764cd08c..c2883db5 100644 --- a/kadi/scripts/update_cmds_v2.py +++ b/kadi/scripts/update_cmds_v2.py @@ -54,11 +54,6 @@ def get_opt(args=None): type=int, help=f"Matching block size (default={conf.matching_block_size})", ) - dev_group.add_argument( - "--add-scheduled-obsid-commands", - action="store_true", - help="Add scheduled OBSID commands to commands archive", - ) dev_group.add_argument( "--match-from-rltt-start", action="store_true", @@ -81,7 +76,6 @@ def main(args=None): for attr in ( "no_match_prev_cmds", "matching_block_size", - "add_scheduled_obsid_commands", "match_from_rltt_start", ): if (value := getattr(opt, attr)) is not None: From c34ec97cdf4de2f4fcd9221740406e0cfecb6faf Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Sat, 31 Jan 2026 15:36:04 -0500 Subject: [PATCH 13/31] Only include obsid_sched state if it makes sense; passing tests! --- kadi/commands/commands_v2.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/kadi/commands/commands_v2.py b/kadi/commands/commands_v2.py index cf0a16f8..a08cd9ba 100644 --- a/kadi/commands/commands_v2.py +++ b/kadi/commands/commands_v2.py @@ -1003,7 +1003,8 @@ def get_cmds_obs_final( elif tlmsid == "OBS": obs_params = cmd["params"] obs_params["obsid"] = obsid - obs_params["obsid_sched"] = obsid_sched + if check_add_scheduled_obsid_cmds(): + obs_params["obsid_sched"] = obsid_sched obs_params["simpos"] = sim_pos # matches states 'simpos' obs_params["obs_start"] = cmd["date"] if obs_params["npnt_enab"]: @@ -1715,11 +1716,6 @@ def get_list_for_matching(cmds: CommandTable) -> list[tuple]: keys = ("date", "type", "tlmsid", "scs", "step", "source", "vcdu") rows = [] for cmd in cmds: - if cmd["type"] == "LOAD_EVENT" and cmd["tlmsid"] == "OBSID": - # Ignore OBSID LOAD_EVENT commands (aka scheduled obsid) for matching - # because the commands archive may or may not include these commands. - continue - row = tuple( cmd[key].decode("ascii") if isinstance(cmd[key], bytes) else str(cmd[key]) for key in keys @@ -1733,9 +1729,7 @@ def get_list_for_matching(cmds: CommandTable) -> list[tuple]: # commands are not mutable in this way we just apply this for OBS commands. if cmd["tlmsid"] == "OBS": row_params = tuple( - (key, cmd["params"][key]) - for key in sorted(cmd["params"]) - if key != "obsid_sched" + (key, cmd["params"][key]) for key in sorted(cmd["params"]) ) row += row_params rows.append(row) From 7197255bdbc4a945d872d5f4f86da1f1f42ab6a7 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Sat, 31 Jan 2026 15:48:54 -0500 Subject: [PATCH 14/31] Change tlmsid from OBSID to OBSID_SCH --- kadi/commands/commands_v2.py | 16 +++++++++------- kadi/commands/states.py | 2 +- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/kadi/commands/commands_v2.py b/kadi/commands/commands_v2.py index a08cd9ba..aff00d1c 100644 --- a/kadi/commands/commands_v2.py +++ b/kadi/commands/commands_v2.py @@ -719,7 +719,7 @@ def get_state_cmds(cmds): "AOUPTARQ", "AONM2NPE", "AONM2NPD", - "OBSID", + "OBSID_SCH", ] if cmds["tlmsid"].dtype.kind == "S": @@ -972,7 +972,7 @@ def get_cmds_obs_final( else: starcat_idx = cmd["idx"] - elif tlmsid == "OBSID": + elif tlmsid == "OBSID_SCH": obsid_sched = cmd["params"]["id"] elif tlmsid == "COAOSQID": @@ -1475,15 +1475,17 @@ def check_add_scheduled_obsid_cmds() -> bool: where original COAOSQID commands in observing loads are dropped. """ if conf.match_from_rltt_start: - logger.info('Adding "OBSID" commands due to conf.match_from_rltt_start=True') + logger.info( + 'Adding "OBSID_SCH" commands due to conf.match_from_rltt_start=True' + ) out = True - elif np.any(IDX_CMDS["tlmsid"] == "OBSID"): + elif np.any(IDX_CMDS["tlmsid"] == "OBSID_SCH"): logger.info( - 'Adding "OBSID" commands due to existing "OBSID" commands in archive' + 'Adding "OBSID_SCH" commands due to existing "OBSID_SCH" commands in archive' ) out = True else: - logger.info('Not adding "OBSID" commands') + logger.info('Not adding "OBSID_SCH" commands') out = False return out @@ -1509,7 +1511,7 @@ def add_scheduled_obsid_cmds(cmds: CommandTable) -> CommandTable: """ cmds_obsid = cmds[cmds["tlmsid"] == "COAOSQID"] cmds_obsid["type"] = "LOAD_EVENT" - cmds_obsid["tlmsid"] = "OBSID" + cmds_obsid["tlmsid"] = "OBSID_SCH" cmds_obsid["scs"] -= 3 logger.info(f"Adding {len(cmds_obsid)} OBSID load event commands") diff --git a/kadi/commands/states.py b/kadi/commands/states.py index 648c8ac4..083f84cc 100644 --- a/kadi/commands/states.py +++ b/kadi/commands/states.py @@ -883,7 +883,7 @@ class ObsidTransition(ParamTransition): class ObsidSchedTransition(ParamTransition): """Scheduled Obsid update""" - command_attributes = {"tlmsid": "OBSID"} + command_attributes = {"tlmsid": "OBSID_SCH"} state_keys = ["obsid_sched"] transition_key = "obsid_sched" cmd_param_key = "id" From a276d63b0a29f72e61dce0e624b50c91d805b681 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Mon, 2 Feb 2026 06:46:06 -0500 Subject: [PATCH 15/31] Support different cmds archive versions Version 3 now includes scheduled obsid --- kadi/commands/commands_v2.py | 60 ++++++---------------------------- kadi/commands/core.py | 43 ++++++++++++++++++++++-- kadi/scripts/update_cmds_v2.py | 15 ++++++++- 3 files changed, 64 insertions(+), 54 deletions(-) diff --git a/kadi/commands/commands_v2.py b/kadi/commands/commands_v2.py index aff00d1c..edc6a439 100644 --- a/kadi/commands/commands_v2.py +++ b/kadi/commands/commands_v2.py @@ -36,6 +36,7 @@ get_cmds_from_backstop, get_cxotime_now, get_par_idx_update_pars_dict, + kadi_cmds_version, load_idx_cmds, load_name_to_cxotime, load_pars_dict, @@ -62,8 +63,8 @@ # Cached values of the full mission commands archive (cmds_v2.h5, cmds_v2.pkl). # These are loaded on demand. -IDX_CMDS = LazyVal(functools.partial(load_idx_cmds, version=2)) -PARS_DICT = LazyVal(functools.partial(load_pars_dict, version=2)) +IDX_CMDS = LazyVal(functools.partial(load_idx_cmds)) +PARS_DICT = LazyVal(functools.partial(load_pars_dict)) REV_PARS_DICT = LazyVal(lambda: {v: k for k, v in PARS_DICT.items()}) # Cache of recent commands keyed by scenario @@ -491,7 +492,7 @@ def update_cmd_events_and_loads_and_get_cmds_recent( loads_backstop_path = paths.LOADS_BACKSTOP_PATH(load_name) with gzip.open(loads_backstop_path, "rb") as fh: cmds: CommandTable = pickle.load(fh) - if check_add_scheduled_obsid_cmds(): + if kadi_cmds_version() >= 3: cmds = add_scheduled_obsid_cmds(cmds) # Filter commands if loads (vehicle and/or observing) were approved but @@ -1003,7 +1004,7 @@ def get_cmds_obs_final( elif tlmsid == "OBS": obs_params = cmd["params"] obs_params["obsid"] = obsid - if check_add_scheduled_obsid_cmds(): + if kadi_cmds_version() >= 3: obs_params["obsid_sched"] = obsid_sched obs_params["simpos"] = sim_pos # matches states 'simpos' obs_params["obs_start"] = cmd["date"] @@ -1423,7 +1424,7 @@ def get_load_cmds_from_occweb_or_local( if not in_work: write_backstop(cmds, cmds_filename) - if check_add_scheduled_obsid_cmds(): + if kadi_cmds_version() >= 3: cmds = add_scheduled_obsid_cmds(cmds) return cmds @@ -1446,51 +1447,6 @@ def parse_backstop(load_name: str, backstop_text: str): return cmds -@functools.cache -def check_add_scheduled_obsid_cmds() -> bool: - """Determine whether to add scheduled OBSID commands to the commands archive. - - This function evaluates the configuration setting - `conf.add_scheduled_obsid_commands` to determine if scheduled OBSID commands should - be added. The logic is: - - - If the config is explicitly set to True or False, return that value - - If the config is None (default), auto-detect based on whether existing commands - archive already contains OBSID commands - - Returns - ------- - bool - True if scheduled OBSID commands should be added, False otherwise. - - Raises - ------ - ValueError - If conf.add_scheduled_obsid_commands is not a bool or None. - - Notes - ----- - This function is cached to avoid repeated evaluation during processing. Scheduled - OBSID commands allow tracking of the scheduled OBSID in the event of an SCS-107 - where original COAOSQID commands in observing loads are dropped. - """ - if conf.match_from_rltt_start: - logger.info( - 'Adding "OBSID_SCH" commands due to conf.match_from_rltt_start=True' - ) - out = True - elif np.any(IDX_CMDS["tlmsid"] == "OBSID_SCH"): - logger.info( - 'Adding "OBSID_SCH" commands due to existing "OBSID_SCH" commands in archive' - ) - out = True - else: - logger.info('Not adding "OBSID_SCH" commands') - out = False - - return out - - def add_scheduled_obsid_cmds(cmds: CommandTable) -> CommandTable: """Add OBSID commands in vehicle loads corresponding to COAOSQID commands. @@ -1554,6 +1510,10 @@ def update_cmds_archive( This updates the archive though ``stop`` date, where is required that the ``stop`` date is within ``lookback`` days of existing data in the archive. + By default this updates the latest version of the archive. This can be changed by + setting the KADI_CMDS_VERSION environment variable. Note that the version is + cached via kadi.commands.core.kadi_cmds_version(). + Parameters ---------- lookback : int, None diff --git a/kadi/commands/core.py b/kadi/commands/core.py index 9ebd6369..d18b6f0a 100644 --- a/kadi/commands/core.py +++ b/kadi/commands/core.py @@ -18,7 +18,7 @@ from ska_helpers import retry from kadi.config import conf -from kadi.paths import IDX_CMDS_PATH, PARS_DICT_PATH +from kadi.paths import DATA_DIR, IDX_CMDS_PATH, PARS_DICT_PATH __all__ = [ "read_backstop", @@ -29,6 +29,7 @@ "SCS107_EVENTS", "filter_scs107_events", "set_time_now", + "kadi_cmds_version", ] @@ -82,14 +83,16 @@ def load_idx_cmds(version=None, file=None): File "H5FDsec2.c", line 941, in H5FD_sec2_lock unable to lock file, errno = 11, error message = 'Resource temporarily unavailable' """ # noqa: E501 + if version is None: + version = kadi_cmds_version() if file is None: file = IDX_CMDS_PATH(version) with tables.open_file(file, mode="r") as h5: idx_cmds = CommandTable(h5.root.data[:]) logger.info(f"Loaded {Path(file).absolute()} with {len(idx_cmds)} commands") - # For V2 add the params column here to make IDX_CMDS be same as regular cmds - if version == 2: + # For V2 or later add params column here to make IDX_CMDS be same as regular cmds + if version >= 2: idx_cmds["params"] = None return idx_cmds @@ -97,6 +100,8 @@ def load_idx_cmds(version=None, file=None): @retry.retry(tries=4, delay=0.5, backoff=4) def load_pars_dict(version=None, file=None): + if version is None: + version = kadi_cmds_version() if file is None: file = PARS_DICT_PATH(version) with open(file, "rb") as fh: @@ -105,6 +110,38 @@ def load_pars_dict(version=None, file=None): return pars_dict +@functools.cache +def kadi_cmds_version() -> int: + """Determine the kadi commands version by checking for cmds*.h5 files. + + The version number can be an integer. Return the highest version number that + where the corresponding PARS_DICT_PATH(version) file also exists. + + Returns + ------- + int + Highest kadi commands version number found. + """ + if version := os.environ.get("KADI_CMDS_VERSION"): + return int(version) + + versions = set() + for path in DATA_DIR().glob("cmds*.h5"): + ver_str = path.stem.replace("cmds", "") + try: + ver = int(ver_str) + except ValueError: + continue + else: + if PARS_DICT_PATH(ver).exists(): + versions.add(ver) + + if not versions: + raise RuntimeError(f"no valid kadi commands versions found in {DATA_DIR()}") + + return max(versions) + + @functools.lru_cache def load_name_to_cxotime(name): """Convert load name to date""" diff --git a/kadi/scripts/update_cmds_v2.py b/kadi/scripts/update_cmds_v2.py index c2883db5..76f6acc4 100644 --- a/kadi/scripts/update_cmds_v2.py +++ b/kadi/scripts/update_cmds_v2.py @@ -1,5 +1,6 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst import argparse +import os from ska_helpers.run_info import log_run_info @@ -15,6 +16,11 @@ def get_opt(args=None): parser = argparse.ArgumentParser( description="Update HDF5 cmds v2 table", ) + parser.add_argument( + "--data-root", + default=".", + help="Data root (default='.')", + ) parser.add_argument( "--lookback", type=int, @@ -34,7 +40,11 @@ def get_opt(args=None): "--scenario", help="Scenario for loads and command events outputs (default=None)", ) - parser.add_argument("--data-root", default=".", help="Data root (default='.')") + parser.add_argument( + "--kadi-cmds-version", + type=int, + help="Kadi cmds version (default=latest)", + ) parser.add_argument( "--version", action="version", @@ -82,6 +92,9 @@ def main(args=None): setattr(conf, attr, value) print(f"Set conf.{attr} = {value}") + if opt.kadi_cmds_version is not None: + os.environ["KADI_CMDS_VERSION"] = str(opt.kadi_cmds_version) + update_cmds_archive( lookback=opt.lookback, stop=opt.stop, From 6c16a07e65e0192c967d611faddc36559dff3621 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Mon, 2 Feb 2026 06:53:25 -0500 Subject: [PATCH 16/31] Fix leftover version=2 in _update_cmds_archive --- kadi/commands/commands_v2.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/kadi/commands/commands_v2.py b/kadi/commands/commands_v2.py index edc6a439..ff8e7c50 100644 --- a/kadi/commands/commands_v2.py +++ b/kadi/commands/commands_v2.py @@ -1553,12 +1553,13 @@ def _update_cmds_archive(lookback, stop_loads, scenario, data_root): # Either no-match-prev-cmds or matching RLTT start disables matching previous cmds match_prev_cmds = not (conf.no_match_prev_cmds or conf.match_from_rltt_start) - idx_cmds_path = Path(data_root) / "cmds2.h5" - pars_dict_path = Path(data_root) / "cmds2.pkl" + version = kadi_cmds_version() + idx_cmds_path = Path(data_root) / f"cmds{version}.h5" + pars_dict_path = Path(data_root) / f"cmds{version}.pkl" if idx_cmds_path.exists(): - cmds_arch = load_idx_cmds(version=2, file=idx_cmds_path) - pars_dict = load_pars_dict(version=2, file=pars_dict_path) + cmds_arch = load_idx_cmds(file=idx_cmds_path) + pars_dict = load_pars_dict(file=pars_dict_path) else: # Make an empty cmds archive table and pars dict cmds_arch = CommandTable( From c748f95cd59bddcfadd687f24fa63d89faad3ee8 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Mon, 2 Feb 2026 07:04:37 -0500 Subject: [PATCH 17/31] Update documentation --- docs/commands_states/commands_details.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/commands_states/commands_details.rst b/docs/commands_states/commands_details.rst index ee1b9e00..c1903bf0 100644 --- a/docs/commands_states/commands_details.rst +++ b/docs/commands_states/commands_details.rst @@ -261,6 +261,12 @@ Environment variables application that is not aware of kadi scenarios, effectively a back door to override the flight commands. +``KADI_CMDS_VERSION`` + Set the kadi commands archive version to use. By default, kadi uses the highest + version found in the data directory. The version is the integer value in the kadi + ``cmds.h5`` and ``cmds.pkl`` files. This environment variable can be set to an + integer value (e.g. ``2`` or ``3``) to force kadi to use a specific version. + Mocking the current time ------------------------ Setting the ``CXOTIME_NOW`` environment variable allows you to pretend that the current From 2b9ecbf540de07d5d1998bc1fa076781808e43ed Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Mon, 2 Feb 2026 07:05:09 -0500 Subject: [PATCH 18/31] Maintain alphabetical order in docs update --- docs/commands_states/commands_details.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/commands_states/commands_details.rst b/docs/commands_states/commands_details.rst index c1903bf0..e1c356ce 100644 --- a/docs/commands_states/commands_details.rst +++ b/docs/commands_states/commands_details.rst @@ -256,17 +256,17 @@ Environment variables Override the default location of kadi flight data files ``cmds2.h5`` and ``cmds2.pkl``. -``KADI_SCENARIO`` - Set the default scenario. This can be used to set the scenario in an - application that is not aware of kadi scenarios, effectively a back door to - override the flight commands. - ``KADI_CMDS_VERSION`` Set the kadi commands archive version to use. By default, kadi uses the highest version found in the data directory. The version is the integer value in the kadi ``cmds.h5`` and ``cmds.pkl`` files. This environment variable can be set to an integer value (e.g. ``2`` or ``3``) to force kadi to use a specific version. +``KADI_SCENARIO`` + Set the default scenario. This can be used to set the scenario in an + application that is not aware of kadi scenarios, effectively a back door to + override the flight commands. + Mocking the current time ------------------------ Setting the ``CXOTIME_NOW`` environment variable allows you to pretend that the current From 33ab5360389e3f4030f3537abc34bd176b28ac00 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Mon, 2 Feb 2026 10:45:24 -0500 Subject: [PATCH 19/31] WIP some stuff --- kadi/commands/commands_v2.py | 4 ---- kadi/scripts/update_cmds_v2.py | 36 +++++++++++++++++++++++++++++++++- 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/kadi/commands/commands_v2.py b/kadi/commands/commands_v2.py index ff8e7c50..1a9d02ea 100644 --- a/kadi/commands/commands_v2.py +++ b/kadi/commands/commands_v2.py @@ -1528,10 +1528,6 @@ def update_cmds_archive( Scenario name for loads and command events data_root : str, Path Root directory where cmds2.h5 and cmds2.pkl are stored. Default is '.'. - match_prev_cmds : bool - One-time use flag set to True to update the cmds archive near the v1/v2 - transition of APR1420B. See ``utils/migrate_cmds_to_cmds2.py`` for - details. """ # For testing allow override of default `stop` value if stop is None: diff --git a/kadi/scripts/update_cmds_v2.py b/kadi/scripts/update_cmds_v2.py index 76f6acc4..2d6b073c 100644 --- a/kadi/scripts/update_cmds_v2.py +++ b/kadi/scripts/update_cmds_v2.py @@ -2,10 +2,12 @@ import argparse import os +import astropy.units as u +from cxotime import CxoTime from ska_helpers.run_info import log_run_info from kadi import __version__ -from kadi.commands.commands_v2 import update_cmds_archive +from kadi.commands.commands_v2 import RLTT_ERA_START, logger, update_cmds_archive from kadi.config import conf @@ -71,6 +73,12 @@ def get_opt(args=None): "(APR1420B). This implies --no-match-prev-cmds.", ) + dev_group.add_argument( + "--reprocess-from-rltt-start", + action="store_true", + help="Reprocess cmds archive from the start of the RLTT era", + ) + args = parser.parse_args(args) return args @@ -81,7 +89,33 @@ def main(args=None): """ opt = get_opt(args) log_run_info(log_func=print, opt=opt) + if opt.reprocess_from_rltt_start: + reprocess_from_rltt_start(opt) + else: + process_one_update(opt) + + +def reprocess_from_rltt_start(opt): + # Start the V2 updates a week and a day after CMDS_V2_START + step = 365 * u.day + date = RLTT_ERA_START + step + stop = CxoTime.now() + opt.lookback = step.to_value(u.day) + 30 + + while date < stop: + logger.info("*" * 80) + logger.info(f"Updating cmds2 to {date}") + logger.info("*" * 80) + opt.stop = date.date + process_one_update(opt) + date += step + + # Final catchup to `stop` + opt.stop = date.date + process_one_update(opt) + +def process_one_update(opt): # Transfer these developer-only options to conf for attr in ( "no_match_prev_cmds", From f1af31c4708a4b5654e9b7abd1979c7249082f15 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Tue, 3 Feb 2026 05:52:33 -0500 Subject: [PATCH 20/31] Define a max kadi_cmds_version and check it --- kadi/commands/core.py | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/kadi/commands/core.py b/kadi/commands/core.py index d18b6f0a..9a68347a 100644 --- a/kadi/commands/core.py +++ b/kadi/commands/core.py @@ -32,6 +32,8 @@ "kadi_cmds_version", ] +# Maximum supported version of kadi commands, used by kadi_cmds_version() +KADI_CMDS_VERSION_MAX = 3 # Events to filter for getting as-planned commands after SCS-107 SCS107_EVENTS = ["SCS-107", "Observing not run", "Obsid", "RTS"] @@ -112,18 +114,28 @@ def load_pars_dict(version=None, file=None): @functools.cache def kadi_cmds_version() -> int: - """Determine the kadi commands version by checking for cmds*.h5 files. + """Determine the kadi commands version by checking for cmds.h5,pkl files. - The version number can be an integer. Return the highest version number that - where the corresponding PARS_DICT_PATH(version) file also exists. + If the environment variable KADI_CMDS_VERSION is set then that version number is + returned instead. + + The version number can be an integer. Return the highest version number satisfying: + - cmds.h5 and cmds.pkl files exist in the data directory. + + In all cases the version must be less than or equal to KADI_CMDS_VERSION_MAX. Returns ------- int - Highest kadi commands version number found. + Highest allowed kadi commands version number found. """ - if version := os.environ.get("KADI_CMDS_VERSION"): - return int(version) + if ver_str := os.environ.get("KADI_CMDS_VERSION"): + version = int(ver_str) + if version > KADI_CMDS_VERSION_MAX: + raise ValueError( + f"KADI_CMDS_VERSION={version} env var exceeds maximum supported " + f"version {KADI_CMDS_VERSION_MAX}" + ) versions = set() for path in DATA_DIR().glob("cmds*.h5"): @@ -133,7 +145,7 @@ def kadi_cmds_version() -> int: except ValueError: continue else: - if PARS_DICT_PATH(ver).exists(): + if ver <= KADI_CMDS_VERSION_MAX and PARS_DICT_PATH(ver).exists(): versions.add(ver) if not versions: From 5d25bd8668b1723b6b1ac56575e305f8f11a9de8 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Tue, 3 Feb 2026 10:56:28 -0500 Subject: [PATCH 21/31] Simplify and improve updating cmds archive, supporting processing from RLTT start --- kadi/commands/commands_v2.py | 49 +++++++------ kadi/config.py | 12 ---- kadi/scripts/update_cmds_v2.py | 123 +++++++++++++++------------------ 3 files changed, 81 insertions(+), 103 deletions(-) diff --git a/kadi/commands/commands_v2.py b/kadi/commands/commands_v2.py index 1a9d02ea..eeb111d6 100644 --- a/kadi/commands/commands_v2.py +++ b/kadi/commands/commands_v2.py @@ -1504,6 +1504,7 @@ def update_cmds_archive( log_level=logging.INFO, scenario=None, data_root=".", + truncate_from_rltt_start=False, ): """Update cmds2.h5 and cmds2.pkl archive files. @@ -1528,6 +1529,9 @@ def update_cmds_archive( Scenario name for loads and command events data_root : str, Path Root directory where cmds2.h5 and cmds2.pkl are stored. Default is '.'. + truncate_from_rltt_start : bool + If True, truncate the commands archive starting at the RLTT era from + load RLTT_ERA_START_LOAD (APR1420B). Default is False. """ # For testing allow override of default `stop` value if stop is None: @@ -1539,16 +1543,21 @@ def update_cmds_archive( with conf.set_temp("commands_dir", data_root), temp_env_var("KADI", data_root): try: kadi_logger.setLevel(log_level) - _update_cmds_archive(lookback, stop, scenario, data_root) + _update_cmds_archive( + lookback, stop, scenario, data_root, truncate_from_rltt_start + ) finally: kadi_logger.setLevel(log_level_orig) -def _update_cmds_archive(lookback, stop_loads, scenario, data_root): +def _update_cmds_archive( + lookback, + stop_loads, + scenario, + data_root, + truncate_from_rltt_start, +): """Do the real work of updating the cmds archive""" - # Either no-match-prev-cmds or matching RLTT start disables matching previous cmds - match_prev_cmds = not (conf.no_match_prev_cmds or conf.match_from_rltt_start) - version = kadi_cmds_version() idx_cmds_path = Path(data_root) / f"cmds{version}.h5" pars_dict_path = Path(data_root) / f"cmds{version}.pkl" @@ -1564,7 +1573,6 @@ def _update_cmds_archive(lookback, stop_loads, scenario, data_root): ) del cmds_arch["timeline_id"] pars_dict = {} - match_prev_cmds = False # No match of previous commands since there are none. cmds_recent = update_cmd_events_and_loads_and_get_cmds_recent( scenario=scenario, @@ -1573,23 +1581,20 @@ def _update_cmds_archive(lookback, stop_loads, scenario, data_root): pars_dict=pars_dict, ) - if match_prev_cmds: - idx0_arch, idx0_recent = get_matching_block_idx(cmds_arch, cmds_recent) - else: + if truncate_from_rltt_start: + # Special case for reprocessing the commands archive starting at the RLTT + # era from load RLTT_ERA_START_LOAD (APR1420B). Find the index of the first + # command in these loads. Note np.argmax() returns the first matching index + # in the case of multiple matches. idx0_recent = 0 - if conf.match_from_rltt_start: - # Special case for reprocessing the commands archive starting at the RLTT - # era from load RLTT_ERA_START_LOAD (APR1420B). Find the index of the first - # command in these loads. Note np.argmax() returns the first matching index - # in the case of multiple matches. - idx0_arch = np.argmax(cmds_arch["source"] == RLTT_ERA_START_LOAD) - logger.info( - f"Matching from RLTT start load {RLTT_ERA_START_LOAD}, " - f"idx0_arch={idx0_arch}, idx0_recent={idx0_recent}" - ) - else: - # Append to end of existing cmds archive file - idx0_arch = len(cmds_arch) + idx0_arch = np.argmax(cmds_arch["source"] == RLTT_ERA_START_LOAD) + logger.info( + f"Matching from RLTT start load {RLTT_ERA_START_LOAD}, " + f"idx0_arch={idx0_arch}, idx0_recent={idx0_recent}" + ) + else: + # Normal processing: find the matching block between recent and archive cmds + idx0_arch, idx0_recent = get_matching_block_idx(cmds_arch, cmds_recent) # Convert from `params` col of dicts to index into same params in pars_dict. for cmd in cmds_recent: diff --git a/kadi/config.py b/kadi/config.py index 0134bb69..1c7235d6 100644 --- a/kadi/config.py +++ b/kadi/config.py @@ -102,23 +102,11 @@ class Conf(ConfigNamespace): ), ) - no_match_prev_cmds = ConfigItem( - False, - "Do not match previous command block when updating cmds v2. " - "Setting to True can produce an invalid commands table (experts only).", - ) - matching_block_size = ConfigItem( 500, "Matching block size for command blocks.", ) - match_from_rltt_start = ConfigItem( - False, - "Match previous commands exactly from the start of the RLTT era (APR1420B). " - "Developers only.", - ) - # Create a configuration instance for the user conf = Conf() diff --git a/kadi/scripts/update_cmds_v2.py b/kadi/scripts/update_cmds_v2.py index 2d6b073c..12c5fa40 100644 --- a/kadi/scripts/update_cmds_v2.py +++ b/kadi/scripts/update_cmds_v2.py @@ -7,8 +7,11 @@ from ska_helpers.run_info import log_run_info from kadi import __version__ -from kadi.commands.commands_v2 import RLTT_ERA_START, logger, update_cmds_archive -from kadi.config import conf +from kadi.commands.commands_v2 import ( + RLTT_ERA_START, + clear_caches, + update_cmds_archive, +) def get_opt(args=None): @@ -53,30 +56,10 @@ def get_opt(args=None): version="%(prog)s {version}".format(version=__version__), ) - # Developer-only options - dev_group = parser.add_argument_group("Developer-only options") - dev_group.add_argument( - "--no-match-prev-cmds", - action="store_true", - help="Do not enforce matching previous command block when updating cmds v2 " - "(experts only, this can produce an invalid commands table)", - ) - dev_group.add_argument( - "--matching-block-size", - type=int, - help=f"Matching block size (default={conf.matching_block_size})", - ) - dev_group.add_argument( - "--match-from-rltt-start", - action="store_true", - help="Match previous commands exactly from the start of the RLTT era " - "(APR1420B). This implies --no-match-prev-cmds.", - ) - - dev_group.add_argument( - "--reprocess-from-rltt-start", + parser.add_argument( + "--truncate-from-rltt-start", action="store_true", - help="Reprocess cmds archive from the start of the RLTT era", + help="Truncate cmds archive from the start of the RLTT era (APR1420B)", ) args = parser.parse_args(args) @@ -88,54 +71,56 @@ def main(args=None): Main function for update_cmds_v2 """ opt = get_opt(args) - log_run_info(log_func=print, opt=opt) - if opt.reprocess_from_rltt_start: - reprocess_from_rltt_start(opt) - else: - process_one_update(opt) - - -def reprocess_from_rltt_start(opt): - # Start the V2 updates a week and a day after CMDS_V2_START - step = 365 * u.day - date = RLTT_ERA_START + step - stop = CxoTime.now() - opt.lookback = step.to_value(u.day) + 30 - - while date < stop: - logger.info("*" * 80) - logger.info(f"Updating cmds2 to {date}") - logger.info("*" * 80) - opt.stop = date.date - process_one_update(opt) - date += step - - # Final catchup to `stop` - opt.stop = date.date - process_one_update(opt) - - -def process_one_update(opt): - # Transfer these developer-only options to conf - for attr in ( - "no_match_prev_cmds", - "matching_block_size", - "match_from_rltt_start", - ): - if (value := getattr(opt, attr)) is not None: - setattr(conf, attr, value) - print(f"Set conf.{attr} = {value}") if opt.kadi_cmds_version is not None: os.environ["KADI_CMDS_VERSION"] = str(opt.kadi_cmds_version) - update_cmds_archive( - lookback=opt.lookback, - stop=opt.stop, - log_level=opt.log_level, - scenario=opt.scenario, - data_root=opt.data_root, - ) + log_run_info(log_func=print, opt=opt) + + if opt.truncate_from_rltt_start: + process_from_rltt_start(opt) + else: + update_cmds_archive( + opt.lookback, + opt.stop, + opt.log_level, + opt.scenario, + opt.data_root, + opt.truncate_from_rltt_start, + ) + + +def process_from_rltt_start(opt): + # Final processing stop + stop0 = RLTT_ERA_START + 21 * u.day + stop1 = CxoTime(opt.stop) if opt.stop else CxoTime.now() + 21 * u.day + step = 365 * u.day + lookback0 = 30 * u.day + stops = CxoTime.linspace(stop0, stop1, step_max=step) + dt = stops[1] - stops[0] + + for stop in stops: + first_update = stop == stop0 + lookback = lookback0 if first_update else dt + lookback0 + + print() + print("*" * 80) + print( + f"Updating cmds archive to {stop} with " + f"lookback={lookback.to_value(u.day):.1f} days" + ) + print("*" * 80) + print() + + update_cmds_archive( + lookback=lookback.to_value(u.day), + stop=stop, + log_level=opt.log_level, + scenario=opt.scenario, + data_root=opt.data_root, + truncate_from_rltt_start=first_update, + ) + clear_caches() if __name__ == "__main__": From 1b45862435d56dc3a1a38b9a0bd8b0a55568aec6 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Tue, 3 Feb 2026 11:06:56 -0500 Subject: [PATCH 22/31] Some tidy and fix a problem in normal update processing --- kadi/scripts/update_cmds_v2.py | 39 +++++++++++++++++++++++----------- 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/kadi/scripts/update_cmds_v2.py b/kadi/scripts/update_cmds_v2.py index 12c5fa40..84ad9107 100644 --- a/kadi/scripts/update_cmds_v2.py +++ b/kadi/scripts/update_cmds_v2.py @@ -19,7 +19,7 @@ def get_opt(args=None): Get options for command line interface to update. """ parser = argparse.ArgumentParser( - description="Update HDF5 cmds v2 table", + description="Update cmds archive HDF5 and pickle files", ) parser.add_argument( "--data-root", @@ -37,9 +37,9 @@ def get_opt(args=None): ) parser.add_argument( "--log-level", - type=int, - default=10, - help="Log level (10=debug, 20=info, 30=warnings)", + type=str, + default="DEBUG", + help="Log level ('DEBUG', 'INFO', 'WARNING', etc.)", ) parser.add_argument( "--scenario", @@ -55,7 +55,6 @@ def get_opt(args=None): action="version", version="%(prog)s {version}".format(version=__version__), ) - parser.add_argument( "--truncate-from-rltt-start", action="store_true", @@ -81,16 +80,32 @@ def main(args=None): process_from_rltt_start(opt) else: update_cmds_archive( - opt.lookback, - opt.stop, - opt.log_level, - opt.scenario, - opt.data_root, - opt.truncate_from_rltt_start, + lookback=opt.lookback, + stop=opt.stop, + log_level=opt.log_level, + scenario=opt.scenario, + data_root=opt.data_root, + truncate_from_rltt_start=False, ) -def process_from_rltt_start(opt): +def process_from_rltt_start(opt: argparse.Namespace) -> None: + """ + Process commands archive from the start of the RLTT era in chunks. + + This function updates the commands archive from the RLTT era start date + (APR1420B) to the specified stop date (or Now+21 days) by breaking the + time range into manageable chunks to avoid memory issues. + + Parameters + ---------- + opt : argparse.Namespace + Command line options containing: + - stop: Optional stop date for processing + - log_level: Logging level + - scenario: Scenario for loads and command events + - data_root: Root directory for data + """ # Final processing stop stop0 = RLTT_ERA_START + 21 * u.day stop1 = CxoTime(opt.stop) if opt.stop else CxoTime.now() + 21 * u.day From ac45aa83ae14bbfd4528ecdb294c2e503a0aec1f Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Tue, 3 Feb 2026 11:10:22 -0500 Subject: [PATCH 23/31] Remove scenario option for archive update This option makes no sense and has not ever been tested. The only meaningful scenario for updating the flight cmds archive is the default None. --- kadi/commands/commands_v2.py | 9 +-------- kadi/scripts/update_cmds_v2.py | 7 ------- 2 files changed, 1 insertion(+), 15 deletions(-) diff --git a/kadi/commands/commands_v2.py b/kadi/commands/commands_v2.py index eeb111d6..3575f89e 100644 --- a/kadi/commands/commands_v2.py +++ b/kadi/commands/commands_v2.py @@ -1502,7 +1502,6 @@ def update_cmds_archive( lookback=None, stop=None, log_level=logging.INFO, - scenario=None, data_root=".", truncate_from_rltt_start=False, ): @@ -1525,8 +1524,6 @@ def update_cmds_archive( events are included. log_level : int Logging level. Default is ``logging.INFO``. - scenario : str, None - Scenario name for loads and command events data_root : str, Path Root directory where cmds2.h5 and cmds2.pkl are stored. Default is '.'. truncate_from_rltt_start : bool @@ -1543,9 +1540,7 @@ def update_cmds_archive( with conf.set_temp("commands_dir", data_root), temp_env_var("KADI", data_root): try: kadi_logger.setLevel(log_level) - _update_cmds_archive( - lookback, stop, scenario, data_root, truncate_from_rltt_start - ) + _update_cmds_archive(lookback, stop, data_root, truncate_from_rltt_start) finally: kadi_logger.setLevel(log_level_orig) @@ -1553,7 +1548,6 @@ def update_cmds_archive( def _update_cmds_archive( lookback, stop_loads, - scenario, data_root, truncate_from_rltt_start, ): @@ -1575,7 +1569,6 @@ def _update_cmds_archive( pars_dict = {} cmds_recent = update_cmd_events_and_loads_and_get_cmds_recent( - scenario=scenario, stop_loads=stop_loads, lookback=lookback, pars_dict=pars_dict, diff --git a/kadi/scripts/update_cmds_v2.py b/kadi/scripts/update_cmds_v2.py index 84ad9107..d9191748 100644 --- a/kadi/scripts/update_cmds_v2.py +++ b/kadi/scripts/update_cmds_v2.py @@ -41,10 +41,6 @@ def get_opt(args=None): default="DEBUG", help="Log level ('DEBUG', 'INFO', 'WARNING', etc.)", ) - parser.add_argument( - "--scenario", - help="Scenario for loads and command events outputs (default=None)", - ) parser.add_argument( "--kadi-cmds-version", type=int, @@ -83,7 +79,6 @@ def main(args=None): lookback=opt.lookback, stop=opt.stop, log_level=opt.log_level, - scenario=opt.scenario, data_root=opt.data_root, truncate_from_rltt_start=False, ) @@ -103,7 +98,6 @@ def process_from_rltt_start(opt: argparse.Namespace) -> None: Command line options containing: - stop: Optional stop date for processing - log_level: Logging level - - scenario: Scenario for loads and command events - data_root: Root directory for data """ # Final processing stop @@ -131,7 +125,6 @@ def process_from_rltt_start(opt: argparse.Namespace) -> None: lookback=lookback.to_value(u.day), stop=stop, log_level=opt.log_level, - scenario=opt.scenario, data_root=opt.data_root, truncate_from_rltt_start=first_update, ) From 935041b7a5a5a2314423a5917e06a4a1a7d9d64d Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Tue, 3 Feb 2026 12:03:21 -0500 Subject: [PATCH 24/31] Fix regression test failure --- kadi/commands/commands_v2.py | 8 ++++++-- kadi/commands/core.py | 2 ++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/kadi/commands/commands_v2.py b/kadi/commands/commands_v2.py index 3575f89e..7f760902 100644 --- a/kadi/commands/commands_v2.py +++ b/kadi/commands/commands_v2.py @@ -1556,7 +1556,7 @@ def _update_cmds_archive( idx_cmds_path = Path(data_root) / f"cmds{version}.h5" pars_dict_path = Path(data_root) / f"cmds{version}.pkl" - if idx_cmds_path.exists(): + if idx_cmds_path_exists := idx_cmds_path.exists(): cmds_arch = load_idx_cmds(file=idx_cmds_path) pars_dict = load_pars_dict(file=pars_dict_path) else: @@ -1574,7 +1574,11 @@ def _update_cmds_archive( pars_dict=pars_dict, ) - if truncate_from_rltt_start: + if not idx_cmds_path_exists: + # New archive, so use all recent commands with zero-length existing archive + idx0_recent = 0 + idx0_arch = 0 + elif truncate_from_rltt_start: # Special case for reprocessing the commands archive starting at the RLTT # era from load RLTT_ERA_START_LOAD (APR1420B). Find the index of the first # command in these loads. Note np.argmax() returns the first matching index diff --git a/kadi/commands/core.py b/kadi/commands/core.py index 9a68347a..54646c38 100644 --- a/kadi/commands/core.py +++ b/kadi/commands/core.py @@ -136,6 +136,8 @@ def kadi_cmds_version() -> int: f"KADI_CMDS_VERSION={version} env var exceeds maximum supported " f"version {KADI_CMDS_VERSION_MAX}" ) + else: + return version versions = set() for path in DATA_DIR().glob("cmds*.h5"): From 163241394c2b935fb8bde8bade7805f90d9a723e Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Tue, 3 Feb 2026 12:18:49 -0500 Subject: [PATCH 25/31] Better implementation of kadi_cmds_version and add a test --- kadi/commands/core.py | 26 +++++++++++--------------- kadi/commands/tests/test_commands.py | 6 ++++++ 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/kadi/commands/core.py b/kadi/commands/core.py index 54646c38..d7e14a87 100644 --- a/kadi/commands/core.py +++ b/kadi/commands/core.py @@ -129,6 +129,9 @@ def kadi_cmds_version() -> int: int Highest allowed kadi commands version number found. """ + # Allowed versions in descending order down to version=2. + versions_allowed = tuple(range(KADI_CMDS_VERSION_MAX, 1, -1)) + if ver_str := os.environ.get("KADI_CMDS_VERSION"): version = int(ver_str) if version > KADI_CMDS_VERSION_MAX: @@ -137,23 +140,16 @@ def kadi_cmds_version() -> int: f"version {KADI_CMDS_VERSION_MAX}" ) else: - return version - - versions = set() - for path in DATA_DIR().glob("cmds*.h5"): - ver_str = path.stem.replace("cmds", "") - try: - ver = int(ver_str) - except ValueError: - continue - else: - if ver <= KADI_CMDS_VERSION_MAX and PARS_DICT_PATH(ver).exists(): - versions.add(ver) + versions_allowed = (version,) - if not versions: - raise RuntimeError(f"no valid kadi commands versions found in {DATA_DIR()}") + for version in versions_allowed: + if IDX_CMDS_PATH(version).exists() and PARS_DICT_PATH(version).exists(): + return version - return max(versions) + raise RuntimeError( + f"no valid kadi commands versions found in {DATA_DIR()}. " + f"Allowed versions: {versions_allowed}" + ) @functools.lru_cache diff --git a/kadi/commands/tests/test_commands.py b/kadi/commands/tests/test_commands.py index fd96d9d9..f7eb6cd3 100644 --- a/kadi/commands/tests/test_commands.py +++ b/kadi/commands/tests/test_commands.py @@ -245,6 +245,12 @@ def test_get_cmds_from_backstop_and_add_cmds(): assert np.all(bs_cmds["params"][ok] != {}) +def test_kadi_cmds_version(): + """Test that kadi commands version matches expectation.""" + version_exp = os.environ.get("KADI_CMDS_VERSION", str(core.KADI_CMDS_VERSION_MAX)) + assert core.kadi_cmds_version() == int(version_exp) + + @pytest.mark.skipif("not HAS_MPDIR") @pytest.mark.skipif(not HAS_INTERNET, reason="No internet connection") def test_commands_create_archive_regress( From 20e7dbf3d1ff06ea6e327161020d09661b84faa8 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Wed, 4 Feb 2026 05:46:13 -0500 Subject: [PATCH 26/31] Duplicate commands testing into two modules for V2 and >= V3 --- kadi/commands/tests/conftest.py | 7 +- kadi/commands/tests/test_commands.py | 5 + kadi/commands/tests/test_commands_v2.py | 2042 +++++++++++++++++++++++ 3 files changed, 2052 insertions(+), 2 deletions(-) create mode 100644 kadi/commands/tests/test_commands_v2.py diff --git a/kadi/commands/tests/conftest.py b/kadi/commands/tests/conftest.py index e7451c82..96d121bd 100644 --- a/kadi/commands/tests/conftest.py +++ b/kadi/commands/tests/conftest.py @@ -2,16 +2,19 @@ import ska_sun import kadi.commands as kc +import kadi.commands.core as kcc @pytest.fixture() def fast_sun_position_method(monkeypatch: pytest.MonkeyPatch): - monkeypatch.setattr(ska_sun.conf, "sun_position_method_default", "fast") + if kcc.kadi_cmds_version() == 2: + monkeypatch.setattr(ska_sun.conf, "sun_position_method_default", "fast") @pytest.fixture() def disable_hrc_scs107_commanding(monkeypatch: pytest.MonkeyPatch): - monkeypatch.setattr(kc.conf, "disable_hrc_scs107_commanding", True) + if kcc.kadi_cmds_version() == 2: + monkeypatch.setattr(kc.conf, "disable_hrc_scs107_commanding", True) @pytest.fixture(scope="module", autouse=True) diff --git a/kadi/commands/tests/test_commands.py b/kadi/commands/tests/test_commands.py index f7eb6cd3..a738824e 100644 --- a/kadi/commands/tests/test_commands.py +++ b/kadi/commands/tests/test_commands.py @@ -37,6 +37,11 @@ HAS_MPDIR = Path(os.environ["SKA"], "data", "mpcrit1", "mplogs", "2020").exists() HAS_INTERNET = has_internet() +KADI_CMDS_VERSION = core.kadi_cmds_version() + +pytestmark = pytest.mark.skipif( + KADI_CMDS_VERSION < 3, reason="requires KADI_CMDS_VERSION >= 3" +) try: agasc.get_agasc_filename(version="1p8") diff --git a/kadi/commands/tests/test_commands_v2.py b/kadi/commands/tests/test_commands_v2.py new file mode 100644 index 00000000..706f0361 --- /dev/null +++ b/kadi/commands/tests/test_commands_v2.py @@ -0,0 +1,2042 @@ +import os +import re +from pathlib import Path + +# Use data file from parse_cm.test for get_cmds_from_backstop test. +# This package is a dependency +import agasc +import astropy.units as u +import numpy as np +import parse_cm.paths +import parse_cm.tests +import pytest +import ska_helpers.utils +import ska_sun +from astropy.table import Table, vstack +from chandra_time import secs2date +from cxotime import CxoTime +from Quaternion import Quat +from testr.test_helper import has_internet + +import kadi +import kadi.commands.states as kcs +from kadi import commands +from kadi.commands import ( + CommandTable, + commands_v2, + conf, + core, + get_observations, + get_starcats, + get_starcats_as_table, + read_backstop, +) +from kadi.commands.command_sets import get_cmds_from_event +from kadi.commands.commands_v2 import read_cmd_events_from_sheet +from kadi.scripts import update_cmds_v2 + +HAS_MPDIR = Path(os.environ["SKA"], "data", "mpcrit1", "mplogs", "2020").exists() +HAS_INTERNET = has_internet() +KADI_CMDS_VERSION = core.kadi_cmds_version() + + +pytestmark = pytest.mark.skipif( + KADI_CMDS_VERSION > 2, reason="requires KADI_CMDS_VERSION <= 2" +) + +try: + agasc.get_agasc_filename(version="1p8") + HAS_AGASC_1P8 = True +except FileNotFoundError: + HAS_AGASC_1P8 = False + + +def get_cmds_from_cmd_evts_text(cmd_evts_text: str): + """Get commands from a cmd_events text string. + + This helper function can make it easier to test scenarios without creating a + scenario file. + """ + cmd_evts = Table.read(cmd_evts_text, format="ascii.csv", fill_values=[]) + cmds_list = [ + get_cmds_from_event(cmd_evt["Date"], cmd_evt["Event"], cmd_evt["Params"]) + for cmd_evt in cmd_evts + ] + cmds: CommandTable = core.vstack_exact(cmds_list) + cmds.sort_in_backstop_order() + + return cmds + + +def test_find(): + idx_cmds = commands_v2.IDX_CMDS + pars_dict = commands_v2.PARS_DICT + + cs = core._find( + "2012:029:12:00:00", "2012:030:12:00:00", idx_cmds=idx_cmds, pars_dict=pars_dict + ) + assert isinstance(cs, Table) + assert len(cs) == 151 + assert np.all(cs["source"][:10] == "JAN2612A") + assert np.all(cs["source"][-10:] == "JAN3012C") + assert cs["date"][0] == "2012:029:13:00:00.000" + assert cs["date"][-1] == "2012:030:11:00:01.285" + assert cs["tlmsid"][-1] == "CTXBON" + + cs = core._find( + "2012:029:12:00:00", + "2012:030:12:00:00", + type="simtrans", + idx_cmds=idx_cmds, + pars_dict=pars_dict, + ) + assert len(cs) == 2 + assert np.all(cs["date"] == ["2012:030:02:00:00.000", "2012:030:08:27:02.000"]) + + cs = core._find( + "2012:015:12:00:00", + "2012:030:12:00:00", + idx_cmds=idx_cmds, + pars_dict=pars_dict, + type="acispkt", + tlmsid="wsvidalldn", + ) + assert len(cs) == 3 + assert np.all( + cs["date"] + == ["2012:018:01:16:15.798", "2012:020:16:51:17.713", "2012:026:05:28:09.000"] + ) + + cs = core._find( + "2011:001:12:00:00", + "2014:001:12:00:00", + msid="aflcrset", + idx_cmds=idx_cmds, + pars_dict=pars_dict, + ) + assert len(cs) == 2494 + + +def test_get_cmds(): + cs = commands.get_cmds("2012:029:12:00:00", "2012:030:12:00:00") + assert isinstance(cs, commands.CommandTable) + assert len(cs) == 151 # OBS commands in v2 only + assert np.all(cs["source"][:10] == "JAN2612A") + assert np.all(cs["source"][-10:] == "JAN3012C") + assert cs["date"][0] == "2012:029:13:00:00.000" + assert cs["date"][-1] == "2012:030:11:00:01.285" + assert cs["tlmsid"][-1] == "CTXBON" + + cs = commands.get_cmds("2012:029:12:00:00", "2012:030:12:00:00", type="simtrans") + assert len(cs) == 2 + assert np.all(cs["date"] == ["2012:030:02:00:00.000", "2012:030:08:27:02.000"]) + assert np.all(cs["pos"] == [75624, 73176]) # from params + + cmd = cs[1] + + assert repr(cmd).startswith("" + ) + assert str(cmd).endswith("scs=133 step=161 source=JAN3012C vcdu=15639968 pos=73176") + + assert cmd["pos"] == 73176 + assert cmd["step"] == 161 + + +def test_get_cmds_zero_length_result(): + cmds = commands.get_cmds(date="2017:001:12:00:00") + assert len(cmds) == 0 + source_name = "source" + assert cmds.colnames == [ + "idx", + "date", + "type", + "tlmsid", + "scs", + "step", + "time", + source_name, + "vcdu", + "params", + ] + + +def test_get_cmds_inclusive_stop(): + # get_cmds returns start <= date < stop for inclusive_stop=False (default) + # or start <= date <= stop for inclusive_stop=True. + # Query over a range that includes two commands at exactly start and stop. + start, stop = "2020:001:15:50:00.000", "2020:001:15:50:00.257" + cmds = commands.get_cmds(start, stop) + assert np.all(cmds["date"] == [start]) + + cmds = commands.get_cmds(start, stop, inclusive_stop=True) + assert np.all(cmds["date"] == [start, stop]) + + +def test_cmds_as_list_of_dict(): + cmds = commands.get_cmds("2020:140", "2020:141") + cmds_list = cmds.as_list_of_dict() + assert isinstance(cmds_list, list) + assert isinstance(cmds_list[0], dict) + cmds_rt = commands.CommandTable(cmds) + assert set(cmds_rt.colnames) == set(cmds.colnames) + for name in cmds.colnames: + assert np.all(cmds_rt[name] == cmds[name]) + + +def test_cmds_as_list_of_dict_ska_parsecm(): + """Test the ska_parsecm=True compatibility mode for list_of_dict""" + cmds = commands.get_cmds("2020:140", "2020:141") + cmds_list = cmds.as_list_of_dict(ska_parsecm=True) + assert isinstance(cmds_list, list) + assert isinstance(cmds_list[0], dict) + exp = { + "cmd": "COMMAND_HW", # Cmd parameter exists and matches type + "date": "2020:140:00:00:00.000", + "idx": 21387, + "params": {"HEX": "7C063C0", "MSID": "CIU1024T"}, # Keys are upper case + "scs": 129, + "step": 496, + "time": 706233669.184, + "tlmsid": "CIMODESL", + "type": "COMMAND_HW", + "vcdu": 12516929, + "source": "MAY1820A", + } + + assert cmds_list[0] == exp + + for cmd in cmds_list: + assert cmd.get("cmd") == cmd.get("type") + assert all(param.upper() == param for param in cmd["params"]) + + +def test_get_cmds_from_backstop_and_add_cmds(): + bs_file = Path(parse_cm.tests.__file__).parent / "data" / "CR182_0803.backstop" + bs_cmds = commands.get_cmds_from_backstop(bs_file, remove_starcat=True) + + cmds = commands.get_cmds(start="2018:182:00:00:00", stop="2018:182:08:00:00") + + assert len(bs_cmds) == 674 + assert len(cmds) == 57 + + # Get rid of source and timeline_id columns which can vary between v1 and v2 + for cs in bs_cmds, cmds: + if "source" in cs.colnames: + del cs["source"] + if "timeline_id" in cs.colnames: + del cs["timeline_id"] + + assert bs_cmds.colnames == cmds.colnames + for bs_col, col in zip(bs_cmds.itercols(), cmds.itercols()): + assert bs_col.dtype == col.dtype + + assert np.all(secs2date(cmds["time"]) == cmds["date"]) + assert np.all(secs2date(bs_cmds["time"]) == bs_cmds["date"]) + + new_cmds = cmds.add_cmds(bs_cmds) + assert len(new_cmds) == len(cmds) + len(bs_cmds) + + # No MP_STARCAT command parameters by default + ok = bs_cmds["type"] == "MP_STARCAT" + assert np.count_nonzero(ok) == 15 + assert np.all(bs_cmds["params"][ok] == {}) + + # Accept MP_STARCAT commands (also check read_backstop command) + bs_cmds = commands.read_backstop(bs_file) + ok = bs_cmds["type"] == "MP_STARCAT" + assert np.count_nonzero(ok) == 15 + assert np.all(bs_cmds["params"][ok] != {}) + + +def test_kadi_cmds_version(): + """Test that kadi commands version matches expectation.""" + version_exp = os.environ.get("KADI_CMDS_VERSION", str(core.KADI_CMDS_VERSION_MAX)) + assert core.kadi_cmds_version() == int(version_exp) + + +@pytest.mark.skipif("not HAS_MPDIR") +@pytest.mark.skipif(not HAS_INTERNET, reason="No internet connection") +def test_commands_create_archive_regress( + tmpdir, fast_sun_position_method, disable_hrc_scs107_commanding +): + """Create cmds archive from scratch and test that it matches flight + + This tests over an eventful month that includes IU reset/NSM, SCS-107 + (radiation), fast replan, loads approved but not uplinked, etc. + """ + kadi_orig = os.environ.get("KADI") + start = CxoTime("2021:290") + stop = start + 30 * u.day + cmds_flight = commands.get_cmds(start + 3 * u.day, stop - 3 * u.day) + cmds_flight.fetch_params() + + sched_stop_flight: np.ndarray = (cmds_flight["type"] == "LOAD_EVENT") & ( + cmds_flight["event_type"] == "SCHEDULED_STOP_TIME" + ) + + with conf.set_temp("commands_dir", str(tmpdir)): + try: + os.environ["KADI"] = str(tmpdir) + update_cmds_v2.main( + ( + f"--stop={stop.date}", + f"--data-root={tmpdir}", + ) + ) + # Force reload of LazyVal + del commands_v2.IDX_CMDS._val + del commands_v2.PARS_DICT._val + del commands_v2.REV_PARS_DICT._val + + # Make sure we are seeing the temporary cmds archive + cmds_empty = commands.get_cmds(start - 60 * u.day, start - 50 * u.day) + cmds_empty = commands.get_cmds(start - 60 * u.day, start - 50 * u.day) + assert len(cmds_empty) == 0 + + cmds_local = commands.get_cmds(start + 3 * u.day, stop - 3 * u.day) + + cmds_local.fetch_params() + if len(cmds_flight) != len(cmds_local): + # Code to debug problems, leave commented for production + # out = "\n".join(cmds_flight.pformat_like_backstop()) + # Path("cmds_flight.txt").write_text(out) + # out = "\n".join(cmds_local.pformat_like_backstop()) + # Path("cmds_local.txt").write_text(out) + assert len(cmds_flight) == len(cmds_local) + + sched_stop_local: np.ndarray = (cmds_local["type"] == "LOAD_EVENT") & ( + cmds_local["event_type"] == "SCHEDULED_STOP_TIME" + ) + + # PR#364 changed the time stamp of SCHEDULED_STOP_TIME commands when there + # is an interrupt, which in turn changes the order. First check that the + # sources of such commands are the same (we have the same sched stop + # commands but ignore timestamps). + assert sorted(cmds_flight[sched_stop_flight]["source"]) == sorted( + cmds_local[sched_stop_local]["source"] + ) + # Now remove such commands from both for the rest of the comparison. + cmds_flight = cmds_flight[~sched_stop_flight] + cmds_local = cmds_local[~sched_stop_local] + + # Validate quaternions using numeric comparison and then remove + # from the == comparison below. This is both appropriate for these + # numeric values and also necessary to deal with architecture + # differences when run on fido vs kady for example. + # In reality, the numeric differences only occur on "calculated" + # quaternions such as the NSM quaternions. + for idx in range(len(cmds_flight)): + for att_name in ["targ_att", "prev_att"]: + if att_name not in cmds_flight[idx]["params"]: + continue + flight_q = Quat(q=cmds_flight[idx]["params"][att_name]) + local_q = Quat(q=cmds_local[idx]["params"][att_name]) + dq = flight_q.dq(local_q) + for attr in ("roll0", "pitch", "yaw"): + assert abs(getattr(dq, attr)) < 1e-6 + del cmds_flight[idx]["params"][att_name] + del cmds_local[idx]["params"][att_name] + + # 'starcat_idx' param in OBS cmd does not match since the pickle files + # are different, so remove it. + for cmds in (cmds_local, cmds_flight): + for cmd in cmds: + if cmd["tlmsid"] == "OBS" and "starcat_idx" in cmd["params"]: + del cmd["params"]["starcat_idx"] + + for attr in ("tlmsid", "date", "params"): + assert np.all(cmds_flight[attr] == cmds_local[attr]) + + finally: + if kadi_orig is None: + del os.environ["KADI"] + else: + os.environ["KADI"] = kadi_orig + + commands.clear_caches() + + +def stop_date_fixture_factory(stop_date): + @pytest.fixture() + def stop_date_fixture(monkeypatch): + commands.clear_caches() + monkeypatch.setenv("CXOTIME_NOW", stop_date) + cmds_dir = Path(conf.commands_dir) / CxoTime(stop_date).iso[:9] + with commands.conf.set_temp("commands_dir", str(cmds_dir)): + yield + commands.clear_caches() + + return stop_date_fixture + + +# 2021:297 0300z just after recovery maneuver following 2021:296 NSM +stop_date_2021_10_24 = stop_date_fixture_factory("2021-10-24T03:00:00") +stop_date_2020_12_03 = stop_date_fixture_factory("2020-12-03") +stop_date_2023_203 = stop_date_fixture_factory("2023:203") +stop_date_2025_10_25 = stop_date_fixture_factory("2025-10-25") +stop_date_2024_035_23_00_00 = stop_date_fixture_factory("2024:035:23:00:00") + + +@pytest.mark.skipif(not HAS_INTERNET, reason="No internet connection") +def test_get_scheduled_stop_time_commands(stop_date_2025_10_25): + """Test a time frame with two load interrupts. + + These provide test coverage of the get_rltt_cmd() and get_scheduled_stop_time_cmd() + methods. + """ + cmds = commands.get_cmds("2025:290", "2025:300") + cok = cmds[(cmds["type"] == "LOAD_EVENT") & (cmds["tlmsid"] == "None")] + params_list = cok["params"].tolist() + for date, params in zip(cok["date"], params_list): + params["date"] = date + assert params_list == [ + {"event_type": "SCHEDULED_STOP_TIME", "date": "2025:292:21:47:50.679"}, + { + "event_type": "RUNNING_LOAD_TERMINATION_TIME", + "date": "2025:292:21:47:50.679", + }, + { + "event_type": "SCHEDULED_STOP_TIME", + "scheduled_stop_time_orig": "2025:299:22:52:08.292", + "interrupt_load": "OCT2125A", + "date": "2025:294:18:45:00.000", + }, + { + "event_type": "RUNNING_LOAD_TERMINATION_TIME", + "date": "2025:294:18:45:00.000", + }, + { + "event_type": "SCHEDULED_STOP_TIME", + "scheduled_stop_time_orig": "2025:299:22:52:08.282", + "interrupt_load": "OCT2225A", + "date": "2025:295:21:30:00.000", + }, + { + "event_type": "RUNNING_LOAD_TERMINATION_TIME", + "date": "2025:295:21:30:00.000", + }, + {"event_type": "SCHEDULED_STOP_TIME", "date": "2025:299:22:52:08.292"}, + ] + + +@pytest.mark.skipif(not HAS_INTERNET, reason="No internet connection") +def test_nsm_safe_mode_pitch_offsets_state_constraints(stop_date_2023_203): + """Test NSM, Safe mode transitions with pitch offsets along with state constraints. + + State constraints testing means that an NMAN maneuver with auto-NPNT transition + is properly interrupted by a NSM or Safe Mode. In this case the NMAN is interrupted + and NPNT never happens. + """ + scenario = "test-nsm-safe-mode" + cmd_events_path = kadi.paths.CMD_EVENTS_PATH(scenario) + cmd_events_path.parent.mkdir(parents=True, exist_ok=True) + # Maneuver attitude: ska_sun.get_att_for_sun_pitch_yaw(pitch=170, yaw=0, time="2023:199") + text = """ + State,Date,Event,Params,Author,Reviewer,Comment + Definitive,2023:199:00:00:00.000,Safe mode,120,,, + Definitive,2023:199:02:00:00.000,NSM,,,, + Definitive,2023:199:03:00:00.000,Maneuver,-0.84752928 0.52176697 0.08279618 0.05097206,,, + Definitive,2023:199:03:17:00.000,NSM,50,,, + Definitive,2023:199:03:30:00.000,Safe mode,,,, + Definitive,2023:199:04:30:00.000,NSM,100,,, + """ + cmd_events_path.write_text(text) + states = kcs.get_states( + "2023:198:23:00:00", + "2023:199:05:00:00", + state_keys=["pitch", "pcad_mode"], + scenario=scenario, + ) + states["pitch"].info.format = ".1f" + out = states["datestart", "pitch", "pcad_mode"].pformat() + + exp = [ + " datestart pitch pcad_mode", + "--------------------- ----- ---------", + "2023:198:23:00:00.000 144.6 NPNT", + "2023:199:00:00:00.000 143.9 STBY", # Safe mode to 120 + "2023:199:00:04:31.439 135.7 STBY", + "2023:199:00:09:02.879 123.7 STBY", + "2023:199:00:13:34.318 120.0 STBY", + "2023:199:02:00:00.000 119.2 NSUN", # NSM to default 90 + "2023:199:02:04:57.883 109.2 NSUN", + "2023:199:02:09:55.766 94.5 NSUN", + "2023:199:02:14:53.648 90.0 NSUN", + "2023:199:03:00:00.000 90.0 NMAN", # Maneuver to 170 + "2023:199:03:00:10.250 90.3 NMAN", + "2023:199:03:05:02.010 93.9 NMAN", + "2023:199:03:09:53.770 103.1 NMAN", + "2023:199:03:14:45.530 115.6 NMAN", + "2023:199:03:17:00.000 114.8 NSUN", # NMAN interrupted by NSM to 50 + "2023:199:03:21:48.808 106.0 NSUN", + "2023:199:03:26:37.617 87.8 NSUN", + "2023:199:03:30:00.000 87.9 STBY", # NSM to 50 interrupted by Safe mode + "2023:199:03:35:06.588 89.3 STBY", # to default 90 + "2023:199:03:40:13.175 90.0 STBY", + "2023:199:04:30:00.000 90.6 NSUN", # NSM to 100 + "2023:199:04:34:11.100 96.9 NSUN", + "2023:199:04:38:22.200 100.0 NSUN", + ] + + assert out == exp + + +@pytest.mark.skipif(not HAS_INTERNET, reason="No internet connection") +def test_get_cmds_v2_arch_only(stop_date_2020_12_03): # noqa: ARG001 + cmds = commands.get_cmds(start="2020-01-01", stop="2020-01-02") + cmds = cmds[cmds["tlmsid"] != "OBS"] + assert len(cmds) == 153 + assert np.all(cmds["idx"] != -1) + # Also do a zero-length query + cmds = commands.get_cmds(start="2020-01-01", stop="2020-01-01") + assert len(cmds) == 0 + commands.clear_caches() + + +@pytest.mark.skipif(not HAS_INTERNET, reason="No internet connection") +def test_get_cmds_v2_arch_recent(stop_date_2020_12_03): # noqa: ARG001 + cmds = commands.get_cmds(start="2020-09-01", stop="2020-12-01") + cmds = cmds[cmds["tlmsid"] != "OBS"] + + # Since recent matches arch in the past, even though the results are a mix + # of arch and recent, they commands actually come from the arch because of + # how the matching block is used (commands come from arch up through the end + # of the matching block). + assert np.all(cmds["idx"] != -1) + # PR #248: made this change from 17640 to 17644 + assert 17640 <= len(cmds) <= 17644 + + commands.clear_caches() + + +@pytest.mark.skipif(not HAS_INTERNET, reason="No internet connection") +def test_get_cmds_v2_recent_only(stop_date_2020_12_03): # noqa: ARG001 + # This query stop is well beyond the default stop date, so it should get + # only commands out to the end of the NOV3020A loads (~ Dec 7). + cmds = commands.get_cmds(start="2020-12-01", stop="2021-01-01") + cmds = cmds[cmds["tlmsid"] != "OBS"] + assert len(cmds) == 1523 + assert np.all(cmds["idx"] == -1) + # fmt: off + assert cmds[:5].pformat_like_backstop() == [ + "2020:336:00:08:38.610 | COMMAND_HW | CNOOP | NOV3020A | hex=7E00000, msid=CNOOPLR, scs=128", + "2020:336:00:08:39.635 | COMMAND_HW | CNOOP | NOV3020A | hex=7E00000, msid=CNOOPLR, scs=128", + "2020:336:00:12:55.214 | ACISPKT | AA00000000 | NOV3020A | cmds=3, words=3, scs=131", + "2020:336:00:12:55.214 | ORBPOINT | None | NOV3020A | event_type=XEF1000, scs=0", + "2020:336:00:12:59.214 | ACISPKT | AA00000000 | NOV3020A | cmds=3, words=3, scs=131", + ] + assert cmds[-5:].pformat_like_backstop() == [ + "2020:342:03:15:02.313 | COMMAND_SW | OFMTSNRM | NOV3020A | hex=8010A00, msid=OFMTSNRM, scs=130", + "2020:342:03:15:02.313 | COMMAND_SW | COSCSEND | NOV3020A | hex=C800000, msid=OBC_END_SCS, scs=130", + "2020:342:06:04:34.287 | ACISPKT | AA00000000 | NOV3020A | cmds=3, words=3, scs=133", + "2020:342:06:04:34.287 | COMMAND_SW | COSCSEND | NOV3020A | hex=C800000, msid=OBC_END_SCS, scs=133", + "2020:342:06:04:34.287 | LOAD_EVENT | None | NOV3020A | event_type=SCHEDULED_STOP_TIME, scs=0", + ] + # fmt: on + # Same for no stop date + cmds = commands.get_cmds(start="2020-12-01", stop=None) + cmds = cmds[cmds["tlmsid"] != "OBS"] + assert len(cmds) == 1523 + assert np.all(cmds["idx"] == -1) + + # zero-length query + cmds = commands.get_cmds(start="2020-12-01", stop="2020-12-01") + assert len(cmds) == 0 + commands.clear_caches() + + +@pytest.mark.skipif(not HAS_INTERNET, reason="No internet connection") +def test_get_cmds_nsm_2021(stop_date_2021_10_24, disable_hrc_scs107_commanding): + """NSM at ~2021:296:10:41. This tests non-load commands from cmd_events.""" + cmds = commands.get_cmds("2021:296:10:35:00") # , '2021:298:01:58:00') + cmds = cmds[cmds["tlmsid"] != "OBS"] + exp = [ + "2021:296:10:35:00.000 | COMMAND_HW | CIMODESL | OCT1821A | " + "hex=7C067C0, msid=CIU1024X, scs=128", + "2021:296:10:35:00.257 | COMMAND_HW | CTXAOF | OCT1821A | " + "hex=780000C, msid=CTXAOF, scs=128", + "2021:296:10:35:00.514 | COMMAND_HW | CPAAOF | OCT1821A | " + "hex=780001E, msid=CPAAOF, scs=128", + "2021:296:10:35:00.771 | COMMAND_HW | CTXBOF | OCT1821A | " + "hex=780004C, msid=CTXBOF, scs=128", + "2021:296:10:35:01.028 | COMMAND_HW | CPABON | OCT1821A | " + "hex=7800056, msid=CPABON, scs=128", + "2021:296:10:35:01.285 | COMMAND_HW | CTXBON | OCT1821A | " + "hex=7800044, msid=CTXBON, scs=128", + "2021:296:10:41:57.000 | LOAD_EVENT | None | CMD_EVT | " + "event=Load_not_run, event_date=2021:296:10:41:57, event_type=LOAD_NOT_RUN, " + "load=OCT2521A, scs=0", + "2021:296:10:41:57.000 | COMMAND_SW | AONSMSAF | CMD_EVT | " + "event=NSM, event_date=2021:296:10:41:57, scs=0", + "2021:296:10:41:57.000 | COMMAND_SW | CODISASX | CMD_EVT | " + "event=NSM, event_date=2021:296:10:41:57, msid=CODISASX, codisas1=128 , scs=0", + "2021:296:10:41:57.000 | COMMAND_SW | CODISASX | CMD_EVT | " + "event=NSM, event_date=2021:296:10:41:57, msid=CODISASX, codisas1=129 , scs=0", + "2021:296:10:41:57.000 | COMMAND_SW | CODISASX | CMD_EVT | " + "event=NSM, event_date=2021:296:10:41:57, msid=CODISASX, codisas1=130 , scs=0", + "2021:296:10:41:57.000 | COMMAND_SW | CODISASX | CMD_EVT | " + "event=NSM, event_date=2021:296:10:41:57, msid=CODISASX, codisas1=131 , scs=0", + "2021:296:10:41:57.000 | COMMAND_SW | CODISASX | CMD_EVT | " + "event=NSM, event_date=2021:296:10:41:57, msid=CODISASX, codisas1=132 , scs=0", + "2021:296:10:41:57.000 | COMMAND_SW | CODISASX | CMD_EVT | " + "event=NSM, event_date=2021:296:10:41:57, msid=CODISASX, codisas1=133 , scs=0", + "2021:296:10:41:57.000 | COMMAND_SW | OORMPDS | CMD_EVT | " + "event=NSM, event_date=2021:296:10:41:57, scs=0", + "2021:296:10:41:58.025 | COMMAND_HW | AFIDP | CMD_EVT | " + "event=NSM, event_date=2021:296:10:41:57, msid=AFLCRSET, scs=0", + "2021:296:10:41:58.025 | SIMTRANS | None | CMD_EVT | " + "event=NSM, event_date=2021:296:10:41:57, pos=-99616, scs=0", + "2021:296:10:42:20.000 | MP_OBSID | COAOSQID | CMD_EVT | " + "event=Obsid, event_date=2021:296:10:42:20, id=0, scs=0", + "2021:296:10:43:03.685 | ACISPKT | AA00000000 | CMD_EVT | " + "event=NSM, event_date=2021:296:10:41:57, scs=0", + "2021:296:10:43:04.710 | ACISPKT | AA00000000 | CMD_EVT | " + "event=NSM, event_date=2021:296:10:41:57, scs=0", + "2021:296:10:43:14.960 | ACISPKT | WSPOW0002A | CMD_EVT | " + "event=NSM, event_date=2021:296:10:41:57, scs=0", + "2021:296:10:43:14.960 | COMMAND_SW | AODSDITH | CMD_EVT | " + "event=NSM, event_date=2021:296:10:41:57, scs=0", + "2021:297:01:41:01.000 | COMMAND_SW | AONMMODE | CMD_EVT | " + "event=Maneuver, event_date=2021:297:01:41:01, msid=AONMMODE, scs=0", + "2021:297:01:41:01.256 | COMMAND_SW | AONM2NPE | CMD_EVT | " + "event=Maneuver, event_date=2021:297:01:41:01, msid=AONM2NPE, scs=0", + "2021:297:01:41:05.356 | MP_TARGQUAT | AOUPTARQ | CMD_EVT | " + "event=Maneuver, event_date=2021:297:01:41:01, q1=7.05469070e-01, " + "q2=3.29883070e-01, q3=5.34409010e-01, q4=3.28477660e-01, scs=0", + "2021:297:01:41:11.250 | COMMAND_SW | AOMANUVR | CMD_EVT | " + "event=Maneuver, event_date=2021:297:01:41:01, msid=AOMANUVR, scs=0", + "2021:297:02:12:42.886 | ORBPOINT | None | OCT1821A | " + "event_type=EQF003M, scs=0", + "2021:297:03:40:42.886 | ORBPOINT | None | OCT1821A | " + "event_type=EQF005M, scs=0", + "2021:297:03:40:42.886 | ORBPOINT | None | OCT1821A | " + "event_type=EQF015M, scs=0", + "2021:297:04:43:26.016 | ORBPOINT | None | OCT1821A | " + "event_type=EALT1, scs=0", + "2021:297:04:43:27.301 | ORBPOINT | None | OCT1821A | " + "event_type=XALT1, scs=0", + "2021:297:12:42:42.886 | ORBPOINT | None | OCT1821A | " + "event_type=EQF013M, scs=0", + "2021:297:13:59:39.602 | ORBPOINT | None | OCT1821A | " + "event_type=EEF1000, scs=0", + "2021:297:14:01:00.000 | LOAD_EVENT | None | OCT1821A | " + "event_type=SCHEDULED_STOP_TIME, scs=0", + ] + + assert cmds.pformat_like_backstop(max_params_width=200) == exp + commands.clear_caches() + + +@pytest.mark.skipif(not HAS_INTERNET, reason="No internet connection") +def test_cmds_scenario(stop_date_2020_12_03): # noqa: ARG001 + """Test custom scenario with a couple of ACIS commands""" + # First make the cmd_events.csv file for the scenario + scenario = "test_acis" + cmds_dir = Path(commands.conf.commands_dir) / scenario + cmds_dir.mkdir(exist_ok=True, parents=True) + # Note variation in format of date, since this comes from humans. + # This also does not have a State column, which tests code to put that in. + cmd_evts_text = """\ +Date,Event,Params,Author,Comment +2020-12-01T00:08:30,Command,ACISPKT | TLMSID=WSPOW00000",Tom Aldcroft, +2020-12-01 00:08:39,Command,"ACISPKT | TLMSID=WSVIDALLDN",Tom Aldcroft, +""" + (cmds_dir / "cmd_events.csv").write_text(cmd_evts_text) + + # Now get commands in a time range that includes the new command events + cmds = commands.get_cmds( + "2020-12-01 00:08:00", "2020-12-01 00:09:00", scenario=scenario + ) + cmds = cmds[cmds["tlmsid"] != "OBS"] + exp = [ + "2020:336:00:08:30.000 | ACISPKT | WSPOW00000 | CMD_EVT |" + " event=Command, event_date=2020:336:00:08:30, scs=0", + "2020:336:00:08:38.610 | COMMAND_HW | CNOOP | NOV3020A |" + " hex=7E00000, msid=CNOOPLR, scs=128", + "2020:336:00:08:39.000 | ACISPKT | WSVIDALLDN | CMD_EVT |" + " event=Command, event_date=2020:336:00:08:39, scs=0", + "2020:336:00:08:39.635 | COMMAND_HW | CNOOP | NOV3020A |" + " hex=7E00000, msid=CNOOPLR, scs=128", + ] + assert cmds.pformat_like_backstop() == exp + commands.clear_caches() + + +stop_date_2024_01_30 = stop_date_fixture_factory("2024-01-30") + + +@pytest.mark.skipif(not HAS_INTERNET, reason="No internet connection") +@pytest.mark.parametrize("d_rasl", [-12, 12]) +def test_nsm_offset_pitch_rasl_with_rate_command_events(d_rasl, stop_date_2024_01_30): # noqa: ARG001 + """Test custom scenario with NSM offset pitch load event command""" + # First make the cmd_events.csv file for the scenario + scenario = "test_nsm_offset_pitch" + cmds_dir = Path(commands.conf.commands_dir) / scenario + cmds_dir.mkdir(exist_ok=True, parents=True) + # Note variation in format of date, since this comes from humans. + cmd_evts_text = f"""\ +State,Date,Event,Params,Author,Reviewer,Comment +Definitive,2024:024:08:00:00,NSM,120,,, +Definitive,2024:024:10:00:00,Maneuver sun rasl,{d_rasl} 0.02,,, +""" + (cmds_dir / "cmd_events.csv").write_text(cmd_evts_text) + + # Now get commands in a time range that includes the new command events + cmds = commands.get_cmds( + "2024-01-24 09:00:00", "2024-01-25 12:00:00", scenario=scenario + ) + cmds = cmds[(cmds["tlmsid"] != "OBS") & (cmds["type"] != "ORBPOINT")] + exp = [ + f"2024:024:10:00:00.000 | LOAD_EVENT | SUN_RASL | CMD_EVT | event=Maneuver_sun_rasl, event_date=2024:024:10:00:00, rasl={d_rasl}, rate=2.00000000e-02, scs=0", + ] + + assert cmds.pformat_like_backstop(max_params_width=200) == exp + + # Now get states in a time range that includes the new command events + states = kcs.get_states( + "2024:024:10:00:00", + "2024:024:11:00:00", + state_keys=["pitch", "rasl"], + scenario=scenario, + ) + if d_rasl == 12: + exp_text = """ + datestart datestop pitch rasl +2024:024:10:00:00.000 2024:024:10:02:00.000 120.048 312.622 +2024:024:10:02:00.000 2024:024:10:04:00.000 120.049 315.022 +2024:024:10:04:00.000 2024:024:10:06:00.000 120.049 317.422 +2024:024:10:06:00.000 2024:024:10:08:00.000 120.050 319.822 +2024:024:10:08:00.000 2024:024:10:10:00.000 120.050 322.222 +2024:024:10:10:00.000 2024:024:11:00:00.000 120.057 324.626 +""" + else: + exp_text = """ + datestart datestop pitch rasl +2024:024:10:00:00.000 2024:024:10:02:00.000 120.0476 312.621 +2024:024:10:02:00.000 2024:024:10:04:00.000 120.0484 310.221 +2024:024:10:04:00.000 2024:024:10:06:00.000 120.0492 307.8219 +2024:024:10:06:00.000 2024:024:10:08:00.000 120.0501 305.4220 +2024:024:10:08:00.000 2024:024:10:10:00.000 120.0511 303.0221 +2024:024:10:10:00.000 2024:024:11:00:00.000 120.0644 300.6232 +""" + + exp = Table.read(exp_text, format="ascii", guess=False, delimiter=" ") + # Total RASL change is 12 degrees. + rasls = states["rasl"] + n_rasl = len(rasls) + np.testing.assert_allclose(states["pitch"], exp["pitch"], atol=1e-2, rtol=0) + np.testing.assert_allclose(rasls, exp["rasl"], atol=1e-2, rtol=0) + np.testing.assert_allclose(rasls[-1] - rasls[0], d_rasl, atol=1e-2, rtol=0) + np.testing.assert_allclose(np.diff(rasls), d_rasl / (n_rasl - 1), atol=1e-2, rtol=0) + np.testing.assert_equal(states["datestart"], exp["datestart"]) + np.testing.assert_equal(states["datestop"], exp["datestop"]) + + +@pytest.mark.skipif(not HAS_INTERNET, reason="No internet connection") +def test_nsm_offset_pitch_rasl_command_events(stop_date_2024_01_30): # noqa: ARG001 + """Test custom scenario with NSM offset pitch load event command""" + # First make the cmd_events.csv file for the scenario + scenario = "test_nsm_offset_pitch" + cmds_dir = Path(commands.conf.commands_dir) / scenario + cmds_dir.mkdir(exist_ok=True, parents=True) + # Note variation in format of date, since this comes from humans. + cmd_evts_text = """\ +State,Date,Event,Params,Author,Reviewer,Comment +Definitive,2024:025:04:00:00,Maneuver sun rasl,90,,, +Definitive,2024:025:00:00:00,Maneuver sun pitch,160,,, +Definitive,2024:024:09:44:06,NSM,,,, +""" + (cmds_dir / "cmd_events.csv").write_text(cmd_evts_text) + + # Now get commands in a time range that includes the new command events + cmds = commands.get_cmds( + "2024-01-24 12:00:00", "2024-01-25 05:00:00", scenario=scenario + ) + cmds = cmds[(cmds["tlmsid"] != "OBS") & (cmds["type"] != "ORBPOINT")] + exp = [ + "2024:025:00:00:00.000 | LOAD_EVENT | SUN_PITCH | CMD_EVT | event=Maneuver_sun_pitch, event_date=2024:025:00:00:00, pitch=160, scs=0", + "2024:025:04:00:00.000 | LOAD_EVENT | SUN_RASL | CMD_EVT | event=Maneuver_sun_rasl, event_date=2024:025:04:00:00, rasl=90, rate=2.50000000e-02, scs=0", + ] + + assert cmds.pformat_like_backstop(max_params_width=200) == exp + + states = kcs.get_states( + "2024:024:09:00:00", + "2024:025:02:00:00", + state_keys=["pitch", "pcad_mode"], + scenario=scenario, + ) + exp = [ + " datestart pitch pcad_mode", + "--------------------- ----- ---------", + "2024:024:09:00:00.000 172.7 NPNT", + "2024:024:09:13:49.112 172.7 NMAN", + "2024:024:09:13:59.363 172.1 NMAN", + "2024:024:09:18:25.800 165.1 NMAN", + "2024:024:09:22:52.238 149.7 NMAN", + "2024:024:09:27:18.675 131.3 NMAN", + "2024:024:09:31:45.112 113.7 NMAN", + "2024:024:09:36:11.550 102.7 NMAN", + "2024:024:09:40:37.987 99.6 NMAN", + "2024:024:09:42:51.205 99.6 NPNT", + "2024:024:09:44:06.000 99.0 NSUN", + "2024:024:09:48:17.979 93.0 NSUN", + "2024:024:09:52:29.957 90.2 NSUN", + "2024:025:00:00:00.000 91.3 NSUN", + "2024:025:00:04:48.673 100.6 NSUN", + "2024:025:00:09:37.346 119.8 NSUN", + "2024:025:00:14:26.019 141.3 NSUN", + "2024:025:00:19:14.692 155.8 NSUN", + "2024:025:00:24:03.365 160.0 NSUN", + ] + + out = states["datestart", "pitch", "pcad_mode"] + out["pitch"].format = ".1f" + assert out.pformat() == exp + + states = kcs.get_states( + "2024:024:09:00:00", + "2024:025:08:00:00", + state_keys=["q1", "q2", "q3", "q4"], + scenario=scenario, + ) + + # Interpolate states at two times just after the pitch maneuver and just after the + # roll about sun line (rasl) maneuver. + dates = ["2024:025:04:00:00", "2024:025:05:00:01"] + sts = kcs.interpolate_states(states, dates) + q1 = Quat([sts["q1"][0], sts["q2"][0], sts["q3"][0], sts["q4"][0]]) + q2 = Quat([sts["q1"][1], sts["q2"][1], sts["q3"][1], sts["q4"][1]]) + pitch1, rasl1 = ska_sun.get_sun_pitch_yaw(q1.ra, q1.dec, dates[0]) + pitch2, rasl2 = ska_sun.get_sun_pitch_yaw(q2.ra, q2.dec, dates[1]) + assert np.isclose(pitch1, 160, atol=0.2) + assert np.isclose(pitch2, 160, atol=0.5) + assert np.isclose((rasl2 - rasl1) % 360, 90, atol=0.2) + + commands.clear_caches() + + +def test_command_set_bsh(): + cmds = get_cmds_from_event("2000:001", "Bright star hold", "") + exp = """\ +2000:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | event=Bright_star_hold, event_date=2000:001:00:00:00, msid=CODISASX, codisas1=128 , scs=0 +2000:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | event=Bright_star_hold, event_date=2000:001:00:00:00, msid=CODISASX, codisas1=129 , scs=0 +2000:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | event=Bright_star_hold, event_date=2000:001:00:00:00, msid=CODISASX, codisas1=130 , scs=0 +2000:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | event=Bright_star_hold, event_date=2000:001:00:00:00, msid=CODISASX, codisas1=131 , scs=0 +2000:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | event=Bright_star_hold, event_date=2000:001:00:00:00, msid=CODISASX, codisas1=132 , scs=0 +2000:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | event=Bright_star_hold, event_date=2000:001:00:00:00, msid=CODISASX, codisas1=133 , scs=0 +2000:001:00:00:00.000 | COMMAND_SW | OORMPDS | CMD_EVT | event=Bright_star_hold, event_date=2000:001:00:00:00, scs=0 +2000:001:00:00:01.025 | COMMAND_HW | AFIDP | CMD_EVT | event=Bright_star_hold, event_date=2000:001:00:00:00, msid=AFLCRSET, scs=0 +2000:001:00:00:01.025 | SIMTRANS | None | CMD_EVT | event=Bright_star_hold, event_date=2000:001:00:00:00, pos=-99616, scs=0 +2000:001:00:01:06.685 | ACISPKT | AA00000000 | CMD_EVT | event=Bright_star_hold, event_date=2000:001:00:00:00, scs=0 +2000:001:00:01:07.710 | ACISPKT | AA00000000 | CMD_EVT | event=Bright_star_hold, event_date=2000:001:00:00:00, scs=0 +2000:001:00:01:17.960 | ACISPKT | WSPOW00000 | CMD_EVT | event=Bright_star_hold, event_date=2000:001:00:00:00, scs=0 +2000:001:00:01:17.960 | COMMAND_HW | 215PCAOF | CMD_EVT | event=Bright_star_hold, event_date=2000:001:00:00:00, scs=0 +2000:001:00:01:19.165 | COMMAND_HW | 2IMHVOF | CMD_EVT | event=Bright_star_hold, event_date=2000:001:00:00:00, scs=0 +2000:001:00:01:20.190 | COMMAND_HW | 2SPHVOF | CMD_EVT | event=Bright_star_hold, event_date=2000:001:00:00:00, scs=0 +2000:001:00:01:21.215 | COMMAND_HW | 2S2STHV | CMD_EVT | event=Bright_star_hold, event_date=2000:001:00:00:00, scs=0 +2000:001:00:01:22.240 | COMMAND_HW | 2S1STHV | CMD_EVT | event=Bright_star_hold, event_date=2000:001:00:00:00, scs=0 +2000:001:00:01:23.265 | COMMAND_HW | 2S2HVOF | CMD_EVT | event=Bright_star_hold, event_date=2000:001:00:00:00, scs=0 +2000:001:00:01:24.290 | COMMAND_HW | 2S1HVOF | CMD_EVT | event=Bright_star_hold, event_date=2000:001:00:00:00, scs=0""" + + assert cmds.pformat_like_backstop(max_params_width=None) == exp.splitlines() + commands.clear_caches() + + +def test_command_set_safe_mode(): + cmds = get_cmds_from_event("2000:001", "Safe mode", "") + exp = """\ +2000:001:00:00:00.000 | COMMAND_SW | ACPCSFSU | CMD_EVT | event=Safe_mode, event_date=2000:001:00:00:00, scs=0 +2000:001:00:00:00.000 | COMMAND_SW | CSELFMT5 | CMD_EVT | event=Safe_mode, event_date=2000:001:00:00:00, scs=0 +2000:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | event=Safe_mode, event_date=2000:001:00:00:00, msid=CODISASX, codisas1=128 , scs=0 +2000:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | event=Safe_mode, event_date=2000:001:00:00:00, msid=CODISASX, codisas1=129 , scs=0 +2000:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | event=Safe_mode, event_date=2000:001:00:00:00, msid=CODISASX, codisas1=130 , scs=0 +2000:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | event=Safe_mode, event_date=2000:001:00:00:00, msid=CODISASX, codisas1=131 , scs=0 +2000:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | event=Safe_mode, event_date=2000:001:00:00:00, msid=CODISASX, codisas1=132 , scs=0 +2000:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | event=Safe_mode, event_date=2000:001:00:00:00, msid=CODISASX, codisas1=133 , scs=0 +2000:001:00:00:00.000 | COMMAND_SW | OORMPDS | CMD_EVT | event=Safe_mode, event_date=2000:001:00:00:00, scs=0 +2000:001:00:00:01.025 | COMMAND_HW | AFIDP | CMD_EVT | event=Safe_mode, event_date=2000:001:00:00:00, msid=AFLCRSET, scs=0 +2000:001:00:00:01.025 | SIMTRANS | None | CMD_EVT | event=Safe_mode, event_date=2000:001:00:00:00, pos=-99616, scs=0 +2000:001:00:01:06.685 | ACISPKT | AA00000000 | CMD_EVT | event=Safe_mode, event_date=2000:001:00:00:00, scs=0 +2000:001:00:01:07.710 | ACISPKT | AA00000000 | CMD_EVT | event=Safe_mode, event_date=2000:001:00:00:00, scs=0 +2000:001:00:01:17.960 | ACISPKT | WSPOW00000 | CMD_EVT | event=Safe_mode, event_date=2000:001:00:00:00, scs=0 +2000:001:00:01:17.960 | COMMAND_HW | 215PCAOF | CMD_EVT | event=Safe_mode, event_date=2000:001:00:00:00, scs=0 +2000:001:00:01:19.165 | COMMAND_HW | 2IMHVOF | CMD_EVT | event=Safe_mode, event_date=2000:001:00:00:00, scs=0 +2000:001:00:01:20.190 | COMMAND_HW | 2SPHVOF | CMD_EVT | event=Safe_mode, event_date=2000:001:00:00:00, scs=0 +2000:001:00:01:21.215 | COMMAND_HW | 2S2STHV | CMD_EVT | event=Safe_mode, event_date=2000:001:00:00:00, scs=0 +2000:001:00:01:22.240 | COMMAND_HW | 2S1STHV | CMD_EVT | event=Safe_mode, event_date=2000:001:00:00:00, scs=0 +2000:001:00:01:23.265 | COMMAND_HW | 2S2HVOF | CMD_EVT | event=Safe_mode, event_date=2000:001:00:00:00, scs=0 +2000:001:00:01:24.290 | COMMAND_HW | 2S1HVOF | CMD_EVT | event=Safe_mode, event_date=2000:001:00:00:00, scs=0 +2000:001:00:01:25.315 | COMMAND_SW | AODSDITH | CMD_EVT | event=Safe_mode, event_date=2000:001:00:00:00, scs=0""" + assert cmds.pformat_like_backstop(max_params_width=None) == exp.splitlines() + commands.clear_caches() + + +@pytest.mark.skipif(not HAS_INTERNET, reason="No internet connection") +def test_bright_star_hold_event( + cmds_dir, stop_date_2020_12_03, disable_hrc_scs107_commanding +): + """Make a scenario with a bright star hold event. + + Confirm that this inserts expected commands and interrupts all load commands. + """ + bsh_dir = Path(conf.commands_dir) / "bsh" + bsh_dir.mkdir(parents=True, exist_ok=True) + cmd_events_file = bsh_dir / "cmd_events.csv" + cmd_events_file.write_text( + """\ +State,Date,Event,Params,Author,Comment +Definitive,2020:337:00:00:00,Bright star hold,,Tom Aldcroft, +""" + ) + cmds = commands.get_cmds(start="2020:336:21:48:00", stop="2020:338", scenario="bsh") + exp = [ + "2020:336:21:48:03.312 | LOAD_EVENT | OBS | NOV3020A | " + "manvr_start=2020:336:21:09:24.361, prev_att=(-0.242373434, -0.348723922, " + "0.42827", + "2020:336:21:48:06.387 | COMMAND_SW | CODISASX | NOV3020A | " + "hex=8456200, msid=CODISASX, codisas1=98 , scs=128", + "2020:336:21:48:07.412 | COMMAND_SW | AOFUNCEN | NOV3020A | " + "hex=803031E, msid=AOFUNCEN, aopcadse=30 , scs=128", + "2020:336:21:54:23.061 | COMMAND_SW | AOFUNCEN | NOV3020A | " + "hex=8030320, msid=AOFUNCEN, aopcadse=32 , scs=128", + "2020:336:21:55:23.061 | COMMAND_SW | AOFUNCEN | NOV3020A | " + "hex=8030315, msid=AOFUNCEN, aopcadse=21 , scs=128", + # BSH interrupt at 2020:337 + "2020:337:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | " + "event=Bright_star_hold, event_date=2020:337:00:00:00, msid=CODISASX, " + "codisas1=12", + "2020:337:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | " + "event=Bright_star_hold, event_date=2020:337:00:00:00, msid=CODISASX, " + "codisas1=12", + "2020:337:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | " + "event=Bright_star_hold, event_date=2020:337:00:00:00, msid=CODISASX, " + "codisas1=13", + "2020:337:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | " + "event=Bright_star_hold, event_date=2020:337:00:00:00, msid=CODISASX, " + "codisas1=13", + "2020:337:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | " + "event=Bright_star_hold, event_date=2020:337:00:00:00, msid=CODISASX, " + "codisas1=13", + "2020:337:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | " + "event=Bright_star_hold, event_date=2020:337:00:00:00, msid=CODISASX, " + "codisas1=13", + "2020:337:00:00:00.000 | COMMAND_SW | OORMPDS | CMD_EVT | " + "event=Bright_star_hold, event_date=2020:337:00:00:00, scs=0", + "2020:337:00:00:01.025 | COMMAND_HW | AFIDP | CMD_EVT | " + "event=Bright_star_hold, event_date=2020:337:00:00:00, msid=AFLCRSET, scs=0", + "2020:337:00:00:01.025 | SIMTRANS | None | CMD_EVT | " + "event=Bright_star_hold, event_date=2020:337:00:00:00, pos=-99616, scs=0", + "2020:337:00:01:06.685 | ACISPKT | AA00000000 | CMD_EVT | " + "event=Bright_star_hold, event_date=2020:337:00:00:00, scs=0", + "2020:337:00:01:07.710 | ACISPKT | AA00000000 | CMD_EVT | " + "event=Bright_star_hold, event_date=2020:337:00:00:00, scs=0", + "2020:337:00:01:17.960 | ACISPKT | WSPOW00000 | CMD_EVT | " + "event=Bright_star_hold, event_date=2020:337:00:00:00, scs=0", + # Only ORBPOINT from here on + "2020:337:02:07:03.790 | ORBPOINT | None | NOV3020A | " + "event_type=EAPOGEE, scs=0", + "2020:337:21:15:45.455 | ORBPOINT | None | NOV3020A | " + "event_type=EALT1, scs=0", + "2020:337:21:15:46.227 | ORBPOINT | None | NOV3020A | " + "event_type=XALT1, scs=0", + ] + assert cmds.pformat_like_backstop() == exp + commands.clear_caches() + + +@pytest.mark.skipif(not HAS_INTERNET, reason="No internet connection") +def test_get_observations_by_obsid_single(): + obss = get_observations(obsid=8008) + assert len(obss) == 1 + del obss[0]["starcat_idx"] + assert obss == [ + { + "obsid": 8008, + "simpos": 92904, + "obs_stop": "2007:002:18:04:28.965", + "manvr_start": "2007:002:04:31:48.216", + "targ_att": (0.149614271, 0.490896707, 0.831470649, 0.21282047), + "npnt_enab": True, + "obs_start": "2007:002:04:46:58.056", + "prev_att": (0.319214732, 0.535685207, 0.766039803, 0.155969017), + "starcat_date": "2007:002:04:31:43.965", + "source": "DEC2506C", + } + ] + + +def test_get_observations_by_obsid_multi(): + # Following ACA high background NSM 2019:248 + obss = get_observations(obsid=47912, scenario="flight") + # Don't compare starcat_idx because it might change with a repro + for obs in obss: + obs.pop("starcat_idx", None) + + assert obss == [ + { + "obsid": 47912, + "simpos": -99616, + "obs_stop": "2019:248:16:51:18.000", + "manvr_start": "2019:248:14:52:35.407", + "targ_att": (-0.564950617, 0.252299958, -0.165669121, 0.767938327), + "npnt_enab": True, + "obs_start": "2019:248:15:27:35.289", + "prev_att": (-0.218410783, 0.748632452, -0.580771797, 0.233560059), + "starcat_date": "2019:248:14:52:31.156", + "source": "SEP0219B", + }, + { + "obsid": 47912, + "simpos": -99616, + "obs_stop": "2019:249:01:59:00.000", + "manvr_start": "2019:248:16:51:18.000", + "targ_att": ( + -0.3594375808951632, + 0.6553454859043244, + -0.4661410647781301, + 0.47332803366853643, + ), + "npnt_enab": False, + "obs_start": "2019:248:17:18:17.732", + "prev_att": (-0.564950617, 0.252299958, -0.165669121, 0.767938327), + "source": "CMD_EVT", + }, + { + "obsid": 47912, + "simpos": -99616, + "obs_stop": "2019:249:23:30:00.000", + "manvr_start": "2019:249:01:59:10.250", + "targ_att": (-0.54577727, 0.27602874, -0.17407247, 0.77177334), + "npnt_enab": True, + "obs_start": "2019:249:02:25:31.907", + "prev_att": ( + -0.3594375808951632, + 0.6553454859043244, + -0.4661410647781301, + 0.47332803366853643, + ), + "source": "CMD_EVT", + }, + ] + + +def test_get_observations_by_start_date(): + # Test observations from a 6 months ago onward + obss = get_observations(start=CxoTime.now() - 180 * u.day, scenario="flight") + assert len(obss) > 500 + # Latest obs should also be no less than 14 days old + assert obss[-1]["obs_start"] > (CxoTime.now() - 14 * u.day).date + + +def test_get_observations_by_start_stop_date_with_scenario(): + # Test observations in a range and use the scenario keyword + obss = get_observations(start="2022:001", stop="2022:002", scenario="flight") + assert len(obss) == 7 + assert obss[1]["obsid"] == 45814 + assert obss[1]["obs_start"] == "2022:001:05:48:44.808" + assert obss[-1]["obsid"] == 23800 + assert obss[-1]["obs_start"] == "2022:001:17:33:53.255" + + +def test_get_observations_no_match(): + with pytest.raises(ValueError, match="No matching observations for obsid=8008"): + get_observations( + obsid=8008, start="2022:001", stop="2022:002", scenario="flight" + ) + + +def test_get_observations_start_stop_inclusion(): + # Covers time from the middle of obsid 8008 to the middle of obsid 8009 + obss = get_observations("2007:002:05:00:00", "2007:002:20:00:01", scenario="flight") + assert len(obss) == 2 + + # One second in the middle of obsid 8008 + obss = get_observations("2007:002:05:00:00", "2007:002:05:00:01", scenario="flight") + assert len(obss) == 1 + + # During a maneuver + obss = get_observations("2007:002:18:05:00", "2007:002:18:08:00", scenario="flight") + assert len(obss) == 0 + + +years = np.arange(2003, 2025) + + +@pytest.mark.parametrize("year", years) +def test_get_starcats_each_year(year): + starcats = get_starcats(start=f"{year}:001", stop=f"{year}:004", scenario="flight") + assert len(starcats) > 2 + for starcat in starcats: + # Make sure fids and stars are all ID'd + ok = starcat["type"] != "MON" + assert np.all(starcat["id"][ok] != -999) + + +def test_get_starcat_only_agasc1p7(): + """ + For obsids 3829 and 2576, try AGASC 1.7 only and show successful star + identification. + """ + with ( + conf.set_temp("cache_starcats", False), + conf.set_temp("date_start_agasc1p8", "2003:001"), + ): + starcat = get_starcats( + "2002:365:18:00:00", "2002:365:19:00:00", scenario="flight" + )[0] + assert np.all(starcat["id"] != -999) + assert np.all(starcat["mag"] != -999) + + +@pytest.mark.skipif(not HAS_AGASC_1P8, reason="AGASC 1.8 not available") +def test_get_starcat_only_agasc1p8(): + """For obsids 3829 and 2576, try AGASC 1.8 only + + For 3829 star identification should succeed, for 2576 it fails. + """ + with ( + conf.set_temp("cache_starcats", False), + conf.set_temp("date_start_agasc1p8", "1994:001"), + ): + # Force AGASC 1.8 and show that star identification fails + with ska_helpers.utils.set_log_level(kadi.logger, "CRITICAL"): + starcats = get_starcats( + "2002:365:16:00:00", "2002:365:19:00:00", scenario="flight" + ) + assert np.count_nonzero(starcats[0]["id"] == -999) == 0 + assert np.count_nonzero(starcats[0]["mag"] == -999) == 0 + assert np.count_nonzero(starcats[1]["id"] == -999) == 3 + assert np.count_nonzero(starcats[1]["mag"] == -999) == 3 + + +def test_get_starcats_with_cmds(): + start, stop = "2021:365:19:00:00", "2022:002:01:25:00" + cmds = commands.get_cmds(start, stop, scenario="flight") + starcats0 = get_starcats(start, stop) + starcats1 = get_starcats(cmds=cmds) + assert len(starcats0) == len(starcats1) + for starcat0, starcat1 in zip(starcats0, starcats1): + eq = starcat0.values_equal(starcat1) + for col in eq.itercols(): + assert np.all(col) + + +def test_get_starcats_obsid(): + from mica.starcheck import get_starcat + + sc_kadi = get_starcats(obsid=26330, scenario="flight")[0] + sc_mica = get_starcat(26330) + assert len(sc_kadi) == len(sc_mica) + assert sc_kadi.colnames == [ + "slot", + "idx", + "id", + "type", + "sz", + "mag", + "maxmag", + "yang", + "zang", + "dim", + "res", + "halfw", + ] + for name in sc_kadi.colnames: + if name == "mag": + continue # kadi mag is latest from agasc, could change + elif name == "maxmag": + assert np.allclose(sc_kadi[name], sc_mica[name], atol=0.001, rtol=0) + elif name in ("yang", "zang"): + assert np.all(np.abs(sc_kadi[name] - sc_mica[name]) < 1) + else: + assert np.all(sc_kadi[name] == sc_mica[name]) + + +def test_get_starcats_date(): + """Test that the starcat `date` is set to obs `starcat_date`. + + And that this matches the time of the corresponding MP_STARCAT AOSTRCAT + command. + + Note: from https://icxc.harvard.edu//mp/mplogs/2006/DEC2506/oflsc/starcheck.html#obsid8008 + MP_STARCAT at 2007:002:04:31:43.965 (VCDU count = 7477935) + """ # noqa: E501 + sc = get_starcats(obsid=8008, scenario="flight")[0] + obs = get_observations(obsid=8008, scenario="flight")[0] + assert sc.date == obs["starcat_date"] == "2007:002:04:31:43.965" + cmds = commands.get_cmds("2007:002", "2007:003") + sc_cmd = cmds[cmds["date"] == obs["starcat_date"]][0] + assert sc_cmd["type"] == "MP_STARCAT" + + +def test_get_starcats_by_date(): + # Test that the getting a starcat using the starcat_date as argument + # returns the same catalog as using the OBSID. + sc = get_starcats(obsid=8008, scenario="flight")[0] + sc_by_date = get_starcats(starcat_date="2007:002:04:31:43.965", scenario="flight")[ + 0 + ] + assert np.all(sc == sc_by_date) + with pytest.raises(ValueError, match="No matching observations for starcat_date"): + get_starcats(starcat_date="2007:002:04:31:43.966", scenario="flight") + + +def test_get_starcats_as_table(): + """Test that get_starcats_as_table returns the same as vstacked get_starcats""" + start, stop = "2020:001", "2020:002" + starcats = get_starcats(start, stop, scenario="flight") + obsids = [] + dates = [] + for starcat in starcats: + obsids.extend([starcat.obsid] * len(starcat)) + dates.extend([starcat.date] * len(starcat)) + # Meta causes warnings in vstack, just ignore here + starcat.meta = {} + aces = get_starcats_as_table(start, stop, scenario="flight") + aces_from_starcats = vstack(starcats) + assert np.all(aces["obsid"] == obsids) + assert np.all(aces["starcat_date"] == dates) + for name in aces_from_starcats.colnames: + assert np.all(aces[name] == aces_from_starcats[name]) + + +def patched_read_cmd_events_from_sheet(doc_id): + """Monkey patch function for test_custom_scenario below""" + if doc_id == conf.cmd_events_custom_id: + evts = Table( + [ + { + "State": "Definitive", + "Date": "2024:035:22:00:00", + "Event": "Load in backstop", + "Params": "FOT/mission_planning/Backstop/Archive/2024/02_feb/FEB0524A", + "Author": "", + "Reviewer": "", + "Comment": "", + } + ] + ) + else: + evts = read_cmd_events_from_sheet(doc_id) + return evts + + +@pytest.mark.skipif(not HAS_INTERNET, reason="No internet connection") +def test_flight_scenario_sheet_access(): + """Test that flight scenario does not access the google command events sheet""" + with kadi.commands.conf.set_temp("cmd_events_flight_id", "id-does-not-exist"): + commands.clear_caches() + commands.get_cmds("-7d", scenario="flight") # succeeds, no sheet access + match = re.escape( + "Failed to get cmd events sheet: 404 for " + "https://docs.google.com/spreadsheets/d/id-does-not-exist/export?format=csv" + ) + commands.clear_caches() + with pytest.raises(ValueError, match=match): + commands.get_cmds("-7d") # fails, bad sheet URL + + +@pytest.mark.skipif(not HAS_INTERNET, reason="No internet connection") +def test_custom_scenario(monkeypatch, stop_date_2024_035_23_00_00): + """Test "custom" scenario with a carefully constructed sequence of events. + + Flight events include NSM at 2024:035:03:27:11.000 and a number of other command + events. + + The flight FEB0524A loads starting at 2024:036:00:45:17.999 were approved but never + run due to the NSM. + + With "flight+custom", we expect the NSM and then the FEB0524A loads from the + archive. + + With only "custom", there was no NSM but we still get the FEB0524A loads from the + archive. + """ + monkeypatch.setattr( + kadi.commands.commands_v2, + "read_cmd_events_from_sheet", + patched_read_cmd_events_from_sheet, + ) + exps = { + "flight+custom": { + "feb0524a": 2543, + "jan2924a": 2105, + "jan2624a": 220, + "cmd_evt": 22, + }, + "custom": { # No NSM, so more JAN2924A load commands and no CMD_EVT's + "feb0524a": 2543, + "jan2924a": 2377, + "jan2624a": 220, + "cmd_evt": 0, + }, + } + for scenario in ("flight+custom", "custom"): + exp = exps[scenario] + cmds = commands.get_cmds("2024:028", "2024:050", scenario=scenario) + sources = {"FEB0524A", "JAN2924A", "JAN2624A"} + if scenario == "flight+custom": + sources.add("CMD_EVT") + assert set(cmds["source"]) == sources + assert len(cmds[cmds["source"] == "FEB0524A"]) == exp["feb0524a"] + assert len(cmds[cmds["source"] == "JAN2924A"]) == exp["jan2924a"] + assert len(cmds[cmds["source"] == "JAN2624A"]) == exp["jan2624a"] + assert len(cmds[cmds["source"] == "CMD_EVT"]) == exp["cmd_evt"] + + # Select RLTT and SST cmds, then get RLTT cmd + ok = ( + (cmds["source"] == "FEB0524A") + & (cmds["tlmsid"] == "None") + & (cmds["type"] == "LOAD_EVENT") + ) + cmd_rltt = cmds[ok][0] + assert cmd_rltt["params"] == { + "event_type": "RUNNING_LOAD_TERMINATION_TIME", + "load_path": "FOT/mission_planning/Backstop/Archive/2024/02_feb/FEB0524A", + } + + +@pytest.mark.parametrize( + "par_str", + [ + "ACISPKT| TLmSID= aa0000000, par1 = 1 , par2=-1.0", + "AcisPKT|TLmSID=AA0000000 ,par1=1, par2=-1.0", + "ACISPKT| TLmSID = aa0000000 , par1 =1, par2 = -1.0", + ], +) +def test_get_cmds_from_event_case(par_str): + cmds = get_cmds_from_event("2022:001", "Command", par_str) + assert len(cmds) == 1 + cmd = cmds[0] + assert cmd["type"] == "ACISPKT" + assert cmd["params"] == { + "event": "Command", + "event_date": "2022:001:00:00:00", + "par1": 1, + "par2": -1.0, + } + + +cmd_events_all_text = """\ + Event,Params + Observing not run,FEB1422A + Load not run,OCT2521A + Command,"ACISPKT | TLMSID= AA00000000, CMDS= 3, WORDS= 3, PACKET(40)= D80000300030603001300" + Command not run,"COMMAND_SW | TLMSID=4OHETGIN, HEX= 8050300, MSID= 4OHETGIN" + Obsid,65527 + Maneuver,0.70546907 0.32988307 0.53440901 0.32847766 + Safe mode, + NSM, + SCS-107, + Bright star hold, + Dither,ON + """ +cmd_events_all = Table.read( + cmd_events_all_text, format="ascii.csv", fill_values=[], converters={"Params": str} +) +cmd_events_all_exps = [ + [ + "2020:001:00:00:00.000 | LOAD_EVENT | None | CMD_EVT | event=Observing_not_run, event_date=2020:001:00:00:00, event_type=OBSERVING_NOT_RUN, load=FEB1422A, scs=0" + ], + [ + "2020:001:00:00:00.000 | LOAD_EVENT | None | CMD_EVT | event=Load_not_run, event_date=2020:001:00:00:00, event_type=LOAD_NOT_RUN, load=OCT2521A, scs=0" + ], + [ + "2020:001:00:00:00.000 | ACISPKT | AA00000000 | CMD_EVT | event=Command, event_date=2020:001:00:00:00, cmds=3, words=3, scs=0" + ], + [ + "2020:001:00:00:00.000 | NOT_RUN | 4OHETGIN | CMD_EVT | event=Command_not_run, event_date=2020:001:00:00:00, hex=8050300, msid=4OHETGIN, __type__=COMMAND_SW, scs=0" + ], + [ + "2020:001:00:00:00.000 | MP_OBSID | COAOSQID | CMD_EVT | event=Obsid, event_date=2020:001:00:00:00, id=65527, scs=0" + ], + [ + "2020:001:00:00:00.000 | COMMAND_SW | AONMMODE | CMD_EVT |" + " event=Maneuver, event_date=2020:001:00:00:00, msid=AONMMODE, scs=0", + "2020:001:00:00:00.256 | COMMAND_SW | AONM2NPE | CMD_EVT |" + " event=Maneuver, event_date=2020:001:00:00:00, msid=AONM2NPE, scs=0", + "2020:001:00:00:04.356 | MP_TARGQUAT | AOUPTARQ | CMD_EVT |" + " event=Maneuver, event_date=2020:001:00:00:00, q1=7.05469070e-01," + " q2=3.29883070e-01, q3=5.34409010e-01, q4=3.28477660e-01, scs=0", + "2020:001:00:00:10.250 | COMMAND_SW | AOMANUVR | CMD_EVT |" + " event=Maneuver, event_date=2020:001:00:00:00, msid=AOMANUVR, scs=0", + ], + [ + "2020:001:00:00:00.000 | COMMAND_SW | ACPCSFSU | CMD_EVT |" + " event=Safe_mode, event_date=2020:001:00:00:00, scs=0", + "2020:001:00:00:00.000 | COMMAND_SW | CSELFMT5 | CMD_EVT |" + " event=Safe_mode, event_date=2020:001:00:00:00, scs=0", + "2020:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT |" + " event=Safe_mode, event_date=2020:001:00:00:00, msid=CODISASX, codisas1=128 ," + " scs=0", + "2020:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT |" + " event=Safe_mode, event_date=2020:001:00:00:00, msid=CODISASX, codisas1=129 ," + " scs=0", + "2020:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT |" + " event=Safe_mode, event_date=2020:001:00:00:00, msid=CODISASX, codisas1=130 ," + " scs=0", + "2020:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT |" + " event=Safe_mode, event_date=2020:001:00:00:00, msid=CODISASX, codisas1=131 ," + " scs=0", + "2020:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT |" + " event=Safe_mode, event_date=2020:001:00:00:00, msid=CODISASX, codisas1=132 ," + " scs=0", + "2020:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT |" + " event=Safe_mode, event_date=2020:001:00:00:00, msid=CODISASX, codisas1=133 ," + " scs=0", + "2020:001:00:00:00.000 | COMMAND_SW | OORMPDS | CMD_EVT |" + " event=Safe_mode, event_date=2020:001:00:00:00, scs=0", + "2020:001:00:00:01.025 | COMMAND_HW | AFIDP | CMD_EVT |" + " event=Safe_mode, event_date=2020:001:00:00:00, msid=AFLCRSET, scs=0", + "2020:001:00:00:01.025 | SIMTRANS | None | CMD_EVT |" + " event=Safe_mode, event_date=2020:001:00:00:00, pos=-99616, scs=0", + "2020:001:00:01:06.685 | ACISPKT | AA00000000 | CMD_EVT |" + " event=Safe_mode, event_date=2020:001:00:00:00, scs=0", + "2020:001:00:01:07.710 | ACISPKT | AA00000000 | CMD_EVT |" + " event=Safe_mode, event_date=2020:001:00:00:00, scs=0", + "2020:001:00:01:17.960 | ACISPKT | WSPOW00000 | CMD_EVT |" + " event=Safe_mode, event_date=2020:001:00:00:00, scs=0", + "2020:001:00:01:17.960 | COMMAND_SW | AODSDITH | CMD_EVT |" + " event=Safe_mode, event_date=2020:001:00:00:00, scs=0", + ], + [ + "2020:001:00:00:00.000 | COMMAND_SW | AONSMSAF | CMD_EVT | event=NSM," + " event_date=2020:001:00:00:00, scs=0", + "2020:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | event=NSM," + " event_date=2020:001:00:00:00, msid=CODISASX, codisas1=128 , scs=0", + "2020:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | event=NSM," + " event_date=2020:001:00:00:00, msid=CODISASX, codisas1=129 , scs=0", + "2020:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | event=NSM," + " event_date=2020:001:00:00:00, msid=CODISASX, codisas1=130 , scs=0", + "2020:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | event=NSM," + " event_date=2020:001:00:00:00, msid=CODISASX, codisas1=131 , scs=0", + "2020:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | event=NSM," + " event_date=2020:001:00:00:00, msid=CODISASX, codisas1=132 , scs=0", + "2020:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT | event=NSM," + " event_date=2020:001:00:00:00, msid=CODISASX, codisas1=133 , scs=0", + "2020:001:00:00:00.000 | COMMAND_SW | OORMPDS | CMD_EVT | event=NSM," + " event_date=2020:001:00:00:00, scs=0", + "2020:001:00:00:01.025 | COMMAND_HW | AFIDP | CMD_EVT | event=NSM," + " event_date=2020:001:00:00:00, msid=AFLCRSET, scs=0", + "2020:001:00:00:01.025 | SIMTRANS | None | CMD_EVT | event=NSM," + " event_date=2020:001:00:00:00, pos=-99616, scs=0", + "2020:001:00:01:06.685 | ACISPKT | AA00000000 | CMD_EVT | event=NSM," + " event_date=2020:001:00:00:00, scs=0", + "2020:001:00:01:07.710 | ACISPKT | AA00000000 | CMD_EVT | event=NSM," + " event_date=2020:001:00:00:00, scs=0", + "2020:001:00:01:17.960 | ACISPKT | WSPOW00000 | CMD_EVT | event=NSM," + " event_date=2020:001:00:00:00, scs=0", + "2020:001:00:01:17.960 | COMMAND_SW | AODSDITH | CMD_EVT | event=NSM," + " event_date=2020:001:00:00:00, scs=0", + ], + [ + "2020:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT |" + " event=SCS-107, event_date=2020:001:00:00:00, msid=CODISASX, codisas1=131 ," + " scs=0", + "2020:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT |" + " event=SCS-107, event_date=2020:001:00:00:00, msid=CODISASX, codisas1=132 ," + " scs=0", + "2020:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT |" + " event=SCS-107, event_date=2020:001:00:00:00, msid=CODISASX, codisas1=133 ," + " scs=0", + "2020:001:00:00:00.000 | COMMAND_SW | OORMPDS | CMD_EVT |" + " event=SCS-107, event_date=2020:001:00:00:00, scs=0", + "2020:001:00:00:01.025 | COMMAND_HW | AFIDP | CMD_EVT |" + " event=SCS-107, event_date=2020:001:00:00:00, msid=AFLCRSET, scs=0", + "2020:001:00:00:01.025 | SIMTRANS | None | CMD_EVT |" + " event=SCS-107, event_date=2020:001:00:00:00, pos=-99616, scs=0", + "2020:001:00:01:06.685 | ACISPKT | AA00000000 | CMD_EVT |" + " event=SCS-107, event_date=2020:001:00:00:00, scs=0", + "2020:001:00:01:07.710 | ACISPKT | AA00000000 | CMD_EVT |" + " event=SCS-107, event_date=2020:001:00:00:00, scs=0", + "2020:001:00:01:17.960 | ACISPKT | WSPOW00000 | CMD_EVT |" + " event=SCS-107, event_date=2020:001:00:00:00, scs=0", + ], + [ + "2020:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT |" + " event=Bright_star_hold, event_date=2020:001:00:00:00, msid=CODISASX," + " codisas1=128 , scs=0", + "2020:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT |" + " event=Bright_star_hold, event_date=2020:001:00:00:00, msid=CODISASX," + " codisas1=129 , scs=0", + "2020:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT |" + " event=Bright_star_hold, event_date=2020:001:00:00:00, msid=CODISASX," + " codisas1=130 , scs=0", + "2020:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT |" + " event=Bright_star_hold, event_date=2020:001:00:00:00, msid=CODISASX," + " codisas1=131 , scs=0", + "2020:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT |" + " event=Bright_star_hold, event_date=2020:001:00:00:00, msid=CODISASX," + " codisas1=132 , scs=0", + "2020:001:00:00:00.000 | COMMAND_SW | CODISASX | CMD_EVT |" + " event=Bright_star_hold, event_date=2020:001:00:00:00, msid=CODISASX," + " codisas1=133 , scs=0", + "2020:001:00:00:00.000 | COMMAND_SW | OORMPDS | CMD_EVT |" + " event=Bright_star_hold, event_date=2020:001:00:00:00, scs=0", + "2020:001:00:00:01.025 | COMMAND_HW | AFIDP | CMD_EVT |" + " event=Bright_star_hold, event_date=2020:001:00:00:00, msid=AFLCRSET, scs=0", + "2020:001:00:00:01.025 | SIMTRANS | None | CMD_EVT |" + " event=Bright_star_hold, event_date=2020:001:00:00:00, pos=-99616, scs=0", + "2020:001:00:01:06.685 | ACISPKT | AA00000000 | CMD_EVT |" + " event=Bright_star_hold, event_date=2020:001:00:00:00, scs=0", + "2020:001:00:01:07.710 | ACISPKT | AA00000000 | CMD_EVT |" + " event=Bright_star_hold, event_date=2020:001:00:00:00, scs=0", + "2020:001:00:01:17.960 | ACISPKT | WSPOW00000 | CMD_EVT |" + " event=Bright_star_hold, event_date=2020:001:00:00:00, scs=0", + ], + [ + "2020:001:00:00:00.000 | COMMAND_SW | AOENDITH | CMD_EVT | event=Dither, event_date=2020:001:00:00:00, scs=0" + ], +] + + +@pytest.mark.parametrize("idx", range(len(cmd_events_all_exps))) +def test_get_cmds_from_event_all(idx, disable_hrc_scs107_commanding): + """Test getting commands from every event type in the Command Events sheet""" + cevt = cmd_events_all[idx] + exp = cmd_events_all_exps[idx] + cmds = get_cmds_from_event("2020:001:00:00:00", cevt["Event"], cevt["Params"]) + if cmds is not None: + cmds = cmds.pformat_like_backstop(max_params_width=None) + assert cmds == exp + + +cmd_events_rts_text = """\ + Event,Params + RTS,"RTSLOAD,1_4_CTI,NUM_HOURS=39:00:00,SCS_NUM=135" + """ +cmd_events_rts = Table.read( + cmd_events_rts_text, format="ascii.csv", fill_values=[], converters={"Params": str} +) +cmd_events_rts_exps = [ + [ + "2020:001:00:00:00.000 | COMMAND_SW | OORMPEN | CMD_EVT | event=RTS," + " event_date=2020:001:00:00:00, msid=OORMPEN, scs=135", + "2020:001:00:00:01.000 | ACISPKT | WSVIDALLDN | CMD_EVT | event=RTS," + " event_date=2020:001:00:00:00, scs=135", + "2020:001:00:00:02.000 | COMMAND_HW | 2S2STHV | CMD_EVT | event=RTS," + " event_date=2020:001:00:00:00, 2s2sthv2=0 , msid=2S2STHV, scs=135", + "2020:001:00:00:03.000 | COMMAND_HW | 2S2HVON | CMD_EVT | event=RTS," + " event_date=2020:001:00:00:00, msid=2S2HVON, scs=135", + "2020:001:00:00:13.000 | COMMAND_HW | 2S2STHV | CMD_EVT | event=RTS," + " event_date=2020:001:00:00:00, 2s2sthv2=4 , msid=2S2STHV, scs=135", + "2020:001:00:00:23.000 | COMMAND_HW | 2S2STHV | CMD_EVT | event=RTS," + " event_date=2020:001:00:00:00, 2s2sthv2=8 , msid=2S2STHV, scs=135", + "2020:001:00:00:24.000 | ACISPKT | WSPOW08E1E | CMD_EVT | event=RTS," + " event_date=2020:001:00:00:00, scs=135", + "2020:001:00:01:27.000 | ACISPKT | WT00C62014 | CMD_EVT | event=RTS," + " event_date=2020:001:00:00:00, scs=135", + "2020:001:00:01:31.000 | ACISPKT | XTZ0000005 | CMD_EVT | event=RTS," + " event_date=2020:001:00:00:00, scs=135", + "2020:001:00:01:35.000 | ACISPKT | RS_0000001 | CMD_EVT | event=RTS," + " event_date=2020:001:00:00:00, scs=135", + "2020:001:00:01:39.000 | ACISPKT | RH_0000001 | CMD_EVT | event=RTS," + " event_date=2020:001:00:00:00, scs=135", + "2020:002:15:01:39.000 | COMMAND_HW | 2S2HVOF | CMD_EVT | event=RTS," + " event_date=2020:001:00:00:00, msid=2S2HVOF, scs=135", + "2020:002:15:01:39.000 | COMMAND_SW | OORMPDS | CMD_EVT | event=RTS," + " event_date=2020:001:00:00:00, msid=OORMPDS, scs=135", + "2020:002:15:01:40.000 | COMMAND_HW | 2S2STHV | CMD_EVT | event=RTS," + " event_date=2020:001:00:00:00, 2s2sthv2=0 , msid=2S2STHV, scs=135", + "2020:002:17:40:00.000 | ACISPKT | AA00000000 | CMD_EVT | event=RTS," + " event_date=2020:001:00:00:00, scs=135", + "2020:002:17:40:10.000 | ACISPKT | AA00000000 | CMD_EVT | event=RTS," + " event_date=2020:001:00:00:00, scs=135", + "2020:002:17:40:14.000 | ACISPKT | WSPOW00000 | CMD_EVT | event=RTS," + " event_date=2020:001:00:00:00, scs=135", + "2020:002:17:40:18.000 | ACISPKT | RS_0000001 | CMD_EVT | event=RTS," + " event_date=2020:001:00:00:00, scs=135", + ], +] + + +@pytest.mark.skipif(not HAS_INTERNET, reason="No internet connection") +@pytest.mark.parametrize("idx", range(len(cmd_events_rts_exps))) +def test_get_cmds_from_event_rts(idx, disable_hrc_scs107_commanding): + """Test getting commands from every event type in the Command Events sheet""" + cevt = cmd_events_rts[idx] + exp = cmd_events_rts_exps[idx] + cmds = get_cmds_from_event("2020:001:00:00:00", cevt["Event"], cevt["Params"]) + if cmds is not None: + cmds = cmds.pformat_like_backstop(max_params_width=None) + assert cmds == exp + + +@pytest.mark.skipif(not HAS_INTERNET, reason="No internet connection") +def test_scenario_with_rts(monkeypatch, fast_sun_position_method): + # Test a custom scenario with RTS. This is basically the same as the + # example in the documentation. + from kadi import paths + + monkeypatch.setenv("CXOTIME_NOW", "2021:299") + + # Ensure local cmd_events.csv is up to date by requesting "recent" commands + # relative to the default stop. + cmds = commands.get_cmds(start="2021:299") + + path_flight = paths.CMD_EVENTS_PATH() + path_cti = paths.CMD_EVENTS_PATH(scenario="nsm-cti") + path_cti.parent.mkdir(exist_ok=True, parents=True) + + # Make a new custom scenario from the flight version + events_flight = Table.read(path_flight) + cti_event = { + "State": "Definitive", + "Date": "2021:297:13:00:00", + "Event": "RTS", + "Params": "RTSLOAD,1_CTI06,NUM_HOURS=12:00:00,SCS_NUM=135", + "Author": "Tom Aldcroft", + "Reviewer": "John Scott", + "Comment": "", + } + events_cti = events_flight.copy() + events_cti.add_row(cti_event) + events_cti.write(path_cti, overwrite=True) + + # Now read the commands from the custom scenario + cmds = commands.get_cmds( + "2021:296:10:35:00", "2021:298:01:58:00", scenario="nsm-cti" + ) + cmds.fetch_params() + for cmd in cmds: + if "hex" in cmd["params"]: + del cmd["params"]["hex"] + + exp = """\ +2021:296:10:35:00.000 | COMMAND_HW | CIMODESL | OCT1821A | msid=CIU1024X, scs=128 +2021:296:10:35:00.257 | COMMAND_HW | CTXAOF | OCT1821A | msid=CTXAOF, scs=128 +2021:296:10:35:00.514 | COMMAND_HW | CPAAOF | OCT1821A | msid=CPAAOF, scs=128 +2021:296:10:35:00.771 | COMMAND_HW | CTXBOF | OCT1821A | msid=CTXBOF, scs=128 +2021:296:10:35:01.028 | COMMAND_HW | CPABON | OCT1821A | msid=CPABON, scs=128 +2021:296:10:35:01.285 | COMMAND_HW | CTXBON | OCT1821A | msid=CTXBON, scs=128 +2021:296:10:41:57.000 | LOAD_EVENT | None | CMD_EVT | event=Load_not_run, event_date=2021:296:10:41:57, event_type +2021:296:10:41:57.000 | COMMAND_SW | AONSMSAF | CMD_EVT | event=NSM, event_date=2021:296:10:41:57, scs=0 +2021:296:10:41:57.000 | COMMAND_SW | CODISASX | CMD_EVT | event=NSM, event_date=2021:296:10:41:57, msid=CODISASX, codi +2021:296:10:41:57.000 | COMMAND_SW | CODISASX | CMD_EVT | event=NSM, event_date=2021:296:10:41:57, msid=CODISASX, codi +2021:296:10:41:57.000 | COMMAND_SW | CODISASX | CMD_EVT | event=NSM, event_date=2021:296:10:41:57, msid=CODISASX, codi +2021:296:10:41:57.000 | COMMAND_SW | CODISASX | CMD_EVT | event=NSM, event_date=2021:296:10:41:57, msid=CODISASX, codi +2021:296:10:41:57.000 | COMMAND_SW | CODISASX | CMD_EVT | event=NSM, event_date=2021:296:10:41:57, msid=CODISASX, codi +2021:296:10:41:57.000 | COMMAND_SW | CODISASX | CMD_EVT | event=NSM, event_date=2021:296:10:41:57, msid=CODISASX, codi +2021:296:10:41:57.000 | COMMAND_SW | OORMPDS | CMD_EVT | event=NSM, event_date=2021:296:10:41:57, scs=0 +2021:296:10:41:58.025 | COMMAND_HW | AFIDP | CMD_EVT | event=NSM, event_date=2021:296:10:41:57, msid=AFLCRSET, scs= +2021:296:10:41:58.025 | SIMTRANS | None | CMD_EVT | event=NSM, event_date=2021:296:10:41:57, pos=-99616, scs=0 +2021:296:10:42:20.000 | MP_OBSID | COAOSQID | CMD_EVT | event=Obsid, event_date=2021:296:10:42:20, id=0, scs=0 +2021:296:10:43:03.685 | ACISPKT | AA00000000 | CMD_EVT | event=NSM, event_date=2021:296:10:41:57, scs=0 +2021:296:10:43:04.710 | ACISPKT | AA00000000 | CMD_EVT | event=NSM, event_date=2021:296:10:41:57, scs=0 +2021:296:10:43:14.960 | ACISPKT | WSPOW0002A | CMD_EVT | event=NSM, event_date=2021:296:10:41:57, scs=0 +2021:296:10:43:14.960 | COMMAND_HW | 215PCAOF | CMD_EVT | event=NSM, event_date=2021:296:10:41:57, scs=0 +2021:296:10:43:16.165 | COMMAND_HW | 2IMHVOF | CMD_EVT | event=NSM, event_date=2021:296:10:41:57, scs=0 +2021:296:10:43:17.190 | COMMAND_HW | 2SPHVOF | CMD_EVT | event=NSM, event_date=2021:296:10:41:57, scs=0 +2021:296:10:43:18.215 | COMMAND_HW | 2S2STHV | CMD_EVT | event=NSM, event_date=2021:296:10:41:57, scs=0 +2021:296:10:43:19.240 | COMMAND_HW | 2S1STHV | CMD_EVT | event=NSM, event_date=2021:296:10:41:57, scs=0 +2021:296:10:43:20.265 | COMMAND_HW | 2S2HVOF | CMD_EVT | event=NSM, event_date=2021:296:10:41:57, scs=0 +2021:296:10:43:21.290 | COMMAND_HW | 2S1HVOF | CMD_EVT | event=NSM, event_date=2021:296:10:41:57, scs=0 +2021:296:10:43:22.315 | COMMAND_SW | AODSDITH | CMD_EVT | event=NSM, event_date=2021:296:10:41:57, scs=0 +2021:296:11:08:12.966 | LOAD_EVENT | OBS | CMD_EVT | manvr_start=2021:296:10:41:57.000, prev_att=(0.594590732, 0. +2021:297:01:41:01.000 | COMMAND_SW | AONMMODE | CMD_EVT | event=Maneuver, event_date=2021:297:01:41:01, msid=AONMMODE, +2021:297:01:41:01.256 | COMMAND_SW | AONM2NPE | CMD_EVT | event=Maneuver, event_date=2021:297:01:41:01, msid=AONM2NPE, +2021:297:01:41:05.356 | MP_TARGQUAT | AOUPTARQ | CMD_EVT | event=Maneuver, event_date=2021:297:01:41:01, q1=7.05469070e +2021:297:01:41:11.250 | COMMAND_SW | AOMANUVR | CMD_EVT | event=Maneuver, event_date=2021:297:01:41:01, msid=AOMANUVR, +2021:297:02:05:11.042 | LOAD_EVENT | OBS | CMD_EVT | manvr_start=2021:297:01:41:11.250, prev_att=(0.2854059718181 +2021:297:02:12:42.886 | ORBPOINT | None | OCT1821A | event_type=EQF003M, scs=0 +2021:297:03:40:42.886 | ORBPOINT | None | OCT1821A | event_type=EQF005M, scs=0 +2021:297:03:40:42.886 | ORBPOINT | None | OCT1821A | event_type=EQF015M, scs=0 +2021:297:04:43:26.016 | ORBPOINT | None | OCT1821A | event_type=EALT1, scs=0 +2021:297:04:43:27.301 | ORBPOINT | None | OCT1821A | event_type=XALT1, scs=0 +2021:297:12:42:42.886 | ORBPOINT | None | OCT1821A | event_type=EQF013M, scs=0 +2021:297:13:00:00.000 | COMMAND_SW | OORMPEN | CMD_EVT | event=RTS, event_date=2021:297:13:00:00, msid=OORMPEN, scs=1 +2021:297:13:00:01.000 | ACISPKT | WSVIDALLDN | CMD_EVT | event=RTS, event_date=2021:297:13:00:00, scs=135 +2021:297:13:00:02.000 | COMMAND_HW | 2S2STHV | CMD_EVT | event=RTS, event_date=2021:297:13:00:00, 2s2sthv2=0 , msid=2 +2021:297:13:00:03.000 | COMMAND_HW | 2S2HVON | CMD_EVT | event=RTS, event_date=2021:297:13:00:00, msid=2S2HVON, scs=1 +2021:297:13:00:13.000 | COMMAND_HW | 2S2STHV | CMD_EVT | event=RTS, event_date=2021:297:13:00:00, 2s2sthv2=4 , msid=2 +2021:297:13:00:23.000 | COMMAND_HW | 2S2STHV | CMD_EVT | event=RTS, event_date=2021:297:13:00:00, 2s2sthv2=8 , msid=2 +2021:297:13:00:24.000 | ACISPKT | WSPOW0CF3F | CMD_EVT | event=RTS, event_date=2021:297:13:00:00, scs=135 +2021:297:13:01:27.000 | ACISPKT | WT007AC024 | CMD_EVT | event=RTS, event_date=2021:297:13:00:00, scs=135 +2021:297:13:01:31.000 | ACISPKT | XTZ0000005 | CMD_EVT | event=RTS, event_date=2021:297:13:00:00, scs=135 +2021:297:13:01:35.000 | ACISPKT | RS_0000001 | CMD_EVT | event=RTS, event_date=2021:297:13:00:00, scs=135 +2021:297:13:01:39.000 | ACISPKT | RH_0000001 | CMD_EVT | event=RTS, event_date=2021:297:13:00:00, scs=135 +2021:297:13:59:39.602 | ORBPOINT | None | OCT2521A | event_type=EEF1000, scs=0 +2021:297:14:01:00.000 | LOAD_EVENT | None | OCT1821A | event_type=SCHEDULED_STOP_TIME, scs=0 +2021:297:14:37:39.602 | ORBPOINT | None | OCT2521A | event_type=EPF1000, scs=0 +2021:297:15:01:42.681 | ORBPOINT | None | OCT2521A | event_type=EALT0, scs=0 +2021:297:15:01:43.574 | ORBPOINT | None | OCT2521A | event_type=XALT0, scs=0 +2021:297:16:30:13.364 | ORBPOINT | None | OCT2521A | event_type=EPERIGEE, scs=0 +2021:297:17:58:42.322 | ORBPOINT | None | OCT2521A | event_type=EALT0, scs=0 +2021:297:17:58:43.505 | ORBPOINT | None | OCT2521A | event_type=XALT0, scs=0 +2021:297:20:09:39.602 | ORBPOINT | None | OCT2521A | event_type=XPF1000, scs=0 +2021:297:20:16:57.284 | ORBPOINT | None | OCT2521A | event_type=EASCNCR, scs=0 +2021:297:21:37:39.602 | ORBPOINT | None | OCT2521A | event_type=XEF1000, scs=0 +2021:297:22:32:42.886 | ORBPOINT | None | OCT2521A | event_type=XQF015M, scs=0 +2021:297:23:00:42.886 | ORBPOINT | None | OCT2521A | event_type=XQF005M, scs=0 +2021:298:00:12:42.886 | ORBPOINT | None | OCT2521A | event_type=XQF003M, scs=0 +2021:298:00:12:42.886 | ORBPOINT | None | OCT2521A | event_type=XQF013M, scs=0 +2021:298:01:01:39.000 | COMMAND_HW | 2S2HVOF | CMD_EVT | event=RTS, event_date=2021:297:13:00:00, msid=2S2HVOF, scs=1 +2021:298:01:01:39.000 | COMMAND_SW | OORMPDS | CMD_EVT | event=RTS, event_date=2021:297:13:00:00, msid=OORMPDS, scs=1 +2021:298:01:01:40.000 | COMMAND_HW | 2S2STHV | CMD_EVT | event=RTS, event_date=2021:297:13:00:00, 2s2sthv2=0 , msid=2 +2021:298:01:57:00.000 | LOAD_EVENT | None | OCT2521B | event_type=RUNNING_LOAD_TERMINATION_TIME, scs=0 +2021:298:01:57:00.000 | COMMAND_SW | AOACRSTD | OCT2521B | msid=AOACRSTD, scs=128 +2021:298:01:57:00.000 | ACISPKT | AA00000000 | OCT2521B | cmds=3, words=3, scs=131 +2021:298:01:57:03.000 | ACISPKT | AA00000000 | OCT2521B | cmds=3, words=3, scs=131 +2021:298:01:57:33.000 | COMMAND_SW | CODISASX | OCT2521B | msid=CODISASX, codisas1=135 , scs=131 +2021:298:01:57:34.000 | COMMAND_SW | COCLRSX | OCT2521B | msid=COCLRSX, coclrs1=135 , scs=131""" + + out = "\n".join(cmds.pformat_like_backstop(max_params_width=60)) + assert out == exp + + # 11 RTS commands. Note that the ACIS stop science commands from the RTS + # are NOT evident because they are cut by the RLTT at 2021:298:01:57:00. + # TODO: (someday?) instead of the RLTT notice the disable SCS 135 command + # CODISASX. + ok = cmds["event"] == "RTS" + assert np.count_nonzero(ok) == 14 + + +stop_date_2022_236 = stop_date_fixture_factory("2022-08-23") + + +@pytest.mark.skipif(not HAS_INTERNET, reason="No internet connection") +def test_no_rltt_for_not_run_load(stop_date_2022_236): # noqa: ARG001 + """The AUG2122A loads were never run but they contain an RLTT that had been + stopping the 2022:232:03:09 ACIS ECS that was previously running. This tests + the fix. + """ # noqa: D205 + exp = [ + " date tlmsid scs", + "--------------------- ---------- ---", + "2022:232:03:09:00.000 AA00000000 135", + "2022:232:03:09:04.000 WSPOW00000 135", + "2022:232:03:09:28.000 WSPOW08E1E 135", + "2022:232:03:10:31.000 WT00C62014 135", + "2022:232:03:10:35.000 XTZ0000005 135", + "2022:232:03:10:39.000 RS_0000001 135", + "2022:232:03:10:43.000 RH_0000001 135", + "2022:233:18:10:43.000 AA00000000 135", # <== After the AUG2122A RLTT + "2022:233:18:10:53.000 AA00000000 135", + "2022:233:18:10:57.000 WSPOW0002A 135", + "2022:233:18:12:00.000 RS_0000001 135", + ] + cmds = commands.get_cmds("2022:232:03:00:00", "2022:233:18:30:00") + cmds = cmds[cmds["type"] == "ACISPKT"] + assert cmds["date", "tlmsid", "scs"].pformat() == exp + + +stop_date_2022_352 = stop_date_fixture_factory("2022-12-17") + + +@pytest.mark.skipif(not HAS_INTERNET, reason="No internet connection") +def test_30_day_lookback_issue(stop_date_2022_352): # noqa: ARG001 + """Test for fix in PR #265 of somewhat obscure issue where a query + within the default 30-day lookback could give zero commands. Prior to + the fix the query below would give zero commands (with the default stop date + set accordingly).""" # noqa: D205, D209 + cmds = commands.get_cmds("2022:319", "2022:324") + assert len(cmds) > 200 + + # Hit the CMDS_RECENT cache as well + cmds = commands.get_cmds("2022:319:00:00:01", "2022:324:00:00:01") + assert len(cmds) > 200 + + +def test_fill_gaps(): + from kadi.commands.utils import fill_gaps_with_nan + + times = [1, 20, 21, 200, 300] + vals = [0, 1, 2, 3, 4] + times_out, vals_out = fill_gaps_with_nan(times, vals, max_gap=2) + times_exp = [1, 1.001, 19.999, 20, 21, 21.001, 199.999, 200, 200.001, 299.999, 300] + assert np.allclose(times_out, times_exp) + vals_exp = np.array( + [0.0, np.nan, np.nan, 1.0, 2.0, np.nan, np.nan, 3.0, np.nan, np.nan, 4.0] + ) + is_nan = np.isnan(vals_out) + assert np.all(is_nan == np.isnan(vals_exp)) + assert np.all(vals_out[~is_nan] == vals_exp[~is_nan]) + + +def test_get_rltt_scheduled_stop_time(): + """RLTT and scheduled stop time are both 2023:009:04:14:00.000.""" + cmds = commands.get_cmds("2023:009", "2023:010") + rltt = cmds.get_rltt() + assert rltt == "2023:009:04:14:00.000" + + stt = cmds.get_scheduled_stop_time() + assert stt == "2023:009:04:14:00.000" + + cmds = commands.get_cmds("2023:009:12:00:00", "2023:010") + assert cmds.get_rltt() is None + assert cmds.get_scheduled_stop_time() is None + + +# For HRC not run testing +stop_date_2023200 = stop_date_fixture_factory("2023:200") + + +@pytest.mark.skipif(not HAS_INTERNET, reason="No internet connection") +def test_hrc_not_run_scenario(stop_date_2023200): # noqa: ARG001 + """Test custom scenario with HRC not run""" + from kadi.commands.states import get_states + + # Baseline states WITHOUT the HRC not run command events + states_exp = [ + " datestart hrc_i hrc_s hrc_24v hrc_15v", + "--------------------- ----- ----- ------- -------", + "2023:183:00:00:00.000 OFF OFF OFF OFF", + "2023:184:18:42:15.224 OFF OFF OFF ON", # JUL0323A + "2023:184:18:43:41.224 ON OFF OFF ON", + "2023:184:22:43:49.224 OFF OFF OFF ON", + "2023:184:22:44:01.224 OFF OFF OFF OFF", + "2023:190:23:39:43.615 OFF OFF OFF ON", + "2023:190:23:39:44.615 OFF OFF ON ON", # Note 24V transition + "2023:190:23:41:17.615 OFF OFF OFF ON", + "2023:190:23:42:45.615 OFF ON OFF ON", + "2023:191:03:46:13.615 OFF OFF OFF ON", + "2023:191:03:46:26.615 OFF OFF OFF OFF", + "2023:194:20:03:50.666 OFF OFF OFF ON", # JUL1023A + "2023:194:20:03:51.666 OFF OFF ON ON", + "2023:194:20:05:24.666 OFF OFF OFF ON", + "2023:194:20:06:52.666 ON OFF OFF ON", + "2023:194:23:13:40.666 OFF OFF OFF ON", + "2023:194:23:13:52.666 OFF OFF OFF OFF", + ] + + keys = ["hrc_i", "hrc_s", "hrc_24v", "hrc_15v"] + states = get_states( + start="2023:183", + stop="2023:195", + state_keys=keys, + merge_identical=True, + ) + states_out = states[["datestart"] + keys].pformat() + assert states_out == states_exp + + # First make the cmd_events.csv file for the scenario where F_HRC_SAFING is run at + # 2023:184:20:00:00.000. + # Note that JUL0323A runs from 2023:183:16:39:00.000 to 2023:191:03:46:28.615. + # We expect the HRC to be off from 2023:184 2000z until the first observation in the + # JUL1023A loads which start at 2023:191:03:43:28.615 + + scenario = "hrc_not_run" + cmds_dir = Path(commands.conf.commands_dir) / scenario + cmds_dir.mkdir(exist_ok=True, parents=True) + # Note variation in format of date, since this comes from humans. + cmd_evts_text = """\ +State,Date,Event,Params,Author,Reviewer,Comment +Definitive,2023:184:20:00:00.000,HRC not run,JUL0323A,Tom,Jean,F_HRC_SAFING 2023:184:20:00:00 +""" + (cmds_dir / "cmd_events.csv").write_text(cmd_evts_text) + + # Now get states in same time range for the HRC not run scenario. + keys = ["hrc_i", "hrc_s", "hrc_24v", "hrc_15v"] + states = get_states( + start="2023:183", + stop="2023:195", + state_keys=keys, + merge_identical=True, + scenario=scenario, + ) + states_exp = [ + " datestart hrc_i hrc_s hrc_24v hrc_15v", + "--------------------- ----- ----- ------- -------", + "2023:183:00:00:00.000 OFF OFF OFF OFF", + "2023:184:18:42:15.224 OFF OFF OFF ON", # JUL0323A + "2023:184:18:43:41.224 ON OFF OFF ON", + "2023:184:20:00:00.000 OFF OFF OFF OFF", # Shut off by HRC not run + "2023:194:20:03:50.666 OFF OFF OFF ON", # JUL1023A + "2023:194:20:03:51.666 OFF OFF ON ON", + "2023:194:20:05:24.666 OFF OFF OFF ON", + "2023:194:20:06:52.666 ON OFF OFF ON", + "2023:194:23:13:40.666 OFF OFF OFF ON", + "2023:194:23:13:52.666 OFF OFF OFF OFF", + ] + + states_out = states[["datestart"] + keys].pformat() + assert states_out == states_exp + + commands.clear_caches() + + +test_command_not_run_cases = [ + { + # Matches multiple commands + "event": { + "date": "2023:351:13:30:33.849", + "event": "Command not run", + "params_str": "COMMAND_SW | TLMSID= COACTSX", + }, + "removed": [3, 4], + }, + { + # Matches one command with multiple criteria + "event": { + "date": "2023:351:13:30:33.849", + "event": "Command not run", + "params_str": ( + "COMMAND_SW | TLMSID= COACTSX, HEX= 840B100, " + "MSID= COACTSX, COACTS1=177 , COACTS2=0 , SCS= 128, STEP= 690" + ), + }, + "removed": [3], + }, + { + # Wrong TLMSID + "event": { + "date": "2023:351:13:30:33.849", + "event": "Command not run", + "params_str": ( + "COMMAND_SW | TLMSID= XXXXXXX, HEX= 840B100, " + "MSID= COACTSX, COACTS1=177 , COACTS2=0 , SCS= 128, STEP= 690" + ), + }, + "removed": [], + }, + { + # Wrong SCS + "event": { + "date": "2023:351:13:30:33.849", + "event": "Command not run", + "params_str": ( + "COMMAND_SW | TLMSID= XXXXXXX, HEX= 840B100, " + "MSID= COACTSX, COACTS1=177 , COACTS2=0 , SCS= 133, STEP= 690" + ), + }, + "removed": [], + }, + { + # Wrong Step + "event": { + "date": "2023:351:13:30:33.849", + "event": "Command not run", + "params_str": ( + "COMMAND_SW | TLMSID= XXXXXXX, HEX= 840B100, " + "MSID= COACTSX, COACTS1=177 , COACTS2=0 , SCS= 128, STEP= 111" + ), + }, + "removed": [], + }, + { + # No TLMSID + "event": { + "date": "2023:351:19:38:41.550", + "event": "Command not run", + "params_str": "SIMTRANS | POS= 92904, SCS= 131, STEP= 1191", + }, + "removed": [6], + }, +] + + +@pytest.mark.parametrize("case", test_command_not_run_cases) +def test_command_not_run(case): + backstop_text = """ +2023:351:13:30:32.824 | 0 0 | COMMAND_SW | TLMSID= AOMANUVR, HEX= 8034101, MSID= AOMANUVR, SCS= 128, STEP= 686 +2023:351:13:30:33.849 | 1 0 | COMMAND_SW | TLMSID= AOACRSTE, HEX= 8032001, MSID= AOACRSTE, SCS= 128, STEP= 688 +2023:351:13:30:33.849 | 2 0 | COMMAND_SW | TLMSID= COENASX, HEX= 844B100, MSID= COENASX, COENAS1=177 , SCS= 128, STEP= 689 +2023:351:13:30:33.849 | 3 0 | COMMAND_SW | TLMSID= COACTSX, HEX= 840B100, MSID= COACTSX, COACTS1=177 , COACTS2=0 , SCS= 128, STEP= 690 +2023:351:13:30:33.849 | 4 0 | COMMAND_SW | TLMSID= COACTSX, HEX= 8402600, MSID= COACTSX, COACTS1=38 , COACTS2=0 , SCS= 128, STEP= 691 +2023:351:13:30:55.373 | 5 0 | COMMAND_HW | TLMSID= 4MC5AEN, HEX= 4800012, MSID= 4MC5AEN, SCS= 131, STEP= 892 +2023:351:19:38:41.550 | 6 0 | SIMTRANS | POS= 92904, SCS= 131, STEP= 1191 + """ + cmds = commands.read_backstop(backstop_text.strip().splitlines()) + cmds["source"] = "DEC1123A" + cmds_exp = cmds.copy() + cmds_exp.remove_rows(case["removed"]) + cmds_from_event = get_cmds_from_event(**case["event"]) + cmds_with_event = cmds.add_cmds(cmds_from_event) + cmds_with_event.sort_in_backstop_order() + cmds_with_event.remove_not_run_cmds() + assert cmds_with_event.pformat_like_backstop() == cmds_exp.pformat_like_backstop() + + +def test_add_cmds(): + """Test add_cmds with and without RLTT""" + cmds1_text = """ +2023:344:23:30:00.000 | 16200421 0 | COMMAND_SW | TLMSID= CMD1, SCS= 128, STEP= 0 +2023:344:23:32:59.999 | 16200421 0 | COMMAND_SW | TLMSID= CMD2, SCS= 128, STEP= 0 +2023:344:23:33:00.000 | 16201123 0 | COMMAND_SW | TLMSID= CMD3, SCS= 128, STEP= 0 +2023:344:23:33:00.000 | 16201123 0 | COMMAND_SW | TLMSID= CMD4, SCS= 128, STEP= 0 +2023:344:23:33:00.001 | 16201124 0 | COMMAND_SW | TLMSID= CMD5, SCS= 128, STEP= 0 +2023:344:23:34:00.000 | 16201124 0 | COMMAND_SW | TLMSID= CMD6, SCS= 128, STEP= 0 +""" + + cmds2_text = """ +2023:344:23:30:00.000 | 16200421 0 | COMMAND_SW | TLMSID= CMD1_2, SCS= 130, STEP= 0 +2023:344:23:32:59.999 | 16200421 0 | COMMAND_SW | TLMSID= CMD2_2, SCS= 130, STEP= 0 +2023:344:23:33:00.000 | 0 0 | LOAD_EVENT | TYPE= RUNNING_LOAD_TERMINATION_TIME, SCS=0, STEP=0 +2023:344:23:33:00.000 | 16201123 0 | COMMAND_SW | TLMSID= CMD3_2, SCS= 130, STEP= 0 +2023:344:23:33:00.000 | 16201123 0 | COMMAND_SW | TLMSID= CMD4_2, SCS= 130, STEP= 0 +2023:344:23:33:00.001 | 16201124 0 | COMMAND_SW | TLMSID= CMD5_2, SCS= 130, STEP= 0 +2023:344:23:34:00.000 | 16201124 0 | COMMAND_SW | TLMSID= CMD6_2, SCS= 130, STEP= 0 +""" + cmds1 = commands.read_backstop(cmds1_text.strip().splitlines()) + cmds2 = commands.read_backstop(cmds2_text.strip().splitlines()) + cmds12_rltt = cmds1.add_cmds(cmds2, rltt=cmds2.get_rltt()) + # Commands from cmds1 with date > 2023:344:23:33:00:000 (RLTT) are removed. + exp_rltt = [ + "2023:344:23:30:00.000 | COMMAND_SW | CMD1 | 0 | scs=128", + "2023:344:23:30:00.000 | COMMAND_SW | CMD1_2 | 0 | scs=130", + "2023:344:23:32:59.999 | COMMAND_SW | CMD2 | 0 | scs=128", + "2023:344:23:32:59.999 | COMMAND_SW | CMD2_2 | 0 | scs=130", + "2023:344:23:33:00.000 | COMMAND_SW | CMD3 | 0 | scs=128", + "2023:344:23:33:00.000 | COMMAND_SW | CMD4 | 0 | scs=128", + "2023:344:23:33:00.000 | LOAD_EVENT | None | 0 | event_type=RUNNING_LOAD_TERMINATION_TIME, scs=0", + "2023:344:23:33:00.000 | COMMAND_SW | CMD3_2 | 0 | scs=130", + "2023:344:23:33:00.000 | COMMAND_SW | CMD4_2 | 0 | scs=130", + "2023:344:23:33:00.001 | COMMAND_SW | CMD5_2 | 0 | scs=130", + "2023:344:23:34:00.000 | COMMAND_SW | CMD6_2 | 0 | scs=130", + ] + assert cmds12_rltt.pformat_like_backstop() == exp_rltt + + cmds12_no_rltt = cmds1.add_cmds(cmds2) + # All commands from both cmds1 and cmds2 are included. + exp_no_rltt = [ + "2023:344:23:30:00.000 | COMMAND_SW | CMD1 | 0 | scs=128", + "2023:344:23:30:00.000 | COMMAND_SW | CMD1_2 | 0 | scs=130", + "2023:344:23:32:59.999 | COMMAND_SW | CMD2 | 0 | scs=128", + "2023:344:23:32:59.999 | COMMAND_SW | CMD2_2 | 0 | scs=130", + "2023:344:23:33:00.000 | COMMAND_SW | CMD3 | 0 | scs=128", + "2023:344:23:33:00.000 | COMMAND_SW | CMD4 | 0 | scs=128", + "2023:344:23:33:00.000 | LOAD_EVENT | None | 0 | event_type=RUNNING_LOAD_TERMINATION_TIME, scs=0", + "2023:344:23:33:00.000 | COMMAND_SW | CMD3_2 | 0 | scs=130", + "2023:344:23:33:00.000 | COMMAND_SW | CMD4_2 | 0 | scs=130", + "2023:344:23:33:00.001 | COMMAND_SW | CMD5 | 0 | scs=128", + "2023:344:23:33:00.001 | COMMAND_SW | CMD5_2 | 0 | scs=130", + "2023:344:23:34:00.000 | COMMAND_SW | CMD6 | 0 | scs=128", + "2023:344:23:34:00.000 | COMMAND_SW | CMD6_2 | 0 | scs=130", + ] + assert cmds12_no_rltt.pformat_like_backstop() == exp_no_rltt + + +def test_read_backstop_with_observations(): + """Test reading backstop with observations in it. + + This tests reading the DEC1123A loads and showing that the observations and + star catalogs are exactly the same as in the flight commands archive. + """ + try: + path = parse_cm.paths.load_file_path("DEC1123A", "CR*.backstop", "backstop") + except FileNotFoundError: + pytest.skip("No backstop file found") + + cmds = read_backstop(path, add_observations=True) + obss = get_observations(cmds=cmds) + starcats = get_starcats(cmds=cmds) + + cmds_flight = commands.get_cmds(source="DEC1123A") + obss_cmds_flight = get_observations(cmds=cmds_flight) + starcats_cmds_flight = get_starcats(cmds=cmds_flight) + + assert len(obss) == len(obss_cmds_flight) + assert len(starcats) == len(starcats_cmds_flight) + + for obs, obs_flight in zip(obss, obss_cmds_flight): + # starcat idx will be different + obs.pop("starcat_idx", None) + obs_flight.pop("starcat_idx", None) + assert obs == obs_flight + + for starcat, starcat_flight in zip(starcats, starcats_cmds_flight): + assert starcat.pformat() == starcat_flight.pformat() From b2bc17c2720af62e07f835298ec1e9f547ee8d30 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Wed, 4 Feb 2026 05:48:06 -0500 Subject: [PATCH 27/31] Remove checks on having AGASC 1.8 --- kadi/commands/tests/test_commands.py | 8 -------- kadi/commands/tests/test_commands_v2.py | 8 -------- 2 files changed, 16 deletions(-) diff --git a/kadi/commands/tests/test_commands.py b/kadi/commands/tests/test_commands.py index a738824e..a3625d30 100644 --- a/kadi/commands/tests/test_commands.py +++ b/kadi/commands/tests/test_commands.py @@ -4,7 +4,6 @@ # Use data file from parse_cm.test for get_cmds_from_backstop test. # This package is a dependency -import agasc import astropy.units as u import numpy as np import parse_cm.paths @@ -43,12 +42,6 @@ KADI_CMDS_VERSION < 3, reason="requires KADI_CMDS_VERSION >= 3" ) -try: - agasc.get_agasc_filename(version="1p8") - HAS_AGASC_1P8 = True -except FileNotFoundError: - HAS_AGASC_1P8 = False - def get_cmds_from_cmd_evts_text(cmd_evts_text: str): """Get commands from a cmd_events text string. @@ -1091,7 +1084,6 @@ def test_get_starcat_only_agasc1p7(): assert np.all(starcat["mag"] != -999) -@pytest.mark.skipif(not HAS_AGASC_1P8, reason="AGASC 1.8 not available") def test_get_starcat_only_agasc1p8(): """For obsids 3829 and 2576, try AGASC 1.8 only diff --git a/kadi/commands/tests/test_commands_v2.py b/kadi/commands/tests/test_commands_v2.py index 706f0361..e8889bfe 100644 --- a/kadi/commands/tests/test_commands_v2.py +++ b/kadi/commands/tests/test_commands_v2.py @@ -4,7 +4,6 @@ # Use data file from parse_cm.test for get_cmds_from_backstop test. # This package is a dependency -import agasc import astropy.units as u import numpy as np import parse_cm.paths @@ -44,12 +43,6 @@ KADI_CMDS_VERSION > 2, reason="requires KADI_CMDS_VERSION <= 2" ) -try: - agasc.get_agasc_filename(version="1p8") - HAS_AGASC_1P8 = True -except FileNotFoundError: - HAS_AGASC_1P8 = False - def get_cmds_from_cmd_evts_text(cmd_evts_text: str): """Get commands from a cmd_events text string. @@ -1092,7 +1085,6 @@ def test_get_starcat_only_agasc1p7(): assert np.all(starcat["mag"] != -999) -@pytest.mark.skipif(not HAS_AGASC_1P8, reason="AGASC 1.8 not available") def test_get_starcat_only_agasc1p8(): """For obsids 3829 and 2576, try AGASC 1.8 only From 31828f1c63296bca0e9ed3f7c7ec3ee7b355d978 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Wed, 4 Feb 2026 06:37:49 -0500 Subject: [PATCH 28/31] Don't require files to exist if version is specified --- kadi/commands/core.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/kadi/commands/core.py b/kadi/commands/core.py index d7e14a87..4695930b 100644 --- a/kadi/commands/core.py +++ b/kadi/commands/core.py @@ -129,9 +129,6 @@ def kadi_cmds_version() -> int: int Highest allowed kadi commands version number found. """ - # Allowed versions in descending order down to version=2. - versions_allowed = tuple(range(KADI_CMDS_VERSION_MAX, 1, -1)) - if ver_str := os.environ.get("KADI_CMDS_VERSION"): version = int(ver_str) if version > KADI_CMDS_VERSION_MAX: @@ -140,8 +137,10 @@ def kadi_cmds_version() -> int: f"version {KADI_CMDS_VERSION_MAX}" ) else: - versions_allowed = (version,) + return version + # Allowed versions in descending order down to version=2. + versions_allowed = tuple(range(KADI_CMDS_VERSION_MAX, 1, -1)) for version in versions_allowed: if IDX_CMDS_PATH(version).exists() and PARS_DICT_PATH(version).exists(): return version From 9a4af98b04dfa7385624df8dabb581e9dd49e8c6 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Wed, 4 Feb 2026 06:38:56 -0500 Subject: [PATCH 29/31] Use stable sort for cmds_list Does not change effective results but this fixes a unit test issue in command ordering. --- kadi/commands/commands_v2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kadi/commands/commands_v2.py b/kadi/commands/commands_v2.py index 7f760902..9213360d 100644 --- a/kadi/commands/commands_v2.py +++ b/kadi/commands/commands_v2.py @@ -553,7 +553,7 @@ def update_cmd_events_and_loads_and_get_cmds_recent( # Sort cmds_list and rltts by the date of the first cmd in each cmds Table cmds_starts = np.array([cmds["date"][0] for cmds in cmds_list]) - idx_sort = np.argsort(cmds_starts) + idx_sort = np.argsort(cmds_starts, kind="stable") cmds_list = [cmds_list[ii] for ii in idx_sort] # Apply RLTT and any END SCS commands (CODISAXS) to loads. RLTT is applied From ff1dd79033388df9c444d5b809163c657c6ceb2a Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Wed, 4 Feb 2026 12:26:40 -0500 Subject: [PATCH 30/31] Include obsid_sch in obs continuity and core util to add observations to commands --- kadi/commands/commands_v2.py | 9 ++++++++- kadi/commands/core.py | 16 ++++++++++++---- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/kadi/commands/commands_v2.py b/kadi/commands/commands_v2.py index 9213360d..d01cd434 100644 --- a/kadi/commands/commands_v2.py +++ b/kadi/commands/commands_v2.py @@ -631,6 +631,7 @@ def add_obs_cmds( prev_att=None, prev_obsid=None, prev_simpos=None, + prev_obsid_sched=None, ): """Add 'type=LOAD_EVENT tlmsid=OBS' commands with info about observations. @@ -661,6 +662,8 @@ def add_obs_cmds( Previous obsid, default is -1. prev_simpos : int, optional Previous SIM position, default is -99616. + prev_obsid_sched : int, optional + Previous scheduled obsid, default is -1. Returns ------- @@ -692,6 +695,7 @@ def add_obs_cmds( schedule_stop_time, prev_obsid=prev_obsid, prev_simpos=prev_simpos, + prev_obsid_sched=prev_obsid_sched, ) # Finally add the OBS cmds to the recent cmds table. @@ -917,6 +921,7 @@ def get_cmds_obs_final( *, prev_obsid=None, prev_simpos=None, + prev_obsid_sched=None, ): """Fill in the rest of params for each OBS command. @@ -940,6 +945,8 @@ def get_cmds_obs_final( Previous obsid, default is -1. prev_simpos : int, optional Previous SIM position, default is -99616. + prev_obsid_sched : int, optional + Previous scheduled obsid, default is -1. Returns ------- @@ -952,7 +959,7 @@ def get_cmds_obs_final( # use values that are not None to avoid errors. For `sim_pos`, if the SIM # has not been commanded in a long time then it will be at -99616. obsid = -1 if prev_obsid is None else prev_obsid - obsid_sched = -1 if prev_obsid is None else prev_obsid + obsid_sched = -1 if prev_obsid_sched is None else prev_obsid_sched starcat_idx = None starcat_date = None sim_pos = -99616 if prev_simpos is None else prev_simpos diff --git a/kadi/commands/core.py b/kadi/commands/core.py index 4695930b..a39bf410 100644 --- a/kadi/commands/core.py +++ b/kadi/commands/core.py @@ -303,11 +303,18 @@ def add_observations_to_cmds( cmds = cmds.copy() cmds["source"] = str(load_name) - # Get continuity for attitude, obsid, and SIM position. These are part of OBS cmd. + # For kadi commands version >= 3, we need to add the OBSID_SCH commands so that + # they are right in the OBS commands. + if kadi_cmds_version() >= 3: + cmds = kcc2.add_scheduled_obsid_cmds(cmds) + + # Get continuity for attitude, obsid, obsid_sched, and SIM position. These are part + # of OBS cmd. rltt = cmds.get_rltt() or cmds["date"][0] - cont = kcs.get_continuity( - date=rltt, state_keys=["simpos", "obsid", "q1", "q2", "q3", "q4"] - ) + state_keys = ["simpos", "obsid", "q1", "q2", "q3", "q4"] + if kadi_cmds_version() >= 3: + state_keys.append("obsid_sched") + cont = kcs.get_continuity(date=rltt, state_keys=state_keys) prev_att = tuple(cont[f"q{ii}"] for ii in range(1, 5)) # Add observations to cmds. This uses the global PARS_DICT and REV_PARS_DICT from @@ -321,6 +328,7 @@ def add_observations_to_cmds( prev_att=prev_att, prev_simpos=cont["simpos"], prev_obsid=cont["obsid"], + prev_obsid_sched=cont["obsid_sched"] if kadi_cmds_version() >= 3 else None, ) # General implementation note: Using a weakref means that it is not possible to From 8b9570c05e5492a48f4d76cd7b2b69cbb1d1d609 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Wed, 4 Feb 2026 12:28:00 -0500 Subject: [PATCH 31/31] Fix some tests --- kadi/commands/tests/conftest.py | 7 ++----- kadi/commands/tests/test_commands.py | 16 +++++++++------- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/kadi/commands/tests/conftest.py b/kadi/commands/tests/conftest.py index 96d121bd..e7451c82 100644 --- a/kadi/commands/tests/conftest.py +++ b/kadi/commands/tests/conftest.py @@ -2,19 +2,16 @@ import ska_sun import kadi.commands as kc -import kadi.commands.core as kcc @pytest.fixture() def fast_sun_position_method(monkeypatch: pytest.MonkeyPatch): - if kcc.kadi_cmds_version() == 2: - monkeypatch.setattr(ska_sun.conf, "sun_position_method_default", "fast") + monkeypatch.setattr(ska_sun.conf, "sun_position_method_default", "fast") @pytest.fixture() def disable_hrc_scs107_commanding(monkeypatch: pytest.MonkeyPatch): - if kcc.kadi_cmds_version() == 2: - monkeypatch.setattr(kc.conf, "disable_hrc_scs107_commanding", True) + monkeypatch.setattr(kc.conf, "disable_hrc_scs107_commanding", True) @pytest.fixture(scope="module", autouse=True) diff --git a/kadi/commands/tests/test_commands.py b/kadi/commands/tests/test_commands.py index a3625d30..716d6aae 100644 --- a/kadi/commands/tests/test_commands.py +++ b/kadi/commands/tests/test_commands.py @@ -251,9 +251,7 @@ def test_kadi_cmds_version(): @pytest.mark.skipif("not HAS_MPDIR") @pytest.mark.skipif(not HAS_INTERNET, reason="No internet connection") -def test_commands_create_archive_regress( - tmpdir, fast_sun_position_method, disable_hrc_scs107_commanding -): +def test_commands_create_archive_regress(tmpdir): """Create cmds archive from scratch and test that it matches flight This tests over an eventful month that includes IU reset/NSM, SCS-107 @@ -499,7 +497,9 @@ def test_get_cmds_v2_arch_recent(stop_date_2020_12_03): # noqa: ARG001 # of the matching block). assert np.all(cmds["idx"] != -1) # PR #248: made this change from 17640 to 17644 - assert 17640 <= len(cmds) <= 17644 + # PR #368: changed to 18112 with addition of OBSID_SCH commands + assert len(cmds) == 18112 + assert np.count_nonzero(cmds["tlmsid"] != "OBSID_SCH") == 17644 commands.clear_caches() @@ -510,7 +510,7 @@ def test_get_cmds_v2_recent_only(stop_date_2020_12_03): # noqa: ARG001 # only commands out to the end of the NOV3020A loads (~ Dec 7). cmds = commands.get_cmds(start="2020-12-01", stop="2021-01-01") cmds = cmds[cmds["tlmsid"] != "OBS"] - assert len(cmds) == 1523 + assert len(cmds) == 1523 + 34 # 34 for OBSID_SCH assert np.all(cmds["idx"] == -1) # fmt: off assert cmds[:5].pformat_like_backstop() == [ @@ -530,7 +530,7 @@ def test_get_cmds_v2_recent_only(stop_date_2020_12_03): # noqa: ARG001 # fmt: on # Same for no stop date cmds = commands.get_cmds(start="2020-12-01", stop=None) - cmds = cmds[cmds["tlmsid"] != "OBS"] + cmds = cmds[(cmds["tlmsid"] != "OBS") & (cmds["tlmsid"] != "OBSID_SCH")] assert len(cmds) == 1523 assert np.all(cmds["idx"] == -1) @@ -544,7 +544,7 @@ def test_get_cmds_v2_recent_only(stop_date_2020_12_03): # noqa: ARG001 def test_get_cmds_nsm_2021(stop_date_2021_10_24, disable_hrc_scs107_commanding): """NSM at ~2021:296:10:41. This tests non-load commands from cmd_events.""" cmds = commands.get_cmds("2021:296:10:35:00") # , '2021:298:01:58:00') - cmds = cmds[cmds["tlmsid"] != "OBS"] + cmds = cmds[(cmds["tlmsid"] != "OBS") & (cmds["tlmsid"] != "OBSID_SCH")] exp = [ "2021:296:10:35:00.000 | COMMAND_HW | CIMODESL | OCT1821A | " "hex=7C067C0, msid=CIU1024X, scs=128", @@ -885,6 +885,7 @@ def test_bright_star_hold_event( """ ) cmds = commands.get_cmds(start="2020:336:21:48:00", stop="2020:338", scenario="bsh") + cmds = cmds[cmds["tlmsid"] != "OBSID_SCH"] exp = [ "2020:336:21:48:03.312 | LOAD_EVENT | OBS | NOV3020A | " "manvr_start=2020:336:21:09:24.361, prev_att=(-0.242373434, -0.348723922, " @@ -1269,6 +1270,7 @@ def test_custom_scenario(monkeypatch, stop_date_2024_035_23_00_00): for scenario in ("flight+custom", "custom"): exp = exps[scenario] cmds = commands.get_cmds("2024:028", "2024:050", scenario=scenario) + cmds = cmds[cmds["tlmsid"] != "OBSID_SCH"] sources = {"FEB0524A", "JAN2924A", "JAN2624A"} if scenario == "flight+custom": sources.add("CMD_EVT")