From e49cd2fdbf8bcb3f7e77a804c99b36cdc92928cc Mon Sep 17 00:00:00 2001 From: Josh Walawender Date: Fri, 5 Dec 2025 10:54:58 -1000 Subject: [PATCH 01/64] Fix branch name code --- kpf/OB_GUI/KPF_OB_GUI.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/kpf/OB_GUI/KPF_OB_GUI.py b/kpf/OB_GUI/KPF_OB_GUI.py index 6a47c326..39af1a38 100755 --- a/kpf/OB_GUI/KPF_OB_GUI.py +++ b/kpf/OB_GUI/KPF_OB_GUI.py @@ -131,8 +131,10 @@ def __init__(self, clargs, *args, **kwargs): # Determine git branch try: - cmd = 'git branch --show-current' + cmd = f'cd {Path(__file__).parent} ; git branch --show-current' result = subprocess.run(cmd, shell=True, stdout=subprocess.PIPE) + self.log.debug('git branch --show-current') + self.log.debug(result.stdout.decode()) self.branch = result.stdout.decode().strip().strip('\n') self.log.debug(f'Got git branch result: {self.branch}') except: From 41ffa3cee6c0b59b9625061919f43c85b4686f01 Mon Sep 17 00:00:00 2001 From: Josh Walawender Date: Fri, 5 Dec 2025 16:02:55 -1000 Subject: [PATCH 02/64] =?UTF-8?q?Fix=20for=20time=20ranges=20not=20based?= =?UTF-8?q?=20on=20=E2=80=9Cnow=E2=80=9D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kpf/calbench/ListLFCRuns.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/kpf/calbench/ListLFCRuns.py b/kpf/calbench/ListLFCRuns.py index 726c1636..718b257e 100644 --- a/kpf/calbench/ListLFCRuns.py +++ b/kpf/calbench/ListLFCRuns.py @@ -36,14 +36,15 @@ class ListLFCRuns(KPFFunction): @classmethod def perform(cls, args): ndays = args.get('ndays', 1) - now = datetime.datetime.now() if args.get('date', '') in ['', 'now']: - start = now - datetime.timedelta(days=ndays) + start = datetime.datetime.now() - datetime.timedelta(days=ndays) + end = datetime.datetime.now() else: start = datetime.datetime.strptime(args.get('date'), '%Y-%m-%d') - datetime.timedelta(days=ndays) + end = datetime.datetime.strptime(args.get('date'), '%Y-%m-%d') LFC_history = keygrabber.retrieve({'kpfcal': ['OPERATIONMODE']}, begin=start.timestamp(), - end=now.timestamp()) + end=end.timestamp()) astrocomb = [x for x in LFC_history if x['ascvalue'] == 'AstroComb'] # kws = {'kpfcal': ['OPERATIONMODE', 'POS_INTENSITY', 'SPECFLATIR', From 8a51e2684fe4afc1dfc55f53b38c8759d4825522 Mon Sep 17 00:00:00 2001 From: Josh Walawender Date: Mon, 8 Dec 2025 15:32:28 -1000 Subject: [PATCH 03/64] Remove blank line --- kpf/socal/IsSoCalShutDown.py | 1 - 1 file changed, 1 deletion(-) diff --git a/kpf/socal/IsSoCalShutDown.py b/kpf/socal/IsSoCalShutDown.py index f2350426..f42b5a75 100644 --- a/kpf/socal/IsSoCalShutDown.py +++ b/kpf/socal/IsSoCalShutDown.py @@ -55,5 +55,4 @@ def add_cmdline_args(cls, parser): parser.add_argument('--email', dest="email", default=False, action="store_true", help='Send email if SoCal is not shut down?') - return super().add_cmdline_args(parser) From 88d5d6f422e5bb51d3f639e3f32d1b16eef33354 Mon Sep 17 00:00:00 2001 From: Josh Walawender Date: Mon, 8 Dec 2025 15:47:00 -1000 Subject: [PATCH 04/64] Add tool to check if KPF-CC schedule files exist --- kpf/linking_table.yml | 2 + kpf/utils/ScheduleFilesCheck.py | 112 ++++++++++++++++++++++++++++++++ 2 files changed, 114 insertions(+) create mode 100644 kpf/utils/ScheduleFilesCheck.py diff --git a/kpf/linking_table.yml b/kpf/linking_table.yml index cf50c00b..76c78679 100644 --- a/kpf/linking_table.yml +++ b/kpf/linking_table.yml @@ -333,6 +333,8 @@ links: cmd: utils.CheckAllowScheduledCals.CheckAllowScheduledCals EndOfNight: cmd: utils.EndOfNight.EndOfNight + ScheduleFilesCheck: + cmd: utils.ScheduleFilesCheck.ScheduleFilesCheck SetObserverFromSchedule: cmd: utils.SetObserverFromSchedule.SetObserverFromSchedule SetOutdirs: diff --git a/kpf/utils/ScheduleFilesCheck.py b/kpf/utils/ScheduleFilesCheck.py new file mode 100644 index 00000000..d235fa51 --- /dev/null +++ b/kpf/utils/ScheduleFilesCheck.py @@ -0,0 +1,112 @@ +from pathlib import Path +import datetime + +from kpf import log, cfg +from kpf.exceptions import * +from kpf.KPFTranslatorFunction import KPFFunction, KPFScript +from kpf.observatoryAPIs import get_semester_dates +from kpf.observatoryAPIs.GetScheduledPrograms import GetScheduledPrograms +from kpf.utils.SendEmail import SendEmail + + +##----------------------------------------------------------------------------- +## ScheduleFilesCheck +##----------------------------------------------------------------------------- +class ScheduleFilesCheck(KPFFunction): + ''' + + Args: + progname (str): The program name to set if a choice is needed. + + Functions Called: + + - `kpf.observatoryAPIs.GetScheduledPrograms` + - `kpf.utils.SendEmail` + ''' + @classmethod + def pre_condition(cls, args): + pass + + @classmethod + def perform(cls, args): + errors = [] + utnow = datetime.datetime.utcnow() + semester, s_start, s_end = get_semester_dates(utnow) + + band_names = ['full-band1', 'full-band2', 'full-band3'] + classical, cadence = GetScheduledPrograms.execute({'semester': 'tonight'}) + if len(cadence) > 0: + band_names.extend(['band1', 'band2', 'band3']) + + base_path = Path('/s/sdata1701/Schedules') + semester_path = base_path / semester + date_string = (utnow - datetime.timedelta(hours=24)).strftime('%Y-%m-%d') + date_path = semester_path / date_string + if date_path.exists() is False: + err = f"{str(date_path)} does not exist" + errors.append(err) + log.error(err) + + band_paths = [date_path / band for band in band_names] + line_counts = {} + for band_path in band_paths: + if band_path.exists() is False: + err = f"{str(band_path)} does not exist" + errors.append(err) + log.error(err) + output_path = band_path / 'output' + if output_path.exists() is False: + err = f"{str(output_path)} does not exist" + errors.append(err) + log.error(err) + output_file = output_path / 'night_plan.csv' + if output_file.exists() is False: + err = f"{str(output_file)} does not exist" + errors.append(err) + log.error(err) + try: + with open(output_file, 'r') as f: + lines = f.readlines() + nlines = len(lines) + line_counts[band_path.name] = nlines + if nlines <= 1: + err = f"{str(output_file)} has only {nlines} lines" + errors.append(err) + log.error(err) + except: + err = f'Failed to read {str(output_file)}' + errors.append(err) + log.error(err) + + # Results + result_str = 'Band Line Count\n' + for band in sorted(line_counts.keys()): + if line_counts[band] <= 1: + result_str += f'{band:11s} {line_counts[band]:d} <-- Low target count!\n' + else: + result_str += f'{band:11s} {line_counts[band]:d}\n' + print(result_str) + + # Send Email + if len(errors) > 0: + msg = 'KPF-CC Schedule May Be Bad\n\n' + if args.get('email', False) == True: + try: + SendEmail.execute({'Subject': f'KPF-CC Schedule May Be Bad', + 'Message': msg+result_str, + 'To': 'jwalawender@keck.hawaii.edu'}) + except Exception as email_err: + log.error(f'Sending email failed') + log.error(email_err) + + + @classmethod + def post_condition(cls, args): + pass + + @classmethod + def add_cmdline_args(cls, parser): + parser.add_argument('--email', dest="email", + default=False, action="store_true", + help='Send email if SoCal is not shut down?') + return super().add_cmdline_args(parser) From 142ed6df1ce994b51d0996c20e47e9e30ed49b79 Mon Sep 17 00:00:00 2001 From: Josh Walawender Date: Mon, 8 Dec 2025 16:27:32 -1000 Subject: [PATCH 05/64] Update docstring --- kpf/utils/ScheduleFilesCheck.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/kpf/utils/ScheduleFilesCheck.py b/kpf/utils/ScheduleFilesCheck.py index d235fa51..9d68c564 100644 --- a/kpf/utils/ScheduleFilesCheck.py +++ b/kpf/utils/ScheduleFilesCheck.py @@ -13,10 +13,11 @@ ## ScheduleFilesCheck ##----------------------------------------------------------------------------- class ScheduleFilesCheck(KPFFunction): - ''' + '''Check whether all expected schedule files are present and count the + number of lines in each file and see if it is larger than 1. Args: - progname (str): The program name to set if a choice is needed. + email (bool): Send an email if a problem is detected? Functions Called: From 2bfbada9f6d783f40aa0279e251a150f0b34de1c Mon Sep 17 00:00:00 2001 From: Josh Walawender Date: Wed, 10 Dec 2025 10:34:31 -1000 Subject: [PATCH 06/64] Log results for later reference --- kpf/utils/ScheduleFilesCheck.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/kpf/utils/ScheduleFilesCheck.py b/kpf/utils/ScheduleFilesCheck.py index 9d68c564..f9ec59a3 100644 --- a/kpf/utils/ScheduleFilesCheck.py +++ b/kpf/utils/ScheduleFilesCheck.py @@ -80,12 +80,17 @@ def perform(cls, args): log.error(err) # Results - result_str = 'Band Line Count\n' + log.info(f'# KPF-CC Schedule File Check') + log.info(f'# {str(date_path)}') + result_str = 'Band Line Count' + log.info(f'# {result_str}') + result_str += '\n' for band in sorted(line_counts.keys()): + newline = f'{band:11s} {line_counts[band]:d}' if line_counts[band] <= 1: - result_str += f'{band:11s} {line_counts[band]:d} <-- Low target count!\n' - else: - result_str += f'{band:11s} {line_counts[band]:d}\n' + newline += ' <-- Low target count!' + log.info(f'# {newline}') + result_str += f"{newline}\n" print(result_str) # Send Email From 50e8ae460fdbd130a02dee260c9443a67f86f917 Mon Sep 17 00:00:00 2001 From: Josh Walawender Date: Thu, 11 Dec 2025 09:40:45 -1000 Subject: [PATCH 07/64] Documentation: add SoCal EKO troubleshooting --- docs/troubleshooting.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index c5db4d66..8ac2fce9 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -32,6 +32,7 @@ * SoCal * [Enclosure Lid Does not Move 1](#enclosure-lid-does-not-move-1) * [Enclosure Lid Does not Move 2](#enclosure-lid-does-not-move-2) + * [EKO Sun Tracker is not on Sun](#eko-sun-tracker-is-not-on-sun) # General Principles @@ -557,3 +558,20 @@ The `~/grep_for_dome_error` script will exclude many of the noisy, not useful li Solution: Reboot the controller (raspberry pi): `sudo reboot` and restart the kpfsocal3 dispatcher: `kpf restart kpfsocal3`. Multiple reboots may be required. + +## EKO Sun Tracker is not on Sun + +Symptom: + +The EKO solar tracker is not pointed at the sun, it may be parked. + +Problem: + +The tracker is not in the right mode. + +Solution: + +* Run `modify -s kpfsocal EKOCMD=2` which tells EKO to "guide" on the Sun. +* Run `modify -s kpfsocal EKOMODE=3` which sets the EKO in to "Sun-sensor with Learning" mode. + +If that fails, restart the EKO dispatcher: `kpf restart kpfsocal1` and run the commands again. From 2d6eb8c62dda6ade67f9bf5d9d53cee81da5187e Mon Sep 17 00:00:00 2001 From: Josh Walawender Date: Thu, 11 Dec 2025 09:59:29 -1000 Subject: [PATCH 08/64] Add verbose option to print lots of OB info to terminal --- kpf/observatoryAPIs/GetObservingBlocks.py | 16 ++++++++++++++- kpf/observatoryAPIs/__init__.py | 24 ++++++++++++++++++++++- 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/kpf/observatoryAPIs/GetObservingBlocks.py b/kpf/observatoryAPIs/GetObservingBlocks.py index 3fdc352c..cac129bb 100644 --- a/kpf/observatoryAPIs/GetObservingBlocks.py +++ b/kpf/observatoryAPIs/GetObservingBlocks.py @@ -18,8 +18,9 @@ def pre_condition(cls, args): @classmethod def perform(cls, args): + verbose = args.get('verbose', False) params = {'id': args.get('OBid', '')} - OBs, failure_messages = get_OBs_from_KPFCC_API(params) + OBs, failure_messages = get_OBs_from_KPFCC_API(params, verbose=verbose) if args.get('show_history', False): print(f'# Observing History for {OBs[0].summary()}') for i,h in enumerate(OBs[0].History): @@ -36,6 +37,16 @@ def perform(cls, args): if len(failure_messages) > 0: for msg in failure_messages: print(msg) + + # Print if verbose + if verbose: + print(f"# Failure Messages:") + print(failure_messages) + for i,OB in enumerate(OBs): + print(f"# Parsed OB {i+1}:") + print(OB.__repr__()) + + return OBs, failure_messages @classmethod @@ -49,6 +60,9 @@ def add_cmdline_args(cls, parser): parser.add_argument('--history', '--show_history', dest="show_history", default=False, action="store_true", help='Print history to screen?') + parser.add_argument('-v', '--verbose', dest="verbose", + default=False, action="store_true", + help='Show verbose OB information?') return super().add_cmdline_args(parser) diff --git a/kpf/observatoryAPIs/__init__.py b/kpf/observatoryAPIs/__init__.py index 4cca8d7c..dcebf1b1 100644 --- a/kpf/observatoryAPIs/__init__.py +++ b/kpf/observatoryAPIs/__init__.py @@ -137,7 +137,7 @@ def setKPFJunkValue(OBid, timestamp, junk=True): return query_observatoryAPI('proposal', 'setKPFJunkValue', params, post=True) -def get_OBs_from_KPFCC_API(params): +def get_OBs_from_KPFCC_API(params, verbose=False): result = query_observatoryAPI('proposal', 'getKPFObservingBlock', params) if result is None: return [] @@ -146,6 +146,28 @@ def get_OBs_from_KPFCC_API(params): n = len(result) for i,entry in enumerate(result): log.debug(f'Parsing entry {i+1} of {n}') + # Verbose Printing + if verbose: + print(f'# API Result {i+1} of {n}:') + OBentry = copy.deepcopy(entry) + target = OBentry.pop('Target', None) + if target is not None: + print(f'# Target:') + print(target) + observations = OBentry.pop('Observations', []) + for iobs,obs in enumerate(observations): + print(f'# Observation {iobs+1}:') + print(obs) + calibrations = OBentry.pop('Calibrations', []) + for ical,cal in enumerate(calibrations): + print(f'# Calibration {ical+1}:') + print(cal) + history = OBentry.pop('History', None) + for key in OBentry.keys(): + print(f"# {key}: {OBentry[key]}") + if history is not None: + print(f'# History:') + print(history) if not isinstance(entry, dict): failures.append([f'{entry}', 'Entry is not dict']) else: From ae1fd44cc6b22f4997128be9d5d191fa2cdc7e54 Mon Sep 17 00:00:00 2001 From: Josh Walawender Date: Mon, 15 Dec 2025 09:52:42 -1000 Subject: [PATCH 09/64] tweaks to read noise plots --- kpf/engineering/plot_ffts.py | 15 ++++++++--- kpf/engineering/plot_read_noise.py | 43 +++++++++++++++--------------- 2 files changed, 33 insertions(+), 25 deletions(-) diff --git a/kpf/engineering/plot_ffts.py b/kpf/engineering/plot_ffts.py index 7935248a..a62b106f 100755 --- a/kpf/engineering/plot_ffts.py +++ b/kpf/engineering/plot_ffts.py @@ -1,6 +1,7 @@ #!python3 ## Import General Tools +import sys from pathlib import Path import argparse import datetime @@ -97,6 +98,8 @@ def main(): fileinfo = {} for file in args.files: fnmatch = re.match('([GR])[a-zA-Z]*_(.*)_4188x4110_(\d+)\.(\w+)', Path(file).name) + if not fnmatch: + fnmatch = re.match('([GR])[a-zA-Z]*_(.*)_(\d+)\.(\w+)', Path(file).name) if fnmatch: timestamp = datetime.datetime.fromtimestamp(Path(file).stat().st_mtime) fileinfo[file] = {'det': fnmatch.group(1), @@ -117,8 +120,9 @@ def main(): txtfiles = [file for file in fileinfo.keys() if fileinfo[file]['ext'] == 'txt'] num_colors = len(txtfiles) - colors = ['r', 'g', 'b', 'c', 'm', 'y', 'k'] - +# colors = ['b', 'g', 'r', 'c', 'm', 'y', 'k'] + colors = {'R': ['r', 'm', 'y'], + 'G': ['b', 'c', 'g']} plt.figure(figsize=(12,8)) for i,file in enumerate(txtfiles): print(f'Getting FFT of {file}') @@ -127,11 +131,13 @@ def main(): allnoise = np.std(alldata[200:1153])#[14:146]) # exclude the spike from parallel read out xf, yfft = do_fft(xs[100000:]*1e-9, ys[100000:]) label = f"{fileinfo[file]['det']} {fileinfo[file]['frameno']}: {fileinfo[file]['label']}" + color = colors[fileinfo[file].get('det')].pop(0) + print(color) multiplier = 1#e3**i - plt.loglog(xf, yfft*multiplier, f'{colors[i]}-', alpha=0.25) +# plt.loglog(xf, yfft*multiplier, f'{colors[i]}-', alpha=0.25) # Bin data and plot mean in each bin means, bins, binnumber = scipy.stats.binned_statistic(xf, yfft, statistic='mean', bins=10000) - plt.loglog(bins[1:], means*multiplier, f'{colors[i]}-', alpha=0.75, + plt.loglog(bins[1:], means*multiplier, f'{color}-', alpha=0.75, drawstyle='steps-pre', label=label) if args.marker: freq = args.marker*1e3 @@ -142,6 +148,7 @@ def main(): plt.legend() plt.xlabel("Freq (Hz)") plt.xlim(1e4,5e7) + plt.ylim(3e2,3e5) plt.ylabel('Amplitude') plt.grid() plt.savefig(args.outfile, bbox_inches='tight', pad_inches=0.1) diff --git a/kpf/engineering/plot_read_noise.py b/kpf/engineering/plot_read_noise.py index 96e67b92..e43acd58 100644 --- a/kpf/engineering/plot_read_noise.py +++ b/kpf/engineering/plot_read_noise.py @@ -85,30 +85,31 @@ def main(): rotation='vertical') plt.ylabel('Read Noise (e-)') # plt.ylim(min(G_RN)*0.98, max(G_RN)*1.05) - plt.ylim(3.5, max(G_RN)+3) + plt.ylim(min(G_RN)-1, min([15, max(G_RN)+3])) plt.xticks(range(len(G_frameno)), G_frameno) plt.grid() - print('# Red Noise Measurements') - R_offset = 0.1 - plt.subplot(2,1,2) - R_RN = [fileinfo[file]['rn_e'] for file in redfiles] - R_RNos = [fileinfo[file]['rn_oscan'] for file in redfiles] - R_timestamp = [fileinfo[file]['timestamp'] for file in redfiles] - R_label = [fileinfo[file]['label'] for file in redfiles] - R_frameno = [fileinfo[file]['frameno'] for file in redfiles] - plt.plot(R_RN, 'ro') - plt.plot(R_RNos, 'ro', alpha=0.5) - for i,file in enumerate(redfiles): - print(f"{fileinfo[file]['timestr']} {file:50s}: {fileinfo[file]['rn_e']:5.2f} {fileinfo[file]['rn_oscan']:5.2f}") - plt.text(i, fileinfo[file]['rn_e']+R_offset, R_label[i], - rotation='vertical') - plt.ylabel('Read Noise (e-)') -# plt.ylim(min(R_RN)*0.98, max(R_RN)*1.05) - plt.ylim(1.5, max(R_RN)+1) - plt.xlabel('Frame Number') - plt.xticks(range(len(R_frameno)), R_frameno) - plt.grid() + if len(redfiles) > 0: + print('# Red Noise Measurements') + R_offset = 0.1 + plt.subplot(2,1,2) + R_RN = [fileinfo[file]['rn_e'] for file in redfiles] + R_RNos = [fileinfo[file]['rn_oscan'] for file in redfiles] + R_timestamp = [fileinfo[file]['timestamp'] for file in redfiles] + R_label = [fileinfo[file]['label'] for file in redfiles] + R_frameno = [fileinfo[file]['frameno'] for file in redfiles] + plt.plot(R_RN, 'ro') + plt.plot(R_RNos, 'ro', alpha=0.5) + for i,file in enumerate(redfiles): + print(f"{fileinfo[file]['timestr']} {file:50s}: {fileinfo[file]['rn_e']:5.2f} {fileinfo[file]['rn_oscan']:5.2f}") + plt.text(i, fileinfo[file]['rn_e']+R_offset, R_label[i], + rotation='vertical') + plt.ylabel('Read Noise (e-)') + # plt.ylim(min(R_RN)*0.98, max(R_RN)*1.05) + plt.ylim(min(R_RN)-0.5, min([9, max(R_RN)+1])) + plt.xlabel('Frame Number') + plt.xticks(range(len(R_frameno)), R_frameno) + plt.grid() plt.savefig('ReadNoise.png', bbox_inches='tight', pad_inches=0.1) plt.show() From 55e7a84e52b0da43c332a58a6b3f1426a1ce1cbf Mon Sep 17 00:00:00 2001 From: Josh Walawender Date: Wed, 17 Dec 2025 18:23:58 -1000 Subject: [PATCH 10/64] Update LFC defaults --- kpf/ObservingBlocks/exampleOBs/Calibrations.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kpf/ObservingBlocks/exampleOBs/Calibrations.yaml b/kpf/ObservingBlocks/exampleOBs/Calibrations.yaml index 75869515..68035334 100644 --- a/kpf/ObservingBlocks/exampleOBs/Calibrations.yaml +++ b/kpf/ObservingBlocks/exampleOBs/Calibrations.yaml @@ -34,12 +34,12 @@ Calibrations: - CalSource: 'LFCFiber' Object: 'LFC' nExp: 1 - ExpTime: 60 + ExpTime: 10 TriggerCaHK: False TriggerGreen: True TriggerRed: True IntensityMonitor: False - CalND1: 'OD 1.0' + CalND1: 'OD 1.3' CalND2: 'OD 0.1' OpenScienceShutter: True OpenSkyShutter: True From f97885e4c7fdcc3e8c8fbf6fbaed7d0db709d748 Mon Sep 17 00:00:00 2001 From: Josh Walawender Date: Fri, 19 Dec 2025 15:40:34 -1000 Subject: [PATCH 11/64] Critical bug fix for star list --- kpf/ObservingBlocks/Target.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/kpf/ObservingBlocks/Target.py b/kpf/ObservingBlocks/Target.py index 47e78053..53ece4d0 100644 --- a/kpf/ObservingBlocks/Target.py +++ b/kpf/ObservingBlocks/Target.py @@ -110,7 +110,11 @@ def to_star_list(self): except: rastr = str(self.RA).replace(':', ' ') decstr = str(self.Dec).replace(':', ' ') - out = f"{self.TargetName.value:15s} {rastr} {decstr}" + if len(self.TargetName.value) > 15: + short_name = self.TargetName.value[:15] + else: + short_name = self.TargetName.value + out = f"{short_name:15s} {rastr} {decstr}" if str(self.Equinox) == 'J2000': out += f" 2000" else: From f1dd3edfb0ad7dedb93d429b363d91860c0ecdb2 Mon Sep 17 00:00:00 2001 From: Josh Walawender Date: Tue, 23 Dec 2025 10:08:08 -1000 Subject: [PATCH 12/64] Update precision on ExpMeterThreshold for faint stars --- kpf/ObservingBlocks/ObservationProperties.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kpf/ObservingBlocks/ObservationProperties.yaml b/kpf/ObservingBlocks/ObservationProperties.yaml index 44284d0a..cda14240 100644 --- a/kpf/ObservingBlocks/ObservationProperties.yaml +++ b/kpf/ObservingBlocks/ObservationProperties.yaml @@ -48,7 +48,7 @@ comment: '[Mphotons/angstrom] Flux at the science detector at peak of order at which to terminate the exposure' valuetype: float defaultvalue: 50000 - precision: 0 + precision: 1 - name: 'TakeSimulCal' comment: 'Inject simultaneous calibration light on to the detector during exposure?' valuetype: bool From 7310e8b10285ff579b672d2418fc4a6b03687356 Mon Sep 17 00:00:00 2001 From: Josh Walawender Date: Tue, 23 Dec 2025 10:27:35 -1000 Subject: [PATCH 13/64] Update default and increase precision to 2 for ExpMeterThreshold --- kpf/ObservingBlocks/ObservationProperties.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kpf/ObservingBlocks/ObservationProperties.yaml b/kpf/ObservingBlocks/ObservationProperties.yaml index cda14240..54383a1f 100644 --- a/kpf/ObservingBlocks/ObservationProperties.yaml +++ b/kpf/ObservingBlocks/ObservationProperties.yaml @@ -47,8 +47,8 @@ - name: 'ExpMeterThreshold' comment: '[Mphotons/angstrom] Flux at the science detector at peak of order at which to terminate the exposure' valuetype: float - defaultvalue: 50000 - precision: 1 + defaultvalue: 1 + precision: 2 - name: 'TakeSimulCal' comment: 'Inject simultaneous calibration light on to the detector during exposure?' valuetype: bool From 0e4c6e7daa2474a69b6bdc689e76c29114fce596 Mon Sep 17 00:00:00 2001 From: Josh Walawender Date: Wed, 31 Dec 2025 14:03:08 -1000 Subject: [PATCH 14/64] Bugfix for unparsed cordinates --- kpf/OB_GUI/KPF_OB_GUI.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kpf/OB_GUI/KPF_OB_GUI.py b/kpf/OB_GUI/KPF_OB_GUI.py index 39af1a38..0c8b5a02 100755 --- a/kpf/OB_GUI/KPF_OB_GUI.py +++ b/kpf/OB_GUI/KPF_OB_GUI.py @@ -1233,8 +1233,8 @@ def set_SOB_Target(self, SOB): self.log.error(e) RA_str = SOB.Target.get('RA') Dec_str = SOB.Target.get('Dec') - self.SOB_TargetRALabel.setText('RA (Epoch=?):') - self.SOB_TargetDecLabel.setText('Dec (Epoch=?):') + RAlabel = 'RA (epoch=?):' + DecLabel = 'Dec (epoch=?):' # If proper motion values are set, try to propagate proper motions # if abs(SOB.Target.PMRA.value) > 0.001 or abs(SOB.Target.PMDEC.value) > 0.001: # try: From eeedfe3f5498bac3f74e0cd0f7ccddc57a7fc33a Mon Sep 17 00:00:00 2001 From: Josh Walawender Date: Wed, 31 Dec 2025 14:05:22 -1000 Subject: [PATCH 15/64] Handle getting epoch more robustly --- kpf/OB_GUI/KPF_OB_GUI.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/kpf/OB_GUI/KPF_OB_GUI.py b/kpf/OB_GUI/KPF_OB_GUI.py index 0c8b5a02..c2eabff1 100755 --- a/kpf/OB_GUI/KPF_OB_GUI.py +++ b/kpf/OB_GUI/KPF_OB_GUI.py @@ -1233,6 +1233,10 @@ def set_SOB_Target(self, SOB): self.log.error(e) RA_str = SOB.Target.get('RA') Dec_str = SOB.Target.get('Dec') + if SOB.Target.get('Epoch') is not None: + RAlabel = f"RA (epoch={SOB.Target.Epoch}):" + DecLabel = f"Dec (epoch={SOB.Target.Epoch}):" + else: RAlabel = 'RA (epoch=?):' DecLabel = 'Dec (epoch=?):' # If proper motion values are set, try to propagate proper motions From f796d5a151dc4692a1003f5c03d978841ab9c6d4 Mon Sep 17 00:00:00 2001 From: Josh Walawender Date: Wed, 31 Dec 2025 15:00:00 -1000 Subject: [PATCH 16/64] Better handling of messaging for OB retrival errors --- kpf/OB_GUI/KPF_OB_GUI.py | 17 ++++++++++++++++- kpf/OB_GUI/Popups.py | 2 -- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/kpf/OB_GUI/KPF_OB_GUI.py b/kpf/OB_GUI/KPF_OB_GUI.py index c2eabff1..43e6ff92 100755 --- a/kpf/OB_GUI/KPF_OB_GUI.py +++ b/kpf/OB_GUI/KPF_OB_GUI.py @@ -1126,6 +1126,11 @@ def load_OBs_from_schedule(self, nonCCnight=False): retrievedOBcount += 1 else: errs += failure_messages + try: + if type(errs[-1]) == list: + errs[-1] = errs[-1] + [entry['Target']] + except: + self.log.warning(f'Unable to append Target info to error: {entry["Target"]}') self.ProgressBar.setValue(int(scheduledOBcount/Nsched*100)) # Append a slewcal OB for convienience if self.OBcache['slewcal'] is not None: @@ -1138,7 +1143,17 @@ def load_OBs_from_schedule(self, nonCCnight=False): self.set_SortOrWeather() # Pop up for any errors if len(errs) > 0: - ConfirmationPopup('Errors retrieving OBs:', errs, info_only=True, warning=True).exec_() + if type(errs) == list: + msg = '' + for err in errs: + if type(err) == list: + msg += " ".join(err) + "\n" + else: + msg += f"{str(err)}\n" + else: + msg = str(errs) + self.log.warning('Errors when retrieving OBs:\n'+msg) + ConfirmationPopup('Errors retrieving OBs:', msg, info_only=True, warning=True).exec_() def refresh_history(self): self.log.debug(f"refresh_history") diff --git a/kpf/OB_GUI/Popups.py b/kpf/OB_GUI/Popups.py index 1735d0bd..fdc18420 100644 --- a/kpf/OB_GUI/Popups.py +++ b/kpf/OB_GUI/Popups.py @@ -22,8 +22,6 @@ def __init__(self, window_title, msg, info_only=False, warning=False, *args, **kwargs): QtWidgets.QMessageBox.__init__(self, *args, **kwargs) self.setWindowTitle(window_title) - if type(msg) == list: - msg = "\n".join(msg) self.setText(msg) if info_only == True: self.setIcon(QtWidgets.QMessageBox.Information) From 487bca043de02353365ac035e1f8ee72f424bf9e Mon Sep 17 00:00:00 2001 From: Josh Walawender Date: Wed, 31 Dec 2025 15:03:17 -1000 Subject: [PATCH 17/64] Reduce log file size between rotations --- kpf/OB_GUI/KPF_OB_GUI.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kpf/OB_GUI/KPF_OB_GUI.py b/kpf/OB_GUI/KPF_OB_GUI.py index 43e6ff92..3601a12f 100755 --- a/kpf/OB_GUI/KPF_OB_GUI.py +++ b/kpf/OB_GUI/KPF_OB_GUI.py @@ -82,7 +82,7 @@ def create_GUI_log(verbose=False): logdir.mkdir(mode=0o777, parents=True) LogFileName = logdir / 'OB_GUI_v2.log' LogFileHandler = RotatingFileHandler(LogFileName, - maxBytes=100*1024*1024, # 100 MB + maxBytes=20*1024*1024, # 20 MB backupCount=1000) # Keep old files LogFileHandler.setLevel(logging.DEBUG) LogFileHandler.setFormatter(LogFormat) From 9f2a986bbf47eccf4e7718073d3eaf858f7f2acd Mon Sep 17 00:00:00 2001 From: Josh Walawender Date: Thu, 1 Jan 2026 00:54:45 -1000 Subject: [PATCH 18/64] Bugfix --- kpf/OB_GUI/Popups.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kpf/OB_GUI/Popups.py b/kpf/OB_GUI/Popups.py index fdc18420..03622706 100644 --- a/kpf/OB_GUI/Popups.py +++ b/kpf/OB_GUI/Popups.py @@ -22,6 +22,8 @@ def __init__(self, window_title, msg, info_only=False, warning=False, *args, **kwargs): QtWidgets.QMessageBox.__init__(self, *args, **kwargs) self.setWindowTitle(window_title) + if type(msg) == list: + msg = ' '.join(msg) self.setText(msg) if info_only == True: self.setIcon(QtWidgets.QMessageBox.Information) From 48ed1e8ce72969dfb2080bb16b1335beecbf200b Mon Sep 17 00:00:00 2001 From: Josh Walawender Date: Fri, 2 Jan 2026 10:32:59 -1000 Subject: [PATCH 19/64] Fix date problem with the schedule files check --- kpf/utils/ScheduleFilesCheck.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/kpf/utils/ScheduleFilesCheck.py b/kpf/utils/ScheduleFilesCheck.py index f9ec59a3..bdc0d49e 100644 --- a/kpf/utils/ScheduleFilesCheck.py +++ b/kpf/utils/ScheduleFilesCheck.py @@ -4,8 +4,7 @@ from kpf import log, cfg from kpf.exceptions import * from kpf.KPFTranslatorFunction import KPFFunction, KPFScript -from kpf.observatoryAPIs import get_semester_dates -from kpf.observatoryAPIs.GetScheduledPrograms import GetScheduledPrograms +from kpf.observatoryAPIs import get_semester_dates, query_observatoryAPI from kpf.utils.SendEmail import SendEmail @@ -32,16 +31,27 @@ def pre_condition(cls, args): def perform(cls, args): errors = [] utnow = datetime.datetime.utcnow() + date_string = utnow.strftime('%Y-%m-%d') + log.info(f"# Checking for schedule for the night of {date_string} HST") semester, s_start, s_end = get_semester_dates(utnow) + log.info(f'# KPF-CC Schedule File Check') band_names = ['full-band1', 'full-band2', 'full-band3'] - classical, cadence = GetScheduledPrograms.execute({'semester': 'tonight'}) - if len(cadence) > 0: + + params = {'date': date_string, 'numdays': 1, 'telnr': 1, 'instrument': 'KPF'} + all_programs = query_observatoryAPI('schedule', 'getSchedule', params) + classical = [p for p in all_programs if p['Instrument'] == 'KPF'] + cadence = [p for p in all_programs if p['Instrument'] == 'KPF-CC'] + cadence_projects = [p['ProjCode'] for p in cadence] + + if len(cadence_projects) > 0: + log.info(f"# Found cadence programs: {cadence_projects}") band_names.extend(['band1', 'band2', 'band3']) + log.info(f"# Checking for {len(band_names)} schedules: {band_names}") base_path = Path('/s/sdata1701/Schedules') semester_path = base_path / semester - date_string = (utnow - datetime.timedelta(hours=24)).strftime('%Y-%m-%d') + date_path = semester_path / date_string if date_path.exists() is False: err = f"{str(date_path)} does not exist" @@ -80,7 +90,6 @@ def perform(cls, args): log.error(err) # Results - log.info(f'# KPF-CC Schedule File Check') log.info(f'# {str(date_path)}') result_str = 'Band Line Count' log.info(f'# {result_str}') @@ -91,7 +100,6 @@ def perform(cls, args): newline += ' <-- Low target count!' log.info(f'# {newline}') result_str += f"{newline}\n" - print(result_str) # Send Email if len(errors) > 0: From 994465a6587e5965a110f2f6c6fd2feeb780558e Mon Sep 17 00:00:00 2001 From: Josh Walawender Date: Sat, 3 Jan 2026 19:18:44 -1000 Subject: [PATCH 20/64] Fix to handle Foul Weather --- kpf/observatoryAPIs/GetTelescopeRelease.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kpf/observatoryAPIs/GetTelescopeRelease.py b/kpf/observatoryAPIs/GetTelescopeRelease.py index bb54056c..5b156f05 100644 --- a/kpf/observatoryAPIs/GetTelescopeRelease.py +++ b/kpf/observatoryAPIs/GetTelescopeRelease.py @@ -25,7 +25,7 @@ def perform(cls, args): params = {'telnr': args.get('telnr', 1)} result = query_observatoryAPI('schedule', 'getTelescopeReadyState', params) log.debug(f'getTelescopeReadyState returned {result}') - return result.get('State', '') == 'Ready' + return result.get('State', '') in ['Ready', 'Foul Weather'] @classmethod def post_condition(cls, args): From 7f1e0c5b00e7ab66dd696501717269bf2ef3e61a Mon Sep 17 00:00:00 2001 From: Josh Walawender Date: Tue, 6 Jan 2026 10:37:19 -1000 Subject: [PATCH 21/64] Fix table format --- kpf/OB_GUI/KPF_OB_GUI.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kpf/OB_GUI/KPF_OB_GUI.py b/kpf/OB_GUI/KPF_OB_GUI.py index 3601a12f..f103d82c 100755 --- a/kpf/OB_GUI/KPF_OB_GUI.py +++ b/kpf/OB_GUI/KPF_OB_GUI.py @@ -1422,7 +1422,7 @@ def prepare_execution_history_file(self): self.execution_history_file = logdir / f'KPFCC_executions_{semester}.csv' if self.execution_history_file.exists() is False: with open(self.execution_history_file, 'w') as f: - contents = ['# timestamp', 'decimalUT', 'executedID', 'OB summary', + contents = ['timestamp', 'decimalUT', 'executedID', 'OB summary', 'executed_line', 'scheduleUT', 'schedule_current_line', 'scheduleUT_current', 'schedule_next_line', 'scheduleUT_next', From b67bab8fce04e0db1d5c12ab5ec553a83436a44f Mon Sep 17 00:00:00 2001 From: Josh Walawender Date: Tue, 6 Jan 2026 10:37:39 -1000 Subject: [PATCH 22/64] Documentation: Update after filter replacement in SM4 --- docs/ObservationProperties.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ObservationProperties.md b/docs/ObservationProperties.md index dace8ffd..9d5a7b18 100644 --- a/docs/ObservationProperties.md +++ b/docs/ObservationProperties.md @@ -46,7 +46,7 @@ Cal Bench filter 1 position. Throughput = 10^-OD. Values: OD 0.1, OD 1.0, OD 1.3, OD 2.0, OD 3.0, OD 4.0 **CalND2**: `str` - Cal Bench filter 2 position. Throughput = 10^-OD. Values: OD 0.1, OD 0.3, OD 0.5, OD 0.8, OD 1.0, OD 4.0 + Cal Bench filter 2 position. Throughput = 10^-OD. Values: OD 0.1, OD 0.3, OD 0.5, OD 1.0, OD 1.3, OD 4.0 **NodN**: `float` [arcseconds] Distance to nod the telescope North before starting exposure. From b9a8b5c94aef456685933e6501a0e378aa02aaad Mon Sep 17 00:00:00 2001 From: Josh Walawender Date: Tue, 6 Jan 2026 10:41:01 -1000 Subject: [PATCH 23/64] Documentation: one more update to filters --- docs/ObservationProperties.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ObservationProperties.md b/docs/ObservationProperties.md index 9d5a7b18..1ff7d485 100644 --- a/docs/ObservationProperties.md +++ b/docs/ObservationProperties.md @@ -46,7 +46,7 @@ Cal Bench filter 1 position. Throughput = 10^-OD. Values: OD 0.1, OD 1.0, OD 1.3, OD 2.0, OD 3.0, OD 4.0 **CalND2**: `str` - Cal Bench filter 2 position. Throughput = 10^-OD. Values: OD 0.1, OD 0.3, OD 0.5, OD 1.0, OD 1.3, OD 4.0 + Cal Bench filter 2 position. Throughput = 10^-OD. Values: OD 0.1, OD 0.3, OD 0.5, OD 1.0, OD 1.3, OD 2.0 **NodN**: `float` [arcseconds] Distance to nod the telescope North before starting exposure. From 0ce4dd5cff19dc6de4c18001f153d854abf8166f Mon Sep 17 00:00:00 2001 From: Josh Walawender Date: Tue, 6 Jan 2026 10:56:11 -1000 Subject: [PATCH 24/64] Documentation: fix links --- docs/buildingOBs.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/buildingOBs.md b/docs/buildingOBs.md index 02a4e3db..803cad01 100644 --- a/docs/buildingOBs.md +++ b/docs/buildingOBs.md @@ -10,18 +10,18 @@ Observers can create OBs in 3 ways: The data in an OB can be divided in to three categories: -**Target**: The OB will contain information about the target beyond what is in a typical Keck Star List entry in order to flow that information to the FITS header and the data reduction pipeline (DRP). The target section is only needed if the OB has observations (i.e. it is not purely a calibration OB). Here is a description of all [Target Properties](../TargetProperties). +**Target**: The OB will contain information about the target beyond what is in a typical Keck Star List entry in order to flow that information to the FITS header and the data reduction pipeline (DRP). The target section is only needed if the OB has observations (i.e. it is not purely a calibration OB). Here is a description of all [Target Properties](TargetProperties.md). -**Calibrations**: An OB can contain calibrations, these are not typically used by the observer (slewcals are handled separately). Here is a description of all [Calibration Properties](../CalibrationProperties). The Calibrations section of an ON is a list of Calibration entries. +**Calibrations**: An OB can contain calibrations, these are not typically used by the observer (slewcals are handled separately). Here is a description of all [Calibration Properties](CalibrationProperties.md). The Calibrations section of an ON is a list of Calibration entries. -**Observations**: Finally, the OB will contain a list of observations to be made of the target. For typical KPF observers, this will only have one entry, but multiple entries are supported. Each entry describes a set of exposures on the target and contains the information on how those exposures should be executed. Here is a description of all [Observation Properties](../ObservationProperties). The Observations section of an ON is a list of Observation entries. +**Observations**: Finally, the OB will contain a list of observations to be made of the target. For typical KPF observers, this will only have one entry, but multiple entries are supported. Each entry describes a set of exposures on the target and contains the information on how those exposures should be executed. Here is a description of all [Observation Properties](ObservationProperties.md). The Observations section of an ON is a list of Observation entries. Note that not all properties are needed in every case. For example, an observation with `ExpMeterMode: 'monitor'` will not need values for `ExpMeterBin` and `ExpMeterThreshold`. ## Example On Sky Science OB -This is an example of what the text file form of an OB might look like. The file is a `yaml` format which resolves in to a python dict with keys for "Target", "Calibrations" and "Observations" (not all are required). The "Target" entry is a dict with the various [Target Properties](../TargetProperties). The "Calibrations" entry (if present) is a **list** of dictionaries, each with the various [Calibration Properties](../CalibrationProperties). Similarly, the "Observations" entry is a **list** of dictionaries, each with the various [Observation Properties](../ObservationProperties). +This is an example of what the text file form of an OB might look like. The file is a `yaml` format which resolves in to a python dict with keys for "Target", "Calibrations" and "Observations" (not all are required). The "Target" entry is a dict with the various [Target Properties](TargetProperties.md). The "Calibrations" entry (if present) is a **list** of dictionaries, each with the various [Calibration Properties](CalibrationProperties.md). Similarly, the "Observations" entry is a **list** of dictionaries, each with the various [Observation Properties](ObservationProperties.md). The example below has a Target, no Calibrations, and a single Observaton: From 3b4cf7fbb05a3eab899b77b6b2dad101c3493a53 Mon Sep 17 00:00:00 2001 From: Josh Walawender Date: Mon, 12 Jan 2026 17:56:38 -1000 Subject: [PATCH 25/64] Add AnalyzeKPFCCExecutions.py --- .../analysis/AnalyzeKPFCCExecutions.py | 154 ++++++++++++++++++ 1 file changed, 154 insertions(+) create mode 100644 kpf/engineering/analysis/AnalyzeKPFCCExecutions.py diff --git a/kpf/engineering/analysis/AnalyzeKPFCCExecutions.py b/kpf/engineering/analysis/AnalyzeKPFCCExecutions.py new file mode 100644 index 00000000..85fe840f --- /dev/null +++ b/kpf/engineering/analysis/AnalyzeKPFCCExecutions.py @@ -0,0 +1,154 @@ +import sys +from pathlib import Path +from datetime import datetime, timedelta +import re + +from astropy.table import Table, Column +import numpy as np + +from matplotlib import pyplot as plt +from matplotlib import ticker + + +semester = '2025B' +logdir = Path(f'/s/sdata1701/KPFTranslator_logs/') +execution_history_file = logdir / f'KPFCC_executions_{semester}.csv' +executions = Table.read(execution_history_file, format='ascii.csv') + +remove_rows = [] +timestamps = [] +line_deltas = [] +time_deltas = [] +UTnight_strings = [] +CNdelta = [] +for r,ex in enumerate(executions): + + # Remove Calibration or filler rows + if re.search('Calibration', ex['OB summary']) is not None: + remove_rows.append(r) +# elif int(ex['scheduleUT']) in [0, 24]: +# print(f"Removing {ex['OB summary']} because scheduled time is {ex['scheduleUT']}") +# remove_rows.append(r) + else: + # Add timestamps + timestamps.append(datetime.strptime(ex['timestamp'], '%Y-%m-%d %H:%M:%S UT')) + # Add line delta + line_deltas.append(ex['executed_line'] - ex['schedule_current_line'] - 0.5) + # Add time deltas + time_deltas.append(ex['decimalUT']-ex['scheduleUT']) + # Add night string + UTnight_strings.append(ex['timestamp'].split()[0]) + # Check current, next lines are 1 apart + CNdelta.append(ex['schedule_next_line'] - ex['schedule_current_line']) + +print(f"Removing {len(remove_rows)} entries") +executions.remove_rows(remove_rows) +executions.add_column(Column(timestamps, name='datetime')) +executions.add_column(Column(line_deltas, name='line_delta')) +executions.add_column(Column(time_deltas, name='time_delta')) +executions.add_column(Column(UTnight_strings, name='UTnight')) +executions.add_column(Column(CNdelta, name='CNdelta')) + +# print(executions.keys()) +# print(executions['executed_line', 'schedule_current_line', 'schedule_next_line', 'on_schedule'][:9]) +# print(executions['decimalUT', 'scheduleUT', 'scheduleUT_current', 'scheduleUT_next', 'on_schedule'][:9]) +# print(executions['datetime', 'line_delta', 'time_delta', 'UTnight', 'on_schedule'][:9]) + +on_schedule_groups = executions.group_by('on_schedule') +off_schedule = on_schedule_groups.groups[0] +off_schedule = off_schedule[off_schedule['scheduleUT'] < 23.99] +on_schedule = on_schedule_groups.groups[1] +unscheduled = executions[executions['scheduleUT'] > 23.99] +PctOffSched = len(off_schedule)/len(executions) +PctOnSched = len(on_schedule)/len(executions) +PctUnSched = len(unscheduled)/len(executions) +print(f"In {semester}, {len(executions)} KPF-CC science OBs have been successfully executed.") +print(f" {len(on_schedule):4d} scheduled OBs were executed according to the schedule ({PctOnSched:.1%})") +print(f" {len(off_schedule):4d} scheduled OBs were executed off the prescribed schedule ({PctOffSched:.1%})") +print(f" {len(unscheduled):4d} unscheduled OBs were executed ({PctUnSched:.1%})") + + +oddities = executions[(executions['CNdelta'] != 1) & (executions['schedule_next_line'] > -1)] +# print(oddities['UTnight', 'decimalUT', 'schedule_current_line', 'schedule_next_line']) +# print(np.median(oddities['CNdelta']), np.min(oddities['CNdelta']), np.max(oddities['CNdelta'])) + +# Time Delta Plot +plt.figure(figsize=(10,6)) +plt.title('Distribution of Time Offsets from Schedule') +plt.subplot(2,1,1) +timebins_on = np.arange(-3,+3,0.10) +plt.hist(on_schedule['time_delta'], bins=timebins_on, + color='g', alpha=0.4, label='On Schedule') +plt.axvline(0, color='k') +plt.legend(loc='best') +plt.ylabel('N Executions') + +plt.subplot(2,1,2) +timebins_off = np.arange(-3,+3,0.10) +plt.hist(off_schedule['time_delta'], bins=timebins_off, + color='r', alpha=0.4, label='Off Schedule') +plt.axvline(0, color='k') +plt.legend(loc='best') +plt.ylabel('N Executions') + +plt.xlabel('Time Delta (hours) [actual-scheduled]') +plt.savefig('KPF-CC_TimeOffsetDistribution.png', bbox_inches='tight', pad_inches=0.1) + + +# Line Delta Plot +# plt.figure(figsize=(10,6)) +# plt.title('Distribution of Schedule Line Offsets') +# plt.subplot(2,1,1) +# timebins_on = np.arange(-15.5,+14.5,1) +# plt.hist(on_schedule['line_delta'], bins=timebins_on, color='g', alpha=0.4) +# plt.axvline(0, color='k') +# plt.ylabel('N Executions') +# +# plt.subplot(2,1,2) +# timebins_off = np.arange(-15.5,+14.5,1) +# plt.hist(off_schedule['line_delta'], bins=timebins_off, color='r', alpha=0.4) +# plt.axvline(0, color='k') +# plt.ylabel('N Executions') +# +# plt.xlabel('Schedule Delta (lines) [actual-scheduled]') +# plt.savefig('KPF-CC_LineOffsetDistribution.png', bbox_inches='tight', pad_inches=0.1) +# plt.show() + + +# On Schedule Rate Over Semester +frac_on = [] +frac_off = [] +frac_un = [] +Ntot = [] +UTnights = [] +UTnight_groups = executions.group_by('UTnight') +for key, UTnight_group in zip(UTnight_groups.groups.keys, UTnight_groups.groups): + UTnights.append(key['UTnight']) + Ntot.append(len(UTnight_group)) + + N_on_schedule = len(UTnight_group[UTnight_group['on_schedule'] == 'True']) + N_off_schedule = len(UTnight_group[UTnight_group['on_schedule'] == 'False']) + N_unscheduled = len(UTnight_group[UTnight_group['scheduleUT'] > 23.99]) + N_off_schedule -= N_unscheduled + + frac_off.append(N_off_schedule/len(UTnight_group)) + frac_on.append(N_on_schedule/len(UTnight_group)) + frac_un.append(N_unscheduled/len(UTnight_group)) + +plt.figure(figsize=(10,6)) +plt.title('Fraction of On Schedule (green), Off Schedule (red), and Unscheduled (Magenta) OBs') +plt.bar(range(1,len(UTnights)+1,1), frac_on, color='g') +plt.bar(range(1,len(UTnights)+1,1), frac_off, color='r', bottom=frac_on) +plt.bar(range(1,len(UTnights)+1,1), frac_un, color='m', + bottom=np.array(frac_on)+np.array(frac_off)) + +for i,N in enumerate(Ntot): + plt.text(i+0.60, 1.03, f"N={N}", rotation=90) + +plt.ylim(0,1.15) +plt.xlim(0,len(UTnights)+1) +plt.gca().xaxis.set_major_locator(ticker.MultipleLocator(1.0)) +tick_labels = ['', ''] + [UTN[5:] for UTN in UTnights] +plt.gca().set_xticklabels(tick_labels, rotation=90) +plt.savefig('KPF-CC_OnScheduleRate.png', bbox_inches='tight', pad_inches=0.1) +# plt.show() \ No newline at end of file From c7a9fb13c744cdbc3d474cb1da58c9d769bc2d41 Mon Sep 17 00:00:00 2001 From: Josh Walawender Date: Mon, 12 Jan 2026 18:03:50 -1000 Subject: [PATCH 26/64] Updates to plots --- kpf/engineering/analysis/AnalyzeKPFCCExecutions.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/kpf/engineering/analysis/AnalyzeKPFCCExecutions.py b/kpf/engineering/analysis/AnalyzeKPFCCExecutions.py index 85fe840f..a7dce804 100644 --- a/kpf/engineering/analysis/AnalyzeKPFCCExecutions.py +++ b/kpf/engineering/analysis/AnalyzeKPFCCExecutions.py @@ -137,16 +137,22 @@ plt.figure(figsize=(10,6)) plt.title('Fraction of On Schedule (green), Off Schedule (red), and Unscheduled (Magenta) OBs') -plt.bar(range(1,len(UTnights)+1,1), frac_on, color='g') -plt.bar(range(1,len(UTnights)+1,1), frac_off, color='r', bottom=frac_on) -plt.bar(range(1,len(UTnights)+1,1), frac_un, color='m', +plt.bar(range(1,len(UTnights)+1,1), frac_on, color='g', alpha=0.6) +plt.bar(range(1,len(UTnights)+1,1), frac_off, color='r', alpha=0.6, + bottom=frac_on) +plt.bar(range(1,len(UTnights)+1,1), frac_un, color='m', alpha=0.6, bottom=np.array(frac_on)+np.array(frac_off)) +plt.axhline(PctOnSched, color='g', label=f'On {PctOnSched:.1%}') +plt.axhline(PctOffSched, color='r', label=f'Off {PctOffSched:.1%}') +plt.axhline(PctUnSched, color='m', label=f'Un {PctUnSched:.1%}') + for i,N in enumerate(Ntot): plt.text(i+0.60, 1.03, f"N={N}", rotation=90) plt.ylim(0,1.15) -plt.xlim(0,len(UTnights)+1) +plt.xlim(0,len(UTnights)+10) +plt.legend(loc='best') plt.gca().xaxis.set_major_locator(ticker.MultipleLocator(1.0)) tick_labels = ['', ''] + [UTN[5:] for UTN in UTnights] plt.gca().set_xticklabels(tick_labels, rotation=90) From 93fca5f992445322af976c731ebb60a98720d893 Mon Sep 17 00:00:00 2001 From: sherry yeh <54916067+syeh19@users.noreply.github.com> Date: Wed, 21 Jan 2026 10:37:01 -1000 Subject: [PATCH 27/64] Update troubleshooting.md modify ao keywords on k1aoserver-new when necessary --- docs/troubleshooting.md | 78 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 8ac2fce9..fadc3237 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -7,6 +7,8 @@ * Scripts * [Existing Script is Running](#existing-script-is-running) * [Agitator Use is Disabled](#agitator-use-is-disabled) + * [Start of Night Script Failed](#start-of-night-script-failed) + * [End of Night Script Failed](#end-of-night-script-failed) * Calibrations * [Calibration Source is Not Working](#calibration-source-is-not-working) * [SlewCal or Simultaneous Calibration Source is Wrong](#slewcal-or-simultaneous-calibration-source-is-wrong) @@ -110,6 +112,82 @@ This means that the `kpfconfig.USEAGITATOR` keyword is set to “No”. This ke The agitator can be reenabled by simply setting the keyword to “Yes”. **This should only be done by WMKO staff** and should only be done if the agitator is fully functional. A broken or misbehaving agitator mechanism presents a significant danger to the science fibers. +## Start of Night Script Failed + +Symptom: + +When executing the start of night script, the script failed to read and set AO keywords. + +Problem: + +There is some kind of AO gateway communicaiton problem, and so far we don't know what the root causee is. This happens intermittently. + +Solution: + +Modify AO keywords as k1obsao in a k1aoserver-new terminal. + +* Open AO hatch and check status + +``` +modify -s ao aohatchcmd=open +show -s ao aohatchsts +``` + +* Send PCU to KPF + +``` +modify -s ao pcuname=KPF +show -s ao pcuname +``` + +* Send AO rotator to 0 deg + +``` +modify -s ao obrt=0 +show -s ao obrt +``` + +* Set rotator to stationary + +``` +modify -s dcs rotmode=stationary +show -s dcs rotmode +``` + +## End of Night Script Failed + +Symptom: + +When executing the end of night script, the script failed to read and set AO keywords. + +Problem: + +There is some kind of AO gateway communicaiton problem, and so far we don't know what the root causee is. This happens intermittently. + +Solution: + +Modify AO keywords as k1obsao in a k1aoserver-new terminal. + +* Close AO hatch and check status + +``` +modify -s ao aohatchcmd=close +show -s ao aohatchsts +``` + +* Send PCU to home + +``` +modify -s ao pcuname=home +show -s ao pcuname +``` + +* Send AO rotator to 45 deg + +``` +modify -s ao obrt=45 +show -s ao obrt +``` # Calibrations From b53152baa69a220237e14d08b802e79e93e34743 Mon Sep 17 00:00:00 2001 From: Josh Walawender Date: Fri, 30 Jan 2026 15:07:07 -1000 Subject: [PATCH 28/64] Draft announcement for 26A --- docs/announcements/2026-02-01_26A-status.md | 23 +++++++++++++++++++++ mkdocs.yml | 7 ++++++- mkdocs_input.yml | 4 +++- 3 files changed, 32 insertions(+), 2 deletions(-) create mode 100644 docs/announcements/2026-02-01_26A-status.md diff --git a/docs/announcements/2026-02-01_26A-status.md b/docs/announcements/2026-02-01_26A-status.md new file mode 100644 index 00000000..464e26e5 --- /dev/null +++ b/docs/announcements/2026-02-01_26A-status.md @@ -0,0 +1,23 @@ +# Semester 26A Stability Announcement + +Precision radial velocity measurements which are comparable across long time scales (months to years) require a stable, well calibrated instrument. KPF has had several problems in the past which have impacted stability and calibrations and these will likely impact data taken in the 26A semester. + +We discuss a few details below, but the high level summary is that we currently do not have confidence in the instrument sub-systems to maintain temperature stability and provide reliable calibrations. As a result, users of KPF who have science goals which require long term stability of the instrument should take this into account when choosing which science programs to execute with their time in 26A. + +## Stability + +The most impactful stability factor for KPF has been the detector temperatures. In order for RV measurements at different times to be compared, the system must not change temperature significantly during that time span as the hysteresis induced by a change and return to temperature results in a zero point change for the RV measurements. The goal is for the system to be stable at the ~1 mK level, while changes of ~1 K are almost certain to break the RV zero point. + +Over its history, KPF has undergone four servicing missions, all of which attempted to address temperature stability problems in one way or another. KPF currently uses a pair of Closed Cycle Refrigerators (CCRs). The CCR for the Green detector has been problematic since it was installed despite multiple interventions. In contrast the Red side CCR has been more reliable, however we are seeing signs of degradation of stability on the red side as well. + +We currently have significant doubts about the current CCRs ability to maintain detector temperatures at their targets without occasional excursions of 1 K or more. Temperature excursions at that level would disrupt long term RV time series. + +The WMKO and KPF Build Teams are currently planning for an additional servicing mission to address the cooling problems, however that will take time to plan and implement. + +## Calibrations + +In addition to stability, the instrument also requires frequent, high precision calibrations. The most fundamental calibrator for KPF is the Laser Frequency Comb (LFC). The LFC has had periods of significant unreliability and the wavelength coverage has been inconsistent. This has resulted in long periods of poor calibrations which requires bootstrapping the wavelength solution from other calibrators or over longer time periods which impacts RV measurement precision. + +## Detector Noise + +In addition to the stability concerns, KPF has also faced challenges with detector noise. The elevated read noise present since early 2024 has been partially mitigated. Currently, the Red detector is performing at a nominal level (4.3 electrons read noise) and although the Green detector noise has been significantly reduced from the 25-35 electrons seen in the past, it remains elevated at about 10 electrons. An intervention to address this is being planned. diff --git a/mkdocs.yml b/mkdocs.yml index 0f9f8ed5..83558547 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -24,7 +24,9 @@ plugins: nav: - "KPF Home": index.md - - "Instrument Status": status.md + - "Instrument Status": + - "Instrument Status": status.md + - "26A Stability Announcement": announcements/2026-02-01_26A-status.md - "Instrument Info": - "Instrument Subsystems": - "Spectrograph": spectrograph.md @@ -107,6 +109,7 @@ nav: - "kpf.calbench": - "CalLampPower": scripts/CalLampPower.md - "IsCalSourceEnabled": scripts/IsCalSourceEnabled.md + - "ListLFCRuns": scripts/ListLFCRuns.md - "PredictNDFilters": scripts/PredictNDFilters.md - "SetCalSource": scripts/SetCalSource.md - "SetFlatFieldFiberPos": scripts/SetFlatFieldFiberPos.md @@ -198,6 +201,7 @@ nav: - "SelectTarget": scripts/SelectTarget.md - "SetTargetList": scripts/SetTargetList.md - "kpf.observatoryAPIs": + - "GetCurrentScheduledProgram": scripts/GetCurrentScheduledProgram.md - "GetExecutionHistory": scripts/GetExecutionHistory.md - "GetKPFCCObservingBlocks": scripts/GetKPFCCObservingBlocks.md - "GetObservingBlocks": scripts/GetObservingBlocks.md @@ -258,6 +262,7 @@ nav: - "BuildCalOB": scripts/BuildCalOB.md - "CheckAllowScheduledCals": scripts/CheckAllowScheduledCals.md - "EndOfNight": scripts/EndOfNight.md + - "ScheduleFilesCheck": scripts/ScheduleFilesCheck.md - "SetObserverFromSchedule": scripts/SetObserverFromSchedule.md - "SetOutdirs": scripts/SetOutdirs.md - "StartGUIs": scripts/StartGUIs.md diff --git a/mkdocs_input.yml b/mkdocs_input.yml index 102d094d..91f50fc8 100644 --- a/mkdocs_input.yml +++ b/mkdocs_input.yml @@ -24,7 +24,9 @@ plugins: nav: - "KPF Home": index.md - - "Instrument Status": status.md + - "Instrument Status": + - "Instrument Status": status.md + - "26A Stability Announcement": announcements/2026-02-01_26A-status.md - "Instrument Info": - "Instrument Subsystems": - "Spectrograph": spectrograph.md From 6ceb7bfdc19bc09d94570d505a0008b422db6294 Mon Sep 17 00:00:00 2001 From: Josh Walawender Date: Mon, 2 Feb 2026 11:00:20 -1000 Subject: [PATCH 29/64] Tweak menu ordering --- kpf/OB_GUI/KPF_OB_GUI.ui | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kpf/OB_GUI/KPF_OB_GUI.ui b/kpf/OB_GUI/KPF_OB_GUI.ui index c315729a..d3394ade 100644 --- a/kpf/OB_GUI/KPF_OB_GUI.ui +++ b/kpf/OB_GUI/KPF_OB_GUI.ui @@ -1746,11 +1746,11 @@ + - - + From a02d3901529133b6a562cea0a32ce892fae4b45f Mon Sep 17 00:00:00 2001 From: Josh Walawender Date: Mon, 2 Feb 2026 11:00:36 -1000 Subject: [PATCH 30/64] New day begins at 7am HST (for schedule loading) --- kpf/OB_GUI/KPF_OB_GUI.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kpf/OB_GUI/KPF_OB_GUI.py b/kpf/OB_GUI/KPF_OB_GUI.py index f103d82c..867447a0 100755 --- a/kpf/OB_GUI/KPF_OB_GUI.py +++ b/kpf/OB_GUI/KPF_OB_GUI.py @@ -1046,7 +1046,7 @@ def load_OBs_from_schedule(self, nonCCnight=False): else: semester, start, end = get_semester_dates(datetime.datetime.now()) utnow = datetime.datetime.utcnow() - date = utnow-datetime.timedelta(hours=20) # Switch dates at 10am HST, 2000UT + date = utnow-datetime.timedelta(hours=17) # Switch dates at 7am HST, 1700UT date_str = date.strftime('%Y-%m-%d').lower() if nonCCnight: schedule_files = [self.schedule_path / semester / date_str / f'full-{WB}' / 'output' / 'night_plan.csv' From fc1a3eb8af310980a2211b7b1727e300a3c9b81f Mon Sep 17 00:00:00 2001 From: Josh Walawender Date: Mon, 2 Feb 2026 11:04:10 -1000 Subject: [PATCH 31/64] Add separator --- kpf/OB_GUI/KPF_OB_GUI.ui | 1 + 1 file changed, 1 insertion(+) diff --git a/kpf/OB_GUI/KPF_OB_GUI.ui b/kpf/OB_GUI/KPF_OB_GUI.ui index d3394ade..15290069 100644 --- a/kpf/OB_GUI/KPF_OB_GUI.ui +++ b/kpf/OB_GUI/KPF_OB_GUI.ui @@ -1745,6 +1745,7 @@ OB List + From b630f0dd4b22dd050cec84a34cef1ca3f0e5c89c Mon Sep 17 00:00:00 2001 From: Josh Walawender Date: Mon, 2 Feb 2026 12:32:30 -1000 Subject: [PATCH 32/64] Update with new values after filter change --- kpf/ObservingBlocks/ObservationProperties.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kpf/ObservingBlocks/ObservationProperties.yaml b/kpf/ObservingBlocks/ObservationProperties.yaml index 54383a1f..b04d9bb2 100644 --- a/kpf/ObservingBlocks/ObservationProperties.yaml +++ b/kpf/ObservingBlocks/ObservationProperties.yaml @@ -62,7 +62,7 @@ valuetype: str defaultvalue: 'OD 0.1' - name: 'CalND2' - comment: 'Cal Bench filter 2 position. Throughput = 10^-OD. Values: OD 0.1, OD 0.3, OD 0.5, OD 0.8, OD 1.0, OD 4.0' + comment: 'Cal Bench filter 2 position. Throughput = 10^-OD. Values: OD 0.1, OD 0.3, OD 0.5, OD 1.0, OD 1.3, OD 2.0' valuetype: str defaultvalue: 'OD 0.1' - name: 'NodN' From d3894e1ad70fa117cb1c05eceb1e18f2e103eebd Mon Sep 17 00:00:00 2001 From: Josh Walawender Date: Mon, 2 Feb 2026 12:36:14 -1000 Subject: [PATCH 33/64] Docs: tweak announcement language --- docs/announcements/2026-02-01_26A-status.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/announcements/2026-02-01_26A-status.md b/docs/announcements/2026-02-01_26A-status.md index 464e26e5..0ac31806 100644 --- a/docs/announcements/2026-02-01_26A-status.md +++ b/docs/announcements/2026-02-01_26A-status.md @@ -4,7 +4,7 @@ Precision radial velocity measurements which are comparable across long time sca We discuss a few details below, but the high level summary is that we currently do not have confidence in the instrument sub-systems to maintain temperature stability and provide reliable calibrations. As a result, users of KPF who have science goals which require long term stability of the instrument should take this into account when choosing which science programs to execute with their time in 26A. -## Stability +## Temperature Stability The most impactful stability factor for KPF has been the detector temperatures. In order for RV measurements at different times to be compared, the system must not change temperature significantly during that time span as the hysteresis induced by a change and return to temperature results in a zero point change for the RV measurements. The goal is for the system to be stable at the ~1 mK level, while changes of ~1 K are almost certain to break the RV zero point. @@ -14,9 +14,9 @@ We currently have significant doubts about the current CCRs ability to maintain The WMKO and KPF Build Teams are currently planning for an additional servicing mission to address the cooling problems, however that will take time to plan and implement. -## Calibrations +## Calibration Reliability -In addition to stability, the instrument also requires frequent, high precision calibrations. The most fundamental calibrator for KPF is the Laser Frequency Comb (LFC). The LFC has had periods of significant unreliability and the wavelength coverage has been inconsistent. This has resulted in long periods of poor calibrations which requires bootstrapping the wavelength solution from other calibrators or over longer time periods which impacts RV measurement precision. +In addition to stability, the instrument also requires frequent, high precision calibrations. The most fundamental calibrator for KPF is the Laser Frequency Comb (LFC). The LFC has had periods of significant unreliability and the wavelength coverage for the LFC has been inconsistent. This has resulted in long periods of poor calibrations which requires bootstrapping the wavelength solution from other calibrators or over longer time periods which impacts RV measurement precision. ## Detector Noise From c4e18ed05821b70481a521b663f5eb11f0078eca Mon Sep 17 00:00:00 2001 From: Josh Walawender Date: Mon, 2 Feb 2026 12:39:47 -1000 Subject: [PATCH 34/64] Docs: formatting tweaks and removed PCU to home --- docs/troubleshooting.md | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index fadc3237..6fcea6c2 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -120,34 +120,34 @@ When executing the start of night script, the script failed to read and set AO k Problem: -There is some kind of AO gateway communicaiton problem, and so far we don't know what the root causee is. This happens intermittently. +There is some kind of AO gateway communicaiton problem, and so far we don't know what the root cause is. This happens intermittently. Solution: Modify AO keywords as k1obsao in a k1aoserver-new terminal. -* Open AO hatch and check status +Open AO hatch and check status: ``` modify -s ao aohatchcmd=open show -s ao aohatchsts ``` -* Send PCU to KPF +Send PCU to KPF: ``` modify -s ao pcuname=KPF show -s ao pcuname ``` -* Send AO rotator to 0 deg +Send AO rotator to 0 deg: ``` modify -s ao obrt=0 show -s ao obrt ``` -* Set rotator to stationary +Set rotator to stationary: ``` modify -s dcs rotmode=stationary @@ -168,21 +168,14 @@ There is some kind of AO gateway communicaiton problem, and so far we don't know Modify AO keywords as k1obsao in a k1aoserver-new terminal. -* Close AO hatch and check status +Close AO hatch and check status: ``` modify -s ao aohatchcmd=close show -s ao aohatchsts ``` -* Send PCU to home - -``` -modify -s ao pcuname=home -show -s ao pcuname -``` - -* Send AO rotator to 45 deg +Send AO rotator to 45 deg: ``` modify -s ao obrt=45 From 10c0f3610513a4c9242fe740a54ecd8cfc711936 Mon Sep 17 00:00:00 2001 From: Josh Walawender Date: Mon, 2 Feb 2026 12:42:52 -1000 Subject: [PATCH 35/64] =?UTF-8?q?Remove=20old=20=E2=80=9Cmini=20fill?= =?UTF-8?q?=E2=80=9D=20scripts?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kpf/engineering/TriggerGreenMiniFill.py | 57 ------------------------- kpf/engineering/TriggerRedMiniFill.py | 57 ------------------------- 2 files changed, 114 deletions(-) delete mode 100644 kpf/engineering/TriggerGreenMiniFill.py delete mode 100644 kpf/engineering/TriggerRedMiniFill.py diff --git a/kpf/engineering/TriggerGreenMiniFill.py b/kpf/engineering/TriggerGreenMiniFill.py deleted file mode 100644 index f7cc2503..00000000 --- a/kpf/engineering/TriggerGreenMiniFill.py +++ /dev/null @@ -1,57 +0,0 @@ -import time - -import ktl - -from kpf.KPFTranslatorFunction import KPFTranslatorFunction -from kpf import (log, KPFException, FailedPreCondition, FailedPostCondition, - FailedToReachDestination, check_input) -from kpf.utils.SendEmail import SendEmail - - -##------------------------------------------------------------------------- -## TriggerGreenMiniFill -##------------------------------------------------------------------------- -class TriggerGreenMiniFill(KPFTranslatorFunction): - '''I really hope this is not necessary in the long term. - ''' - @classmethod - def pre_condition(cls, args, logger, cfg): - kpffill = ktl.cache('kpffill') - if kpffill['GREENFILLIP'].read() == 'True': - raise FailedPreCondition('Green fill already in progress') - - @classmethod - def perform(cls, args, logger, cfg): - kpffill = ktl.cache('kpffill') - # Start fill - log.warning(f'Starting green mini fill') - kpffill['GREENSTART'].write(1) - # Wait - sleep_time = args.get('duration', 240) - log.debug(f'Sleeping {sleep_time:.0f} s') - time.sleep(sleep_time) - # Stop fill - if kpffill['GREENFILLIP'].read() == 'True': - log.warning(f'Stopping green mini fill') - kpffill['GREENSTOP'].write(1) - time.sleep(5) - else: - msg = 'Expected green mini fill to be in progress.' - SendEmail.execute({'Subject': 'TriggerGreenMiniFill Failed', - 'Message': f'{msg}'}) - raise KPFException(msg) - - @classmethod - def post_condition(cls, args, logger, cfg): - kpffill = ktl.cache('kpffill') - if kpffill['GREENFILLIP'].read() == 'True': - msg = 'Green still in progress, should be stopped!' - SendEmail.execute({'Subject': 'TriggerGreenMiniFill Failed', - 'Message': f'{msg}'}) - raise FailedPostCondition(msg) - - @classmethod - def add_cmdline_args(cls, parser, cfg=None): - parser.add_argument('duration', type=float, - help='The duration of the fill in seconds (240 recommended)') - return super().add_cmdline_args(parser, cfg) diff --git a/kpf/engineering/TriggerRedMiniFill.py b/kpf/engineering/TriggerRedMiniFill.py deleted file mode 100644 index 1d09a20e..00000000 --- a/kpf/engineering/TriggerRedMiniFill.py +++ /dev/null @@ -1,57 +0,0 @@ -import time - -import ktl - -from kpf.KPFTranslatorFunction import KPFTranslatorFunction -from kpf import (log, KPFException, FailedPreCondition, FailedPostCondition, - FailedToReachDestination, check_input) -from kpf.utils.SendEmail import SendEmail - - -##------------------------------------------------------------------------- -## TriggerRedMiniFill -##------------------------------------------------------------------------- -class TriggerRedMiniFill(KPFTranslatorFunction): - '''I really hope this is not necessary in the long term. - ''' - @classmethod - def pre_condition(cls, args, logger, cfg): - kpffill = ktl.cache('kpffill') - if kpffill['REDFILLIP'].read() == 'True': - raise FailedPreCondition('Red fill already in progress') - - @classmethod - def perform(cls, args, logger, cfg): - kpffill = ktl.cache('kpffill') - # Start fill - log.warning(f'Starting Red mini fill') - kpffill['REDSTART'].write(1) - # Wait - sleep_time = args.get('duration', 240) - log.debug(f'Sleeping {sleep_time:.0f} s') - time.sleep(sleep_time) - # Stop fill - if kpffill['REDFILLIP'].read() == 'True': - log.warning(f'Stopping Red mini fill') - kpffill['REDSTOP'].write(1) - time.sleep(5) - else: - msg = 'Expected Red mini fill to be in progress.' - SendEmail.execute({'Subject': 'TriggerRedMiniFill Failed', - 'Message': f'{msg}'}) - raise KPFException(msg) - - @classmethod - def post_condition(cls, args, logger, cfg): - kpffill = ktl.cache('kpffill') - if kpffill['RedFILLIP'].read() == 'True': - msg = 'Red still in progress, should be stopped!' - SendEmail.execute({'Subject': 'TriggerRedMiniFill Failed', - 'Message': f'{msg}'}) - raise FailedPostCondition(msg) - - @classmethod - def add_cmdline_args(cls, parser, cfg=None): - parser.add_argument('duration', type=float, - help='The duration of the fill in seconds (240 recommended)') - return super().add_cmdline_args(parser, cfg) From da7eb111368578a5044f8c3a090fb499aa99ebe4 Mon Sep 17 00:00:00 2001 From: Josh Walawender Date: Mon, 2 Feb 2026 12:48:25 -1000 Subject: [PATCH 36/64] Put email accounts in config file --- kpf/kpf_inst_config.ini | 5 +++++ kpf/utils/ScheduleFilesCheck.py | 3 ++- kpf/utils/SendEmail.py | 3 ++- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/kpf/kpf_inst_config.ini b/kpf/kpf_inst_config.ini index efab67cf..aec5510b 100644 --- a/kpf/kpf_inst_config.ini +++ b/kpf/kpf_inst_config.ini @@ -1,3 +1,8 @@ +[operations] +lead_sa = jwalawender +deputy_sa = syeh +info_email = kpf_info + [telescope] telnr = 1 max_offset = 900 diff --git a/kpf/utils/ScheduleFilesCheck.py b/kpf/utils/ScheduleFilesCheck.py index bdc0d49e..4e39f7b2 100644 --- a/kpf/utils/ScheduleFilesCheck.py +++ b/kpf/utils/ScheduleFilesCheck.py @@ -106,9 +106,10 @@ def perform(cls, args): msg = 'KPF-CC Schedule May Be Bad\n\n' if args.get('email', False) == True: try: + to_value = cfg.get('operations', 'lead_sa', 'jwalawender') SendEmail.execute({'Subject': f'KPF-CC Schedule May Be Bad', 'Message': msg+result_str, - 'To': 'jwalawender@keck.hawaii.edu'}) + 'To': f'{to_value}@keck.hawaii.edu'}) except Exception as email_err: log.error(f'Sending email failed') log.error(email_err) diff --git a/kpf/utils/SendEmail.py b/kpf/utils/SendEmail.py index 4d654da7..1be0194d 100644 --- a/kpf/utils/SendEmail.py +++ b/kpf/utils/SendEmail.py @@ -24,9 +24,10 @@ def pre_condition(cls, args): @classmethod def perform(cls, args): + default_to = cfg.get('operations', 'info_email', 'kpf_info') msg = MIMEText(args.get('Message', 'Test email. Please ignore.')) msg['To'] = args.get('To', 'kpf_info@keck.hawaii.edu') - msg['From'] = args.get('From', 'kpf_info@keck.hawaii.edu') + msg['From'] = args.get('From', f'{default_to}@keck.hawaii.edu') msg['Subject'] = args.get('Subject', 'KPF Alert') log.warning(f"Sending email, To {msg.get('To')}") log.warning(f"Sending email, Subject {msg.get('Subject')}") From 44b54bcfffa990eb310e4ce21862f6bde2b0441f Mon Sep 17 00:00:00 2001 From: Josh Walawender Date: Wed, 4 Feb 2026 09:15:29 -1000 Subject: [PATCH 37/64] Add paragraph on faint star extractions --- docs/announcements/2026-02-01_26A-status.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/announcements/2026-02-01_26A-status.md b/docs/announcements/2026-02-01_26A-status.md index 0ac31806..0654fb18 100644 --- a/docs/announcements/2026-02-01_26A-status.md +++ b/docs/announcements/2026-02-01_26A-status.md @@ -4,6 +4,8 @@ Precision radial velocity measurements which are comparable across long time sca We discuss a few details below, but the high level summary is that we currently do not have confidence in the instrument sub-systems to maintain temperature stability and provide reliable calibrations. As a result, users of KPF who have science goals which require long term stability of the instrument should take this into account when choosing which science programs to execute with their time in 26A. +In addition, the KPF DRP is not performing well on faint object extractions. Somewhere around Gmag of 15-16 is where the performance degrades significantly, so we advise users to avoid faint stars if they need immediate results. We do expect the DRP to handle this case properly in the long term, so data taken now will likely be useful in the future, but the results coming out at the moment will significantly underperform the ETC. + ## Temperature Stability The most impactful stability factor for KPF has been the detector temperatures. In order for RV measurements at different times to be compared, the system must not change temperature significantly during that time span as the hysteresis induced by a change and return to temperature results in a zero point change for the RV measurements. The goal is for the system to be stable at the ~1 mK level, while changes of ~1 K are almost certain to break the RV zero point. From ce411c0076d17e90f6ef320452f9e89bdfc6c83c Mon Sep 17 00:00:00 2001 From: Josh Walawender Date: Wed, 4 Feb 2026 13:37:59 -1000 Subject: [PATCH 38/64] Draft new failover startegy for AO interaction --- kpf/ao/SetupAOforKPF.py | 53 ++++++++++++++++++++++------------- kpf/ao/ShutdownAOforKPF.py | 57 ++++++++++++++++++++++++++++++++++++++ kpf/linking_table.yml | 2 ++ kpf/utils/EndOfNight.py | 21 ++------------ 4 files changed, 94 insertions(+), 39 deletions(-) create mode 100644 kpf/ao/ShutdownAOforKPF.py diff --git a/kpf/ao/SetupAOforKPF.py b/kpf/ao/SetupAOforKPF.py index e004e514..37458901 100644 --- a/kpf/ao/SetupAOforKPF.py +++ b/kpf/ao/SetupAOforKPF.py @@ -1,4 +1,5 @@ import ktl +import ktl.Exceptions from kpf import log, cfg from kpf.exceptions import * @@ -43,27 +44,39 @@ def pre_condition(cls, args): @classmethod def perform(cls, args): - log.info('Set AO rotator to Manual') - SetAORotatorManual.execute({}) - log.info('Set AO rotator to 0 deg') - SetAORotator.execute({'dest': 0}) - - log.info('Turn off HEPA') - TurnHepaOff.execute({}) - - log.info('Set AO in DCS sim mode') - SetAODCStoSIM.execute({}) - - log.info('Turn K1 AO light source off') - TurnLightSourceOff.execute({}) - - PCSstagekw = ktl.cache('ao', 'PCSFNAME') - if PCSstagekw.read() != 'kpf': - log.info('Move PCU to Home') - SendPCUtoHome.execute({}) - log.info('Move PCU to KPF') - SendPCUtoKPF.execute({}) + try: + assert 2 == 3 + log.info('Set AO rotator to Manual') + SetAORotatorManual.execute({}) + + log.info('Set AO rotator to 0 deg') + SetAORotator.execute({'dest': 0}) + + log.info('Turn off HEPA') + TurnHepaOff.execute({}) + + log.info('Set AO in DCS sim mode') + SetAODCStoSIM.execute({}) + + log.info('Turn K1 AO light source off') + TurnLightSourceOff.execute({}) + + PCSstagekw = ktl.cache('ao', 'PCSFNAME') + if PCSstagekw.read() != 'kpf': + log.info('Move PCU to Home') + SendPCUtoHome.execute({}) + log.info('Move PCU to KPF') + SendPCUtoKPF.execute({}) + except Exception as e: + log.warning('SetupAOforKPF failed.') + log.warning(e) + log.warning(f'SSHing to k1obsao@k1aoserver-new to run kpfStart.csh') + ssh_cmd = 'ssh k1obsao@k1aoserver-new kpfStart.csh' + cmd = ['xterm', '-title', 'SetupAOforKPF', '-name', 'SetupAOforKPF', + '-fn', '10x20', '-bg', 'black', '-fg', 'white', + '-e', f'{ssh_cmd}'] + proc = subprocess.Popen(cmd) @classmethod def post_condition(cls, args): diff --git a/kpf/ao/ShutdownAOforKPF.py b/kpf/ao/ShutdownAOforKPF.py new file mode 100644 index 00000000..874303f3 --- /dev/null +++ b/kpf/ao/ShutdownAOforKPF.py @@ -0,0 +1,57 @@ +import ktl +import ktl.Exceptions + +from kpf import log, cfg +from kpf.exceptions import * +from kpf.KPFTranslatorFunction import KPFFunction, KPFScript +from kpf.ao.ControlAOHatch import ControlAOHatch +from kpf.ao.TurnHepaOn import TurnHepaOn + + +class ShutdownAOforKPF(KPFFunction): + ''' + + KTL Keywords Used: + + - `` + + Functions Called: + + - `` + ''' + @classmethod + def pre_condition(cls, args): + pass + + @classmethod + def perform(cls, args): + log.info('Closing AO Hatch') + try: + assert 2==3 + ControlAOHatch.execute({'destination': 'closed'}) + except Exception as e: + log.warning(f"Failure controlling AO hatch") + log.warning(e) + log.warning(f'SSHing to k1obsao@k1aoserver-new to run modify -s ao aohatchcmd=1') + ssh_cmd = "ssh k1obsao@k1aoserver-new 'modify -s ao aohatchcmd=1'" + cmd = ['xterm', '-title', 'CloseAOHatch', '-name', 'CloseAOHatch', + '-fn', '10x20', '-bg', 'black', '-fg', 'white', + '-e', f'{ssh_cmd}'] + proc = subprocess.Popen(cmd) + log.info('Turning on AO HEPA Filter System') + try: + assert 2==3 + TurnHepaOn.execute({}) + except Exception as e: + log.warning(f"Failure controlling AO HEPA Filter System") + log.warning(e) + log.warning(f'SSHing to k1obsao@k1aoserver-new to run modify -s ao obhpaon=1') + ssh_cmd = "ssh k1obsao@k1aoserver-new 'modify -s ao obhpaon=1'" + cmd = ['xterm', '-title', 'TurnHEPAOn', '-name', 'TurnHEPAOn', + '-fn', '10x20', '-bg', 'black', '-fg', 'white', + '-e', f'{ssh_cmd}'] + proc = subprocess.Popen(cmd) + + @classmethod + def post_condition(cls, args): + pass diff --git a/kpf/linking_table.yml b/kpf/linking_table.yml index 76c78679..82bbf3ed 100644 --- a/kpf/linking_table.yml +++ b/kpf/linking_table.yml @@ -24,6 +24,8 @@ links: cmd: ao.SetupAOforACAM.SetupAOforACAM SetupAOforKPF: cmd: ao.SetupAOforKPF.SetupAOforKPF + ShutdownAOforKPF: + cmd: ao.ShutdownAOforKPF.ShutdownAOforKPF TurnHepaOff: cmd: ao.TurnHepaOff.TurnHepaOff TurnHepaOn: diff --git a/kpf/utils/EndOfNight.py b/kpf/utils/EndOfNight.py index e7e90fd7..82eb8ee6 100644 --- a/kpf/utils/EndOfNight.py +++ b/kpf/utils/EndOfNight.py @@ -8,9 +8,7 @@ from kpf.KPFTranslatorFunction import KPFFunction, KPFScript from kpf.scripts import (register_script, obey_scriptrun, check_scriptstop, add_script_log) -from kpf.ao.ControlAOHatch import ControlAOHatch -from kpf.ao.TurnHepaOn import TurnHepaOn -from kpf.ao.SendPCUtoHome import SendPCUtoHome +from kpf.ao.ShutdownAOforKPF import ShutdownAOforKPF from kpf.fiu.ShutdownTipTilt import ShutdownTipTilt from kpf.fiu.ConfigureFIU import ConfigureFIU from kpf.fiu.WaitForConfigureFIU import WaitForConfigureFIU @@ -159,22 +157,7 @@ def perform(cls, args): user_input = input() if user_input.lower() in ['y', 'yes', '']: log.debug('User chose to shut down AO') - log.info('Closing AO Hatch') - try: - ControlAOHatch.execute({'destination': 'closed'}) - except FailedToReachDestination: - log.error(f"AO hatch did not move successfully") - except Exception as e: - log.error(f"Failure controlling AO hatch") - log.error(e) - log.info('Sending PCU stage to Home position') - try: - SendPCUtoHome.execute({}) - except Exception as e: - log.error(f"Failure sending PCU to home") - log.error(e) - # log.info('Turning on AO HEPA Filter System') - # TurnHepaOn.execute({}) + ShutdownAOforKPF.execute({}) else: log.warning(f'User chose to skip AO shutdown') From fcf47c35216cd7fcb3f65f3480fc51d4ebf38543 Mon Sep 17 00:00:00 2001 From: Josh Walawender Date: Wed, 4 Feb 2026 13:40:35 -1000 Subject: [PATCH 39/64] =?UTF-8?q?Don=E2=80=99t=20send=20PCU=20to=20home=20?= =?UTF-8?q?first?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kpf/ao/SetupAOforKPF.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/kpf/ao/SetupAOforKPF.py b/kpf/ao/SetupAOforKPF.py index 37458901..42924efb 100644 --- a/kpf/ao/SetupAOforKPF.py +++ b/kpf/ao/SetupAOforKPF.py @@ -64,8 +64,6 @@ def perform(cls, args): PCSstagekw = ktl.cache('ao', 'PCSFNAME') if PCSstagekw.read() != 'kpf': - log.info('Move PCU to Home') - SendPCUtoHome.execute({}) log.info('Move PCU to KPF') SendPCUtoKPF.execute({}) except Exception as e: From 404073d855474d52712a9d38798e4107873f731c Mon Sep 17 00:00:00 2001 From: Josh Walawender Date: Wed, 4 Feb 2026 13:41:52 -1000 Subject: [PATCH 40/64] fix import --- kpf/ao/SetupAOforKPF.py | 2 ++ kpf/ao/ShutdownAOforKPF.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/kpf/ao/SetupAOforKPF.py b/kpf/ao/SetupAOforKPF.py index 42924efb..d576ef28 100644 --- a/kpf/ao/SetupAOforKPF.py +++ b/kpf/ao/SetupAOforKPF.py @@ -1,3 +1,5 @@ +import subprocess + import ktl import ktl.Exceptions diff --git a/kpf/ao/ShutdownAOforKPF.py b/kpf/ao/ShutdownAOforKPF.py index 874303f3..8f6e3741 100644 --- a/kpf/ao/ShutdownAOforKPF.py +++ b/kpf/ao/ShutdownAOforKPF.py @@ -1,3 +1,5 @@ +import subprocess + import ktl import ktl.Exceptions From 5895486be180fab089bbbf84722b6f7a1874fc95 Mon Sep 17 00:00:00 2001 From: Josh Walawender Date: Wed, 4 Feb 2026 13:42:35 -1000 Subject: [PATCH 41/64] add -X --- kpf/ao/SetupAOforKPF.py | 2 +- kpf/ao/ShutdownAOforKPF.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/kpf/ao/SetupAOforKPF.py b/kpf/ao/SetupAOforKPF.py index d576ef28..4e9f4f23 100644 --- a/kpf/ao/SetupAOforKPF.py +++ b/kpf/ao/SetupAOforKPF.py @@ -72,7 +72,7 @@ def perform(cls, args): log.warning('SetupAOforKPF failed.') log.warning(e) log.warning(f'SSHing to k1obsao@k1aoserver-new to run kpfStart.csh') - ssh_cmd = 'ssh k1obsao@k1aoserver-new kpfStart.csh' + ssh_cmd = 'ssh -X k1obsao@k1aoserver-new kpfStart.csh' cmd = ['xterm', '-title', 'SetupAOforKPF', '-name', 'SetupAOforKPF', '-fn', '10x20', '-bg', 'black', '-fg', 'white', '-e', f'{ssh_cmd}'] diff --git a/kpf/ao/ShutdownAOforKPF.py b/kpf/ao/ShutdownAOforKPF.py index 8f6e3741..73056469 100644 --- a/kpf/ao/ShutdownAOforKPF.py +++ b/kpf/ao/ShutdownAOforKPF.py @@ -48,7 +48,7 @@ def perform(cls, args): log.warning(f"Failure controlling AO HEPA Filter System") log.warning(e) log.warning(f'SSHing to k1obsao@k1aoserver-new to run modify -s ao obhpaon=1') - ssh_cmd = "ssh k1obsao@k1aoserver-new 'modify -s ao obhpaon=1'" + ssh_cmd = "ssh -X k1obsao@k1aoserver-new 'modify -s ao obhpaon=1'" cmd = ['xterm', '-title', 'TurnHEPAOn', '-name', 'TurnHEPAOn', '-fn', '10x20', '-bg', 'black', '-fg', 'white', '-e', f'{ssh_cmd}'] From e1882f8aaf005310472628359ece90ee6e7b0dd0 Mon Sep 17 00:00:00 2001 From: Josh Walawender Date: Wed, 4 Feb 2026 13:44:28 -1000 Subject: [PATCH 42/64] Add Done and sleep --- kpf/ao/ShutdownAOforKPF.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kpf/ao/ShutdownAOforKPF.py b/kpf/ao/ShutdownAOforKPF.py index 73056469..9a275a8a 100644 --- a/kpf/ao/ShutdownAOforKPF.py +++ b/kpf/ao/ShutdownAOforKPF.py @@ -38,7 +38,7 @@ def perform(cls, args): ssh_cmd = "ssh k1obsao@k1aoserver-new 'modify -s ao aohatchcmd=1'" cmd = ['xterm', '-title', 'CloseAOHatch', '-name', 'CloseAOHatch', '-fn', '10x20', '-bg', 'black', '-fg', 'white', - '-e', f'{ssh_cmd}'] + '-e', f'{ssh_cmd} ; echo Done ; sleep 60'] proc = subprocess.Popen(cmd) log.info('Turning on AO HEPA Filter System') try: @@ -51,7 +51,7 @@ def perform(cls, args): ssh_cmd = "ssh -X k1obsao@k1aoserver-new 'modify -s ao obhpaon=1'" cmd = ['xterm', '-title', 'TurnHEPAOn', '-name', 'TurnHEPAOn', '-fn', '10x20', '-bg', 'black', '-fg', 'white', - '-e', f'{ssh_cmd}'] + '-e', f'{ssh_cmd} ; echo Done ; sleep 60'] proc = subprocess.Popen(cmd) @classmethod From a715dcfd97042d3a7ce85ac2954e30a97726cee2 Mon Sep 17 00:00:00 2001 From: Josh Walawender Date: Wed, 4 Feb 2026 13:55:26 -1000 Subject: [PATCH 43/64] =?UTF-8?q?Get=20the=20=E2=80=9CDone=E2=80=9D=20and?= =?UTF-8?q?=20sleep=20in.=20Remove=20asserts=20to=20force=20failover?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kpf/ao/SetupAOforKPF.py | 6 ++++-- kpf/ao/ShutdownAOforKPF.py | 16 ++++++++++------ 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/kpf/ao/SetupAOforKPF.py b/kpf/ao/SetupAOforKPF.py index 4e9f4f23..e6f6e48d 100644 --- a/kpf/ao/SetupAOforKPF.py +++ b/kpf/ao/SetupAOforKPF.py @@ -48,7 +48,6 @@ def pre_condition(cls, args): def perform(cls, args): try: - assert 2 == 3 log.info('Set AO rotator to Manual') SetAORotatorManual.execute({}) @@ -72,7 +71,10 @@ def perform(cls, args): log.warning('SetupAOforKPF failed.') log.warning(e) log.warning(f'SSHing to k1obsao@k1aoserver-new to run kpfStart.csh') - ssh_cmd = 'ssh -X k1obsao@k1aoserver-new kpfStart.csh' + ssh_cmds = ['ssh -X k1obsao@k1aoserver-new kpfStart.csh', + f'echo "Done!"', + f'sleep 30'] + ssh_cmd = ' ; '.join(ssh_cmds) cmd = ['xterm', '-title', 'SetupAOforKPF', '-name', 'SetupAOforKPF', '-fn', '10x20', '-bg', 'black', '-fg', 'white', '-e', f'{ssh_cmd}'] diff --git a/kpf/ao/ShutdownAOforKPF.py b/kpf/ao/ShutdownAOforKPF.py index 9a275a8a..e00d8fed 100644 --- a/kpf/ao/ShutdownAOforKPF.py +++ b/kpf/ao/ShutdownAOforKPF.py @@ -29,29 +29,33 @@ def pre_condition(cls, args): def perform(cls, args): log.info('Closing AO Hatch') try: - assert 2==3 ControlAOHatch.execute({'destination': 'closed'}) except Exception as e: log.warning(f"Failure controlling AO hatch") log.warning(e) log.warning(f'SSHing to k1obsao@k1aoserver-new to run modify -s ao aohatchcmd=1') - ssh_cmd = "ssh k1obsao@k1aoserver-new 'modify -s ao aohatchcmd=1'" + ssh_cmds = ["ssh k1obsao@k1aoserver-new 'modify -s ao aohatchcmd=1'", + f'echo "Done!"', + f'sleep 30'] + ssh_cmd = ' ; '.join(ssh_cmds) cmd = ['xterm', '-title', 'CloseAOHatch', '-name', 'CloseAOHatch', '-fn', '10x20', '-bg', 'black', '-fg', 'white', - '-e', f'{ssh_cmd} ; echo Done ; sleep 60'] + '-e', f'{ssh_cmd}'] proc = subprocess.Popen(cmd) log.info('Turning on AO HEPA Filter System') try: - assert 2==3 TurnHepaOn.execute({}) except Exception as e: log.warning(f"Failure controlling AO HEPA Filter System") log.warning(e) log.warning(f'SSHing to k1obsao@k1aoserver-new to run modify -s ao obhpaon=1') - ssh_cmd = "ssh -X k1obsao@k1aoserver-new 'modify -s ao obhpaon=1'" + ssh_cmds = ["ssh k1obsao@k1aoserver-new 'modify -s ao obhpaon=1'", + f'echo "Done!"', + f'sleep 30'] + ssh_cmd = ' ; '.join(ssh_cmds) cmd = ['xterm', '-title', 'TurnHEPAOn', '-name', 'TurnHEPAOn', '-fn', '10x20', '-bg', 'black', '-fg', 'white', - '-e', f'{ssh_cmd} ; echo Done ; sleep 60'] + '-e', f'{ssh_cmd}'] proc = subprocess.Popen(cmd) @classmethod From 36520583bcaeed40347b943e31a2676ec6cca506 Mon Sep 17 00:00:00 2001 From: Josh Walawender Date: Wed, 4 Feb 2026 14:04:41 -1000 Subject: [PATCH 44/64] Skip verification step for HEPA filter --- kpf/ao/TurnHepaOff.py | 7 ++++--- kpf/ao/TurnHepaOn.py | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/kpf/ao/TurnHepaOff.py b/kpf/ao/TurnHepaOff.py index de7f02ed..e3dc38a1 100644 --- a/kpf/ao/TurnHepaOff.py +++ b/kpf/ao/TurnHepaOff.py @@ -25,6 +25,7 @@ def perform(cls, args): @classmethod def post_condition(cls, args): - OBHPASTA = ktl.cache('ao', 'OBHPASTA') - if OBHPASTA.waitfor('== "off"', timeout=3) is not True: - raise FailedToReachDestination(OBHPASTA.read(), 'off') + pass +# OBHPASTA = ktl.cache('ao', 'OBHPASTA') +# if OBHPASTA.waitfor('== "off"', timeout=3) is not True: +# raise FailedToReachDestination(OBHPASTA.read(), 'off') diff --git a/kpf/ao/TurnHepaOn.py b/kpf/ao/TurnHepaOn.py index 735adf15..71d47978 100644 --- a/kpf/ao/TurnHepaOn.py +++ b/kpf/ao/TurnHepaOn.py @@ -25,6 +25,7 @@ def perform(cls, args): @classmethod def post_condition(cls, args): - OBHPASTA = ktl.cache('ao', 'OBHPASTA') - if OBHPASTA.waitfor('== "on"', timeout=3) is not True: - raise FailedToReachDestination(OBHPASTA.read(), 'on') + pass +# OBHPASTA = ktl.cache('ao', 'OBHPASTA') +# if OBHPASTA.waitfor('== "on"', timeout=3) is not True: +# raise FailedToReachDestination(OBHPASTA.read(), 'on') From c06d54a075015a030f98c8d871822db73fe56c07 Mon Sep 17 00:00:00 2001 From: Josh Walawender Date: Fri, 6 Feb 2026 10:38:02 -1000 Subject: [PATCH 45/64] Draft tool to find detector temperature excusions --- .../analysis/ListTemperatureExcursions.py | 101 ++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 kpf/engineering/analysis/ListTemperatureExcursions.py diff --git a/kpf/engineering/analysis/ListTemperatureExcursions.py b/kpf/engineering/analysis/ListTemperatureExcursions.py new file mode 100644 index 00000000..a2405144 --- /dev/null +++ b/kpf/engineering/analysis/ListTemperatureExcursions.py @@ -0,0 +1,101 @@ +#!python3 + +## Import General Tools +import sys +import copy +import time +from datetime import datetime, timedelta +from matplotlib import pyplot as plt +import numpy as np +from scipy.signal import find_peaks + +import keygrabber + + +def find_excursions(side, threshold=0.005, min_duration=1*60*60, set_point=-100, + start='2025-11-01'): + excursion_min_duration = timedelta(hours=1) + set_point = -100 + + start_date = datetime.strptime(start, '%Y-%m-%d') + begin = time.mktime(start_date.timetuple()) + end = time.mktime(datetime.now().timetuple()) + +# start_date = datetime.strptime('2025-11-09 05:00:00', '%Y-%m-%d %H:%M:%S') +# begin = time.mktime(start_date.timetuple()) +# end_date = datetime.strptime('2025-11-10 08:00:00', '%Y-%m-%d %H:%M:%S') +# end = time.mktime(end_date.timetuple()) + + det_temp_history = keygrabber.retrieve({f'kpf{side.lower()}': ['STA_CCDVAL']}, + begin=begin, end=end) + det_temp = np.array([abs(float(x['binvalue'])-set_point) + for x in det_temp_history]) + det_time = np.array([datetime.fromtimestamp(x['time']) + for x in det_temp_history]) + + # Find local maxima (peaks) + peaks_indices, _ = find_peaks(det_temp, height=threshold, + distance=1, width=5) + + # Get the actual values of the peaks + peak_values = det_temp[peaks_indices] + peak_times = det_time[peaks_indices] + + + # Print the indices and values of the peaks + remove_inds = [] + for i,peak in enumerate(peak_values): + pt = peak_times[i] + print(f"{peak_times[i]}: {peak:.3f} K") + delta_ts = abs(pt-peak_times) + in_window = delta_ts < excursion_min_duration + in_window_inds = np.where(in_window)[0] + windowed_peak_values = copy.deepcopy(peak_values) + windowed_peak_values[~in_window] = 0 + max_ind = np.argmax(windowed_peak_values) + not_peak = (in_window_inds != max_ind) + remove_inds.extend(in_window_inds[not_peak]) + + + print(f'{side} Final list') + final_times = [] + final_values = [] + for i,peak in enumerate(peak_values): + if i not in remove_inds: + final_times.append(peak_times[i]) + final_values.append(peak_values[i]) + print(f"{peak_times[i]}: {peak*1000:4.0f} mK") + + return det_time, det_temp, final_times, final_values + + +if __name__ == '__main__': + Gdet_time, Gdet_temp, Gfinal_times, Gfinal_values = find_excursions('Green') + Rdet_time, Rdet_temp, Rfinal_times, Rfinal_values = find_excursions('Red') + + # Plot the results to visualize + plt.figure(figsize=(12, 5)) + + plt.subplot(2,1,1) + plt.title('Detector Temperature Excursions') + plt.fill_between([min(Gdet_time), max(Gdet_time)], y1=0.0001, y2=0.005, color='g', alpha=0.3) + plt.plot(Gdet_time, Gdet_temp, 'k-', alpha=0.3, label='Green Temperature') + plt.scatter(Gfinal_times, Gfinal_values, color='red', marker='x', label='Green Excursions') + plt.ylim(0.0005, max([0.010, max(Gdet_temp)*2])) + plt.xlim(min(Gdet_time), max(Gdet_time)) + plt.legend() + plt.gca().set_yscale('log') + plt.ylabel('Temperature Delta (K)') + + plt.subplot(2,1,2) + plt.fill_between([min(Rdet_time), max(Rdet_time)], y1=0.0001, y2=0.005, color='g', alpha=0.3) + plt.plot(Rdet_time, Rdet_temp, 'k-', alpha=0.3, label='Red Temperature') + plt.scatter(Rfinal_times, Rfinal_values, color='red', marker='x', label='Red Excursions') + plt.xlim(min(Rdet_time), max(Rdet_time)) + plt.ylim(0.0005, max([0.010, max(Rdet_temp)*2])) + plt.legend() + plt.gca().set_yscale('log') + + plt.xlabel('Time') + plt.ylabel('Temperature Delta (K)') + plt.show() From ec188f7d9db171c80f5533bf4aee396e4560ece4 Mon Sep 17 00:00:00 2001 From: Josh Walawender Date: Fri, 6 Feb 2026 11:55:37 -1000 Subject: [PATCH 46/64] Add list of temperature excursions --- docs/status.md | 24 ++++++++++++++++++++++++ kpf/linking_table.yml | 4 ---- mkdocs.yml | 3 +-- 3 files changed, 25 insertions(+), 6 deletions(-) diff --git a/docs/status.md b/docs/status.md index d38fb116..e7c43b5e 100644 --- a/docs/status.md +++ b/docs/status.md @@ -2,6 +2,7 @@ ### Current and Past Announcements +* 2026 February: [26A Status Announcement](announcements/2026-02-01_26A-status.md) * 2025 August: [26A Stability Announcement](KPF Stability Statement - August 15 2025.pdf) * 2023 September: [Keck Science Meeting presentation](Keck Science Meeting 2023 Breakout Session.pdf) @@ -23,3 +24,26 @@ Last Status Update: 2025-10-22 - **Simultaneous Calibration (SimulCal)**: Simultaneous calibrations are supported. - **Nod to Sky Observations**: For observations which need a sky measurement other than the built in sky fibers, nodding away to a sky position can be accomplished manually by running separate OBs for the target and sky and asking the OA to offset the telescope as appropriate. We plan to build a separate Nod To Sky observing mode which will accomplish this within a single OB, but that is not yet available. - **Off Target Guiding**: Not yet commissioned. Currently, the tip tilt system must be able to detect the science target in order to position it on the fiber. + +### KPF Era 4.0 Summary + +KPF Era 4.0 began in late-October 2025 after Servicing Mission 4. + +List of Temperature Excursion Events: + +| Side | Date & Time (HST) | Delta T | +| ---- | ------------------- | -------- | +| G | 2025-11-09 21:22:50 | 7 mK | +| G | 2025-11-10 05:48:42 | 109 mK | +| G | 2025-11-10 13:15:18 | 2825 mK | +| G | 2025-11-15 11:50:50 | 250 mK | +| G | 2025-11-16 16:04:50 | 5779 mK | +| G | 2025-11-17 09:17:54 | 5564 mK | +| G | 2025-11-17 10:34:40 | 6147 mK | +| G | 2025-11-17 11:52:54 | 87 mK | +| G | 2025-11-18 05:06:38 | 4200 mK | +| G | 2025-11-19 01:01:22 | 5862 mK | +| G | 2025-11-19 02:15:06 | 90 mK | +| G | 2026-01-23 21:28:10 | 95 mK | +| G | 2026-01-25 16:15:50 | 712 mK | +| G | 2026-01-30 19:52:12 | 129 mK | diff --git a/kpf/linking_table.yml b/kpf/linking_table.yml index 82bbf3ed..4b61e50f 100644 --- a/kpf/linking_table.yml +++ b/kpf/linking_table.yml @@ -89,10 +89,6 @@ links: cmd: engineering.TakeADCOffsetGridData.TakeADCOffsetGridData TakeGuiderSensitivityData: cmd: engineering.TakeGuiderSensitivityData.TakeGuiderSensitivityData - TriggerGreenMiniFill: - cmd: engineering.TriggerGreenMiniFill.TriggerGreenMiniFill - TriggerRedMiniFill: - cmd: engineering.TriggerRedMiniFill.TriggerRedMiniFill # Engineering/Analysis AnalyzeGridSearch: cmd: engineering.analysis.AnalyzeGridSearch.AnalyzeGridSearch diff --git a/mkdocs.yml b/mkdocs.yml index 83558547..88da019d 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -101,6 +101,7 @@ nav: - "SetAORotatorManual": scripts/SetAORotatorManual.md - "SetupAOforACAM": scripts/SetupAOforACAM.md - "SetupAOforKPF": scripts/SetupAOforKPF.md + - "ShutdownAOforKPF": scripts/ShutdownAOforKPF.md - "TurnHepaOff": scripts/TurnHepaOff.md - "TurnHepaOn": scripts/TurnHepaOn.md - "TurnLightSourceOff": scripts/TurnLightSourceOff.md @@ -134,8 +135,6 @@ nav: - "TakeADCGridData": scripts/TakeADCGridData.md - "TakeADCOffsetGridData": scripts/TakeADCOffsetGridData.md - "TakeGuiderSensitivityData": scripts/TakeGuiderSensitivityData.md - - "TriggerGreenMiniFill": scripts/TriggerGreenMiniFill.md - - "TriggerRedMiniFill": scripts/TriggerRedMiniFill.md - "AnalyzeGridSearch": scripts/AnalyzeGridSearch.md - "AnalyzeTipTiltPerformance": scripts/AnalyzeTipTiltPerformance.md - "Fit2DGridSearch": scripts/Fit2DGridSearch.md From 09ef2e8d4f710bf2993235fff9d1d0955155e3d4 Mon Sep 17 00:00:00 2001 From: Josh Walawender Date: Fri, 6 Feb 2026 11:55:52 -1000 Subject: [PATCH 47/64] Output formatted table for web --- .../analysis/ListTemperatureExcursions.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/kpf/engineering/analysis/ListTemperatureExcursions.py b/kpf/engineering/analysis/ListTemperatureExcursions.py index a2405144..49682908 100644 --- a/kpf/engineering/analysis/ListTemperatureExcursions.py +++ b/kpf/engineering/analysis/ListTemperatureExcursions.py @@ -56,15 +56,12 @@ def find_excursions(side, threshold=0.005, min_duration=1*60*60, set_point=-100, not_peak = (in_window_inds != max_ind) remove_inds.extend(in_window_inds[not_peak]) - - print(f'{side} Final list') final_times = [] final_values = [] for i,peak in enumerate(peak_values): if i not in remove_inds: final_times.append(peak_times[i]) final_values.append(peak_values[i]) - print(f"{peak_times[i]}: {peak*1000:4.0f} mK") return det_time, det_temp, final_times, final_values @@ -73,6 +70,22 @@ def find_excursions(side, threshold=0.005, min_duration=1*60*60, set_point=-100, Gdet_time, Gdet_temp, Gfinal_times, Gfinal_values = find_excursions('Green') Rdet_time, Rdet_temp, Rfinal_times, Rfinal_values = find_excursions('Red') + Excursions = [] + GreenExcursions = list(zip(Gfinal_times, Gfinal_values, ['G']*len(Gfinal_values))) + if GreenExcursions: Excursions.extend(GreenExcursions) + RedExcursions = list(zip(Rfinal_times, Rfinal_values, ['R']*len(Gfinal_values))) + if RedExcursions: Excursions.extend(RedExcursions) + Excursions = sorted(Excursions) + + print() + print(f'List of Temperature Excursion Events') + print('| Side | Date & Time (HST) | Delta T |') + print('| ---- | ------------------- | -------- |') + for i,entry in enumerate(Excursions): + time_str = entry[0].strftime('%Y-%m-%d %H:%M:%S') + print(f"| {entry[2]:4s} | {time_str} | {entry[1]*1000:5.0f} mK |") + + # Plot the results to visualize plt.figure(figsize=(12, 5)) From 4c4de2d02522ac6e43eb0164799053a7a8ff5237 Mon Sep 17 00:00:00 2001 From: Josh Walawender Date: Fri, 6 Feb 2026 11:56:40 -1000 Subject: [PATCH 48/64] Language tweak --- docs/status.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/status.md b/docs/status.md index e7c43b5e..54a283a0 100644 --- a/docs/status.md +++ b/docs/status.md @@ -25,7 +25,7 @@ Last Status Update: 2025-10-22 - **Nod to Sky Observations**: For observations which need a sky measurement other than the built in sky fibers, nodding away to a sky position can be accomplished manually by running separate OBs for the target and sky and asking the OA to offset the telescope as appropriate. We plan to build a separate Nod To Sky observing mode which will accomplish this within a single OB, but that is not yet available. - **Off Target Guiding**: Not yet commissioned. Currently, the tip tilt system must be able to detect the science target in order to position it on the fiber. -### KPF Era 4.0 Summary +### KPF Era 4.0 Stability Summary KPF Era 4.0 began in late-October 2025 after Servicing Mission 4. From 79848f93b0e45ef09592ab1c40f51a366b353179 Mon Sep 17 00:00:00 2001 From: Josh Walawender Date: Fri, 6 Feb 2026 13:59:33 -1000 Subject: [PATCH 49/64] Update status summary and Era 4 stability discussion --- docs/status.md | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/docs/status.md b/docs/status.md index 54a283a0..a4ccb3f4 100644 --- a/docs/status.md +++ b/docs/status.md @@ -10,26 +10,23 @@ This is an attempt to summarize the status of various sub-systems of the instrument. Each sub-system name is color coded to indicate the status at a glance: green means functioning normally, orange means mostly normal, but with some caveats or minor issues, and red means the sub-system is compromised in some way. -Last Status Update: 2025-10-22 +Last Status Update: 2026-02-01 -- **Detector Noise**: Starting in November of 2024, additional pattern noise has been present on the detectors. We have been working on eliminating the spurious nose, but we have been unable to completely remove it. As of late-October 2025 the read noise on the Green side remains elevated (~10 electrons), while the red side is near our nominal target (~4.3 electrons). -- **LFC**: We are evaluating the reliability of the LFC after recent service. Initial indications look prominsing on reliability, though the bluest flux (below ~490 nm) is not consistent. +- **Detector Noise**: Starting in November of 2024, additional non-gaussian noise has been present on the detectors. As of late-October 2025 the read noise on the Green side remains elevated (~10 electrons), while the red side is at our target level (~4.3 electrons). +- **LFC**: Initial evaluations of the reliability of the LFC after the recent service look promising, though the bluest flux (below ~490 nm) is not consistent. +- **Detector Cooling Systems**: The green side CCR has very little overhead on maintaining temperature and has quasi-periodic deviations which affect the detector. Red side is performing well, but has shown evidence of a slow degradation of performance. - **Etalon**: Operational. -- **Detector Cooling Systems**: Both detectors are now cooled with closed cycle refrigerators (CCRs). The green side CCR has problems and has very little overhead on maintaining temperature and has quasi-periodic deviations which affect the detector. Red side is performing well. -- **Detector Errors**: The red and green detectors suffer from occasional “start state errors” in which the affected detector remains in the start phase and does not produce a useful exposure. The observing scripts detect this, abort the current exposure (with read out) and start a fresh exposure on both cameras. **No action is necessary on the part of the observer.** The occurrence rate is such that around one in every 180 exposures is affected by one of the two detectors experiencing this error. +- **Detector Errors**: The red and green detectors suffer from occasional “start state errors” in which the affected detector does not produce a useful exposure. The observing scripts detect this, abort the exposure (with read out) and start a fresh exposure on both cameras. **No action is necessary on the part of the observer.** The occurrence rate is such that around one in every 180 exposures is affected by one of the two detectors experiencing this error. - **Ca H&K Detector**: The CA H&K detector is operational. - **Exposure Meter Terminated Exposures**: Operational. - **Tip Tilt Corrections**: The tip tilt axis are currently correcting as expected. - **Double Star Observations**: Operational. - **Simultaneous Calibration (SimulCal)**: Simultaneous calibrations are supported. -- **Nod to Sky Observations**: For observations which need a sky measurement other than the built in sky fibers, nodding away to a sky position can be accomplished manually by running separate OBs for the target and sky and asking the OA to offset the telescope as appropriate. We plan to build a separate Nod To Sky observing mode which will accomplish this within a single OB, but that is not yet available. - **Off Target Guiding**: Not yet commissioned. Currently, the tip tilt system must be able to detect the science target in order to position it on the fiber. -### KPF Era 4.0 Stability Summary +### KPF Era 4.0 Temperature Stability Summary -KPF Era 4.0 began in late-October 2025 after Servicing Mission 4. - -List of Temperature Excursion Events: +KPF Era 4.0 began in late-October 2025 after Servicing Mission 4. We continue to have issues with the cooling systems for the detectors, especially the Green side. We list below all temperature excursions in which one of the detectors deviated temperature by more than 5 mK from the set point. Excursions of order 1 K (1000 mK) or more are expected to induce a radial velocity offset which is not calibratable. We are providing this data as a guide, but users should not assume that past performance is a good indicator of future performance -- we have seen indications that the cooling systems are slowly degrading, so an increasing rate of these temperature excursions is a distinct possibility. | Side | Date & Time (HST) | Delta T | | ---- | ------------------- | -------- | From da20c4b6c205308057f124ea0160ea3f58a6c6ee Mon Sep 17 00:00:00 2001 From: Josh Walawender Date: Fri, 6 Feb 2026 14:01:16 -1000 Subject: [PATCH 50/64] Reorganize announcements --- docs/status.md | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/docs/status.md b/docs/status.md index a4ccb3f4..a8042daf 100644 --- a/docs/status.md +++ b/docs/status.md @@ -1,17 +1,13 @@ # Instrument Status -### Current and Past Announcements +### Current Announcements * 2026 February: [26A Status Announcement](announcements/2026-02-01_26A-status.md) -* 2025 August: [26A Stability Announcement](KPF Stability Statement - August 15 2025.pdf) -* 2023 September: [Keck Science Meeting presentation](Keck Science Meeting 2023 Breakout Session.pdf) ### Status Summary by Subsystem This is an attempt to summarize the status of various sub-systems of the instrument. Each sub-system name is color coded to indicate the status at a glance: green means functioning normally, orange means mostly normal, but with some caveats or minor issues, and red means the sub-system is compromised in some way. -Last Status Update: 2026-02-01 - - **Detector Noise**: Starting in November of 2024, additional non-gaussian noise has been present on the detectors. As of late-October 2025 the read noise on the Green side remains elevated (~10 electrons), while the red side is at our target level (~4.3 electrons). - **LFC**: Initial evaluations of the reliability of the LFC after the recent service look promising, though the bluest flux (below ~490 nm) is not consistent. - **Detector Cooling Systems**: The green side CCR has very little overhead on maintaining temperature and has quasi-periodic deviations which affect the detector. Red side is performing well, but has shown evidence of a slow degradation of performance. @@ -24,6 +20,8 @@ Last Status Update: 2026-02-01 - **Simultaneous Calibration (SimulCal)**: Simultaneous calibrations are supported. - **Off Target Guiding**: Not yet commissioned. Currently, the tip tilt system must be able to detect the science target in order to position it on the fiber. +Last Updated: 2026-02-01 + ### KPF Era 4.0 Temperature Stability Summary KPF Era 4.0 began in late-October 2025 after Servicing Mission 4. We continue to have issues with the cooling systems for the detectors, especially the Green side. We list below all temperature excursions in which one of the detectors deviated temperature by more than 5 mK from the set point. Excursions of order 1 K (1000 mK) or more are expected to induce a radial velocity offset which is not calibratable. We are providing this data as a guide, but users should not assume that past performance is a good indicator of future performance -- we have seen indications that the cooling systems are slowly degrading, so an increasing rate of these temperature excursions is a distinct possibility. @@ -44,3 +42,8 @@ KPF Era 4.0 began in late-October 2025 after Servicing Mission 4. We continue to | G | 2026-01-23 21:28:10 | 95 mK | | G | 2026-01-25 16:15:50 | 712 mK | | G | 2026-01-30 19:52:12 | 129 mK | + +### Past Announcements + +* 2025 August: [26A Stability Announcement](KPF Stability Statement - August 15 2025.pdf) +* 2023 September: [Keck Science Meeting presentation](Keck Science Meeting 2023 Breakout Session.pdf) From 31bf703da4e276302601608486cb6bc1d81ea6b8 Mon Sep 17 00:00:00 2001 From: Josh Walawender Date: Fri, 6 Feb 2026 14:03:39 -1000 Subject: [PATCH 51/64] formatting tweak --- docs/status.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/status.md b/docs/status.md index a8042daf..6f328bcc 100644 --- a/docs/status.md +++ b/docs/status.md @@ -30,14 +30,14 @@ KPF Era 4.0 began in late-October 2025 after Servicing Mission 4. We continue to | ---- | ------------------- | -------- | | G | 2025-11-09 21:22:50 | 7 mK | | G | 2025-11-10 05:48:42 | 109 mK | -| G | 2025-11-10 13:15:18 | 2825 mK | +| G | 2025-11-10 13:15:18 | 2,825 mK | | G | 2025-11-15 11:50:50 | 250 mK | -| G | 2025-11-16 16:04:50 | 5779 mK | -| G | 2025-11-17 09:17:54 | 5564 mK | -| G | 2025-11-17 10:34:40 | 6147 mK | +| G | 2025-11-16 16:04:50 | 5,779 mK | +| G | 2025-11-17 09:17:54 | 5,564 mK | +| G | 2025-11-17 10:34:40 | 6,147 mK | | G | 2025-11-17 11:52:54 | 87 mK | -| G | 2025-11-18 05:06:38 | 4200 mK | -| G | 2025-11-19 01:01:22 | 5862 mK | +| G | 2025-11-18 05:06:38 | 4,200 mK | +| G | 2025-11-19 01:01:22 | 5,862 mK | | G | 2025-11-19 02:15:06 | 90 mK | | G | 2026-01-23 21:28:10 | 95 mK | | G | 2026-01-25 16:15:50 | 712 mK | From a0bfb3b8ab516394e1f49f62ece21ba7107885fe Mon Sep 17 00:00:00 2001 From: Josh Walawender Date: Fri, 6 Feb 2026 14:03:54 -1000 Subject: [PATCH 52/64] Formatting tweak --- kpf/engineering/analysis/ListTemperatureExcursions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kpf/engineering/analysis/ListTemperatureExcursions.py b/kpf/engineering/analysis/ListTemperatureExcursions.py index 49682908..fa500569 100644 --- a/kpf/engineering/analysis/ListTemperatureExcursions.py +++ b/kpf/engineering/analysis/ListTemperatureExcursions.py @@ -83,7 +83,7 @@ def find_excursions(side, threshold=0.005, min_duration=1*60*60, set_point=-100, print('| ---- | ------------------- | -------- |') for i,entry in enumerate(Excursions): time_str = entry[0].strftime('%Y-%m-%d %H:%M:%S') - print(f"| {entry[2]:4s} | {time_str} | {entry[1]*1000:5.0f} mK |") + print(f"| {entry[2]:4s} | {time_str} | {entry[1]*1000:5,.0f} mK |") # Plot the results to visualize From e58adc547550120c848e6f2d7b6fef4ecfc00f4b Mon Sep 17 00:00:00 2001 From: Josh Walawender Date: Fri, 6 Feb 2026 14:10:19 -1000 Subject: [PATCH 53/64] Announcements listed in status page, not menu --- mkdocs.yml | 4 +--- mkdocs_input.yml | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/mkdocs.yml b/mkdocs.yml index 88da019d..3b4f6df8 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -24,9 +24,7 @@ plugins: nav: - "KPF Home": index.md - - "Instrument Status": - - "Instrument Status": status.md - - "26A Stability Announcement": announcements/2026-02-01_26A-status.md + - "Instrument Status": status.md - "Instrument Info": - "Instrument Subsystems": - "Spectrograph": spectrograph.md diff --git a/mkdocs_input.yml b/mkdocs_input.yml index 91f50fc8..102d094d 100644 --- a/mkdocs_input.yml +++ b/mkdocs_input.yml @@ -24,9 +24,7 @@ plugins: nav: - "KPF Home": index.md - - "Instrument Status": - - "Instrument Status": status.md - - "26A Stability Announcement": announcements/2026-02-01_26A-status.md + - "Instrument Status": status.md - "Instrument Info": - "Instrument Subsystems": - "Spectrograph": spectrograph.md From db253a08a59500a6bb72e050b1137e4466e3a244 Mon Sep 17 00:00:00 2001 From: Josh Walawender Date: Fri, 6 Feb 2026 15:36:26 -1000 Subject: [PATCH 54/64] Add tool to plot basement temperatures --- .../analysis/PlotBasementTemperatures.py | 90 +++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 kpf/engineering/analysis/PlotBasementTemperatures.py diff --git a/kpf/engineering/analysis/PlotBasementTemperatures.py b/kpf/engineering/analysis/PlotBasementTemperatures.py new file mode 100644 index 00000000..2b70c4cd --- /dev/null +++ b/kpf/engineering/analysis/PlotBasementTemperatures.py @@ -0,0 +1,90 @@ +#!python3 + +## Import General Tools +import sys +import copy +import time +from datetime import datetime, timedelta +from matplotlib import pyplot as plt +import numpy as np +from scipy.signal import find_peaks +from astropy.table import Table + +import keygrabber + + +def retrieve_basement_temperatures(start='2026-01-01', end='2026-02-01'): + + start_date = datetime.strptime(start, '%Y-%m-%d') + begin = time.mktime(start_date.timetuple()) + end_date = datetime.strptime(end, '%Y-%m-%d') + end = time.mktime(end_date.timetuple()) + +# start_date = datetime.strptime('2025-11-09 05:00:00', '%Y-%m-%d %H:%M:%S') +# begin = time.mktime(start_date.timetuple()) +# end_date = datetime.strptime('2025-11-10 08:00:00', '%Y-%m-%d %H:%M:%S') +# end = time.mktime(end_date.timetuple()) + + keywords = {f'kpfmet': ['TEMP', 'ETALON_AMBIENT', 'CAL_BENCH_BOT', + 'ENCLOSURE_FRONT', 'ENCLOSURE_TOP']} + temp_history = keygrabber.retrieve(keywords, begin=begin, end=end) + + data = {} + for kw in keywords['kpfmet']: + data[f'{kw} value'] = [] + data[f'{kw} times'] = [] + for entry in temp_history: + kw = entry['keyword'] + value = float(entry['binvalue']) + timestamp = datetime.fromtimestamp(entry['time']) + if value > -10 and value < 50: + data[f'{kw} value'].append(value) + data[f'{kw} times'].append(timestamp) + + return data, keywords['kpfmet'] + + +def plot_basement_temperatures(data, kws, start=None, end=None): + if start and end: + start_date = datetime.strptime(start, '%Y-%m-%d') + end_date = datetime.strptime(end, '%Y-%m-%d') + + # Plot the results to visualize + plt.figure(figsize=(14, 8)) + + plt.subplot(2,1,1) + plt.title('Basement Temperatures') + for kw in kws: + plt.plot(data[f'{kw} times'], data[f'{kw} value'], marker=',', label=kw) + plt.ylabel('Temperature (C)') + plt.grid() + plt.legend(loc='best') + plt.ylim(17.5,27.5) + if start and end: + plt.xlim(start_date, end_date) + + plt.subplot(2,1,2) + minmax = [0, 0] + for kw in kws: + mint = min(data[f'{kw} value']-np.median(data[f'{kw} value'])) + maxt = max(data[f'{kw} value']-np.median(data[f'{kw} value'])) + plt.plot(data[f'{kw} times'], data[f'{kw} value']-np.median(data[f'{kw} value']), + marker=',', label=kw, alpha=0.3) + if mint < minmax[0]: minmax[0] = np.floor(mint*4)/4 + if maxt > minmax[1]: minmax[1] = np.ceil(maxt*4)/4 + plt.ylabel('Temperature Changes (C)') + plt.grid() + plt.legend(loc='best') + plt.ylim(*minmax) + if start and end: + plt.xlim(start_date, end_date) + + plt.xlabel('Time') + plt.show() + + +if __name__ == '__main__': + start = '2025-11-01' + end = '2026-03-01' + data, kws = retrieve_basement_temperatures(start=start, end=end) + plot_basement_temperatures(data, kws, start=start, end=end) From d424a95df04154d77e44aacced03bf3542770050 Mon Sep 17 00:00:00 2001 From: Josh Walawender Date: Thu, 12 Feb 2026 15:20:36 -1000 Subject: [PATCH 55/64] Update to bring text closer to 26B InstAvailability text --- docs/announcements/2026-02-01_26A-status.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/announcements/2026-02-01_26A-status.md b/docs/announcements/2026-02-01_26A-status.md index 0654fb18..0fff1e58 100644 --- a/docs/announcements/2026-02-01_26A-status.md +++ b/docs/announcements/2026-02-01_26A-status.md @@ -10,9 +10,7 @@ In addition, the KPF DRP is not performing well on faint object extractions. So The most impactful stability factor for KPF has been the detector temperatures. In order for RV measurements at different times to be compared, the system must not change temperature significantly during that time span as the hysteresis induced by a change and return to temperature results in a zero point change for the RV measurements. The goal is for the system to be stable at the ~1 mK level, while changes of ~1 K are almost certain to break the RV zero point. -Over its history, KPF has undergone four servicing missions, all of which attempted to address temperature stability problems in one way or another. KPF currently uses a pair of Closed Cycle Refrigerators (CCRs). The CCR for the Green detector has been problematic since it was installed despite multiple interventions. In contrast the Red side CCR has been more reliable, however we are seeing signs of degradation of stability on the red side as well. - -We currently have significant doubts about the current CCRs ability to maintain detector temperatures at their targets without occasional excursions of 1 K or more. Temperature excursions at that level would disrupt long term RV time series. +The Green and Red CCDs in the main spectrometer are likely to experience thermal events of a few degrees C (or more) above the -100 C detector setpoints lasting a few hours. While the number and timing of these are difficult to predict, it is likely that KPF will not maintain long-term RV stability over 2026A. Thus, time series RV measurements over long periods (weeks or months) are likely to be compromised by changing RV zero points at the ~10 m/s level. Experience has shown that these zero-point changes are difficult/impossible to calibrate out and are different for each star observed. Science projects that rely on a time series of KPF RVs or spectra over a short timescale (a night to a few nights) are unlikely to be affected by the warmups (which would have to occur during the observing sequence to be a problem). Users who have science goals which require high precision (a few m/s or better) on long time scales should take this into account when choosing which science programs to execute with their time in 26A. The WMKO and KPF Build Teams are currently planning for an additional servicing mission to address the cooling problems, however that will take time to plan and implement. From 085e177adfbca694d29116a7f5477ae2e95df4ae Mon Sep 17 00:00:00 2001 From: Josh Walawender Date: Tue, 17 Feb 2026 14:22:10 -1000 Subject: [PATCH 56/64] Add menu item to bypass telescope release for magiq --- kpf/OB_GUI/KPF_OB_GUI.py | 22 +++++++++++++++++++--- kpf/OB_GUI/KPF_OB_GUI.ui | 6 ++++++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/kpf/OB_GUI/KPF_OB_GUI.py b/kpf/OB_GUI/KPF_OB_GUI.py index 867447a0..e2905c82 100755 --- a/kpf/OB_GUI/KPF_OB_GUI.py +++ b/kpf/OB_GUI/KPF_OB_GUI.py @@ -202,6 +202,7 @@ def __init__(self, clargs, *args, **kwargs): self.fast = False # Tracked values self.disabled_detectors = [] + self.disable_telescope_release_check = False self.telescope_released = GetTelescopeRelease.execute({}) # Get KPF Programs on schedule classical, cadence = GetScheduledPrograms.execute({'semester': 'current'}) @@ -305,9 +306,10 @@ def setupUi(self): self.SendOBListToMagiq = self.findChild(QtWidgets.QAction, 'actionSend_Current_OBs_as_Star_List') self.SendOBListToMagiq.triggered.connect(self.OBListModel.update_star_list) self.SendOBListToMagiq.setEnabled(False) - self.DisableMagiq = self.findChild(QtWidgets.QAction, 'actionDisable_Magiq') self.DisableMagiq.triggered.connect(self.toggle_magiq_enabled) + self.OverrideRelease = self.findChild(QtWidgets.QAction, 'actionOverride_Telescope_Release_Check') + self.OverrideRelease.triggered.connect(self.toggle_telescope_release_check) #------------------------------------------------------------------- # Main Window @@ -741,7 +743,10 @@ def update_LST(self, value): self.log.debug('Updating: SOB info, telescope_released') self.update_counter = 0 self.update_SOB_display() # Updates alt, az - self.telescope_released = GetTelescopeRelease.execute({}) + if self.disable_telescope_release_check is True: + self.telescope_released = True + else: + self.telescope_released = GetTelescopeRelease.execute({}) # Update execution history if we're vaguely near observing times try: UTh = int(self.UTValue.text().split(':')[0]) @@ -751,7 +756,6 @@ def update_LST(self, value): self.log.debug('Updating: execution history') self.refresh_history() - ##------------------------------------------- ## Methods for Observing Menu Actions ##------------------------------------------- @@ -847,6 +851,18 @@ def toggle_magiq_enabled(self): self.DisableMagiq.setText(action_text) self.update_selected_instrument(self.SelectedInstrument.text()) + def toggle_telescope_release_check(self): + self.log.info('Toggling telescope release check') + self.disable_telescope_release_check = not self.disable_telescope_release_check + self.log.debug(f"disable release check = {self.disable_telescope_release_check}") + action = {False: 'Disable', True: 'Enable'}[self.disable_telescope_release_check] + action_text = f"{action} Telescope Release Check" + self.OverrideRelease.setText(action_text) + if self.disable_telescope_release_check is True: + self.telescope_released = True + else: + self.telescope_released = GetTelescopeRelease.execute({}) + ##------------------------------------------- ## Methods to Operate on OB List UI diff --git a/kpf/OB_GUI/KPF_OB_GUI.ui b/kpf/OB_GUI/KPF_OB_GUI.ui index 15290069..7f891406 100644 --- a/kpf/OB_GUI/KPF_OB_GUI.ui +++ b/kpf/OB_GUI/KPF_OB_GUI.ui @@ -1782,6 +1782,7 @@ + @@ -1899,6 +1900,11 @@ Load KPF-CC Schedule for a non-CC Night + + + Disable Telescope Release Check + + From c1ea096dc7a8ff030c25f90f6e11d3401c9bdc3c Mon Sep 17 00:00:00 2001 From: Josh Walawender Date: Wed, 18 Feb 2026 12:54:06 -1000 Subject: [PATCH 57/64] Improved algorithm to find excursion events --- .../analysis/ListTemperatureExcursions.py | 148 ++++++++++++------ 1 file changed, 99 insertions(+), 49 deletions(-) diff --git a/kpf/engineering/analysis/ListTemperatureExcursions.py b/kpf/engineering/analysis/ListTemperatureExcursions.py index fa500569..3ed61c28 100644 --- a/kpf/engineering/analysis/ListTemperatureExcursions.py +++ b/kpf/engineering/analysis/ListTemperatureExcursions.py @@ -23,7 +23,7 @@ def find_excursions(side, threshold=0.005, min_duration=1*60*60, set_point=-100, # start_date = datetime.strptime('2025-11-09 05:00:00', '%Y-%m-%d %H:%M:%S') # begin = time.mktime(start_date.timetuple()) -# end_date = datetime.strptime('2025-11-10 08:00:00', '%Y-%m-%d %H:%M:%S') +# end_date = datetime.strptime('2025-11-10 16:00:00', '%Y-%m-%d %H:%M:%S') # end = time.mktime(end_date.timetuple()) det_temp_history = keygrabber.retrieve({f'kpf{side.lower()}': ['STA_CCDVAL']}, @@ -33,57 +33,103 @@ def find_excursions(side, threshold=0.005, min_duration=1*60*60, set_point=-100, det_time = np.array([datetime.fromtimestamp(x['time']) for x in det_temp_history]) - # Find local maxima (peaks) - peaks_indices, _ = find_peaks(det_temp, height=threshold, - distance=1, width=5) - - # Get the actual values of the peaks - peak_values = det_temp[peaks_indices] - peak_times = det_time[peaks_indices] - - - # Print the indices and values of the peaks - remove_inds = [] - for i,peak in enumerate(peak_values): - pt = peak_times[i] - print(f"{peak_times[i]}: {peak:.3f} K") - delta_ts = abs(pt-peak_times) - in_window = delta_ts < excursion_min_duration - in_window_inds = np.where(in_window)[0] - windowed_peak_values = copy.deepcopy(peak_values) - windowed_peak_values[~in_window] = 0 - max_ind = np.argmax(windowed_peak_values) - not_peak = (in_window_inds != max_ind) - remove_inds.extend(in_window_inds[not_peak]) - - final_times = [] - final_values = [] - for i,peak in enumerate(peak_values): - if i not in remove_inds: - final_times.append(peak_times[i]) - final_values.append(peak_values[i]) - - return det_time, det_temp, final_times, final_values + # Find Our of Range (ooR) time spans + events = [{'Begin': None, 'End': None, 'Side': side}] + ooR = det_temp > threshold + for i,val in enumerate(ooR): + if val == True and events[-1]['Begin'] is None: + events[-1]['Begin'] = det_time[i] + if val == False and events[-1]['End'] is None and events[-1]['Begin'] is not None: + events[-1]['End'] = det_time[i] + events.append({'Begin': None, 'End': None, 'Side': side}) + + if events[-1] == {'Begin': None, 'End': None, 'Side': side}: + events.pop(-1) + + merge = [] + for i,event in enumerate(events): + if i > 0: + ok_duration = (event['Begin']-events[i-1]['End']).total_seconds() + if ok_duration < min_duration: + merge.append(i) + merge.reverse() + for m in merge: + events[m-1]['End'] = events[m]['End'] + events = [e for i,e in enumerate(events) if i not in merge] + + # Find Peak on a Per event basis + for i,event in enumerate(events): + when = (det_time >= event['Begin']) & (det_time <= event['End']) + events[i]['DeltaTemp'] = max(det_temp[when]) + + return det_time, det_temp, events + +# # Find local maxima (peaks) +# peaks_indices, _ = find_peaks(det_temp, height=threshold, +# distance=1, width=5) +# +# # Get the actual values of the peaks +# peak_values = det_temp[peaks_indices] +# peak_times = det_time[peaks_indices] +# +# +# # Print the indices and values of the peaks +# remove_inds = [] +# for i,peak in enumerate(peak_values): +# pt = peak_times[i] +# print(f"{peak_times[i]}: {peak:.3f} K") +# delta_ts = abs(pt-peak_times) +# in_window = delta_ts < excursion_min_duration +# in_window_inds = np.where(in_window)[0] +# windowed_peak_values = copy.deepcopy(peak_values) +# windowed_peak_values[~in_window] = 0 +# max_ind = np.argmax(windowed_peak_values) +# not_peak = (in_window_inds != max_ind) +# remove_inds.extend(in_window_inds[not_peak]) +# +# final_times = [] +# final_values = [] +# for i,peak in enumerate(peak_values): +# if i not in remove_inds: +# final_times.append(peak_times[i]) +# final_values.append(peak_values[i]) +# +# return det_time, det_temp, final_times, final_values, events if __name__ == '__main__': - Gdet_time, Gdet_temp, Gfinal_times, Gfinal_values = find_excursions('Green') - Rdet_time, Rdet_temp, Rfinal_times, Rfinal_values = find_excursions('Red') - - Excursions = [] - GreenExcursions = list(zip(Gfinal_times, Gfinal_values, ['G']*len(Gfinal_values))) - if GreenExcursions: Excursions.extend(GreenExcursions) - RedExcursions = list(zip(Rfinal_times, Rfinal_values, ['R']*len(Gfinal_values))) - if RedExcursions: Excursions.extend(RedExcursions) - Excursions = sorted(Excursions) +# Gdet_time, Gdet_temp, Gfinal_times, Gfinal_values, Gevents = find_excursions('Green') +# Rdet_time, Rdet_temp, Rfinal_times, Rfinal_values, Revents = find_excursions('Red') + Gdet_time, Gdet_temp, Gevents = find_excursions('Green') + Rdet_time, Rdet_temp, Revents = find_excursions('Red') + +# Excursions = [] +# GreenExcursions = list(zip(Gfinal_times, Gfinal_values, ['G']*len(Gfinal_values))) +# if GreenExcursions: Excursions.extend(GreenExcursions) +# RedExcursions = list(zip(Rfinal_times, Rfinal_values, ['R']*len(Gfinal_values))) +# if RedExcursions: Excursions.extend(RedExcursions) +# Excursions = sorted(Excursions) +# +# print() +# print(f'List of Temperature Excursion Events') +# print('| Side | Date & Time (HST) | Delta T |') +# print('| ---- | ------------------- | -------- |') +# for i,entry in enumerate(Excursions): +# time_str = entry[0].strftime('%Y-%m-%d %H:%M:%S') +# print(f"| {entry[2]:4s} | {time_str} | {entry[1]*1000:5,.0f} mK |") + + events = Gevents + Revents + events = sorted(events, key=lambda x: x['Begin']) - print() print(f'List of Temperature Excursion Events') - print('| Side | Date & Time (HST) | Delta T |') - print('| ---- | ------------------- | -------- |') - for i,entry in enumerate(Excursions): - time_str = entry[0].strftime('%Y-%m-%d %H:%M:%S') - print(f"| {entry[2]:4s} | {time_str} | {entry[1]*1000:5,.0f} mK |") + print('| Side | Duration | Delta T | Start (HST) | End (HST) |') + print('| ----- | ------------ | --------- | ------------------- | ------------------- |') + for event in events: + b = event['Begin'].strftime('%Y-%m-%d %H:%M:%S') + e = event['End'].strftime('%Y-%m-%d %H:%M:%S') + d = (event['End']-event['Begin']).total_seconds() / 3600 + dT = event['DeltaTemp']*1000 + print(f"| {event['Side']:4s} | {d:6.2f} hours | {dT:6,.0f} mK | {b:19s} | {e:19s} |") # Plot the results to visualize @@ -93,7 +139,9 @@ def find_excursions(side, threshold=0.005, min_duration=1*60*60, set_point=-100, plt.title('Detector Temperature Excursions') plt.fill_between([min(Gdet_time), max(Gdet_time)], y1=0.0001, y2=0.005, color='g', alpha=0.3) plt.plot(Gdet_time, Gdet_temp, 'k-', alpha=0.3, label='Green Temperature') - plt.scatter(Gfinal_times, Gfinal_values, color='red', marker='x', label='Green Excursions') + for Gevent in Gevents: + plt.fill_between([Gevent['Begin'], Gevent['End']], y1=0.005, y2=Gevent['DeltaTemp'], color='r', alpha=0.3) +# plt.scatter(Gfinal_times, Gfinal_values, color='red', marker='x', label='Green Excursions') plt.ylim(0.0005, max([0.010, max(Gdet_temp)*2])) plt.xlim(min(Gdet_time), max(Gdet_time)) plt.legend() @@ -103,7 +151,9 @@ def find_excursions(side, threshold=0.005, min_duration=1*60*60, set_point=-100, plt.subplot(2,1,2) plt.fill_between([min(Rdet_time), max(Rdet_time)], y1=0.0001, y2=0.005, color='g', alpha=0.3) plt.plot(Rdet_time, Rdet_temp, 'k-', alpha=0.3, label='Red Temperature') - plt.scatter(Rfinal_times, Rfinal_values, color='red', marker='x', label='Red Excursions') + for Revent in Revents: + plt.fill_between([Revent['Begin'], Revent['End']], y1=0.005, y2=0.050, color='r', alpha=0.3) +# plt.scatter(Rfinal_times, Rfinal_values, color='red', marker='x', label='Red Excursions') plt.xlim(min(Rdet_time), max(Rdet_time)) plt.ylim(0.0005, max([0.010, max(Rdet_temp)*2])) plt.legend() From 02a8ac31481803c6ae216c43583be7d27008ca8c Mon Sep 17 00:00:00 2001 From: Josh Walawender Date: Wed, 18 Feb 2026 12:54:18 -1000 Subject: [PATCH 58/64] Documentation: update event table --- docs/status.md | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/docs/status.md b/docs/status.md index 6f328bcc..f3cef7b2 100644 --- a/docs/status.md +++ b/docs/status.md @@ -26,22 +26,20 @@ Last Updated: 2026-02-01 KPF Era 4.0 began in late-October 2025 after Servicing Mission 4. We continue to have issues with the cooling systems for the detectors, especially the Green side. We list below all temperature excursions in which one of the detectors deviated temperature by more than 5 mK from the set point. Excursions of order 1 K (1000 mK) or more are expected to induce a radial velocity offset which is not calibratable. We are providing this data as a guide, but users should not assume that past performance is a good indicator of future performance -- we have seen indications that the cooling systems are slowly degrading, so an increasing rate of these temperature excursions is a distinct possibility. -| Side | Date & Time (HST) | Delta T | -| ---- | ------------------- | -------- | -| G | 2025-11-09 21:22:50 | 7 mK | -| G | 2025-11-10 05:48:42 | 109 mK | -| G | 2025-11-10 13:15:18 | 2,825 mK | -| G | 2025-11-15 11:50:50 | 250 mK | -| G | 2025-11-16 16:04:50 | 5,779 mK | -| G | 2025-11-17 09:17:54 | 5,564 mK | -| G | 2025-11-17 10:34:40 | 6,147 mK | -| G | 2025-11-17 11:52:54 | 87 mK | -| G | 2025-11-18 05:06:38 | 4,200 mK | -| G | 2025-11-19 01:01:22 | 5,862 mK | -| G | 2025-11-19 02:15:06 | 90 mK | -| G | 2026-01-23 21:28:10 | 95 mK | -| G | 2026-01-25 16:15:50 | 712 mK | -| G | 2026-01-30 19:52:12 | 129 mK | +| Side | Duration | Delta T | Start (HST) | End (HST) | +| ----- | ------------ | --------- | ------------------- | ------------------- | +| Green | 0.05 hours | 7 mK | 2025-11-09 21:22:14 | 2025-11-09 21:25:06 | +| Green | 0.36 hours | 109 mK | 2025-11-10 05:37:12 | 2025-11-10 05:58:50 | +| Green | 2.78 hours | 2,825 mK | 2025-11-10 11:09:06 | 2025-11-10 13:55:54 | +| Green | 0.63 hours | 250 mK | 2025-11-15 11:28:36 | 2025-11-15 12:06:32 | +| Green | 2.54 hours | 5,779 mK | 2025-11-16 14:27:20 | 2025-11-16 16:59:54 | +| Green | 4.04 hours | 6,147 mK | 2025-11-17 07:52:06 | 2025-11-17 11:54:40 | +| Green | 2.31 hours | 4,200 mK | 2025-11-18 03:40:28 | 2025-11-18 05:59:16 | +| Green | 3.14 hours | 5,862 mK | 2025-11-18 23:08:20 | 2025-11-19 02:16:52 | +| Green | 0.56 hours | 95 mK | 2026-01-23 21:12:46 | 2026-01-23 21:46:16 | +| Green | 1.20 hours | 712 mK | 2026-01-25 15:25:34 | 2026-01-25 16:37:48 | +| Green | 0.46 hours | 129 mK | 2026-01-30 19:36:24 | 2026-01-30 20:03:54 | +| Green | 13.82 hours | 53,527 mK | 2026-02-17 21:17:12 | 2026-02-18 11:06:40 | ### Past Announcements From c7da8a4932c61649ba61a0175ed7f21babeec402 Mon Sep 17 00:00:00 2001 From: Josh Walawender Date: Thu, 19 Feb 2026 10:29:00 -1000 Subject: [PATCH 59/64] Documentation: add latest thermal excursion --- docs/status.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/status.md b/docs/status.md index f3cef7b2..ec7a6d7f 100644 --- a/docs/status.md +++ b/docs/status.md @@ -40,6 +40,7 @@ KPF Era 4.0 began in late-October 2025 after Servicing Mission 4. We continue to | Green | 1.20 hours | 712 mK | 2026-01-25 15:25:34 | 2026-01-25 16:37:48 | | Green | 0.46 hours | 129 mK | 2026-01-30 19:36:24 | 2026-01-30 20:03:54 | | Green | 13.82 hours | 53,527 mK | 2026-02-17 21:17:12 | 2026-02-18 11:06:40 | +| Green | 4.48 hours | 11,585 mK | 2026-02-19 02:40:56 | 2026-02-19 07:09:32 | ### Past Announcements From a160818a82b26aa557dc7599c2d21032a9b6bdbe Mon Sep 17 00:00:00 2001 From: Josh Walawender Date: Thu, 19 Feb 2026 19:13:02 -1000 Subject: [PATCH 60/64] fix --- kpf/scripts/RunOB.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/kpf/scripts/RunOB.py b/kpf/scripts/RunOB.py index f7d5f670..e8a75564 100644 --- a/kpf/scripts/RunOB.py +++ b/kpf/scripts/RunOB.py @@ -219,12 +219,12 @@ def perform(cls, args, OB=None): log.info(msg) SCRIPTMSG.write(msg) print() - print("###############################################################") - print(" WARNING: The telescope control system target name: {TARGNAME}") - print(" WARNING: does not match the OB target name: {OB.Target.TargetName}") + print(f"###############################################################") + print(f" WARNING: The telescope control system target name: {TARGNAME}") + print(f" WARNING: does not match the OB target name: {OB.Target.TargetName}") print() - print(" Press 'Enter' to begin exposure(s) anyway or 'a' to abort script") - print("###############################################################") + print(f" Press 'Enter' to begin exposure(s) anyway or 'a' to abort script") + print(f"###############################################################") print() user_input = input() log.debug(f'response: "{user_input}"') From f7ed9ef223f2a7b3a6c83027a60162d58c8d3693 Mon Sep 17 00:00:00 2001 From: Josh Walawender Date: Wed, 25 Feb 2026 12:26:48 -1000 Subject: [PATCH 61/64] Update etalon exposure parameters --- kpf/ObservingBlocks/exampleOBs/Calibrations.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kpf/ObservingBlocks/exampleOBs/Calibrations.yaml b/kpf/ObservingBlocks/exampleOBs/Calibrations.yaml index 68035334..75a33554 100644 --- a/kpf/ObservingBlocks/exampleOBs/Calibrations.yaml +++ b/kpf/ObservingBlocks/exampleOBs/Calibrations.yaml @@ -19,13 +19,13 @@ Calibrations: - CalSource: 'EtalonFiber' Object: 'Etalon' nExp: 1 - ExpTime: 20 + ExpTime: 60 TriggerCaHK: False TriggerGreen: True TriggerRed: True IntensityMonitor: False CalND1: 'OD 0.1' - CalND2: 'OD 1.3' + CalND2: 'OD 1.0' OpenScienceShutter: True OpenSkyShutter: True TakeSimulCal: True From 28f30b9bf0e41f173185a7c8b09d2dbf62ada1ef Mon Sep 17 00:00:00 2001 From: Josh Walawender Date: Thu, 5 Mar 2026 11:28:07 -1000 Subject: [PATCH 62/64] Temporary hack to let SoCal operate with disabled enclosure. --- kpf/socal/WaitForSoCalOnTarget.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/kpf/socal/WaitForSoCalOnTarget.py b/kpf/socal/WaitForSoCalOnTarget.py index 46ae6362..fca7bbdb 100644 --- a/kpf/socal/WaitForSoCalOnTarget.py +++ b/kpf/socal/WaitForSoCalOnTarget.py @@ -30,16 +30,20 @@ def pre_condition(cls, args): def perform(cls, args): timeout = cfg.getfloat('SoCal', 'enclosure_status_time', fallback=10) pyrirrad_threshold = cfg.getfloat('SoCal', 'pyrirrad_threshold', fallback=1000) - expr = '($kpfsocal.ENCSTA == 0) ' - expr += 'and ($kpfsocal.EKOONLINE == Online)' - expr += 'and ($kpfsocal.EKOMODE == 3)' - expr += f'and ($kpfsocal.PYRIRRAD > {pyrirrad_threshold})' - expr += 'and ($kpfsocal.AUTONOMOUS == 1)' - expr += 'and ($kpfsocal.CAN_OPEN == True)' - expr += 'and ($kpfsocal.IS_OPEN == True)' - expr += 'and ($kpfsocal.IS_TRACKING == True)' - expr += 'and ($kpfsocal.ONLINE == True)' - expr += 'and ($kpfsocal.STATE == Tracking)' + # EKO Tracker Status + expr = '($kpfsocal.EKOONLINE == Online) ' + expr += 'and ($kpfsocal.EKOMODE == 3) ' + # Pyrheliometer threshold + expr += f'and ($kpfsocal.PYRIRRAD > {pyrirrad_threshold}) ' + # Enclosure Status +# expr += 'and ($kpfsocal.ENCSTA == 0) ' + # Sequencer Status +# expr += 'and ($kpfsocal.AUTONOMOUS == 1) ' +# expr += 'and ($kpfsocal.CAN_OPEN == True) ' +# expr += 'and ($kpfsocal.IS_OPEN == True) ' +# expr += 'and ($kpfsocal.IS_TRACKING == True) ' +# expr += 'and ($kpfsocal.ONLINE == True) ' +# expr += 'and ($kpfsocal.STATE == Tracking) ' on_target = ktl.waitFor(expr, timeout=timeout) msg = {True: 'On Target', False: 'NOT On Target'}[on_target] print(msg) From 2244403e11e2313c8f42bf287e5df79992ef706b Mon Sep 17 00:00:00 2001 From: Josh Walawender Date: Fri, 27 Mar 2026 10:12:03 -1000 Subject: [PATCH 63/64] Remove HK ThAr setup, was confusing CalOBBuilder --- kpf/ObservingBlocks/exampleOBs/Calibrations.yaml | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/kpf/ObservingBlocks/exampleOBs/Calibrations.yaml b/kpf/ObservingBlocks/exampleOBs/Calibrations.yaml index 75a33554..c1664c03 100644 --- a/kpf/ObservingBlocks/exampleOBs/Calibrations.yaml +++ b/kpf/ObservingBlocks/exampleOBs/Calibrations.yaml @@ -60,21 +60,6 @@ Calibrations: OpenSkyShutter: True TakeSimulCal: True WideFlatPos: 'Blank' -# Th Daily (for CaHK): -- CalSource: 'Th_daily' - Object: 'ThAr_forCaHK' - nExp: 1 - ExpTime: 120 - TriggerCaHK: False - TriggerGreen: True - TriggerRed: True - IntensityMonitor: False - CalND1: 'OD 0.1' - CalND2: 'OD 0.1' - OpenScienceShutter: True - OpenSkyShutter: True - TakeSimulCal: True - WideFlatPos: 'Blank' # U Daily: - CalSource: 'U_daily' Object: 'UNe' From c265528ad718e43167bc717d772e64b78b436f9a Mon Sep 17 00:00:00 2001 From: Josh Walawender Date: Fri, 27 Mar 2026 10:55:41 -1000 Subject: [PATCH 64/64] Pass args to RunOB --- kpf/utils/BuildCalOB.py | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/kpf/utils/BuildCalOB.py b/kpf/utils/BuildCalOB.py index d6230af3..c9013762 100644 --- a/kpf/utils/BuildCalOB.py +++ b/kpf/utils/BuildCalOB.py @@ -106,7 +106,9 @@ def perform(cls, args): print(f"# {cal_string}") duration = EstimateOBDuration.execute({'verbose': True}, OB=OB) if args.get('execute', False): - RunOB.execute({}, OB=OB) + runOBargs = {'waitforscript': args.get('waitforscript', False), + 'scheduled': args.get('scheduled', False)} + RunOB.execute(runOBargs, OB=OB) return OB @@ -117,16 +119,22 @@ def post_condition(cls, args): @classmethod def add_cmdline_args(cls, parser): parser.add_argument('calinputs', nargs='*', - help="Calibrations to take in the form ") + help="Calibrations to take in the form ") parser.add_argument("-v", "-t", "--time", "--estimate", dest="estimate", - default=False, action="store_true", - help="Estimate the execution time for this OB?") + default=False, action="store_true", + help="Estimate the execution time for this OB?") parser.add_argument("-s", "--save", dest="save", type=str, default='', - help="Save resulting OB to the specified file.") + help="Save resulting OB to the specified file.") parser.add_argument("-o", "--overwrite", dest="overwrite", - default=False, action="store_true", - help="Overwrite output file if it exists?") - parser.add_argument("--execute", dest="execute", - default=False, action="store_true", - help="Execute the resulting OB?") + default=False, action="store_true", + help="Overwrite output file if it exists?") + parser.add_argument("-x", "--execute", dest="execute", + default=False, action="store_true", + help="Execute the resulting OB?") + parser.add_argument('-w', '--waitforscript', dest="waitforscript", + default=False, action="store_true", + help='Wait for running script to end before starting?') + parser.add_argument('-a', '--scheduled', dest="scheduled", + default=False, action="store_true", + help='Script is scheduled and should obey ALLOWSCHEDULEDCALS keyword') return super().add_cmdline_args(parser)