From d828d17f8f6301f2cd4a56301cfd37dc64522983 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Fri, 27 Mar 2026 08:28:33 +0100 Subject: [PATCH] Update e3sm_update utility for new deployment --- utils/e3sm_update/README.md | 18 ++++--- .../{example.cfg => mpaso_example.cfg} | 6 +-- utils/e3sm_update/omega_example.cfg | 4 +- utils/e3sm_update/test_e3sm_changes.py | 47 +++++++++++++------ 4 files changed, 48 insertions(+), 27 deletions(-) rename utils/e3sm_update/{example.cfg => mpaso_example.cfg} (87%) diff --git a/utils/e3sm_update/README.md b/utils/e3sm_update/README.md index 00c04988b3..53b2a35da8 100644 --- a/utils/e3sm_update/README.md +++ b/utils/e3sm_update/README.md @@ -9,11 +9,15 @@ help of a config file similar to `example.cfg`. ## Instructions -1. Configure the polaris environment and create load scripts with the desiredof - compiler and mpi library, e.g.: +1. Deploy polaris and create the load script for the desired compiler and MPI + library, e.g.: ```shell - ./conda/configure_polaris_env.py --env_name polaris_e3sm_update \ - --compiler intel --mpi impi --conda ~/miniforge3/ + ./deploy.py --machine chrysalis --compiler intel --mpi openmpi + ``` + Then source the generated load script once to confirm the environment is + working, for example: + ```shell + source load_polaris_chrysalis_intel_openmpi.sh ``` 2. Copy `example.cfg` to the base of the branch: @@ -26,10 +30,10 @@ help of a config file similar to `example.cfg`. 4. On a login node, run: ```shell - ./utils/e3sm_update/test_e3sm_update.py -f e3sm_update.cfg + ./utils/e3sm_update/test_e3sm_changes.py -f e3sm_update.cfg ``` - Optionally use the `--submit` flag to submit jobs once each configuration - has been built and set up. + The utility sources the configured load script, sets up each comparison run + and submits the resulting job script automatically. 5. Worktrees will be created for the current and new submodules as well as each relevant E3SM pull request inbetween. A job will be submitted diff --git a/utils/e3sm_update/example.cfg b/utils/e3sm_update/mpaso_example.cfg similarity index 87% rename from utils/e3sm_update/example.cfg rename to utils/e3sm_update/mpaso_example.cfg index e603bfa911..81ca96bd0a 100644 --- a/utils/e3sm_update/example.cfg +++ b/utils/e3sm_update/mpaso_example.cfg @@ -23,7 +23,7 @@ work_base = /lcrc/group/e3sm/ac.xylar/polaris_0.1/chrysalis/test_20230218/update # note: the mpas model, baseline and work directories will be appended # automatically so don't include --clean_build, --build, --quiet_build, # -p, -b or -w flags -setup_command = polaris suite -s -c ocean -t pr -# the absolute or relative path to the load script use to activate the -# polaris environment +setup_command = polaris suite -c ocean -t mpaso_pr --model mpas-ocean +# the absolute or relative path to the deploy-generated load script used to +# activate the polaris environment load_script = load_polaris_e3sm_update_chrysalis_intel_openmpi.sh diff --git a/utils/e3sm_update/omega_example.cfg b/utils/e3sm_update/omega_example.cfg index 022c72a837..91003acaef 100644 --- a/utils/e3sm_update/omega_example.cfg +++ b/utils/e3sm_update/omega_example.cfg @@ -24,6 +24,6 @@ work_base = /lcrc/group/e3sm/ac.xylar/polaris_0.9/chrysalis/test_20260124/omega_ # automatically so don't include --clean_build, --build, --quiet_build, # -p, -b or -w flags setup_command = polaris suite --model omega -c ocean -t omega_pr -# the absolute or relative path to the load script use to activate the -# polaris environment +# the absolute or relative path to the deploy-generated load script used to +# activate the polaris environment load_script = load_polaris_omega_update_chrysalis_intel_openmpi.sh diff --git a/utils/e3sm_update/test_e3sm_changes.py b/utils/e3sm_update/test_e3sm_changes.py index 998e8309e2..6bdbbeab61 100755 --- a/utils/e3sm_update/test_e3sm_changes.py +++ b/utils/e3sm_update/test_e3sm_changes.py @@ -3,6 +3,7 @@ import argparse import configparser import os +import shlex import subprocess from typing import Dict, List @@ -182,31 +183,47 @@ def setup_worktree(submodule, worktree, hash): def setup_and_submit( load_script, setup_command, worktree, workdir, baseline=None ): - if ' -t ' in setup_command: - split = setup_command.split() - index = split.index('-t') - suite = split[index + 1] - elif '--test_suite' in setup_command: - split = setup_command.split() - index = split.index('--test_suite') - suite = split[index + 1] - else: - suite = 'custom' + suite = _get_suite_name(setup_command) full_setup = ( - f'{setup_command} --clean_build --quiet_build --branch {worktree} ' - f'-w {workdir}' + f'{setup_command} --clean_build --quiet_build ' + f'--branch {shlex.quote(worktree)} -w {shlex.quote(workdir)}' ) if baseline is not None: - full_setup = f'{full_setup} -b {baseline}' + full_setup = f'{full_setup} -b {shlex.quote(baseline)}' - commands = f'source {load_script} && {full_setup}' + commands = f'source {shlex.quote(load_script)} && {full_setup}' print_and_run(commands) - commands = f'cd {workdir} && sbatch job_script.{suite}.sh' + commands = ( + f'cd {shlex.quote(workdir)} && ' + f'sbatch {shlex.quote(f"job_script.{suite}.sh")}' + ) print_and_run(commands) +def _get_suite_name(setup_command): + split = shlex.split(setup_command) + + subcommand = split[1] if len(split) > 1 else None + + if subcommand == 'suite': + if '-t' in split: + index = split.index('-t') + return split[index + 1] + + for flag in ('--task_suite', '--test_suite'): + if flag in split: + index = split.index(flag) + return split[index + 1] + + if '--suite_name' in split: + index = split.index('--suite_name') + return split[index + 1] + + return 'custom' + + def print_and_run(commands, get_output=False): print('\nRunning:') print_commands = commands.replace(' && ', '\n ')