diff --git a/dev/ci/gitlab-ci-hosts.yml b/dev/ci/gitlab-ci-hosts.yml index 7fbb74e815d..6542fd72e09 100644 --- a/dev/ci/gitlab-ci-hosts.yml +++ b/dev/ci/gitlab-ci-hosts.yml @@ -248,7 +248,13 @@ finalize_success-ursa: stage: run_tests parallel: matrix: - - CTEST_NAME: ['C48_ATM_gfs_fcst_seg0', 'C48_S2SW_gfs_fcst_seg0'] + - CTEST_NAME: + - C48_ATM-gfs_fcst_seg0 + - C48_ATM-gfs_atmos_prod_f000-f002 + - C48_S2SW-gfs_fcst_seg0 + - C48_S2SW-gfs_ocean_prod_f006 + - C48_S2SW-gfs_ice_prod_f006 + - C48_S2SWA_gefs-gefs_fcst_mem001_seg0 # Host-specific CTest setup jobs setup_ctests-hera: diff --git a/dev/ctests/CMakeLists.txt b/dev/ctests/CMakeLists.txt index 540aad46dbb..35b57e6b807 100644 --- a/dev/ctests/CMakeLists.txt +++ b/dev/ctests/CMakeLists.txt @@ -72,30 +72,30 @@ function(AddJJOBTest) "${novals}" "${singlevals}" "${multivals}" ${ARGN}) - set(TEST_NAME ${ARG_CASE}_${ARG_JOB}) + set(TEST_NAME ${ARG_CASE}-${ARG_JOB}) set(CASE_PATH ${HOMEgfs}/dev/ci/cases/pr) set(CASE_YAML ${CASE_PATH}/${ARG_CASE}.yaml) add_test(NAME test_${TEST_NAME}_setup COMMAND ./setup.sh ${TEST_NAME} ${CASE_YAML} ${ARG_TEST_DATE} WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/scripts) - set_tests_properties(test_${TEST_NAME}_setup PROPERTIES LABELS "${ARG_CASE}_${ARG_JOB}") + set_tests_properties(test_${TEST_NAME}_setup PROPERTIES LABELS "${ARG_CASE}-${ARG_JOB}") add_test(NAME test_${TEST_NAME}_stage COMMAND ./stage.sh ${ARG_CASE} ${TEST_NAME} ${ARG_TEST_DATE} WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/scripts) - set_tests_properties(test_${TEST_NAME}_stage PROPERTIES DEPENDS test_${TEST_NAME}_setup LABELS "${ARG_CASE}_${ARG_JOB}") + set_tests_properties(test_${TEST_NAME}_stage PROPERTIES DEPENDS test_${TEST_NAME}_setup LABELS "${ARG_CASE}-${ARG_JOB}") add_test(NAME test_${TEST_NAME}_execute COMMAND ./execute.sh ${TEST_NAME} ${ARG_JOB} ${ARG_TEST_DATE} WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/scripts) - set_tests_properties(test_${TEST_NAME}_execute PROPERTIES DEPENDS test_${TEST_NAME}_stage LABELS "${ARG_CASE}_${ARG_JOB}") + set_tests_properties(test_${TEST_NAME}_execute PROPERTIES DEPENDS test_${TEST_NAME}_stage LABELS "${ARG_CASE}-${ARG_JOB}") # TODO - This is a stub for the validation step add_test(NAME test_${TEST_NAME}_validate COMMAND ./validate.sh ${ARG_CASE} ${TEST_NAME} ${ARG_TEST_DATE} WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/scripts) - set_tests_properties(test_${TEST_NAME}_validate PROPERTIES DEPENDS test_${TEST_NAME}_execute LABELS "${ARG_CASE}_${ARG_JOB}") + set_tests_properties(test_${TEST_NAME}_validate PROPERTIES DEPENDS test_${TEST_NAME}_execute LABELS "${ARG_CASE}-${ARG_JOB}") endfunction() AddJJOBTest( @@ -104,8 +104,32 @@ AddJJOBTest( TEST_DATE "2021032312" ) +AddJJOBTest( + CASE "C48_ATM" + JOB "gfs_atmos_prod_f000-f002" + TEST_DATE "2021032312" +) + AddJJOBTest( CASE "C48_S2SW" JOB "gfs_fcst_seg0" TEST_DATE "2021032312" ) + +AddJJOBTest( + CASE "C48_S2SW" + JOB "gfs_ice_prod_f006" + TEST_DATE "2021032312" +) + +AddJJOBTest( + CASE "C48_S2SW" + JOB "gfs_ocean_prod_f006" + TEST_DATE "2021032312" +) + +AddJJOBTest( + CASE "C48_S2SWA_gefs" + JOB "gefs_fcst_mem001_seg0" + TEST_DATE "2021032312" +) diff --git a/dev/ctests/README.md b/dev/ctests/README.md index 788903e79bf..95e8ae13872 100644 --- a/dev/ctests/README.md +++ b/dev/ctests/README.md @@ -1,63 +1,263 @@ -# CTest Framework for NOAA Global Workflow +# CTest Framework Quick Start -This directory contains the CTest framework for testing Rocoto JJOBS. The framework allows you to stage, execute, and validate individual JJOBS independently from other jobs in the workflow. Each test requires its own YAML definition of inputs and configurations. +This directory contains the CTest framework for testing Rocoto workflow jobs (JJOBS) independently. Each test runs in an isolated environment with staged input files from nightly baseline runs. -## Overview +> **📖 Complete Documentation**: See the [comprehensive testing documentation](../../docs/source/testing.rst) for detailed information on framework architecture, YAML configuration, test patterns, CI/CD integration, and troubleshooting. -The CTest framework consists of the following scripts: -- **setup.sh.in**: Prepares the environment and creates the experiment. -- **stage.sh.in**: Stages the input files needed to run a JJOB. -- **execute.sh.in**: Executes the JJOB and monitors its status. -- **validate.sh.in**: Validates the results of the JJOB. +## Quick Start Guide -**NOTE:** So far only test C48_ATM *gfs_fcst_set0* has `output_files` for the validation step using a basic chksum for testing. Further development using grib and NETCDF comparison tools is pending. +### Prerequisites -## Usage +The following environment variables must be set (either in your environment or via platform configuration): -### CMake Configuration +```bash +HPC_ACCOUNT # Your HPC allocation account +STAGED_CTESTS # Path to nightly baseline COMROOT +ICSDIR_ROOT # Path to initial condition files +``` -To configure the **CTest** framework using **CMake**, you need to provide several environment variables. Here is an example of how to configure and build the project: +These are typically defined in `$HOMEgfs/dev/ci/platforms/config.$MACHINE_ID`. -**NOTE**: The the specific values for these three enviroment variables can be found in `$HOMEgfs/dev/ci/platforms/config.$MACHINE_ID` and may also be added to the `cmake` command line with the `-D` option +### Configure and Build -# Run CMake to configure the ctest framework -```shell +```bash cd $HOMEgfs/dev/ctests -mkdir build +mkdir -p build cd build -cmake ../.. -``` -### Running Tests with CTest +# Configure using environment variables from config.$MACHINE_ID +cmake ../../.. -Once the project is configured, you can run the tests using CTest. Here are some examples: +# Or configure with explicit command-line options +cmake -DHPC_ACCOUNT=myaccount \ + -DSTAGED_CTESTS=/path/to/baselines/RUNTESTS \ + -DICSDIR_ROOT=/path/to/ics \ + ../../.. +``` -#### Run All Tests +### Run Tests ```bash -cd /path/to/build +# Run all tests ctest + +# Run tests for a specific configuration case +ctest -L C48_ATM + +# Run test for a specific JJOB +cest -L C48_ATM-gfs_atmos_prod_f000-f002 + +# Run a specific test with verbose output +ctest -R test_C48_ATM-gfs_fcst_seg0_execute -V + +# Run tests in parallel (4 concurrent tests) +ctest -j 4 + +# Show test list without running +ctest -N ``` -#### Run Tests for a Specific Case +### Common CTest Options -You can use the `-L` option with CTest to run tests for a specific case. For example, to run tests for the `C48_ATM` case: +| Option | Description | +|--------|-------------| +| `-V` | Verbose output | +| `-VV` | Extra verbose output | +| `-N` | Dry run (list tests without executing) | +| `-L