[WIP] Packages Sync

.gitlab-ci.yml

@@ -11,42 +11,36 @@
 # at Lawrence Livermore National Laboratory (LLNL).
 # This entire pipeline is LLNL-specific
 #
-# Important note: This file is a template provided by llnl/radiuss-shared-ci.
-# Remains to set variable values, change the reference to the radiuss-shared-ci
-# repo, opt-in and out optional features. The project can then extend it with
-# additional stages.
+# This file uses GitLab CI Components from radiuss-shared-ci (requires GitLab 17.0+).
+# Components provide better versioning, type-safe inputs, and cleaner syntax.
 #
-# In addition, each project should copy over and complete:
-# - .gitlab/custom-jobs-and-variables.yml
-# - .gitlab/subscribed-pipelines.yml
-#
-# The jobs should be specified in a file local to the project,
-# - .gitlab/jobs/${CI_MACHINE}.yml
-# or generated (see LLNL/Umpire for an example).
+# In addition, each project should have:
+# - .gitlab/custom-jobs.yml - Job templates for child pipelines
+# - .gitlab/custom-variables.yml - Machine-specific variables
+# - .gitlab/jobs/${CI_MACHINE}.yml - Machine-specific job definitions
+###############################################################################
+
+###############################################################################
+# VARIABLES
 ###############################################################################
 
-# We define the following GitLab pipeline variables:
 variables:
-##### LC GITLAB CONFIGURATION
+
+# LC GITLAB CONFIGURATION
 # Use an LLNL service user to run CI. This prevents from running pipelines as
 # an actual user.
   LLNL_SERVICE_USER: ""
 # Use the service user workspace. Solves permission issues, stores everything
 # at the same location whoever triggers a pipeline.
 #  CUSTOM_CI_BUILDS_DIR: ""
-# Submodules: We don't need to fetch dependencies handled by Spack.
+# Optimize submodules usage: only fetch the ones we need.
+# Note: We don't need to fetch dependencies handled by Spack.
   GIT_SUBMODULE_STRATEGY: normal
   GIT_SUBMODULE_DEPTH: 1
   GIT_SUBMODULE_UPDATE_FLAGS: --jobs 2
   GIT_SUBMODULE_PATHS: scripts/radiuss-spack-configs scripts/uberenv
 
-##### PROJECT VARIABLES
-# We build the projects in the CI clone directory. Used in
-# script/gitlab/build-and-test.sh script.
-# TODO: add a clean-up mechanism.
-  BUILD_ROOT: ${CI_PROJECT_DIR}
-
-##### SHARED_CI CONFIGURATION
+# SHARED_CI CONFIGURATION
 # Required information about GitHub repository
   GITHUB_PROJECT_NAME: "Caliper"
   GITHUB_PROJECT_ORG: "LLNL"
@@ -56,41 +50,172 @@ variables:
   JOB_CMD:
     value: "./scripts/gitlab/build-and-test.sh"
     expand: false
-# Override the pattern describing branches that will skip the "draft PR filter
-# test".  Add protected branches here. See default value in
-# preliminary-ignore-draft-pr.yml.
-  ALWAYS_RUN_PATTERN: "^develop$|^master$|^v[0-9.]*$|^releases/$"
 
+###############################################################################
+# MAIN PIPELINE STAGES
+###############################################################################
+# IMPORTANT: You must define stages yourself to allow customization.
+# The following stages are REQUIRED by RADIUSS Shared CI components:
+stages:
+  - prerequisites           # Required: machine availability checks
+  - build-and-test          # Required: build and test jobs
+
+###############################################################################
+# INCLUDES
+###############################################################################
+
+include:
+  # Sets ID tokens for every job using `default:`
+  - project: 'lc-templates/id_tokens'
+    file: 'id_tokens.yml'
+
+  # Base pipeline templates and utilities
+  - component: $CI_SERVER_FQDN/radiuss/radiuss-shared-ci/base-pipeline@woptim/migrate-to-components
+    inputs:
+      github_project_name: $GITHUB_PROJECT_NAME
+      github_project_org: $GITHUB_PROJECT_ORG
+      github_token: $GITHUB_TOKEN
+
+  # Draft PR filter
+  - component: $CI_SERVER_FQDN/radiuss/radiuss-shared-ci/utility-draft-pr-filter@woptim/migrate-to-components
+    inputs:
+      github_token: $GITHUB_TOKEN
+      github_project_name: $GITHUB_PROJECT_NAME
+      github_project_org: $GITHUB_PROJECT_ORG
+      always_run_pattern: "^develop$|^master$|^v[0-9.]*$|^releases/$"
+
+  # Local custom variables (used for component inputs and forwarded to child pipelines)
+  - local: '.gitlab/custom-variables.yml'
+
+###############################################################################
+# MACHINE PIPELINES
+###############################################################################
 # We organize the build-and-test stage with sub-pipelines. Each sub-pipeline
 # corresponds to a test batch on a given machine.
 
-# High level stages
-stages:
-  - prerequisites
-  - build-and-test
+# Note: .machine-check template is provided by the base-pipeline component
+# and includes better error handling, validation, and GitHub status reporting.
+# Trigger a build-and-test pipeline for each machine.
+# Comment out the machine blocks you don't need.
+
+# One job to generate the job list for all the subpipelines
+generate-job-lists:
+  stage: prerequisites
+  tags: [shell, oslic]
+  variables:
+    RADIUSS_JOBS_PATH: "scripts/radiuss-spack-configs/gitlab/radiuss-jobs"
+    LOCAL_JOBS_PATH: ".gitlab/jobs"
+  script:
+    - cat ${RADIUSS_JOBS_PATH}/dane.yml ${LOCAL_JOBS_PATH}/dane.yml > dane-jobs.yml
+    - cat ${RADIUSS_JOBS_PATH}/matrix.yml ${LOCAL_JOBS_PATH}/matrix.yml > matrix-jobs.yml
+    - cat ${RADIUSS_JOBS_PATH}/corona.yml ${LOCAL_JOBS_PATH}/corona.yml > corona-jobs.yml
+    - cat ${RADIUSS_JOBS_PATH}/tioga.yml ${LOCAL_JOBS_PATH}/tioga.yml > tioga-jobs.yml
+    - cat ${RADIUSS_JOBS_PATH}/tuolumne.yml ${LOCAL_JOBS_PATH}/tuolumne.yml > tuolumne-jobs.yml
+  artifacts:
+    paths:
+      - dane-jobs.yml
+      - matrix-jobs.yml
+      - corona-jobs.yml
+      - tioga-jobs.yml
+      - tuolumne-jobs.yml
 
-# Template for jobs triggering a build-and-test sub-pipeline:
-.build-and-test:
-  stage: build-and-test
+# DANE
+dane-up-check:
+  extends: [.dane, .machine-check]
+
+dane-build-and-test:
+  extends: [.dane, .build-and-test]
+  needs: [dane-up-check, generate-job-lists]
   trigger:
     include:
-      - local: '.gitlab/custom-jobs-and-variables.yml'
-      - project: 'radiuss/radiuss-shared-ci'
-        ref: 'v2025.09.0'
-        file: 'pipelines/${CI_MACHINE}.yml'
-      - artifact: '${CI_MACHINE}-jobs.yml'
+      - component: $CI_SERVER_FQDN/radiuss/radiuss-shared-ci/dane-pipeline@woptim/migrate-to-components
+        inputs:
+          job_cmd: $JOB_CMD
+          shared_alloc: $DANE_SHARED_ALLOC
+          job_alloc: $DANE_JOB_ALLOC
+          github_project_name: $GITHUB_PROJECT_NAME
+          github_project_org: $GITHUB_PROJECT_ORG
+      - local: '.gitlab/custom-jobs.yml'
+      - artifact: 'dane-jobs.yml'
+        job: 'generate-job-lists'
+
+# MATRIX
+matrix-up-check:
+  extends: [.matrix, .machine-check]
+
+matrix-build-and-test:
+  extends: [.matrix, .build-and-test]
+  needs: [matrix-up-check, generate-job-lists]
+  trigger:
+    include:
+      - component: $CI_SERVER_FQDN/radiuss/radiuss-shared-ci/matrix-pipeline@woptim/migrate-to-components
+        inputs:
+          job_cmd: $JOB_CMD
+          shared_alloc: $MATRIX_SHARED_ALLOC
+          job_alloc: $MATRIX_JOB_ALLOC
+          github_project_name: $GITHUB_PROJECT_NAME
+          github_project_org: $GITHUB_PROJECT_ORG
+      - local: '.gitlab/custom-jobs.yml'
+      - artifact: 'matrix-jobs.yml'
+        job: 'generate-job-lists'
+
+# CORONA
+corona-up-check:
+  extends: [.corona, .machine-check]
+
+corona-build-and-test:
+  extends: [.corona, .build-and-test]
+  needs: [corona-up-check, generate-job-lists]
+  trigger:
+    include:
+      - component: $CI_SERVER_FQDN/radiuss/radiuss-shared-ci/corona-pipeline@woptim/migrate-to-components
+        inputs:
+          job_cmd: $JOB_CMD
+          shared_alloc: $CORONA_SHARED_ALLOC
+          job_alloc: $CORONA_JOB_ALLOC
+          github_project_name: $GITHUB_PROJECT_NAME
+          github_project_org: $GITHUB_PROJECT_ORG
+      - local: '.gitlab/custom-jobs.yml'
+      - artifact: 'corona-jobs.yml'
+        job: 'generate-job-lists'
+
+# TIOGA
+tioga-up-check:
+  extends: [.tioga, .machine-check]
+
+tioga-build-and-test:
+  extends: [.tioga, .build-and-test]
+  needs: [tioga-up-check, generate-job-lists]
+  trigger:
+    include:
+      - component: $CI_SERVER_FQDN/radiuss/radiuss-shared-ci/tioga-pipeline@woptim/migrate-to-components
+        inputs:
+          job_cmd: $JOB_CMD
+          shared_alloc: $TIOGA_SHARED_ALLOC
+          job_alloc: $TIOGA_JOB_ALLOC
+          github_project_name: $GITHUB_PROJECT_NAME
+          github_project_org: $GITHUB_PROJECT_ORG
+      - local: '.gitlab/custom-jobs.yml'
+      - artifact: 'tioga-jobs.yml'
+        job: 'generate-job-lists'
+
+# TUOLUMNE
+tuolumne-up-check:
+  extends: [.tuolumne, .machine-check]
+
+tuolumne-build-and-test:
+  extends: [.tuolumne, .build-and-test]
+  needs: [tuolumne-up-check, generate-job-lists]
+  trigger:
+    include:
+      - component: $CI_SERVER_FQDN/radiuss/radiuss-shared-ci/tuolumne-pipeline@woptim/migrate-to-components
+        inputs:
+          job_cmd: $JOB_CMD
+          shared_alloc: $TUOLUMNE_SHARED_ALLOC
+          job_alloc: $TUOLUMNE_JOB_ALLOC
+          github_project_name: $GITHUB_PROJECT_NAME
+          github_project_org: $GITHUB_PROJECT_ORG
+      - local: '.gitlab/custom-jobs.yml'
+      - artifact: 'tuolumne-jobs.yml'
         job: 'generate-job-lists'
-    strategy: depend
-    forward:
-      pipeline_variables: true
 
-include:
-  # Sets ID tokens for every job using `default:`
-  - project: 'lc-templates/id_tokens'
-    file: 'id_tokens.yml'
-  # [Optional] checks preliminary to running the actual CI test
-  - project: 'radiuss/radiuss-shared-ci'
-    ref: 'v2025.09.0'
-    file: 'utilities/preliminary-ignore-draft-pr.yml'
-  # pipelines subscribed by the project
-  - local: '.gitlab/subscribed-pipelines.yml'

.gitlab/custom-jobs-and-variables.yml

@@ -22,6 +22,16 @@ variables:
 # Project specific deps for dane
   PROJECT_DANE_DEPS: ""
 
+# Matrix
+# Arguments for top level allocation
+  MATRIX_SHARED_ALLOC: "--exclusive --partition=pci --time=10 --nodes=1"
+# Arguments for job level allocation
+  MATRIX_JOB_ALLOC: "--partition=pci --overlap --nodes=1"
+# Project specific variants for matrix
+  PROJECT_MATRIX_VARIANTS: "+tests ~papi +cuda cuda_arch=75"
+# Project specific deps for matrix
+  PROJECT_MATRIX_DEPS: ""
+
 # Corona
 # Arguments for top level allocation
   CORONA_SHARED_ALLOC: "--exclusive --time-limit=8m --nodes=1 -o per-resource.count=2"

.gitlab/custom-jobs.yml

@@ -0,0 +1,37 @@
+###############################################################################
+# Copyright (c) 2022-25, Lawrence Livermore National Security, LLC and RADIUSS
+# project contributors. See the COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (MIT)
+###############################################################################
+
+# This file defines JOB TEMPLATES ONLY.
+# It is included in CHILD pipelines (via trigger: include:) to provide
+# templates that your jobs can extend.
+
+###############################################################################
+# JOB CUSTOMIZATION TEMPLATES
+###############################################################################
+
+# Configuration shared by build and test jobs specific to this project.
+# Not all configuration can be shared. Here projects can fine tune the
+# CI behavior.
+.custom_job:
+  artifacts:
+    reports:
+      junit: junit.xml
+    name: "${CI_PROJECT_NAME}-${CI_MACHINE}-${CI_JOB_NAME}-${CI_PIPELINE_ID}"
+    paths:
+      - ./*.cmake
+
+.reproducer_vars:
+  script:
+    - |
+      echo -e "
+      # Required variables \n
+      export MODULE_LIST=\"${MODULE_LIST}\" \n
+      export SPEC=\"${SPEC//\"/\\\"}\" \n
+      # Allow to set job script for debugging (only this differs from CI) \n
+      export DEBUG_MODE=true \n
+      # Using the CI build cache is optional and requires a token. Set it like so: \n
+      # export REGISTRY_TOKEN=\"<your token here>\" \n"

.gitlab/custom-variables.yml

@@ -0,0 +1,67 @@
+###############################################################################
+# Copyright (c) 2022-25, Lawrence Livermore National Security, LLC and RADIUSS
+# project contributors. See the COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (MIT)
+###############################################################################
+
+# This file defines project-specific VARIABLES ONLY.
+# It is included in the PARENT pipeline (.gitlab-ci.yml) to provide
+# variables that are passed as component inputs and forwarded to child pipelines.
+
+###############################################################################
+# MACHINE-SPECIFIC ALLOCATION VARIABLES
+###############################################################################
+
+variables:
+  # Allocation name (used for shared allocations)
+  ALLOC_NAME: ${CI_PROJECT_NAME}_ci_${CI_PIPELINE_ID}
+
+  # Dane (SLURM) allocation settings
+  DANE_SHARED_ALLOC: "--exclusive --reservation=ci --time=10 --nodes=1"
+  DANE_JOB_ALLOC: "--reservation=ci --overlap --nodes=1"
+  # Project specific variants for dane
+  PROJECT_DANE_VARIANTS: "+tests ~papi"
+  # Project specific deps for dane
+  PROJECT_DANE_DEPS: ""
+
+  # Matrix (SLURM) allocation settings
+  MATRIX_SHARED_ALLOC: "--exclusive --time=10 --nodes=1"
+  MATRIX_JOB_ALLOC: "--overlap --nodes=1"
+  # Project specific variants for matrix
+  PROJECT_MATRIX_VARIANTS: "+tests ~papi +cuda cuda_arch=75"
+  # Project specific deps for matrix
+  PROJECT_MATRIX_DEPS: ""
+
+  # Corona (flux) allocation settings
+  CORONA_SHARED_ALLOC: "--exclusive --time-limit=8m --nodes=1 -o per-resource.count=2"
+  CORONA_JOB_ALLOC: "--nodes=1 --begin-time=+5s"
+  # Project specific variants for corona
+  PROJECT_CORONA_VARIANTS: "+tests"
+  # Project specific deps for corona
+  PROJECT_CORONA_DEPS: ""
+
+  # Tioga (flux) allocation settings
+  TIOGA_SHARED_ALLOC: "--queue=pci --exclusive --time-limit=15m --nodes=1 -o per-resource.count=2"
+  TIOGA_JOB_ALLOC: "--nodes=1 --begin-time=+5s"
+  # Project specific variants for tioga
+  PROJECT_TIOGA_VARIANTS: "+tests"
+  # Project specific deps for tioga
+  PROJECT_TIOGA_DEPS: ""
+
+  # Tuolumne (flux) allocation settings
+  TUOLUMNE_SHARED_ALLOC: "--queue=pci --exclusive --time-limit=15m --nodes=1 -o per-resource.count=2"
+  TUOLUMNE_JOB_ALLOC: "--nodes=1 --begin-time=+5s"
+  # Project specific variants for tuolumne
+  PROJECT_TUOLUMNE_VARIANTS: "+tests"
+  # Project specific deps for tuolumne
+  PROJECT_TUOLUMNE_DEPS: ""
+
+  # Lassen (LSF) allocation settings
+  # Lassen uses a different job scheduler (spectrum lsf) that does not
+  # allow pre-allocation the same way slurm does.
+  LASSEN_JOB_ALLOC: "1 -W 10 -q pci"
+  # Project specific variants for lassen
+  PROJECT_LASSEN_VARIANTS: " +tests"
+  # Project specific deps for lassen
+  PROJECT_LASSEN_DEPS: "^adiak ~mpi"

.gitlab/jobs/corona.yml

@@ -14,12 +14,10 @@
 variables:
   INCLUDED_FILE_CANNOT_BE_EMPTY: "True"
 
-# The correct way is now to use llvm-amdgpu compiler directly as done in shared spec
-### Template for extra jobs defined by the project.
-#clang14.0.6-rocm5.4.1:
-#  variables:
-#    SPEC: "%llvm@14.0.6 +rocm amdgpu_target=gfx906 ^hip@5.4.1 "
-#  extends: .job_on_corona
+rocmcc_6_4_2_hip:
+  variables:
+    SPEC: "${PROJECT_CORONA_VARIANTS} +rocm amdgpu_target=gfx906 %llvm-amdgpu@=6.4.2 ^hip@6.4.2 ${PROJECT_CORONA_DEPS}"
+  extends: .job_on_corona
 
 master:
   variables: