From 9beb2d6d8f9c7e3e65edaa589e02c7680c223213 Mon Sep 17 00:00:00 2001 From: Christopher Dilks Date: Wed, 26 Nov 2025 09:20:37 -0500 Subject: [PATCH 01/11] test(ci): use LFS for `validation-data` --- .github/workflows/ci.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 683baee344..c9d1befa8e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -58,6 +58,15 @@ jobs: git clone --depth 1 https://code.jlab.org/hallb/clas12/validation-data cp validation-data/raw/rg-d/clas_018779.evio.00001 . + test_lfs: + runs-on: ubuntu-latest + steps: + - name: download + run: | + git lfs install + git clone --depth 1 https://code.jlab.org/hallb/clas12/validation-data + cp validation-data/raw/rg-d/clas_018779.evio.00001 . + # build ############################################################################# From cd73fa67c3c0720c7e2734160b611491575a0029 Mon Sep 17 00:00:00 2001 From: Christopher Dilks Date: Wed, 26 Nov 2025 09:26:23 -0500 Subject: [PATCH 02/11] ci: steps --- .github/workflows/ci.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c9d1befa8e..8ca87b923a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,11 +61,11 @@ jobs: test_lfs: runs-on: ubuntu-latest steps: - - name: download - run: | - git lfs install - git clone --depth 1 https://code.jlab.org/hallb/clas12/validation-data - cp validation-data/raw/rg-d/clas_018779.evio.00001 . + - run: git lfs install + - name: git clone + run: git clone --depth 1 https://code.jlab.org/hallb/clas12/validation-data + - name: cp + run: cp validation-data/raw/rg-d/clas_018779.evio.00001 . # build ############################################################################# From 721e784107453daeba3f842f3d113d88ea0f7696 Mon Sep 17 00:00:00 2001 From: Christopher Dilks Date: Wed, 26 Nov 2025 09:34:40 -0500 Subject: [PATCH 03/11] test: what do we get? --- .github/workflows/ci.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8ca87b923a..0104b33dc7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -63,7 +63,11 @@ jobs: steps: - run: git lfs install - name: git clone - run: git clone --depth 1 https://code.jlab.org/hallb/clas12/validation-data + run: git clone --depth 1 https://code.jlab.org/hallb/clas12/validation-data || echo "failed" + - run: tree + working-directory: validation-data + - run: git status + working-directory: validation-data - name: cp run: cp validation-data/raw/rg-d/clas_018779.evio.00001 . From f86725d51b86a907e58f7f83ea46db1af09018b0 Mon Sep 17 00:00:00 2001 From: Christopher Dilks Date: Wed, 26 Nov 2025 09:37:29 -0500 Subject: [PATCH 04/11] fix: `.git` --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0104b33dc7..188a559428 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -63,7 +63,7 @@ jobs: steps: - run: git lfs install - name: git clone - run: git clone --depth 1 https://code.jlab.org/hallb/clas12/validation-data || echo "failed" + run: git clone --depth 1 https://code.jlab.org/hallb/clas12/validation-data.git || echo "failed" - run: tree working-directory: validation-data - run: git status From c0c927c2315da2e66232e2f394407e7b4130fcad Mon Sep 17 00:00:00 2001 From: Christopher Dilks Date: Wed, 26 Nov 2025 09:48:23 -0500 Subject: [PATCH 05/11] test: only get `recon/mon` --- .github/workflows/ci.yml | 398 +------------------------------ .github/workflows/validation.yml | 2 +- 2 files changed, 4 insertions(+), 396 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 188a559428..903155ebf7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,407 +31,15 @@ env: jobs: - # download & cache - ############################################################################# - - download_test_data: - runs-on: ubuntu-latest - steps: - - uses: actions/cache@v4 - id: cache - with: - key: raw_test_data # fixed key will always hit; clear cache to trigger cache miss - path: | - clas_018779.evio.00001 - lookup-only: true - - name: install xrootd-client - if: ${{ steps.cache.outputs.cache-hit != 'true' }} - run: | - sudo apt -y update - sudo apt -y install xrootd-client - - name: download - if: ${{ steps.cache.outputs.cache-hit != 'true' }} -# run: | -# xrdcp xroot://sci-xrootd.jlab.org///osgpool/hallb/clas12/validation/raw/rg-d/clas_018779.evio.00001 ./ - run: | - git lfs install - git clone --depth 1 https://code.jlab.org/hallb/clas12/validation-data - cp validation-data/raw/rg-d/clas_018779.evio.00001 . - test_lfs: runs-on: ubuntu-latest + env: + GIT_LFS_SKIP_SMUDGE: 1 # skip automatic LFS checkout steps: - run: git lfs install - name: git clone - run: git clone --depth 1 https://code.jlab.org/hallb/clas12/validation-data.git || echo "failed" + run: git lfs clone --depth 1 -I recon/mon https://code.jlab.org/hallb/clas12/validation-data.git - run: tree working-directory: validation-data - run: git status working-directory: validation-data - - name: cp - run: cp validation-data/raw/rg-d/clas_018779.evio.00001 . - - # build - ############################################################################# - - build: - strategy: - fail-fast: true - matrix: - runner: - - ubuntu-latest - - macos-latest - runs-on: ${{ matrix.runner }} - steps: - - uses: actions/checkout@v6 - - uses: actions/setup-java@v5 - with: - java-version: ${{ env.JAVA_VERSION }} - distribution: ${{ env.java_distribution }} - cache: maven - - name: setup cvmfs - uses: cvmfs-contrib/github-action-cvmfs@v5 - with: - cvmfs_repositories: 'oasis.opensciencegrid.org' - - name: cvmfs - run: ls /cvmfs/oasis.opensciencegrid.org/jlab/hallb/clas12/sw/noarch/data/ccdb/ccdb_latest.sqlite - - name: bump version to tag if tag trigger - if: ${{ github.ref_type == 'tag' }} - run: libexec/version-bump.sh ${{ github.ref_name }} - - name: build - run: | - git lfs install - git submodule update --init ./etc/data/magfield - ./build-coatjava.sh --cvmfs --no-progress -T${{ env.nthreads }} - ./install-clara -b -c ./coatjava ./clara - - name: tar # tarball to preserve permissions - run: | - tar czvf coatjava.tar.gz coatjava - tar czvf clara.tar.gz clara - - uses: actions/upload-artifact@v5 - with: - name: build_${{ matrix.runner }} - retention-days: 1 - path: | - coatjava.tar.gz - clara.tar.gz - - # tests - ############################################################################# - - java_matrix: # make JSON job matrix, to workaround GitHub's lack of `env` var support in job matrix params - runs-on: ubuntu-latest - outputs: - versions: ${{ steps.matrix.outputs.versions }} - steps: - - name: matrix - id: matrix - run: echo versions="[\"${{ env.JAVA_VERSION }}\",\"${{ env.JAVA_VERSION_LATEST }}\"]" | tee -a $GITHUB_OUTPUT - - unit_tests: - needs: - - java_matrix - strategy: - fail-fast: false - matrix: - JAVA_VERSION: ${{ fromJson(needs.java_matrix.outputs.versions) }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - - uses: actions/setup-java@v5 - with: - java-version: ${{ matrix.JAVA_VERSION }} - distribution: ${{ env.java_distribution }} - cache: maven - - uses: cvmfs-contrib/github-action-cvmfs@v5 - with: - cvmfs_repositories: 'oasis.opensciencegrid.org' - - name: unit tests - run: ./build-coatjava.sh --cvmfs --unittests --no-progress -T${{ env.nthreads }} - - name: collect jacoco report - if: ${{ matrix.JAVA_VERSION == env.JAVA_VERSION }} - run: validation/jacoco-aggregate.sh - - name: publish jacoco report - if: ${{ matrix.JAVA_VERSION == env.JAVA_VERSION }} - uses: actions/upload-artifact@v5 - with: - name: jacoco_report - path: publish/ - retention-days: 1 - - spotbugs: - needs: [ build ] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - - name: Set up JDK - uses: actions/setup-java@v5 - with: - java-version: ${{ env.JAVA_VERSION }} - distribution: ${{ env.java_distribution }} - cache: maven - - uses: cvmfs-contrib/github-action-cvmfs@v5 - with: - cvmfs_repositories: 'oasis.opensciencegrid.org' - - uses: actions/download-artifact@v6 - with: - name: build_ubuntu-latest - - name: untar build - run: tar xzvf coatjava.tar.gz - - name: spotbugs - run: ./build-coatjava.sh --spotbugs --nomaps --no-progress - - test_decoder: - needs: [ build, download_test_data ] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - - name: Set up JDK - uses: actions/setup-java@v5 - with: - java-version: ${{ env.JAVA_VERSION }} - distribution: ${{ env.java_distribution }} - cache: maven - - uses: cvmfs-contrib/github-action-cvmfs@v5 - with: - cvmfs_repositories: 'oasis.opensciencegrid.org' - - uses: actions/download-artifact@v6 - with: - name: build_ubuntu-latest - - uses: actions/cache/restore@v4 - with: - key: raw_test_data - path: | - clas_018779.evio.00001 - - name: untar build - run: tar xzvf coatjava.tar.gz - - name: run test - run: | - ls -lhtr - ./coatjava/bin/decoder -n 10000 -o dog.hipo ./clas_018779.evio.00001 - - test_clara: - needs: [ build, download_test_data ] - strategy: - fail-fast: true - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - - name: Set up JDK - uses: actions/setup-java@v5 - with: - java-version: ${{ env.JAVA_VERSION }} - distribution: ${{ env.java_distribution }} - cache: maven - - name: setup cvmfs - uses: cvmfs-contrib/github-action-cvmfs@v5 - with: - cvmfs_repositories: 'oasis.opensciencegrid.org' - - uses: actions/download-artifact@v6 - with: - name: build_ubuntu-latest - - uses: actions/cache/restore@v4 - with: - key: raw_test_data - path: | - clas_018779.evio.00001 - - name: untar build - run: | - tar xzvf clara.tar.gz - - name: run test - run: | - ls -lhtr - ./bin/run-clara -y ./etc/services/rgd-clarode.yml -t 4 -n 500 -c ./clara -o ./tmp ./clas_018779.evio.00001 - ls -lhtr - - test_coatjava: - needs: [ build ] - strategy: - fail-fast: true - matrix: - runner: - - ubuntu-latest - id: - - kpp - - eb-ep - - eb-eg - - eb-epc - - eb-enc - - eb-eftpi - include: - # run all tests on ubuntu - - { id: kpp, cmd: ./run-advanced-tests.sh } - - { id: eb-ep, cmd: ./run-eb-tests.sh -100 electronproton } - - { id: eb-eg, cmd: ./run-eb-tests.sh -100 electrongamma } - - { id: eb-epc, cmd: ./run-eb-tests.sh -100 electronprotonC } - - { id: eb-enc, cmd: ./run-eb-tests.sh -100 electronneutronC } - - { id: eb-eftpi, cmd: ./run-eb-tests.sh -100 electronFTpion } - # run one macos test - - { runner: macos-latest, id: eb-ep, cmd: ./run-eb-tests.sh -100 electronproton } - runs-on: ${{ matrix.runner }} - steps: - - uses: actions/checkout@v6 - - name: Set up JDK - uses: actions/setup-java@v5 - with: - java-version: ${{ env.JAVA_VERSION }} - distribution: ${{ env.java_distribution }} - cache: maven - - uses: actions/download-artifact@v6 - with: - name: build_${{ matrix.runner }} - - uses: cvmfs-contrib/github-action-cvmfs@v5 - with: - cvmfs_repositories: 'oasis.opensciencegrid.org' - - name: untar build - run: | - tar xzvf coatjava.tar.gz - tar xzvf clara.tar.gz - - name: run test - run: | - git lfs install - git submodule update --init validation/advanced-tests/data - cd validation/advanced-tests - echo "COMMAND: ${{ matrix.cmd }}" - ${{ matrix.cmd }} - - test_run-groovy: - needs: [ build ] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - - name: Set up JDK - uses: actions/setup-java@v5 - with: - java-version: ${{ env.JAVA_VERSION }} - distribution: ${{ env.java_distribution }} - cache: maven - - name: setup groovy - uses: wtfjoke/setup-groovy@v3 - with: - groovy-version: ${{ env.groovy_version }} - - uses: actions/download-artifact@v6 - with: - name: build_ubuntu-latest - - name: untar build - run: tar xzvf coatjava.tar.gz - - name: test run-groovy - run: coatjava/bin/run-groovy validation/advanced-tests/test-run-groovy.groovy - - dependency_analysis: - needs: [ build ] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - - name: Set up JDK - uses: actions/setup-java@v5 - with: - java-version: ${{ env.JAVA_VERSION }} - distribution: ${{ env.java_distribution }} - cache: maven - - uses: actions/download-artifact@v6 - with: - name: build_ubuntu-latest - - name: untar build - run: tar xzvf coatjava.tar.gz - - name: print dependency tree - run: libexec/dependency-tree.sh - - name: dependency analysis - run: libexec/dependency-analysis.sh - - # documentation - ############################################################################# - - generate_documentation: - runs-on: ubuntu-latest - needs: [ unit_tests ] - steps: - ### mkdocs - - uses: actions/checkout@v6 - - name: install mkdocs dependencies - run: python -m pip install -r docs/mkdocs/requirements.txt - - name: generate mkdocs - run: docs/mkdocs/generate.sh pages - ### jacoco - - name: download jacoco report artifact - uses: actions/download-artifact@v6 - with: - name: jacoco_report - path: pages/jacoco - ### javadoc - - name: set up JDK - uses: actions/setup-java@v5 - with: - java-version: ${{ env.javadoc_version }} - distribution: ${{ env.java_distribution }} - cache: maven - - name: build coatjava javadocs # javadoc:aggregate output dir cannot be controlled, so assume the latest "standard" path and `mv` it - run: | - libexec/build-javadocs.sh - mv target/reports/apidocs pages/javadoc - ### upload artifacts - - uses: actions/upload-pages-artifact@v4 - with: - retention-days: 7 - path: pages/ - - deploy_web_pages: - if: ${{ github.ref == 'refs/heads/development' }} - needs: [ generate_documentation ] - permissions: - pages: write - id-token: write - environment: - name: github-pages - url: ${{ steps.deployment.outputs.page_url }} - runs-on: ubuntu-latest - steps: - - name: deployment - id: deployment - uses: actions/deploy-pages@v4 - - # finalize - ############################################################################# - - release_build: - if: ${{ github.ref_type == 'tag' }} - needs: [ build ] - runs-on: ubuntu-latest - permissions: - contents: write - issues: write - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - REPO_URL: ${{ github.event.repository.html_url }} - RUN_ID: ${{ github.run_id }} - TAG_NAME: ${{ github.ref_name }} - steps: - - uses: actions/checkout@v6 - - uses: actions/download-artifact@v6 - with: - name: build_ubuntu-latest - - name: rename artifact - run: | - tar xzf coatjava.tar.gz - mv coatjava{,-${{ env.TAG_NAME }}} - tar czf coatjava-${{ env.TAG_NAME }}{.tar.gz,} - - name: release - uses: softprops/action-gh-release@v2 - with: - files: coatjava-${{ env.TAG_NAME }}.tar.gz - - name: open issue if failed - if: ${{ cancelled() || failure() }} - uses: JasonEtco/create-an-issue@v2 - with: - filename: .github/issue_release.md - - final: - needs: - - test_coatjava - - dependency_analysis - - test_run-groovy - - generate_documentation - runs-on: ubuntu-latest - steps: - - name: pass - run: exit 0 diff --git a/.github/workflows/validation.yml b/.github/workflows/validation.yml index 6c42d6d924..9e035dc8d0 100644 --- a/.github/workflows/validation.yml +++ b/.github/workflows/validation.yml @@ -1,7 +1,7 @@ name: Validation on: - pull_request: + # pull_request: push: branches: [ development ] tags: [ '*' ] From 7fb3a7753c9e88113c521d441906524f9f16393e Mon Sep 17 00:00:00 2001 From: Christopher Dilks Date: Wed, 26 Nov 2025 09:53:16 -0500 Subject: [PATCH 06/11] fix: `git-clone-lfs` is deprecated --- .github/workflows/ci.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 903155ebf7..b9e8d4f63f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,7 +38,10 @@ jobs: steps: - run: git lfs install - name: git clone - run: git lfs clone --depth 1 -I recon/mon https://code.jlab.org/hallb/clas12/validation-data.git + run: git clone --depth 1 https://code.jlab.org/hallb/clas12/validation-data.git + - name: git lfs pull + working-directory: validation-data + run: git lfs pull -I recon/mon/validation_files.tar.zst - run: tree working-directory: validation-data - run: git status From 1292937b2a1cd82bd66343881aabc11d2c67bb9d Mon Sep 17 00:00:00 2001 From: Christopher Dilks Date: Wed, 26 Nov 2025 09:58:15 -0500 Subject: [PATCH 07/11] test: pull README --- .github/workflows/ci.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b9e8d4f63f..039da29648 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,9 +39,14 @@ jobs: - run: git lfs install - name: git clone run: git clone --depth 1 https://code.jlab.org/hallb/clas12/validation-data.git - - name: git lfs pull + - run: tree + working-directory: validation-data + - run: git status + working-directory: validation-data + - run: git lfs pull -I README.md + working-directory: validation-data + - run: git lfs pull -I recon/mon/validation_files.tar.zst working-directory: validation-data - run: git lfs pull -I recon/mon/validation_files.tar.zst - run: tree working-directory: validation-data - run: git status From 40d88b07c41753ec0ec10e81a9d67b0e2a79c75a Mon Sep 17 00:00:00 2001 From: Christopher Dilks Date: Wed, 26 Nov 2025 10:05:41 -0500 Subject: [PATCH 08/11] test: what can we pull? --- .github/workflows/ci.yml | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 039da29648..73ea9e7730 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,9 +45,17 @@ jobs: working-directory: validation-data - run: git lfs pull -I README.md working-directory: validation-data - - run: git lfs pull -I recon/mon/validation_files.tar.zst + - name: pull evio working-directory: validation-data - - run: tree - working-directory: validation-data - - run: git status + run: | + for f in $(find raw -type f); do + echo "PULL $f" + git lfs pull -I $f || echo "FAILED TO PULL $f" + done + - name: pull tarballs working-directory: validation-data + run: | + for f in $(find recon -type f -name "*.zst"); do + echo "PULL $f" + git lfs pull -I $f || echo "FAILED TO PULL $f" + done From 71f892ecfa9adc0f929d1c98c59acabccfd71b5f Mon Sep 17 00:00:00 2001 From: Christopher Dilks Date: Wed, 26 Nov 2025 10:09:02 -0500 Subject: [PATCH 09/11] fix: cleanup --- .github/workflows/ci.yml | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 73ea9e7730..fb4ec51aba 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,13 +4,6 @@ name: Coatjava-CI on: pull_request: - push: - branches: [ development ] - tags: [ '*' ] - schedule: - # NOTE: From what I read, the email notification for cron can only go - # to the last committer of this file!!!!! - - cron: '0 22 * * *' concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} @@ -20,15 +13,6 @@ defaults: run: shell: bash -env: - JAVA_VERSION: 21 # must be consistent with POM; see also dependabot config for any version limits - JAVA_VERSION_LATEST: 25 # the latest `JAVA_VERSION` that we test with CI - java_distribution: zulu - javadoc_version: 25 # newer than `JAVA_VERSION` for better javadoc - groovy_version: 4.x - CCDB_CONNECTION: 'sqlite:////cvmfs/oasis.opensciencegrid.org/jlab/hallb/clas12/sw/noarch/data/ccdb/ccdb_latest.sqlite' - nthreads: 1 - jobs: test_lfs: From 4247f92c6684411d9d81f5b0a67bc2570b09c7e6 Mon Sep 17 00:00:00 2001 From: Christopher Dilks Date: Wed, 26 Nov 2025 10:15:34 -0500 Subject: [PATCH 10/11] test: order of operations? --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fb4ec51aba..efacdee68d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,9 +20,10 @@ jobs: env: GIT_LFS_SKIP_SMUDGE: 1 # skip automatic LFS checkout steps: - - run: git lfs install - name: git clone run: git clone --depth 1 https://code.jlab.org/hallb/clas12/validation-data.git + - run: git lfs install + working-directory: validation-data - run: tree working-directory: validation-data - run: git status From ef8aae55b0188141bd546e377c59202797d0fb02 Mon Sep 17 00:00:00 2001 From: Christopher Dilks Date: Wed, 26 Nov 2025 10:37:35 -0500 Subject: [PATCH 11/11] ci: retry