From da22f1478558970873d7c90d1dc500e48ca55247 Mon Sep 17 00:00:00 2001 From: Max Lindqvist Date: Thu, 28 Aug 2025 06:53:24 +0000 Subject: [PATCH 01/35] Update .gitlab-ci.yml file --- .gitlab-ci.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3c051d103..cf2e3899a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -40,6 +40,18 @@ stages: - pages - release + +connect_toki01: + stage: startup + tags: + - maxlin + - private + - toki01 + script: + - echo "Connected to toki01" + - hostname -f + - whoami + # --- images --- .image_gitlab_mpcdf_struphy: From 5371dee322d568d6e859477230e837cf87668503 Mon Sep 17 00:00:00 2001 From: Max Lindqvist Date: Thu, 28 Aug 2025 08:54:34 +0200 Subject: [PATCH 02/35] Commented out all the normal jobs from the ci --- .gitlab-ci.yml | 2020 ++++++++++++++++++++++++------------------------ 1 file changed, 1010 insertions(+), 1010 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cf2e3899a..078b69348 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -52,1013 +52,1013 @@ connect_toki01: - hostname -f - whoami -# --- images --- - -.image_gitlab_mpcdf_struphy: - image: gitlab-registry.mpcdf.mpg.de/struphy/struphy/mpcdf-gcc-openmpi-with-struphy - -.image_ubuntu_struphy: - image: gitlab-registry.mpcdf.mpg.de/struphy/struphy/ubuntu-latest-with-struphy - -.image_gitlab_mpcdf_gcc_openmp: - image: gitlab-registry.mpcdf.mpg.de/mpcdf/ci-module-image/gcc_14-openmpi_5_0:2025 - -.image_ubuntu_latest: - image: gitlab-registry.mpcdf.mpg.de/struphy/struphy/ubuntu-latest - -# --- job variables --- - -.variables_push: - variables: - LANGUAGE: 'c' - OMP: '' - -# --- rules --- - -# Options for CI_PIPELINE_SOURCE: push, merge_request_event, web, schedule, api -.rules_startup: - rules: - - if: $CI_COMMIT_TAG - when: never - - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_REF_PROTECTED == "false" - when: never - - when: always # this is a fix from https://forum.gitlab.com/t/push-to-gitlab-triggers-one-or-two-pipelines/61070/2 - -.rules_common: - skip_pages_and_scheduled: - - if: $TEST_SCHEDULED == "true" - when: never - - if: $MAKE_PAGES == "true" - when: never - - if: $CI_PIPELINE_SOURCE == "schedule" - when: never - - if: $CI_COMMIT_TAG - when: never - - if: $CI_PIPELINE_SOURCE == "push" - when: never - release: - - if: $TEST_SCHEDULED == "true" - when: never - - if: $CI_PIPELINE_SOURCE == "schedule" - when: never - - if: $CI_COMMIT_TAG - when: never - - if: $CI_COMMIT_REF_NAME == "master" - -.rules_mr_to_devel: - rules: - - !reference [.rules_common, skip_pages_and_scheduled] - - if: $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "devel" && $CI_PIPELINE_SOURCE == "merge_request_event" - -.rules_mr_to_master: - rules: - - !reference [.rules_common, skip_pages_and_scheduled] - - if: $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "master" - when: always - -.rules_scheduled: - rules: - - if: $TEST_SCHEDULED == "true" - - if: $CI_PIPELINE_SOURCE == "schedule" - -.rules_update_images: - rules: - - if: $UPDATE_IMAGES == "true" - -.rules_pypi_release: - rules: - - !reference [.rules_common, release] - -.rules_gitlab_release: - rules: - - !reference [.rules_common, release] - - if: $MAKE_GITLAB_RELEASE == "true" - -.rules_pages_release: - rules: - - !reference [.rules_common, release] - - if: $MAKE_PAGES == "true" - -.rules_cleanup_macos: - rules: - - if: $CLEANUP_MACOS_NMPP == "true" - -# --- parallel matrices --- - -.parallel_matrix_install: - parallel: - matrix: - - LANGUAGE: "fortran" - OMP: ["", "--omp-pic"] - # - LANGUAGE: "c" - # OMP: [""] - -.parallel_matrix_scheduled: - parallel: - matrix: - - LANGUAGE: "fortran" - OMP: [""] #, "--omp-pic"] # TODO: Enable when unit tests are working with --omp-pic! - - LANGUAGE: "c" - OMP: [""] - -# --- before scripts --- - -.before_script_load_modules: - before_script: - - date - - pwd - - ls -1a - - uname -a - - module list - - module avail - # struphy modules - - module load gcc/14 openmpi/5.0 python-waterboa/2024.06 git graphviz/8 - # gvec modules and variables - - module load cmake netcdf-serial mkl hdf5-serial - - export FC=`which gfortran` - - export CC=`which gcc` - - export CXX=`which g++` - # verify - - module list - - echo "Fortran compiler is ${FC}" - - make -v - - python3 --version - -.requirements_fedora: - before_script: - - !reference [.scripts, inspect_directory] - # basic - - dnf install -y wget yum-utils make openssl-devel bzip2-devel libffi-devel zlib-devel - - dnf update -y - # compilers and mpi - - dnf install -y gcc - - dnf install -y gfortran - - dnf install -y blas-devel lapack-devel - - dnf install -y openmpi openmpi-devel - # python - - dnf install -y python3-devel - - dnf install -y python3-mpi4py-openmpi - # gvec - - dnf install -y g++ cmake netcdf netcdf-devel netcdf-fortran netcdf-fortran-devel pkgconf - - export FC=`which gfortran` - - export CC=`which gcc` - - export CXX=`which g++` - # additional - - dnf install -y git - - dnf install -y pandoc - - dnf update -y - - export OMPI_ALLOW_RUN_AS_ROOT=1 - - export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 - # virtual env - - python3 -m venv env - - source env/bin/activate - - . /etc/profile.d/modules.sh - - module load mpi/openmpi-$(arch) - - module list - -.requirements_opensuse: - before_script: - - !reference [.scripts, inspect_directory] - # basic - - zypper refresh - # compilers and mpi - - zypper install -y gcc-fortran gcc - - zypper install -y blas-devel lapack-devel - - zypper install -y openmpi openmpi-devel openmpi4-devel - - zypper install -y libgomp1 - # python - - zypper install -y python3 python3-devel - - zypper install -y python3-pip python3-virtualenv python3-pkgconfig - # gvec - - zypper install -y gcc-c++ cmake netcdf - - zypper addrepo -G https://download.opensuse.org/repositories/science/openSUSE_Tumbleweed/science.repo - - zypper install -y netcdf-fortran-devel - - export FC=`which gfortran` - - export CC=`which gcc` - - export CXX=`which g++` - # additional - - zypper install -y git - - zypper install -y pandoc - - zypper install -y vim - - zypper install -y make - - export OMPI_ALLOW_RUN_AS_ROOT=1 - - export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 - - export OMPI_MCA_pml=ob1 - - export OMPI_MCA_btl=tcp,self - - export PATH="/usr/lib64/mpi/gcc/openmpi4/bin/:$PATH" - - export LD_LIBRARY_PATH="/usr/lib64/mpi/gcc/openmpi4/lib64:$LD_LIBRARY_PATH" - - !reference [.scripts, create_venv] - -.requirements_almalinux: - before_script: - - !reference [.scripts, inspect_directory] - # basic - - yum install -y wget yum-utils make openssl-devel bzip2-devel libffi-devel zlib-devel - - yum update -y - - yum clean all - # compilers and mpi - - yum install -y gcc - - yum install -y gfortran - - yum install -y openmpi openmpi-devel - - yum install -y libgomp - - yum install -y environment-modules - # python - - wget https://www.python.org/ftp/python/3.12.8/Python-3.12.8.tgz - - tar xzf Python-3.12.8.tgz - - cd Python-3.12.8 - - ./configure --with-system-ffi --with-computed-gotos --enable-loadable-sqlite-extensions - - make -j ${nproc} - - make altinstall - - alternatives --install /usr/bin/python3 python3 /usr/local/bin/python3.12 1 - - alternatives --set python3 /usr/local/bin/python3.12 - - mv /usr/local/lib/libpython3.12.a libpython3.12.a.bak - - python3 -V - - cd .. - - pwd - - ls - # additional - - yum install -y git - - export OMPI_ALLOW_RUN_AS_ROOT=1 - - export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 - - export OMPI_MCA_pml=ob1 - - export OMPI_MCA_btl=tcp,self - - export PATH="/usr/lib64/openmpi/bin:$PATH" - # gvec - - yum install -y g++ cmake which flexiblas-devel - - yum install -y epel-release - - yum install -y netcdf-devel netcdf-fortran-devel - - export FC=`which gfortran` - - export CC=`which gcc` - - export CXX=`which g++` - # virtual env - - python3 -m pip list - - python3 -m venv env - - source env/bin/activate - -# --- scripts --- - -.scripts: - inspect_directory: - - date - - pwd - - ls -1a - - uname -a - - ls dist/ || true - create_venv: - - python3 -m venv env_${CI_PIPELINE_ID} - - source env_${CI_PIPELINE_ID}/bin/activate - - python3 --version - - pip install -U pip - - python3 -m pip list - source_env_matrix: - - echo env_${CI_PIPELINE_ID}_${LANGUAGE}_${OMP} - - source env_${CI_PIPELINE_ID}_${LANGUAGE}_${OMP}/bin/activate - - python3 -m pip list - - python3 --version - source_env_matrix_push: - - ls / - - echo struphy_${LANGUAGE}_${OMP} - - cd /struphy_${LANGUAGE}_${OMP} - - source env_${LANGUAGE}_${OMP}/bin/activate - - python3 -m pip list - - python3 --version - create_venv_matrix: - - python3 -m ensurepip --upgrade - - python3 -m pip install --upgrade pip - - echo env_${CI_PIPELINE_ID}_${LANGUAGE}_${OMP} - - python3 -m venv env_${CI_PIPELINE_ID}_${LANGUAGE}_${OMP} - - source env_${CI_PIPELINE_ID}_${LANGUAGE}_${OMP}/bin/activate - - pip install -U pip - - python3 -m pip list - - python3 --version - install_pyccel_devel: - - !reference [.scripts, source_env_matrix_push] - - git status - - git fetch origin - - echo $CI_COMMIT_REF_NAME - - git checkout $CI_COMMIT_REF_NAME - - git pull - - pip uninstall psydac pyccel -y - - echo $STRUPHY_SOURCE - - pip install -e $(echo $STRUPHY_SOURCE)[phys] - - struphy compile -d -y - - !reference [.scripts, compile_matrix] - install_on_push: - - !reference [.scripts, source_env_matrix_push] - - git status - - git fetch origin - - echo $CI_COMMIT_REF_NAME - - git checkout $CI_COMMIT_REF_NAME - - git pull - - echo $STRUPHY_SOURCE - - pip install -e $(echo $STRUPHY_SOURCE)[phys] - - !reference [.scripts, compile_matrix] - install: - - echo $STRUPHY_SOURCE - - pip install $(echo $STRUPHY_SOURCE)[phys] - - pip show struphy - - pip list - - struphy -h - install_base: - - echo $STRUPHY_SOURCE - - pip install $(echo $STRUPHY_SOURCE) - - pip show struphy - - pip list - - struphy -h - install_no_binary: - - echo $STRUPHY_SOURCE - - pip install $(echo $STRUPHY_SOURCE)[phys] --no-binary mpi4py - - pip show struphy - - pip list - - struphy -h - install_base_no_binary: - - echo $STRUPHY_SOURCE - - pip install $(echo $STRUPHY_SOURCE) --no-binary mpi4py - - pip show struphy - - pip list - - struphy -h - install_all: - - pip install -U pip - - echo $STRUPHY_SOURCE - - pip install $(echo $STRUPHY_SOURCE)[all] --no-cache-dir - - pip show struphy - - pip list - - struphy -h - compile: - - pyccel --version - - struphy compile --status - - struphy compile -y - - | - struphy compile -y || ( - echo "Initial compile failed. Removing compiled kernels and trying again..." && - struphy compile -d -y && - struphy compile -y - ) - compile_matrix: - - pyccel --version - - struphy compile --status - - echo ${LANGUAGE} && echo ${OMP} - - | - struphy compile --language ${LANGUAGE} ${OMP} -y || ( - echo "Initial compile failed. Removing compiled kernels and trying again..." && - struphy compile -d -y && - struphy compile --language ${LANGUAGE} ${OMP} -y - ) - build: - - pip install -U build - - python3 -m build - inspect_struphy: - - struphy -p - - struphy -h - - struphy run -h - unit_tests: - - struphy compile --status - - struphy --refresh-models - - struphy test unit - model_tests: - - struphy compile --status - - struphy test models --fast - - struphy test models --fast --verification --mpi 1 - - struphy test models --fast --verification --mpi 4 - - struphy test models --fast --verification --mpi 4 --nclones 2 - - struphy test DriftKineticElectrostaticAdiabatic --mpi 2 --nclones 2 - quickstart_tests: - - struphy --set-i . - - struphy --set-o . - - struphy -p - - struphy -h - - struphy run -h - - struphy params VlasovMaxwellOneSpecies -y - - ls -1a - - mv params_VlasovMaxwellOneSpecies.yml test.yml - - struphy run -i test.yml -o my_first_sim - - ls my_first_sim/ - - struphy pproc my_first_sim - - ls my_first_sim/post_processing/fields_data/ - - ls my_first_sim/post_processing/kinetic_data/ - - struphy params VlasovMaxwellOneSpecies --options - - struphy run -i test.yml -o my_first_sim_1clone --mpi 4 --nclones 1 - - struphy run -i test.yml -o my_first_sim_2clone --mpi 4 --nclones 2 - - struphy run -i test.yml -o my_first_sim_4clone --mpi 4 --nclones 4 - - struphy pproc my_first_sim_1clone my_first_sim_2clone my_first_sim_4clone - build_images: - - buildah images - - buildah build -t gitlab-registry.mpcdf.mpg.de/struphy/struphy/almalinux-latest -f docker/almalinux-latest.dockerfile . - - buildah build -t gitlab-registry.mpcdf.mpg.de/struphy/struphy/fedora-latest -f docker/fedora-latest.dockerfile . - - buildah build -t gitlab-registry.mpcdf.mpg.de/struphy/struphy/mpcdf-gcc-openmpi-with-struphy -f docker/mpcdf-gcc-openmpi-with-struphy.dockerfile . - - buildah build -t gitlab-registry.mpcdf.mpg.de/struphy/struphy/opensuse-latest -f docker/opensuse-latest.dockerfile . - - buildah build -t gitlab-registry.mpcdf.mpg.de/struphy/struphy/ubuntu-latest-with-struphy -f docker/ubuntu-latest-with-struphy.dockerfile . - - buildah build -t gitlab-registry.mpcdf.mpg.de/struphy/struphy/ubuntu-latest -f docker/ubuntu-latest.dockerfile . - push_images: - - buildah push gitlab-registry.mpcdf.mpg.de/struphy/struphy/almalinux-latest - - buildah push gitlab-registry.mpcdf.mpg.de/struphy/struphy/fedora-latest - - buildah push gitlab-registry.mpcdf.mpg.de/struphy/struphy/mpcdf-gcc-openmpi-with-struphy - - buildah push gitlab-registry.mpcdf.mpg.de/struphy/struphy/opensuse-latest - - buildah push gitlab-registry.mpcdf.mpg.de/struphy/struphy/ubuntu-latest-with-struphy - - buildah push gitlab-registry.mpcdf.mpg.de/struphy/struphy/ubuntu-latest - -# --- artifacts --- - -.artifacts_scheduled: - artifacts: - name: 'python-env-installed-${LANGUAGE}-${OMP}' - paths: - - env_${CI_PIPELINE_ID}_${LANGUAGE}_${OMP} - expire_in: 1 day - -##################### -### STARTUP STAGE ### -##################### - -update_images_buildah: - tags: [podman] #necessary to run on the correct runner - stage: build_images - image: quay.io/buildah/stable #stable buildah image should suffice - extends: - - .rules_update_images - variables: - BUILDAH_FORMAT: docker #necessary - BUILDAH_ISOLATION: chroot #necessary - before_script: - - !reference [.scripts, inspect_directory] - - buildah login -u $CI_REGISTRY_USER --password $CI_REGISTRY_PASSWORD $CI_REGISTRY - script: - - !reference [.scripts, build_images] - - !reference [.scripts, push_images] - -# Print predefined variables: https://gitlab.mpcdf.mpg.de/help/ci/variables/predefined_variables.md -ci_vars: - stage: startup - extends: - - .rules_startup - - .image_gitlab_mpcdf_gcc_openmp - - .before_script_load_modules - script: - - echo "All environment variables:" - - env - -inspect_repo: - stage: startup - extends: - - .rules_startup - script: - - curl https://raw.githubusercontent.com/AlDanial/cloc/master/cloc > cloc - - chmod +x cloc - - ./cloc --version - - ./cloc $(git ls-files) - -##################### -### PUSH PIPELINE ### -##################### - -install_tests: - stage: test - extends: - - .rules_mr_to_devel - - .image_gitlab_mpcdf_struphy - - .before_script_load_modules - - .parallel_matrix_install - script: - - !reference [.scripts, install_on_push] - -unit_tests: - stage: test - extends: - - .rules_mr_to_devel - - .image_gitlab_mpcdf_struphy - - .before_script_load_modules - - .variables_push - script: - - !reference [.scripts, install_on_push] - - !reference [.scripts, unit_tests] - -tests_pyccel_devel: - stage: test - extends: - - .rules_scheduled - - .image_gitlab_mpcdf_struphy - - .before_script_load_modules - - .variables_push - script: - - !reference [.scripts, install_pyccel_devel] - - !reference [.scripts, unit_tests] - - !reference [.scripts, model_tests] - -model_tests: - stage: test - extends: - - .rules_mr_to_devel - - .image_gitlab_mpcdf_struphy - - .before_script_load_modules - - .variables_push - script: - - !reference [.scripts, install_on_push] - - !reference [.scripts, model_tests] - -quickstart_tests: - stage: test - extends: - - .rules_mr_to_devel - - .image_gitlab_mpcdf_struphy - - .before_script_load_modules - - .variables_push - script: - - !reference [.scripts, install_on_push] - - !reference [.scripts, quickstart_tests] - -pages_tests: - stage: test - extends: - - .rules_mr_to_devel - - .image_gitlab_mpcdf_struphy - - .before_script_load_modules - - .variables_push - script: - - !reference [.scripts, install_on_push] - - !reference [.scripts, inspect_struphy] - - module load pandoc - - module list - - pip install -e .[doc] - - | - struphy compile -y || ( - echo "Initial compile failed. Removing compiled kernels and trying again..." && - struphy compile -d -y && - struphy compile -y - ) - - cd doc ; make html; cd .. - -ubuntu_latest: - stage: test - extends: - - .rules_mr_to_devel - - .image_ubuntu_struphy - - .variables_push - script: - - !reference [.scripts, inspect_directory] - - !reference [.scripts, install_on_push] - - !reference [.scripts, quickstart_tests] - - !reference [.scripts, model_tests] - -macos_nmpp: - stage: test - tags: [macos, nmpp, private] - needs: [] - variables: - # Job ID changes for each job, so reusing the directory becomes impossible, - # therefore, the fetch strategy also becomes unusable - GIT_CLONE_PATH: $CI_BUILDS_DIR/$CI_PROJECT_NAME/$CI_JOB_ID/$CI_CONCURRENT_PROJECT_ID - _JOB_PATH: $CI_BUILDS_DIR/$CI_PROJECT_NAME/$CI_JOB_ID - extends: - - .rules_mr_to_devel - - .variables_push - before_script: - # - brew install cmake - - brew link --overwrite cmake - - cmake --version - - make -v - - printenv - - system_profiler SPHardwareDataType - - export FC=`which gfortran` # for gvec - - export CC=`which gcc` # for gvec - - export CXX=`which g++` # for gvec - script: - - !reference [.scripts, create_venv] - - !reference [.scripts, install] - - !reference [.scripts, compile] - - !reference [.scripts, quickstart_tests] - - !reference [.scripts, model_tests] - after_script: - - pwd - - ls -a - - echo $_JOB_PATH - - ls -a $_JOB_PATH - - rm -rf $_JOB_PATH - -cleanup_macos: - stage: test - tags: [macos, nmpp, private] - needs: [] - extends: - - .rules_cleanup_macos - before_script: - - make -v - - printenv - - system_profiler SPHardwareDataType - script: - - _STRUPHY_PATH=$CI_BUILDS_DIR/$CI_PROJECT_NAME - - pwd - - ls -a - - echo $_STRUPHY_PATH - - ls -a $_STRUPHY_PATH - - rm -rf $_STRUPHY_PATH/* - - ls -a $_STRUPHY_PATH - -########################## -### SCHEDULED PIPELINE ### -########################## - -install_scheduled: - stage: install - needs: [] - extends: - - .rules_scheduled - - .image_gitlab_mpcdf_gcc_openmp - - .before_script_load_modules - - .parallel_matrix_scheduled - - .artifacts_scheduled - script: - - !reference [.scripts, create_venv_matrix] - - !reference [.scripts, install_no_binary] - - !reference [.scripts, compile_matrix] - -compile_timing: - stage: install - needs: [] - extends: - - .image_gitlab_mpcdf_gcc_openmp - - .before_script_load_modules - - .rules_scheduled - script: - - !reference [.scripts, create_venv] - - !reference [.scripts, install_no_binary] - - pyccel --version - - struphy compile --status - - struphy compile --delete - - struphy compile --time-execution -y --language c > timings_c.txt - - struphy compile --delete - - struphy compile --time-execution -y --language fortran > timings_fortran.txt - - | - cat << 'EOF' > check_pyccel.sh - #!/bin/bash - grep 'pyccel --libdir' timings_c.txt | sed -E 's|.*/site-packages/||' | while read filepath; do - echo "========================================" - echo "Kernel: $filepath" - echo "-------------------- Timers -------------------------" - echo "# Language: C" - grep -A 11 "pyccel.*${filepath}" timings_c.txt | tail -n +3 - echo "# Language: Fortran" - grep -A 13 "pyccel.*${filepath}" timings_fortran.txt | tail -n +3 - done - EOF - - cat check_pyccel.sh - - chmod +x check_pyccel.sh - - ./check_pyccel.sh - -test_build: - stage: test - extends: - - .image_gitlab_mpcdf_gcc_openmp - - .rules_scheduled - - .before_script_load_modules - needs: [] - script: - - !reference [.scripts, create_venv] - - !reference [.scripts, build] - - pip install $(echo dist/struphy*.whl) --no-cache-dir --no-binary mpi4py - -unit_tests_scheduled: - stage: test - extends: - - .image_gitlab_mpcdf_gcc_openmp - - .rules_scheduled - - .parallel_matrix_scheduled - - .before_script_load_modules - script: - - !reference [.scripts, source_env_matrix] - - !reference [.scripts, unit_tests] - -model_tests_scheduled: - stage: test - extends: - - .image_gitlab_mpcdf_gcc_openmp - - .rules_scheduled - - .parallel_matrix_scheduled - - .before_script_load_modules - script: - - !reference [.scripts, source_env_matrix] - - !reference [.scripts, model_tests] - -check_release_dependencies: - stage: test - extends: - - .image_gitlab_mpcdf_gcc_openmp - - .before_script_load_modules - - .rules_mr_to_master - script: - - !reference [.scripts, inspect_directory] - - !reference [.scripts, create_venv] - - !reference [.scripts, install_all] - - python src/struphy/utils/set_release_dependencies.py - - | - if git diff --exit-code pyproject.toml; then - echo "pyproject.toml has not changed, dependencies are set correctly!" - else - echo "pyproject.toml has changed, dependencies are set incorrectly!" - echo "Run python src/struphy/utils/set_release_dependencies.py to update pyproject.toml" - exit 1 - fi - -Ubuntu-latest-from-registry: - stage: test - needs: [] - extends: - - .image_ubuntu_latest - - .rules_scheduled - - .parallel_matrix_scheduled - before_script: - - !reference [.scripts, inspect_directory] - - !reference [.scripts, create_venv] - script: - - !reference [.scripts, install] - - !reference [.scripts, compile_matrix] - - !reference [.scripts, model_tests] - -Fedora-latest: - stage: test - needs: [] - extends: - - .rules_scheduled - - .parallel_matrix_scheduled - - .requirements_fedora - image: fedora:latest - script: - - !reference [.scripts, install] - - !reference [.scripts, compile_matrix] - - !reference [.scripts, model_tests] - -Fedora-latest-from-registry: - stage: test - needs: [] - extends: - - .rules_scheduled - - .parallel_matrix_scheduled - image: gitlab-registry.mpcdf.mpg.de/struphy/struphy/fedora-latest - before_script: - - !reference [.scripts, inspect_directory] - - !reference [.scripts, create_venv] - - . /etc/profile.d/modules.sh - - module load mpi/openmpi-$(arch) - - module list - script: - - !reference [.scripts, install] - - !reference [.scripts, compile_matrix] - - !reference [.scripts, model_tests] - -Fedora-latest-from-registry-base: - stage: test - needs: [] - extends: - - .rules_scheduled - - .parallel_matrix_scheduled - image: gitlab-registry.mpcdf.mpg.de/struphy/struphy/fedora-latest - before_script: - - !reference [.scripts, inspect_directory] - - !reference [.scripts, create_venv] - - . /etc/profile.d/modules.sh - - module load mpi/openmpi-$(arch) - - module list - script: - - !reference [.scripts, install_base] - - !reference [.scripts, compile_matrix] - - !reference [.scripts, model_tests] - -OpenSuse-latest: - stage: test - needs: [] - extends: - - .rules_scheduled - - .parallel_matrix_scheduled - - .requirements_opensuse - image: opensuse/tumbleweed:latest - script: - - !reference [.scripts, install] - - !reference [.scripts, compile_matrix] - - !reference [.scripts, model_tests] - -Opensuse-latest-from-registry: - stage: test - needs: [] - extends: - - .rules_scheduled - - .parallel_matrix_scheduled - image: gitlab-registry.mpcdf.mpg.de/struphy/struphy/opensuse-latest - before_script: - - !reference [.scripts, inspect_directory] - - set -x - - !reference [.scripts, create_venv] - script: - - !reference [.scripts, install] - - !reference [.scripts, compile_matrix] - - !reference [.scripts, model_tests] - -Opensuse-latest-from-registry-base: - stage: test - needs: [] - extends: - - .rules_scheduled - - .parallel_matrix_scheduled - image: gitlab-registry.mpcdf.mpg.de/struphy/struphy/opensuse-latest - before_script: - - !reference [.scripts, inspect_directory] - - set -x - - !reference [.scripts, create_venv] - script: - - !reference [.scripts, install_base] - - !reference [.scripts, compile_matrix] - - !reference [.scripts, model_tests] - -AlmaLinux-latest: - stage: test - needs: [] - extends: - - .rules_scheduled - - .parallel_matrix_scheduled - - .requirements_almalinux - image: almalinux:latest - script: - - !reference [.scripts, install_no_binary] - - !reference [.scripts, compile_matrix] - - !reference [.scripts, model_tests] - -Almalinux-latest-from-registry: - stage: test - needs: [] - extends: - - .rules_scheduled - - .parallel_matrix_scheduled - image: gitlab-registry.mpcdf.mpg.de/struphy/struphy/almalinux-latest - before_script: - - !reference [.scripts, inspect_directory] - - python3 -m ensurepip --upgrade - - python3 -m pip install --upgrade pip - - !reference [.scripts, create_venv] - script: - - !reference [.scripts, install_no_binary] - - !reference [.scripts, compile_matrix] - - !reference [.scripts, model_tests] - -Almalinux-latest-from-registry-base: - stage: test - needs: [] - extends: - - .rules_scheduled - - .parallel_matrix_scheduled - image: gitlab-registry.mpcdf.mpg.de/struphy/struphy/almalinux-latest - before_script: - - !reference [.scripts, inspect_directory] - - python3 -m ensurepip --upgrade - - python3 -m pip install --upgrade pip - - !reference [.scripts, create_venv] - script: - - !reference [.scripts, install_base_no_binary] - - !reference [.scripts, compile_matrix] - - !reference [.scripts, model_tests] - -################## -### LINT STAGE ### -################## - -# Run in weekly CI pipeline only -lint_full_repo_report: - stage: lint - needs: [] - extends: - - .rules_scheduled - - .image_ubuntu_latest - script: - - !reference [.scripts, inspect_directory] - - !reference [.scripts, create_venv] - - pip install -e .[dev] # We have to install struphy in editable mode for struphy lint to work - - !reference [.scripts, inspect_struphy] - - struphy lint all --output-format report - - mkdir -p lint_reports - - mv *.html lint_reports/ 2>/dev/null || echo "No HTML files found." - allow_failure: true - artifacts: - paths: - - lint_reports/ - expire_in: 1 month - -# Lint struphy with `struphy lint` -lint_repo: - stage: lint - needs: [] - extends: - - .rules_mr_to_devel - - .image_ubuntu_latest - script: - - !reference [.scripts, inspect_directory] - - !reference [.scripts, create_venv] - - pip install -e .[dev] # We have to install struphy in editable mode for struphy lint to work - - !reference [.scripts, inspect_struphy] - # Lint all files in current branch, FAIL the CI pipeline if any files are incorrectly formatted - - struphy lint all --output-format plain --verbose - -# Lint struphy with `struphy lint` -lint_branch_report: - stage: lint - needs: [] - extends: - - .rules_mr_to_devel - - .image_ubuntu_latest - script: - - !reference [.scripts, inspect_directory] - - !reference [.scripts, create_venv] - - pip install -e .[dev] # We have to install struphy in editable mode for struphy lint to work - - !reference [.scripts, inspect_struphy] - - struphy lint branch --output-format report - - mkdir -p lint_reports - - mv *.html lint_reports/ 2>/dev/null || echo "No HTML files found." - allow_failure: true - artifacts: - paths: - - lint_reports/ - expire_in: 1 month - -################### -### PAGES STAGE ### -################### -pages: - stage: pages - extends: - - .image_ubuntu_latest - - .rules_pages_release - before_script: - - !reference [.scripts, inspect_directory] - - !reference [.scripts, create_venv] - - apt show pandoc - - apt show graphviz - - echo $CI_COMMIT_MESSAGE - - pip install .[doc] - script: - - !reference [.scripts, inspect_struphy] - - !reference [.scripts, compile] - - cd doc ; make html - - mv _build/html/ ../public/ - artifacts: - name: 'pages' - paths: - - public/ - -##################### -### RELEASE STAGE ### -##################### - -deploy: - stage: release - extends: - - .rules_pypi_release - - .image_gitlab_mpcdf_gcc_openmp - - .before_script_load_modules - script: - - !reference [.scripts, create_venv] - - !reference [.scripts, build] - - pip install -U twine - - twine upload dist/* - -vars: - stage: release - extends: - - .rules_gitlab_release - before_script: - - !reference [.scripts, inspect_directory] - script: - - var=$(> vars.env - artifacts: - name: 'vars' - reports: - dotenv: vars.env - expire_in: 1 day - -release_job: - stage: release - image: registry.gitlab.com/gitlab-org/release-cli:latest - extends: - - .rules_gitlab_release - needs: ['vars'] - before_script: - - !reference [.scripts, inspect_directory] - script: - - cat /etc/*-release - - echo $VERSION - release: # See https://docs.gitlab.com/ee/ci/yaml/#release for available properties - tag_name: 'v$VERSION' # The version is incremented per pipeline. - name: 'v$VERSION' - ref: '$CI_COMMIT_SHA' # The tag is created from the pipeline SHA. - description: 'CHANGELOG.md' - assets: - links: - - name: 'Documentation' - url: 'https://struphy.pages.mpcdf.de/struphy/index.html' - - name: 'PyPI' - url: 'https://pypi.org/project/struphy/' +# # --- images --- + +# .image_gitlab_mpcdf_struphy: +# image: gitlab-registry.mpcdf.mpg.de/struphy/struphy/mpcdf-gcc-openmpi-with-struphy + +# .image_ubuntu_struphy: +# image: gitlab-registry.mpcdf.mpg.de/struphy/struphy/ubuntu-latest-with-struphy + +# .image_gitlab_mpcdf_gcc_openmp: +# image: gitlab-registry.mpcdf.mpg.de/mpcdf/ci-module-image/gcc_14-openmpi_5_0:2025 + +# .image_ubuntu_latest: +# image: gitlab-registry.mpcdf.mpg.de/struphy/struphy/ubuntu-latest + +# # --- job variables --- + +# .variables_push: +# variables: +# LANGUAGE: 'c' +# OMP: '' + +# # --- rules --- + +# # Options for CI_PIPELINE_SOURCE: push, merge_request_event, web, schedule, api +# .rules_startup: +# rules: +# - if: $CI_COMMIT_TAG +# when: never +# - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_REF_PROTECTED == "false" +# when: never +# - when: always # this is a fix from https://forum.gitlab.com/t/push-to-gitlab-triggers-one-or-two-pipelines/61070/2 + +# .rules_common: +# skip_pages_and_scheduled: +# - if: $TEST_SCHEDULED == "true" +# when: never +# - if: $MAKE_PAGES == "true" +# when: never +# - if: $CI_PIPELINE_SOURCE == "schedule" +# when: never +# - if: $CI_COMMIT_TAG +# when: never +# - if: $CI_PIPELINE_SOURCE == "push" +# when: never +# release: +# - if: $TEST_SCHEDULED == "true" +# when: never +# - if: $CI_PIPELINE_SOURCE == "schedule" +# when: never +# - if: $CI_COMMIT_TAG +# when: never +# - if: $CI_COMMIT_REF_NAME == "master" + +# .rules_mr_to_devel: +# rules: +# - !reference [.rules_common, skip_pages_and_scheduled] +# - if: $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "devel" && $CI_PIPELINE_SOURCE == "merge_request_event" + +# .rules_mr_to_master: +# rules: +# - !reference [.rules_common, skip_pages_and_scheduled] +# - if: $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "master" +# when: always + +# .rules_scheduled: +# rules: +# - if: $TEST_SCHEDULED == "true" +# - if: $CI_PIPELINE_SOURCE == "schedule" + +# .rules_update_images: +# rules: +# - if: $UPDATE_IMAGES == "true" + +# .rules_pypi_release: +# rules: +# - !reference [.rules_common, release] + +# .rules_gitlab_release: +# rules: +# - !reference [.rules_common, release] +# - if: $MAKE_GITLAB_RELEASE == "true" + +# .rules_pages_release: +# rules: +# - !reference [.rules_common, release] +# - if: $MAKE_PAGES == "true" + +# .rules_cleanup_macos: +# rules: +# - if: $CLEANUP_MACOS_NMPP == "true" + +# # --- parallel matrices --- + +# .parallel_matrix_install: +# parallel: +# matrix: +# - LANGUAGE: "fortran" +# OMP: ["", "--omp-pic"] +# # - LANGUAGE: "c" +# # OMP: [""] + +# .parallel_matrix_scheduled: +# parallel: +# matrix: +# - LANGUAGE: "fortran" +# OMP: [""] #, "--omp-pic"] # TODO: Enable when unit tests are working with --omp-pic! +# - LANGUAGE: "c" +# OMP: [""] + +# # --- before scripts --- + +# .before_script_load_modules: +# before_script: +# - date +# - pwd +# - ls -1a +# - uname -a +# - module list +# - module avail +# # struphy modules +# - module load gcc/14 openmpi/5.0 python-waterboa/2024.06 git graphviz/8 +# # gvec modules and variables +# - module load cmake netcdf-serial mkl hdf5-serial +# - export FC=`which gfortran` +# - export CC=`which gcc` +# - export CXX=`which g++` +# # verify +# - module list +# - echo "Fortran compiler is ${FC}" +# - make -v +# - python3 --version + +# .requirements_fedora: +# before_script: +# - !reference [.scripts, inspect_directory] +# # basic +# - dnf install -y wget yum-utils make openssl-devel bzip2-devel libffi-devel zlib-devel +# - dnf update -y +# # compilers and mpi +# - dnf install -y gcc +# - dnf install -y gfortran +# - dnf install -y blas-devel lapack-devel +# - dnf install -y openmpi openmpi-devel +# # python +# - dnf install -y python3-devel +# - dnf install -y python3-mpi4py-openmpi +# # gvec +# - dnf install -y g++ cmake netcdf netcdf-devel netcdf-fortran netcdf-fortran-devel pkgconf +# - export FC=`which gfortran` +# - export CC=`which gcc` +# - export CXX=`which g++` +# # additional +# - dnf install -y git +# - dnf install -y pandoc +# - dnf update -y +# - export OMPI_ALLOW_RUN_AS_ROOT=1 +# - export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 +# # virtual env +# - python3 -m venv env +# - source env/bin/activate +# - . /etc/profile.d/modules.sh +# - module load mpi/openmpi-$(arch) +# - module list + +# .requirements_opensuse: +# before_script: +# - !reference [.scripts, inspect_directory] +# # basic +# - zypper refresh +# # compilers and mpi +# - zypper install -y gcc-fortran gcc +# - zypper install -y blas-devel lapack-devel +# - zypper install -y openmpi openmpi-devel openmpi4-devel +# - zypper install -y libgomp1 +# # python +# - zypper install -y python3 python3-devel +# - zypper install -y python3-pip python3-virtualenv python3-pkgconfig +# # gvec +# - zypper install -y gcc-c++ cmake netcdf +# - zypper addrepo -G https://download.opensuse.org/repositories/science/openSUSE_Tumbleweed/science.repo +# - zypper install -y netcdf-fortran-devel +# - export FC=`which gfortran` +# - export CC=`which gcc` +# - export CXX=`which g++` +# # additional +# - zypper install -y git +# - zypper install -y pandoc +# - zypper install -y vim +# - zypper install -y make +# - export OMPI_ALLOW_RUN_AS_ROOT=1 +# - export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 +# - export OMPI_MCA_pml=ob1 +# - export OMPI_MCA_btl=tcp,self +# - export PATH="/usr/lib64/mpi/gcc/openmpi4/bin/:$PATH" +# - export LD_LIBRARY_PATH="/usr/lib64/mpi/gcc/openmpi4/lib64:$LD_LIBRARY_PATH" +# - !reference [.scripts, create_venv] + +# .requirements_almalinux: +# before_script: +# - !reference [.scripts, inspect_directory] +# # basic +# - yum install -y wget yum-utils make openssl-devel bzip2-devel libffi-devel zlib-devel +# - yum update -y +# - yum clean all +# # compilers and mpi +# - yum install -y gcc +# - yum install -y gfortran +# - yum install -y openmpi openmpi-devel +# - yum install -y libgomp +# - yum install -y environment-modules +# # python +# - wget https://www.python.org/ftp/python/3.12.8/Python-3.12.8.tgz +# - tar xzf Python-3.12.8.tgz +# - cd Python-3.12.8 +# - ./configure --with-system-ffi --with-computed-gotos --enable-loadable-sqlite-extensions +# - make -j ${nproc} +# - make altinstall +# - alternatives --install /usr/bin/python3 python3 /usr/local/bin/python3.12 1 +# - alternatives --set python3 /usr/local/bin/python3.12 +# - mv /usr/local/lib/libpython3.12.a libpython3.12.a.bak +# - python3 -V +# - cd .. +# - pwd +# - ls +# # additional +# - yum install -y git +# - export OMPI_ALLOW_RUN_AS_ROOT=1 +# - export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 +# - export OMPI_MCA_pml=ob1 +# - export OMPI_MCA_btl=tcp,self +# - export PATH="/usr/lib64/openmpi/bin:$PATH" +# # gvec +# - yum install -y g++ cmake which flexiblas-devel +# - yum install -y epel-release +# - yum install -y netcdf-devel netcdf-fortran-devel +# - export FC=`which gfortran` +# - export CC=`which gcc` +# - export CXX=`which g++` +# # virtual env +# - python3 -m pip list +# - python3 -m venv env +# - source env/bin/activate + +# # --- scripts --- + +# .scripts: +# inspect_directory: +# - date +# - pwd +# - ls -1a +# - uname -a +# - ls dist/ || true +# create_venv: +# - python3 -m venv env_${CI_PIPELINE_ID} +# - source env_${CI_PIPELINE_ID}/bin/activate +# - python3 --version +# - pip install -U pip +# - python3 -m pip list +# source_env_matrix: +# - echo env_${CI_PIPELINE_ID}_${LANGUAGE}_${OMP} +# - source env_${CI_PIPELINE_ID}_${LANGUAGE}_${OMP}/bin/activate +# - python3 -m pip list +# - python3 --version +# source_env_matrix_push: +# - ls / +# - echo struphy_${LANGUAGE}_${OMP} +# - cd /struphy_${LANGUAGE}_${OMP} +# - source env_${LANGUAGE}_${OMP}/bin/activate +# - python3 -m pip list +# - python3 --version +# create_venv_matrix: +# - python3 -m ensurepip --upgrade +# - python3 -m pip install --upgrade pip +# - echo env_${CI_PIPELINE_ID}_${LANGUAGE}_${OMP} +# - python3 -m venv env_${CI_PIPELINE_ID}_${LANGUAGE}_${OMP} +# - source env_${CI_PIPELINE_ID}_${LANGUAGE}_${OMP}/bin/activate +# - pip install -U pip +# - python3 -m pip list +# - python3 --version +# install_pyccel_devel: +# - !reference [.scripts, source_env_matrix_push] +# - git status +# - git fetch origin +# - echo $CI_COMMIT_REF_NAME +# - git checkout $CI_COMMIT_REF_NAME +# - git pull +# - pip uninstall psydac pyccel -y +# - echo $STRUPHY_SOURCE +# - pip install -e $(echo $STRUPHY_SOURCE)[phys] +# - struphy compile -d -y +# - !reference [.scripts, compile_matrix] +# install_on_push: +# - !reference [.scripts, source_env_matrix_push] +# - git status +# - git fetch origin +# - echo $CI_COMMIT_REF_NAME +# - git checkout $CI_COMMIT_REF_NAME +# - git pull +# - echo $STRUPHY_SOURCE +# - pip install -e $(echo $STRUPHY_SOURCE)[phys] +# - !reference [.scripts, compile_matrix] +# install: +# - echo $STRUPHY_SOURCE +# - pip install $(echo $STRUPHY_SOURCE)[phys] +# - pip show struphy +# - pip list +# - struphy -h +# install_base: +# - echo $STRUPHY_SOURCE +# - pip install $(echo $STRUPHY_SOURCE) +# - pip show struphy +# - pip list +# - struphy -h +# install_no_binary: +# - echo $STRUPHY_SOURCE +# - pip install $(echo $STRUPHY_SOURCE)[phys] --no-binary mpi4py +# - pip show struphy +# - pip list +# - struphy -h +# install_base_no_binary: +# - echo $STRUPHY_SOURCE +# - pip install $(echo $STRUPHY_SOURCE) --no-binary mpi4py +# - pip show struphy +# - pip list +# - struphy -h +# install_all: +# - pip install -U pip +# - echo $STRUPHY_SOURCE +# - pip install $(echo $STRUPHY_SOURCE)[all] --no-cache-dir +# - pip show struphy +# - pip list +# - struphy -h +# compile: +# - pyccel --version +# - struphy compile --status +# - struphy compile -y +# - | +# struphy compile -y || ( +# echo "Initial compile failed. Removing compiled kernels and trying again..." && +# struphy compile -d -y && +# struphy compile -y +# ) +# compile_matrix: +# - pyccel --version +# - struphy compile --status +# - echo ${LANGUAGE} && echo ${OMP} +# - | +# struphy compile --language ${LANGUAGE} ${OMP} -y || ( +# echo "Initial compile failed. Removing compiled kernels and trying again..." && +# struphy compile -d -y && +# struphy compile --language ${LANGUAGE} ${OMP} -y +# ) +# build: +# - pip install -U build +# - python3 -m build +# inspect_struphy: +# - struphy -p +# - struphy -h +# - struphy run -h +# unit_tests: +# - struphy compile --status +# - struphy --refresh-models +# - struphy test unit +# model_tests: +# - struphy compile --status +# - struphy test models --fast +# - struphy test models --fast --verification --mpi 1 +# - struphy test models --fast --verification --mpi 4 +# - struphy test models --fast --verification --mpi 4 --nclones 2 +# - struphy test DriftKineticElectrostaticAdiabatic --mpi 2 --nclones 2 +# quickstart_tests: +# - struphy --set-i . +# - struphy --set-o . +# - struphy -p +# - struphy -h +# - struphy run -h +# - struphy params VlasovMaxwellOneSpecies -y +# - ls -1a +# - mv params_VlasovMaxwellOneSpecies.yml test.yml +# - struphy run -i test.yml -o my_first_sim +# - ls my_first_sim/ +# - struphy pproc my_first_sim +# - ls my_first_sim/post_processing/fields_data/ +# - ls my_first_sim/post_processing/kinetic_data/ +# - struphy params VlasovMaxwellOneSpecies --options +# - struphy run -i test.yml -o my_first_sim_1clone --mpi 4 --nclones 1 +# - struphy run -i test.yml -o my_first_sim_2clone --mpi 4 --nclones 2 +# - struphy run -i test.yml -o my_first_sim_4clone --mpi 4 --nclones 4 +# - struphy pproc my_first_sim_1clone my_first_sim_2clone my_first_sim_4clone +# build_images: +# - buildah images +# - buildah build -t gitlab-registry.mpcdf.mpg.de/struphy/struphy/almalinux-latest -f docker/almalinux-latest.dockerfile . +# - buildah build -t gitlab-registry.mpcdf.mpg.de/struphy/struphy/fedora-latest -f docker/fedora-latest.dockerfile . +# - buildah build -t gitlab-registry.mpcdf.mpg.de/struphy/struphy/mpcdf-gcc-openmpi-with-struphy -f docker/mpcdf-gcc-openmpi-with-struphy.dockerfile . +# - buildah build -t gitlab-registry.mpcdf.mpg.de/struphy/struphy/opensuse-latest -f docker/opensuse-latest.dockerfile . +# - buildah build -t gitlab-registry.mpcdf.mpg.de/struphy/struphy/ubuntu-latest-with-struphy -f docker/ubuntu-latest-with-struphy.dockerfile . +# - buildah build -t gitlab-registry.mpcdf.mpg.de/struphy/struphy/ubuntu-latest -f docker/ubuntu-latest.dockerfile . +# push_images: +# - buildah push gitlab-registry.mpcdf.mpg.de/struphy/struphy/almalinux-latest +# - buildah push gitlab-registry.mpcdf.mpg.de/struphy/struphy/fedora-latest +# - buildah push gitlab-registry.mpcdf.mpg.de/struphy/struphy/mpcdf-gcc-openmpi-with-struphy +# - buildah push gitlab-registry.mpcdf.mpg.de/struphy/struphy/opensuse-latest +# - buildah push gitlab-registry.mpcdf.mpg.de/struphy/struphy/ubuntu-latest-with-struphy +# - buildah push gitlab-registry.mpcdf.mpg.de/struphy/struphy/ubuntu-latest + +# # --- artifacts --- + +# .artifacts_scheduled: +# artifacts: +# name: 'python-env-installed-${LANGUAGE}-${OMP}' +# paths: +# - env_${CI_PIPELINE_ID}_${LANGUAGE}_${OMP} +# expire_in: 1 day + +# ##################### +# ### STARTUP STAGE ### +# ##################### + +# update_images_buildah: +# tags: [podman] #necessary to run on the correct runner +# stage: build_images +# image: quay.io/buildah/stable #stable buildah image should suffice +# extends: +# - .rules_update_images +# variables: +# BUILDAH_FORMAT: docker #necessary +# BUILDAH_ISOLATION: chroot #necessary +# before_script: +# - !reference [.scripts, inspect_directory] +# - buildah login -u $CI_REGISTRY_USER --password $CI_REGISTRY_PASSWORD $CI_REGISTRY +# script: +# - !reference [.scripts, build_images] +# - !reference [.scripts, push_images] + +# # Print predefined variables: https://gitlab.mpcdf.mpg.de/help/ci/variables/predefined_variables.md +# ci_vars: +# stage: startup +# extends: +# - .rules_startup +# - .image_gitlab_mpcdf_gcc_openmp +# - .before_script_load_modules +# script: +# - echo "All environment variables:" +# - env + +# inspect_repo: +# stage: startup +# extends: +# - .rules_startup +# script: +# - curl https://raw.githubusercontent.com/AlDanial/cloc/master/cloc > cloc +# - chmod +x cloc +# - ./cloc --version +# - ./cloc $(git ls-files) + +# ##################### +# ### PUSH PIPELINE ### +# ##################### + +# install_tests: +# stage: test +# extends: +# - .rules_mr_to_devel +# - .image_gitlab_mpcdf_struphy +# - .before_script_load_modules +# - .parallel_matrix_install +# script: +# - !reference [.scripts, install_on_push] + +# unit_tests: +# stage: test +# extends: +# - .rules_mr_to_devel +# - .image_gitlab_mpcdf_struphy +# - .before_script_load_modules +# - .variables_push +# script: +# - !reference [.scripts, install_on_push] +# - !reference [.scripts, unit_tests] + +# tests_pyccel_devel: +# stage: test +# extends: +# - .rules_scheduled +# - .image_gitlab_mpcdf_struphy +# - .before_script_load_modules +# - .variables_push +# script: +# - !reference [.scripts, install_pyccel_devel] +# - !reference [.scripts, unit_tests] +# - !reference [.scripts, model_tests] + +# model_tests: +# stage: test +# extends: +# - .rules_mr_to_devel +# - .image_gitlab_mpcdf_struphy +# - .before_script_load_modules +# - .variables_push +# script: +# - !reference [.scripts, install_on_push] +# - !reference [.scripts, model_tests] + +# quickstart_tests: +# stage: test +# extends: +# - .rules_mr_to_devel +# - .image_gitlab_mpcdf_struphy +# - .before_script_load_modules +# - .variables_push +# script: +# - !reference [.scripts, install_on_push] +# - !reference [.scripts, quickstart_tests] + +# pages_tests: +# stage: test +# extends: +# - .rules_mr_to_devel +# - .image_gitlab_mpcdf_struphy +# - .before_script_load_modules +# - .variables_push +# script: +# - !reference [.scripts, install_on_push] +# - !reference [.scripts, inspect_struphy] +# - module load pandoc +# - module list +# - pip install -e .[doc] +# - | +# struphy compile -y || ( +# echo "Initial compile failed. Removing compiled kernels and trying again..." && +# struphy compile -d -y && +# struphy compile -y +# ) +# - cd doc ; make html; cd .. + +# ubuntu_latest: +# stage: test +# extends: +# - .rules_mr_to_devel +# - .image_ubuntu_struphy +# - .variables_push +# script: +# - !reference [.scripts, inspect_directory] +# - !reference [.scripts, install_on_push] +# - !reference [.scripts, quickstart_tests] +# - !reference [.scripts, model_tests] + +# macos_nmpp: +# stage: test +# tags: [macos, nmpp, private] +# needs: [] +# variables: +# # Job ID changes for each job, so reusing the directory becomes impossible, +# # therefore, the fetch strategy also becomes unusable +# GIT_CLONE_PATH: $CI_BUILDS_DIR/$CI_PROJECT_NAME/$CI_JOB_ID/$CI_CONCURRENT_PROJECT_ID +# _JOB_PATH: $CI_BUILDS_DIR/$CI_PROJECT_NAME/$CI_JOB_ID +# extends: +# - .rules_mr_to_devel +# - .variables_push +# before_script: +# # - brew install cmake +# - brew link --overwrite cmake +# - cmake --version +# - make -v +# - printenv +# - system_profiler SPHardwareDataType +# - export FC=`which gfortran` # for gvec +# - export CC=`which gcc` # for gvec +# - export CXX=`which g++` # for gvec +# script: +# - !reference [.scripts, create_venv] +# - !reference [.scripts, install] +# - !reference [.scripts, compile] +# - !reference [.scripts, quickstart_tests] +# - !reference [.scripts, model_tests] +# after_script: +# - pwd +# - ls -a +# - echo $_JOB_PATH +# - ls -a $_JOB_PATH +# - rm -rf $_JOB_PATH + +# cleanup_macos: +# stage: test +# tags: [macos, nmpp, private] +# needs: [] +# extends: +# - .rules_cleanup_macos +# before_script: +# - make -v +# - printenv +# - system_profiler SPHardwareDataType +# script: +# - _STRUPHY_PATH=$CI_BUILDS_DIR/$CI_PROJECT_NAME +# - pwd +# - ls -a +# - echo $_STRUPHY_PATH +# - ls -a $_STRUPHY_PATH +# - rm -rf $_STRUPHY_PATH/* +# - ls -a $_STRUPHY_PATH + +# ########################## +# ### SCHEDULED PIPELINE ### +# ########################## + +# install_scheduled: +# stage: install +# needs: [] +# extends: +# - .rules_scheduled +# - .image_gitlab_mpcdf_gcc_openmp +# - .before_script_load_modules +# - .parallel_matrix_scheduled +# - .artifacts_scheduled +# script: +# - !reference [.scripts, create_venv_matrix] +# - !reference [.scripts, install_no_binary] +# - !reference [.scripts, compile_matrix] + +# compile_timing: +# stage: install +# needs: [] +# extends: +# - .image_gitlab_mpcdf_gcc_openmp +# - .before_script_load_modules +# - .rules_scheduled +# script: +# - !reference [.scripts, create_venv] +# - !reference [.scripts, install_no_binary] +# - pyccel --version +# - struphy compile --status +# - struphy compile --delete +# - struphy compile --time-execution -y --language c > timings_c.txt +# - struphy compile --delete +# - struphy compile --time-execution -y --language fortran > timings_fortran.txt +# - | +# cat << 'EOF' > check_pyccel.sh +# #!/bin/bash +# grep 'pyccel --libdir' timings_c.txt | sed -E 's|.*/site-packages/||' | while read filepath; do +# echo "========================================" +# echo "Kernel: $filepath" +# echo "-------------------- Timers -------------------------" +# echo "# Language: C" +# grep -A 11 "pyccel.*${filepath}" timings_c.txt | tail -n +3 +# echo "# Language: Fortran" +# grep -A 13 "pyccel.*${filepath}" timings_fortran.txt | tail -n +3 +# done +# EOF +# - cat check_pyccel.sh +# - chmod +x check_pyccel.sh +# - ./check_pyccel.sh + +# test_build: +# stage: test +# extends: +# - .image_gitlab_mpcdf_gcc_openmp +# - .rules_scheduled +# - .before_script_load_modules +# needs: [] +# script: +# - !reference [.scripts, create_venv] +# - !reference [.scripts, build] +# - pip install $(echo dist/struphy*.whl) --no-cache-dir --no-binary mpi4py + +# unit_tests_scheduled: +# stage: test +# extends: +# - .image_gitlab_mpcdf_gcc_openmp +# - .rules_scheduled +# - .parallel_matrix_scheduled +# - .before_script_load_modules +# script: +# - !reference [.scripts, source_env_matrix] +# - !reference [.scripts, unit_tests] + +# model_tests_scheduled: +# stage: test +# extends: +# - .image_gitlab_mpcdf_gcc_openmp +# - .rules_scheduled +# - .parallel_matrix_scheduled +# - .before_script_load_modules +# script: +# - !reference [.scripts, source_env_matrix] +# - !reference [.scripts, model_tests] + +# check_release_dependencies: +# stage: test +# extends: +# - .image_gitlab_mpcdf_gcc_openmp +# - .before_script_load_modules +# - .rules_mr_to_master +# script: +# - !reference [.scripts, inspect_directory] +# - !reference [.scripts, create_venv] +# - !reference [.scripts, install_all] +# - python src/struphy/utils/set_release_dependencies.py +# - | +# if git diff --exit-code pyproject.toml; then +# echo "pyproject.toml has not changed, dependencies are set correctly!" +# else +# echo "pyproject.toml has changed, dependencies are set incorrectly!" +# echo "Run python src/struphy/utils/set_release_dependencies.py to update pyproject.toml" +# exit 1 +# fi + +# Ubuntu-latest-from-registry: +# stage: test +# needs: [] +# extends: +# - .image_ubuntu_latest +# - .rules_scheduled +# - .parallel_matrix_scheduled +# before_script: +# - !reference [.scripts, inspect_directory] +# - !reference [.scripts, create_venv] +# script: +# - !reference [.scripts, install] +# - !reference [.scripts, compile_matrix] +# - !reference [.scripts, model_tests] + +# Fedora-latest: +# stage: test +# needs: [] +# extends: +# - .rules_scheduled +# - .parallel_matrix_scheduled +# - .requirements_fedora +# image: fedora:latest +# script: +# - !reference [.scripts, install] +# - !reference [.scripts, compile_matrix] +# - !reference [.scripts, model_tests] + +# Fedora-latest-from-registry: +# stage: test +# needs: [] +# extends: +# - .rules_scheduled +# - .parallel_matrix_scheduled +# image: gitlab-registry.mpcdf.mpg.de/struphy/struphy/fedora-latest +# before_script: +# - !reference [.scripts, inspect_directory] +# - !reference [.scripts, create_venv] +# - . /etc/profile.d/modules.sh +# - module load mpi/openmpi-$(arch) +# - module list +# script: +# - !reference [.scripts, install] +# - !reference [.scripts, compile_matrix] +# - !reference [.scripts, model_tests] + +# Fedora-latest-from-registry-base: +# stage: test +# needs: [] +# extends: +# - .rules_scheduled +# - .parallel_matrix_scheduled +# image: gitlab-registry.mpcdf.mpg.de/struphy/struphy/fedora-latest +# before_script: +# - !reference [.scripts, inspect_directory] +# - !reference [.scripts, create_venv] +# - . /etc/profile.d/modules.sh +# - module load mpi/openmpi-$(arch) +# - module list +# script: +# - !reference [.scripts, install_base] +# - !reference [.scripts, compile_matrix] +# - !reference [.scripts, model_tests] + +# OpenSuse-latest: +# stage: test +# needs: [] +# extends: +# - .rules_scheduled +# - .parallel_matrix_scheduled +# - .requirements_opensuse +# image: opensuse/tumbleweed:latest +# script: +# - !reference [.scripts, install] +# - !reference [.scripts, compile_matrix] +# - !reference [.scripts, model_tests] + +# Opensuse-latest-from-registry: +# stage: test +# needs: [] +# extends: +# - .rules_scheduled +# - .parallel_matrix_scheduled +# image: gitlab-registry.mpcdf.mpg.de/struphy/struphy/opensuse-latest +# before_script: +# - !reference [.scripts, inspect_directory] +# - set -x +# - !reference [.scripts, create_venv] +# script: +# - !reference [.scripts, install] +# - !reference [.scripts, compile_matrix] +# - !reference [.scripts, model_tests] + +# Opensuse-latest-from-registry-base: +# stage: test +# needs: [] +# extends: +# - .rules_scheduled +# - .parallel_matrix_scheduled +# image: gitlab-registry.mpcdf.mpg.de/struphy/struphy/opensuse-latest +# before_script: +# - !reference [.scripts, inspect_directory] +# - set -x +# - !reference [.scripts, create_venv] +# script: +# - !reference [.scripts, install_base] +# - !reference [.scripts, compile_matrix] +# - !reference [.scripts, model_tests] + +# AlmaLinux-latest: +# stage: test +# needs: [] +# extends: +# - .rules_scheduled +# - .parallel_matrix_scheduled +# - .requirements_almalinux +# image: almalinux:latest +# script: +# - !reference [.scripts, install_no_binary] +# - !reference [.scripts, compile_matrix] +# - !reference [.scripts, model_tests] + +# Almalinux-latest-from-registry: +# stage: test +# needs: [] +# extends: +# - .rules_scheduled +# - .parallel_matrix_scheduled +# image: gitlab-registry.mpcdf.mpg.de/struphy/struphy/almalinux-latest +# before_script: +# - !reference [.scripts, inspect_directory] +# - python3 -m ensurepip --upgrade +# - python3 -m pip install --upgrade pip +# - !reference [.scripts, create_venv] +# script: +# - !reference [.scripts, install_no_binary] +# - !reference [.scripts, compile_matrix] +# - !reference [.scripts, model_tests] + +# Almalinux-latest-from-registry-base: +# stage: test +# needs: [] +# extends: +# - .rules_scheduled +# - .parallel_matrix_scheduled +# image: gitlab-registry.mpcdf.mpg.de/struphy/struphy/almalinux-latest +# before_script: +# - !reference [.scripts, inspect_directory] +# - python3 -m ensurepip --upgrade +# - python3 -m pip install --upgrade pip +# - !reference [.scripts, create_venv] +# script: +# - !reference [.scripts, install_base_no_binary] +# - !reference [.scripts, compile_matrix] +# - !reference [.scripts, model_tests] + +# ################## +# ### LINT STAGE ### +# ################## + +# # Run in weekly CI pipeline only +# lint_full_repo_report: +# stage: lint +# needs: [] +# extends: +# - .rules_scheduled +# - .image_ubuntu_latest +# script: +# - !reference [.scripts, inspect_directory] +# - !reference [.scripts, create_venv] +# - pip install -e .[dev] # We have to install struphy in editable mode for struphy lint to work +# - !reference [.scripts, inspect_struphy] +# - struphy lint all --output-format report +# - mkdir -p lint_reports +# - mv *.html lint_reports/ 2>/dev/null || echo "No HTML files found." +# allow_failure: true +# artifacts: +# paths: +# - lint_reports/ +# expire_in: 1 month + +# # Lint struphy with `struphy lint` +# lint_repo: +# stage: lint +# needs: [] +# extends: +# - .rules_mr_to_devel +# - .image_ubuntu_latest +# script: +# - !reference [.scripts, inspect_directory] +# - !reference [.scripts, create_venv] +# - pip install -e .[dev] # We have to install struphy in editable mode for struphy lint to work +# - !reference [.scripts, inspect_struphy] +# # Lint all files in current branch, FAIL the CI pipeline if any files are incorrectly formatted +# - struphy lint all --output-format plain --verbose + +# # Lint struphy with `struphy lint` +# lint_branch_report: +# stage: lint +# needs: [] +# extends: +# - .rules_mr_to_devel +# - .image_ubuntu_latest +# script: +# - !reference [.scripts, inspect_directory] +# - !reference [.scripts, create_venv] +# - pip install -e .[dev] # We have to install struphy in editable mode for struphy lint to work +# - !reference [.scripts, inspect_struphy] +# - struphy lint branch --output-format report +# - mkdir -p lint_reports +# - mv *.html lint_reports/ 2>/dev/null || echo "No HTML files found." +# allow_failure: true +# artifacts: +# paths: +# - lint_reports/ +# expire_in: 1 month + +# ################### +# ### PAGES STAGE ### +# ################### +# pages: +# stage: pages +# extends: +# - .image_ubuntu_latest +# - .rules_pages_release +# before_script: +# - !reference [.scripts, inspect_directory] +# - !reference [.scripts, create_venv] +# - apt show pandoc +# - apt show graphviz +# - echo $CI_COMMIT_MESSAGE +# - pip install .[doc] +# script: +# - !reference [.scripts, inspect_struphy] +# - !reference [.scripts, compile] +# - cd doc ; make html +# - mv _build/html/ ../public/ +# artifacts: +# name: 'pages' +# paths: +# - public/ + +# ##################### +# ### RELEASE STAGE ### +# ##################### + +# deploy: +# stage: release +# extends: +# - .rules_pypi_release +# - .image_gitlab_mpcdf_gcc_openmp +# - .before_script_load_modules +# script: +# - !reference [.scripts, create_venv] +# - !reference [.scripts, build] +# - pip install -U twine +# - twine upload dist/* + +# vars: +# stage: release +# extends: +# - .rules_gitlab_release +# before_script: +# - !reference [.scripts, inspect_directory] +# script: +# - var=$(> vars.env +# artifacts: +# name: 'vars' +# reports: +# dotenv: vars.env +# expire_in: 1 day + +# release_job: +# stage: release +# image: registry.gitlab.com/gitlab-org/release-cli:latest +# extends: +# - .rules_gitlab_release +# needs: ['vars'] +# before_script: +# - !reference [.scripts, inspect_directory] +# script: +# - cat /etc/*-release +# - echo $VERSION +# release: # See https://docs.gitlab.com/ee/ci/yaml/#release for available properties +# tag_name: 'v$VERSION' # The version is incremented per pipeline. +# name: 'v$VERSION' +# ref: '$CI_COMMIT_SHA' # The tag is created from the pipeline SHA. +# description: 'CHANGELOG.md' +# assets: +# links: +# - name: 'Documentation' +# url: 'https://struphy.pages.mpcdf.de/struphy/index.html' +# - name: 'PyPI' +# url: 'https://pypi.org/project/struphy/' From de6c2cbf3d0eb6d38dcb6d371d2951fdac174694 Mon Sep 17 00:00:00 2001 From: Max Lindqvist Date: Thu, 28 Aug 2025 06:56:20 +0000 Subject: [PATCH 03/35] Update .gitlab-ci.yml file --- .gitlab-ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 078b69348..f038fba0f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -51,6 +51,9 @@ connect_toki01: - echo "Connected to toki01" - hostname -f - whoami + - pwd + - module list + - module avail # # --- images --- From 37d44ba0f3943fb8c1fbf367652a5581f984b72b Mon Sep 17 00:00:00 2001 From: Max Lindqvist Date: Thu, 28 Aug 2025 06:58:47 +0000 Subject: [PATCH 04/35] Update .gitlab-ci.yml file --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f038fba0f..f279d73e3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -54,6 +54,7 @@ connect_toki01: - pwd - module list - module avail + - module load gcc/14 openmpi/5.0 python-waterboa/2025.06 # # --- images --- From 482f4adbaba4d0d00cec8dca73c0894ab459b5c7 Mon Sep 17 00:00:00 2001 From: Max Lindqvist Date: Thu, 28 Aug 2025 07:02:05 +0000 Subject: [PATCH 05/35] Update .gitlab-ci.yml file --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f279d73e3..9e6490527 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -55,6 +55,8 @@ connect_toki01: - module list - module avail - module load gcc/14 openmpi/5.0 python-waterboa/2025.06 + - python -m venv env + - source env/bin/activate # # --- images --- From d841a0ef4efd140d0fd387fb573a019fdfe18a7e Mon Sep 17 00:00:00 2001 From: Max Lindqvist Date: Thu, 28 Aug 2025 07:03:39 +0000 Subject: [PATCH 06/35] Update .gitlab-ci.yml file --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9e6490527..b3bd999bb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -57,6 +57,8 @@ connect_toki01: - module load gcc/14 openmpi/5.0 python-waterboa/2025.06 - python -m venv env - source env/bin/activate + - python -m pip install --upgrade pip + - pip install . # # --- images --- From b78a237840917628bca18af928dcdca82b2135da Mon Sep 17 00:00:00 2001 From: Max Lindqvist Date: Thu, 28 Aug 2025 07:07:25 +0000 Subject: [PATCH 07/35] Update .gitlab-ci.yml file --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b3bd999bb..1d9a34759 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -59,6 +59,7 @@ connect_toki01: - source env/bin/activate - python -m pip install --upgrade pip - pip install . + - struphy test unit # # --- images --- From 8ef810974fe08e830e2bd7ef4d9b55346565e016 Mon Sep 17 00:00:00 2001 From: Max Lindqvist Date: Thu, 28 Aug 2025 07:10:35 +0000 Subject: [PATCH 08/35] Update .gitlab-ci.yml file --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1d9a34759..18440b4d5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -52,6 +52,7 @@ connect_toki01: - hostname -f - whoami - pwd + - module purge - module list - module avail - module load gcc/14 openmpi/5.0 python-waterboa/2025.06 From f603987afa40776646015fcd1d14f244317e2657 Mon Sep 17 00:00:00 2001 From: Max Lindqvist Date: Thu, 28 Aug 2025 07:26:20 +0000 Subject: [PATCH 09/35] Update .gitlab-ci.yml file --- .gitlab-ci.yml | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 18440b4d5..03c796b2f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -41,7 +41,7 @@ stages: - release -connect_toki01: +test_toki01: stage: startup tags: - maxlin @@ -56,6 +56,30 @@ connect_toki01: - module list - module avail - module load gcc/14 openmpi/5.0 python-waterboa/2025.06 + - module list + - python -m venv env + - source env/bin/activate + - python -m pip install --upgrade pip + - pip install . + - struphy test unit + + +connect_toki01: + stage: startup + tags: + - maxlin + - private + - viper01 + script: + - echo "Connected to viper01" + - hostname -f + - whoami + - pwd + - module purge + - module list + - module avail + - module load gcc/14 openmpi/5.0 python-waterboa/2025.06 + - module list - python -m venv env - source env/bin/activate - python -m pip install --upgrade pip From 8d779f88f46196e6f09b703af6f12be8ad0d5aff Mon Sep 17 00:00:00 2001 From: Max Lindqvist Date: Thu, 28 Aug 2025 07:26:57 +0000 Subject: [PATCH 10/35] Update .gitlab-ci.yml file --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 03c796b2f..4f04966fa 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -64,7 +64,7 @@ test_toki01: - struphy test unit -connect_toki01: +test_viper01: stage: startup tags: - maxlin From 5c841f58af645972bdb0a4eb93659134980b2106 Mon Sep 17 00:00:00 2001 From: Max Lindqvist Date: Thu, 28 Aug 2025 12:40:05 +0000 Subject: [PATCH 11/35] Update .gitlab-ci.yml file --- .gitlab-ci.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4f04966fa..eff8280ad 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -86,6 +86,28 @@ test_viper01: - pip install . - struphy test unit +test_raven01: + stage: startup + tags: + - maxlin + - private + - raven01 + script: + - echo "Connected to viper01" + - hostname -f + - whoami + - pwd + - module purge + - module list + - module avail + - module load gcc/14 openmpi/5.0 python-waterboa/2025.06 + - module list + - python -m venv env + - source env/bin/activate + - python -m pip install --upgrade pip + - pip install . + - struphy test unit + # # --- images --- # .image_gitlab_mpcdf_struphy: From d3f8a60b2fb4e11da5bf66c9e47f6625f22b5717 Mon Sep 17 00:00:00 2001 From: Max Lindqvist Date: Fri, 29 Aug 2025 06:32:34 +0000 Subject: [PATCH 12/35] Update .gitlab-ci.yml file --- .gitlab-ci.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index eff8280ad..d60349612 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -108,6 +108,28 @@ test_raven01: - pip install . - struphy test unit +test_pitagora05: + stage: startup + tags: + - maxlin + - private + - pitagora05 + script: + - echo "Connected to viper01" + - hostname -f + - whoami + - pwd + - module purge + - module list + - module avail + - module load gcc/14 openmpi/5.0 python-waterboa/2025.06 + - module list + - python -m venv env + - source env/bin/activate + - python -m pip install --upgrade pip + - pip install . + - struphy test unit + # # --- images --- # .image_gitlab_mpcdf_struphy: From 95d6e2631f3b914de5e06f06d9243ee153158dbc Mon Sep 17 00:00:00 2001 From: Max Lindqvist Date: Fri, 29 Aug 2025 06:34:24 +0000 Subject: [PATCH 13/35] Update .gitlab-ci.yml file --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d60349612..1602d188d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -122,7 +122,7 @@ test_pitagora05: - module purge - module list - module avail - - module load gcc/14 openmpi/5.0 python-waterboa/2025.06 + - module load gcc/12.3.0 openmpi/4.1.6--gcc--12.3.0 python/3.11.7 - module list - python -m venv env - source env/bin/activate From ba081832697388ee62aeec9c2a4409f23d413f04 Mon Sep 17 00:00:00 2001 From: Max Lindqvist Date: Fri, 29 Aug 2025 07:49:25 +0000 Subject: [PATCH 14/35] Update .gitlab-ci.yml file --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1602d188d..409b552d7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -106,6 +106,8 @@ test_raven01: - source env/bin/activate - python -m pip install --upgrade pip - pip install . + - struphy -h + - struphy --refresh-models - struphy test unit test_pitagora05: From fc5fd45e72ae8ca94cbb58cc5868628af08c0f12 Mon Sep 17 00:00:00 2001 From: Max Lindqvist Date: Fri, 29 Aug 2025 07:51:06 +0000 Subject: [PATCH 15/35] Update .gitlab-ci.yml file --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 409b552d7..28e57277a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -106,6 +106,7 @@ test_raven01: - source env/bin/activate - python -m pip install --upgrade pip - pip install . + - python -c "from mpi4py import MPI" - struphy -h - struphy --refresh-models - struphy test unit From 3a29b8b318da8e78f4a168985e01218f83d4a9ea Mon Sep 17 00:00:00 2001 From: Max Lindqvist Date: Fri, 29 Aug 2025 09:55:26 +0200 Subject: [PATCH 16/35] Updated .gitlab-ci.yml --- .gitlab-ci.yml | 340 ++++++++++++++++++++++++------------------------- 1 file changed, 167 insertions(+), 173 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 28e57277a..00a77210a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -49,9 +49,7 @@ test_toki01: - toki01 script: - echo "Connected to toki01" - - hostname -f - - whoami - - pwd + - !reference [.scripts, inspect_directory] - module purge - module list - module avail @@ -72,9 +70,7 @@ test_viper01: - viper01 script: - echo "Connected to viper01" - - hostname -f - - whoami - - pwd + - !reference [.scripts, inspect_directory] - module purge - module list - module avail @@ -93,10 +89,8 @@ test_raven01: - private - raven01 script: - - echo "Connected to viper01" - - hostname -f - - whoami - - pwd + - echo "Connected to raven01" + - !reference [.scripts, inspect_directory] - module purge - module list - module avail @@ -118,10 +112,8 @@ test_pitagora05: - private - pitagora05 script: - - echo "Connected to viper01" - - hostname -f - - whoami - - pwd + - echo "Connected to pitagora05" + - !reference [.scripts, inspect_directory] - module purge - module list - module avail @@ -378,165 +370,167 @@ test_pitagora05: # # --- scripts --- -# .scripts: -# inspect_directory: -# - date -# - pwd -# - ls -1a -# - uname -a -# - ls dist/ || true -# create_venv: -# - python3 -m venv env_${CI_PIPELINE_ID} -# - source env_${CI_PIPELINE_ID}/bin/activate -# - python3 --version -# - pip install -U pip -# - python3 -m pip list -# source_env_matrix: -# - echo env_${CI_PIPELINE_ID}_${LANGUAGE}_${OMP} -# - source env_${CI_PIPELINE_ID}_${LANGUAGE}_${OMP}/bin/activate -# - python3 -m pip list -# - python3 --version -# source_env_matrix_push: -# - ls / -# - echo struphy_${LANGUAGE}_${OMP} -# - cd /struphy_${LANGUAGE}_${OMP} -# - source env_${LANGUAGE}_${OMP}/bin/activate -# - python3 -m pip list -# - python3 --version -# create_venv_matrix: -# - python3 -m ensurepip --upgrade -# - python3 -m pip install --upgrade pip -# - echo env_${CI_PIPELINE_ID}_${LANGUAGE}_${OMP} -# - python3 -m venv env_${CI_PIPELINE_ID}_${LANGUAGE}_${OMP} -# - source env_${CI_PIPELINE_ID}_${LANGUAGE}_${OMP}/bin/activate -# - pip install -U pip -# - python3 -m pip list -# - python3 --version -# install_pyccel_devel: -# - !reference [.scripts, source_env_matrix_push] -# - git status -# - git fetch origin -# - echo $CI_COMMIT_REF_NAME -# - git checkout $CI_COMMIT_REF_NAME -# - git pull -# - pip uninstall psydac pyccel -y -# - echo $STRUPHY_SOURCE -# - pip install -e $(echo $STRUPHY_SOURCE)[phys] -# - struphy compile -d -y -# - !reference [.scripts, compile_matrix] -# install_on_push: -# - !reference [.scripts, source_env_matrix_push] -# - git status -# - git fetch origin -# - echo $CI_COMMIT_REF_NAME -# - git checkout $CI_COMMIT_REF_NAME -# - git pull -# - echo $STRUPHY_SOURCE -# - pip install -e $(echo $STRUPHY_SOURCE)[phys] -# - !reference [.scripts, compile_matrix] -# install: -# - echo $STRUPHY_SOURCE -# - pip install $(echo $STRUPHY_SOURCE)[phys] -# - pip show struphy -# - pip list -# - struphy -h -# install_base: -# - echo $STRUPHY_SOURCE -# - pip install $(echo $STRUPHY_SOURCE) -# - pip show struphy -# - pip list -# - struphy -h -# install_no_binary: -# - echo $STRUPHY_SOURCE -# - pip install $(echo $STRUPHY_SOURCE)[phys] --no-binary mpi4py -# - pip show struphy -# - pip list -# - struphy -h -# install_base_no_binary: -# - echo $STRUPHY_SOURCE -# - pip install $(echo $STRUPHY_SOURCE) --no-binary mpi4py -# - pip show struphy -# - pip list -# - struphy -h -# install_all: -# - pip install -U pip -# - echo $STRUPHY_SOURCE -# - pip install $(echo $STRUPHY_SOURCE)[all] --no-cache-dir -# - pip show struphy -# - pip list -# - struphy -h -# compile: -# - pyccel --version -# - struphy compile --status -# - struphy compile -y -# - | -# struphy compile -y || ( -# echo "Initial compile failed. Removing compiled kernels and trying again..." && -# struphy compile -d -y && -# struphy compile -y -# ) -# compile_matrix: -# - pyccel --version -# - struphy compile --status -# - echo ${LANGUAGE} && echo ${OMP} -# - | -# struphy compile --language ${LANGUAGE} ${OMP} -y || ( -# echo "Initial compile failed. Removing compiled kernels and trying again..." && -# struphy compile -d -y && -# struphy compile --language ${LANGUAGE} ${OMP} -y -# ) -# build: -# - pip install -U build -# - python3 -m build -# inspect_struphy: -# - struphy -p -# - struphy -h -# - struphy run -h -# unit_tests: -# - struphy compile --status -# - struphy --refresh-models -# - struphy test unit -# model_tests: -# - struphy compile --status -# - struphy test models --fast -# - struphy test models --fast --verification --mpi 1 -# - struphy test models --fast --verification --mpi 4 -# - struphy test models --fast --verification --mpi 4 --nclones 2 -# - struphy test DriftKineticElectrostaticAdiabatic --mpi 2 --nclones 2 -# quickstart_tests: -# - struphy --set-i . -# - struphy --set-o . -# - struphy -p -# - struphy -h -# - struphy run -h -# - struphy params VlasovMaxwellOneSpecies -y -# - ls -1a -# - mv params_VlasovMaxwellOneSpecies.yml test.yml -# - struphy run -i test.yml -o my_first_sim -# - ls my_first_sim/ -# - struphy pproc my_first_sim -# - ls my_first_sim/post_processing/fields_data/ -# - ls my_first_sim/post_processing/kinetic_data/ -# - struphy params VlasovMaxwellOneSpecies --options -# - struphy run -i test.yml -o my_first_sim_1clone --mpi 4 --nclones 1 -# - struphy run -i test.yml -o my_first_sim_2clone --mpi 4 --nclones 2 -# - struphy run -i test.yml -o my_first_sim_4clone --mpi 4 --nclones 4 -# - struphy pproc my_first_sim_1clone my_first_sim_2clone my_first_sim_4clone -# build_images: -# - buildah images -# - buildah build -t gitlab-registry.mpcdf.mpg.de/struphy/struphy/almalinux-latest -f docker/almalinux-latest.dockerfile . -# - buildah build -t gitlab-registry.mpcdf.mpg.de/struphy/struphy/fedora-latest -f docker/fedora-latest.dockerfile . -# - buildah build -t gitlab-registry.mpcdf.mpg.de/struphy/struphy/mpcdf-gcc-openmpi-with-struphy -f docker/mpcdf-gcc-openmpi-with-struphy.dockerfile . -# - buildah build -t gitlab-registry.mpcdf.mpg.de/struphy/struphy/opensuse-latest -f docker/opensuse-latest.dockerfile . -# - buildah build -t gitlab-registry.mpcdf.mpg.de/struphy/struphy/ubuntu-latest-with-struphy -f docker/ubuntu-latest-with-struphy.dockerfile . -# - buildah build -t gitlab-registry.mpcdf.mpg.de/struphy/struphy/ubuntu-latest -f docker/ubuntu-latest.dockerfile . -# push_images: -# - buildah push gitlab-registry.mpcdf.mpg.de/struphy/struphy/almalinux-latest -# - buildah push gitlab-registry.mpcdf.mpg.de/struphy/struphy/fedora-latest -# - buildah push gitlab-registry.mpcdf.mpg.de/struphy/struphy/mpcdf-gcc-openmpi-with-struphy -# - buildah push gitlab-registry.mpcdf.mpg.de/struphy/struphy/opensuse-latest -# - buildah push gitlab-registry.mpcdf.mpg.de/struphy/struphy/ubuntu-latest-with-struphy -# - buildah push gitlab-registry.mpcdf.mpg.de/struphy/struphy/ubuntu-latest +.scripts: + inspect_directory: + - hostname -f + - whoami + - date + - pwd + - ls -1a + - uname -a + - ls dist/ || true + create_venv: + - python3 -m venv env_${CI_PIPELINE_ID} + - source env_${CI_PIPELINE_ID}/bin/activate + - python3 --version + - pip install -U pip + - python3 -m pip list + source_env_matrix: + - echo env_${CI_PIPELINE_ID}_${LANGUAGE}_${OMP} + - source env_${CI_PIPELINE_ID}_${LANGUAGE}_${OMP}/bin/activate + - python3 -m pip list + - python3 --version + source_env_matrix_push: + - ls / + - echo struphy_${LANGUAGE}_${OMP} + - cd /struphy_${LANGUAGE}_${OMP} + - source env_${LANGUAGE}_${OMP}/bin/activate + - python3 -m pip list + - python3 --version + create_venv_matrix: + - python3 -m ensurepip --upgrade + - python3 -m pip install --upgrade pip + - echo env_${CI_PIPELINE_ID}_${LANGUAGE}_${OMP} + - python3 -m venv env_${CI_PIPELINE_ID}_${LANGUAGE}_${OMP} + - source env_${CI_PIPELINE_ID}_${LANGUAGE}_${OMP}/bin/activate + - pip install -U pip + - python3 -m pip list + - python3 --version + install_pyccel_devel: + - !reference [.scripts, source_env_matrix_push] + - git status + - git fetch origin + - echo $CI_COMMIT_REF_NAME + - git checkout $CI_COMMIT_REF_NAME + - git pull + - pip uninstall psydac pyccel -y + - echo $STRUPHY_SOURCE + - pip install -e $(echo $STRUPHY_SOURCE)[phys] + - struphy compile -d -y + - !reference [.scripts, compile_matrix] + install_on_push: + - !reference [.scripts, source_env_matrix_push] + - git status + - git fetch origin + - echo $CI_COMMIT_REF_NAME + - git checkout $CI_COMMIT_REF_NAME + - git pull + - echo $STRUPHY_SOURCE + - pip install -e $(echo $STRUPHY_SOURCE)[phys] + - !reference [.scripts, compile_matrix] + install: + - echo $STRUPHY_SOURCE + - pip install $(echo $STRUPHY_SOURCE)[phys] + - pip show struphy + - pip list + - struphy -h + install_base: + - echo $STRUPHY_SOURCE + - pip install $(echo $STRUPHY_SOURCE) + - pip show struphy + - pip list + - struphy -h + install_no_binary: + - echo $STRUPHY_SOURCE + - pip install $(echo $STRUPHY_SOURCE)[phys] --no-binary mpi4py + - pip show struphy + - pip list + - struphy -h + install_base_no_binary: + - echo $STRUPHY_SOURCE + - pip install $(echo $STRUPHY_SOURCE) --no-binary mpi4py + - pip show struphy + - pip list + - struphy -h + install_all: + - pip install -U pip + - echo $STRUPHY_SOURCE + - pip install $(echo $STRUPHY_SOURCE)[all] --no-cache-dir + - pip show struphy + - pip list + - struphy -h + compile: + - pyccel --version + - struphy compile --status + - struphy compile -y + - | + struphy compile -y || ( + echo "Initial compile failed. Removing compiled kernels and trying again..." && + struphy compile -d -y && + struphy compile -y + ) + compile_matrix: + - pyccel --version + - struphy compile --status + - echo ${LANGUAGE} && echo ${OMP} + - | + struphy compile --language ${LANGUAGE} ${OMP} -y || ( + echo "Initial compile failed. Removing compiled kernels and trying again..." && + struphy compile -d -y && + struphy compile --language ${LANGUAGE} ${OMP} -y + ) + build: + - pip install -U build + - python3 -m build + inspect_struphy: + - struphy -p + - struphy -h + - struphy run -h + unit_tests: + - struphy compile --status + - struphy --refresh-models + - struphy test unit + model_tests: + - struphy compile --status + - struphy test models --fast + - struphy test models --fast --verification --mpi 1 + - struphy test models --fast --verification --mpi 4 + - struphy test models --fast --verification --mpi 4 --nclones 2 + - struphy test DriftKineticElectrostaticAdiabatic --mpi 2 --nclones 2 + quickstart_tests: + - struphy --set-i . + - struphy --set-o . + - struphy -p + - struphy -h + - struphy run -h + - struphy params VlasovMaxwellOneSpecies -y + - ls -1a + - mv params_VlasovMaxwellOneSpecies.yml test.yml + - struphy run -i test.yml -o my_first_sim + - ls my_first_sim/ + - struphy pproc my_first_sim + - ls my_first_sim/post_processing/fields_data/ + - ls my_first_sim/post_processing/kinetic_data/ + - struphy params VlasovMaxwellOneSpecies --options + - struphy run -i test.yml -o my_first_sim_1clone --mpi 4 --nclones 1 + - struphy run -i test.yml -o my_first_sim_2clone --mpi 4 --nclones 2 + - struphy run -i test.yml -o my_first_sim_4clone --mpi 4 --nclones 4 + - struphy pproc my_first_sim_1clone my_first_sim_2clone my_first_sim_4clone + build_images: + - buildah images + - buildah build -t gitlab-registry.mpcdf.mpg.de/struphy/struphy/almalinux-latest -f docker/almalinux-latest.dockerfile . + - buildah build -t gitlab-registry.mpcdf.mpg.de/struphy/struphy/fedora-latest -f docker/fedora-latest.dockerfile . + - buildah build -t gitlab-registry.mpcdf.mpg.de/struphy/struphy/mpcdf-gcc-openmpi-with-struphy -f docker/mpcdf-gcc-openmpi-with-struphy.dockerfile . + - buildah build -t gitlab-registry.mpcdf.mpg.de/struphy/struphy/opensuse-latest -f docker/opensuse-latest.dockerfile . + - buildah build -t gitlab-registry.mpcdf.mpg.de/struphy/struphy/ubuntu-latest-with-struphy -f docker/ubuntu-latest-with-struphy.dockerfile . + - buildah build -t gitlab-registry.mpcdf.mpg.de/struphy/struphy/ubuntu-latest -f docker/ubuntu-latest.dockerfile . + push_images: + - buildah push gitlab-registry.mpcdf.mpg.de/struphy/struphy/almalinux-latest + - buildah push gitlab-registry.mpcdf.mpg.de/struphy/struphy/fedora-latest + - buildah push gitlab-registry.mpcdf.mpg.de/struphy/struphy/mpcdf-gcc-openmpi-with-struphy + - buildah push gitlab-registry.mpcdf.mpg.de/struphy/struphy/opensuse-latest + - buildah push gitlab-registry.mpcdf.mpg.de/struphy/struphy/ubuntu-latest-with-struphy + - buildah push gitlab-registry.mpcdf.mpg.de/struphy/struphy/ubuntu-latest # # --- artifacts --- From 5d614c9e0d4a8b32cf68e66f03c8115dae93b42a Mon Sep 17 00:00:00 2001 From: Max Lindqvist Date: Fri, 29 Aug 2025 10:00:10 +0200 Subject: [PATCH 17/35] Updated .gitlab-ci.yml --- .gitlab-ci.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 00a77210a..6cccd05f7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -57,6 +57,7 @@ test_toki01: - module list - python -m venv env - source env/bin/activate + - pip install --upgrade pip - python -m pip install --upgrade pip - pip install . - struphy test unit @@ -121,6 +122,7 @@ test_pitagora05: - module list - python -m venv env - source env/bin/activate + - pip install --upgrade pip - python -m pip install --upgrade pip - pip install . - struphy test unit @@ -371,6 +373,15 @@ test_pitagora05: # # --- scripts --- .scripts: + install_struphy_clusters: + - python -m venv env + - source env/bin/activate + - python -m pip install --upgrade pip + - pip install . + - python -c "from mpi4py import MPI" + - struphy -h + - struphy --refresh-models + - struphy test unit inspect_directory: - hostname -f - whoami From bdd8c0ff3d5ff88b25636037cc5c2f979d2f3f6d Mon Sep 17 00:00:00 2001 From: Max Lindqvist Date: Fri, 29 Aug 2025 11:28:52 +0200 Subject: [PATCH 18/35] Updated .gitlab-ci.yml --- .gitlab-ci.yml | 31 ++++++------------------------- 1 file changed, 6 insertions(+), 25 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6cccd05f7..3333a160c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -55,12 +55,7 @@ test_toki01: - module avail - module load gcc/14 openmpi/5.0 python-waterboa/2025.06 - module list - - python -m venv env - - source env/bin/activate - - pip install --upgrade pip - - python -m pip install --upgrade pip - - pip install . - - struphy test unit + - !reference [.scripts, install_struphy_clusters] test_viper01: @@ -77,11 +72,7 @@ test_viper01: - module avail - module load gcc/14 openmpi/5.0 python-waterboa/2025.06 - module list - - python -m venv env - - source env/bin/activate - - python -m pip install --upgrade pip - - pip install . - - struphy test unit + - !reference [.scripts, install_struphy_clusters] test_raven01: stage: startup @@ -97,14 +88,7 @@ test_raven01: - module avail - module load gcc/14 openmpi/5.0 python-waterboa/2025.06 - module list - - python -m venv env - - source env/bin/activate - - python -m pip install --upgrade pip - - pip install . - - python -c "from mpi4py import MPI" - - struphy -h - - struphy --refresh-models - - struphy test unit + - !reference [.scripts, install_struphy_clusters] test_pitagora05: stage: startup @@ -120,12 +104,8 @@ test_pitagora05: - module avail - module load gcc/12.3.0 openmpi/4.1.6--gcc--12.3.0 python/3.11.7 - module list - - python -m venv env - - source env/bin/activate - - pip install --upgrade pip - - python -m pip install --upgrade pip - - pip install . - - struphy test unit + - !reference [.scripts, install_struphy_clusters] + # # --- images --- @@ -377,6 +357,7 @@ test_pitagora05: - python -m venv env - source env/bin/activate - python -m pip install --upgrade pip + - pip install --no-binary=mpi4py mpi4py - pip install . - python -c "from mpi4py import MPI" - struphy -h From 4f200db1e6c64302c71f75bd8b6e246c7d5d1160 Mon Sep 17 00:00:00 2001 From: Max Lindqvist Date: Fri, 29 Aug 2025 11:43:58 +0200 Subject: [PATCH 19/35] Updated .gitlab-ci.yml --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3333a160c..aa8211378 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -362,6 +362,7 @@ test_pitagora05: - python -c "from mpi4py import MPI" - struphy -h - struphy --refresh-models + - struphy compile - struphy test unit inspect_directory: - hostname -f From 4f523406bfaa25b2ce3ec00014a670b78b0b573d Mon Sep 17 00:00:00 2001 From: Max Lindqvist Date: Fri, 29 Aug 2025 13:10:51 +0000 Subject: [PATCH 20/35] Update .gitlab-ci.yml file --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index aa8211378..f252c960b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -358,7 +358,7 @@ test_pitagora05: - source env/bin/activate - python -m pip install --upgrade pip - pip install --no-binary=mpi4py mpi4py - - pip install . + - pip install ".[phys]" - python -c "from mpi4py import MPI" - struphy -h - struphy --refresh-models From c4a31f2b9b845ccce0f396097b204d7bb3daeb44 Mon Sep 17 00:00:00 2001 From: Max Lindqvist Date: Wed, 10 Sep 2025 15:28:25 +0200 Subject: [PATCH 21/35] Install with . --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f252c960b..aa8211378 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -358,7 +358,7 @@ test_pitagora05: - source env/bin/activate - python -m pip install --upgrade pip - pip install --no-binary=mpi4py mpi4py - - pip install ".[phys]" + - pip install . - python -c "from mpi4py import MPI" - struphy -h - struphy --refresh-models From a60ff6d2bbe186982abacd2e20031c8287fd8ea8 Mon Sep 17 00:00:00 2001 From: Max Date: Sun, 14 Sep 2025 14:48:12 +0200 Subject: [PATCH 22/35] Removed commented CI jobs --- .gitlab-ci.yml | 849 ------------------------------------------------- 1 file changed, 849 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index aa8211378..2ef8d6a1c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -107,249 +107,6 @@ test_pitagora05: - !reference [.scripts, install_struphy_clusters] -# # --- images --- - -# .image_gitlab_mpcdf_struphy: -# image: gitlab-registry.mpcdf.mpg.de/struphy/struphy/mpcdf-gcc-openmpi-with-struphy - -# .image_ubuntu_struphy: -# image: gitlab-registry.mpcdf.mpg.de/struphy/struphy/ubuntu-latest-with-struphy - -# .image_gitlab_mpcdf_gcc_openmp: -# image: gitlab-registry.mpcdf.mpg.de/mpcdf/ci-module-image/gcc_14-openmpi_5_0:2025 - -# .image_ubuntu_latest: -# image: gitlab-registry.mpcdf.mpg.de/struphy/struphy/ubuntu-latest - -# # --- job variables --- - -# .variables_push: -# variables: -# LANGUAGE: 'c' -# OMP: '' - -# # --- rules --- - -# # Options for CI_PIPELINE_SOURCE: push, merge_request_event, web, schedule, api -# .rules_startup: -# rules: -# - if: $CI_COMMIT_TAG -# when: never -# - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_REF_PROTECTED == "false" -# when: never -# - when: always # this is a fix from https://forum.gitlab.com/t/push-to-gitlab-triggers-one-or-two-pipelines/61070/2 - -# .rules_common: -# skip_pages_and_scheduled: -# - if: $TEST_SCHEDULED == "true" -# when: never -# - if: $MAKE_PAGES == "true" -# when: never -# - if: $CI_PIPELINE_SOURCE == "schedule" -# when: never -# - if: $CI_COMMIT_TAG -# when: never -# - if: $CI_PIPELINE_SOURCE == "push" -# when: never -# release: -# - if: $TEST_SCHEDULED == "true" -# when: never -# - if: $CI_PIPELINE_SOURCE == "schedule" -# when: never -# - if: $CI_COMMIT_TAG -# when: never -# - if: $CI_COMMIT_REF_NAME == "master" - -# .rules_mr_to_devel: -# rules: -# - !reference [.rules_common, skip_pages_and_scheduled] -# - if: $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "devel" && $CI_PIPELINE_SOURCE == "merge_request_event" - -# .rules_mr_to_master: -# rules: -# - !reference [.rules_common, skip_pages_and_scheduled] -# - if: $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "master" -# when: always - -# .rules_scheduled: -# rules: -# - if: $TEST_SCHEDULED == "true" -# - if: $CI_PIPELINE_SOURCE == "schedule" - -# .rules_update_images: -# rules: -# - if: $UPDATE_IMAGES == "true" - -# .rules_pypi_release: -# rules: -# - !reference [.rules_common, release] - -# .rules_gitlab_release: -# rules: -# - !reference [.rules_common, release] -# - if: $MAKE_GITLAB_RELEASE == "true" - -# .rules_pages_release: -# rules: -# - !reference [.rules_common, release] -# - if: $MAKE_PAGES == "true" - -# .rules_cleanup_macos: -# rules: -# - if: $CLEANUP_MACOS_NMPP == "true" - -# # --- parallel matrices --- - -# .parallel_matrix_install: -# parallel: -# matrix: -# - LANGUAGE: "fortran" -# OMP: ["", "--omp-pic"] -# # - LANGUAGE: "c" -# # OMP: [""] - -# .parallel_matrix_scheduled: -# parallel: -# matrix: -# - LANGUAGE: "fortran" -# OMP: [""] #, "--omp-pic"] # TODO: Enable when unit tests are working with --omp-pic! -# - LANGUAGE: "c" -# OMP: [""] - -# # --- before scripts --- - -# .before_script_load_modules: -# before_script: -# - date -# - pwd -# - ls -1a -# - uname -a -# - module list -# - module avail -# # struphy modules -# - module load gcc/14 openmpi/5.0 python-waterboa/2024.06 git graphviz/8 -# # gvec modules and variables -# - module load cmake netcdf-serial mkl hdf5-serial -# - export FC=`which gfortran` -# - export CC=`which gcc` -# - export CXX=`which g++` -# # verify -# - module list -# - echo "Fortran compiler is ${FC}" -# - make -v -# - python3 --version - -# .requirements_fedora: -# before_script: -# - !reference [.scripts, inspect_directory] -# # basic -# - dnf install -y wget yum-utils make openssl-devel bzip2-devel libffi-devel zlib-devel -# - dnf update -y -# # compilers and mpi -# - dnf install -y gcc -# - dnf install -y gfortran -# - dnf install -y blas-devel lapack-devel -# - dnf install -y openmpi openmpi-devel -# # python -# - dnf install -y python3-devel -# - dnf install -y python3-mpi4py-openmpi -# # gvec -# - dnf install -y g++ cmake netcdf netcdf-devel netcdf-fortran netcdf-fortran-devel pkgconf -# - export FC=`which gfortran` -# - export CC=`which gcc` -# - export CXX=`which g++` -# # additional -# - dnf install -y git -# - dnf install -y pandoc -# - dnf update -y -# - export OMPI_ALLOW_RUN_AS_ROOT=1 -# - export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 -# # virtual env -# - python3 -m venv env -# - source env/bin/activate -# - . /etc/profile.d/modules.sh -# - module load mpi/openmpi-$(arch) -# - module list - -# .requirements_opensuse: -# before_script: -# - !reference [.scripts, inspect_directory] -# # basic -# - zypper refresh -# # compilers and mpi -# - zypper install -y gcc-fortran gcc -# - zypper install -y blas-devel lapack-devel -# - zypper install -y openmpi openmpi-devel openmpi4-devel -# - zypper install -y libgomp1 -# # python -# - zypper install -y python3 python3-devel -# - zypper install -y python3-pip python3-virtualenv python3-pkgconfig -# # gvec -# - zypper install -y gcc-c++ cmake netcdf -# - zypper addrepo -G https://download.opensuse.org/repositories/science/openSUSE_Tumbleweed/science.repo -# - zypper install -y netcdf-fortran-devel -# - export FC=`which gfortran` -# - export CC=`which gcc` -# - export CXX=`which g++` -# # additional -# - zypper install -y git -# - zypper install -y pandoc -# - zypper install -y vim -# - zypper install -y make -# - export OMPI_ALLOW_RUN_AS_ROOT=1 -# - export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 -# - export OMPI_MCA_pml=ob1 -# - export OMPI_MCA_btl=tcp,self -# - export PATH="/usr/lib64/mpi/gcc/openmpi4/bin/:$PATH" -# - export LD_LIBRARY_PATH="/usr/lib64/mpi/gcc/openmpi4/lib64:$LD_LIBRARY_PATH" -# - !reference [.scripts, create_venv] - -# .requirements_almalinux: -# before_script: -# - !reference [.scripts, inspect_directory] -# # basic -# - yum install -y wget yum-utils make openssl-devel bzip2-devel libffi-devel zlib-devel -# - yum update -y -# - yum clean all -# # compilers and mpi -# - yum install -y gcc -# - yum install -y gfortran -# - yum install -y openmpi openmpi-devel -# - yum install -y libgomp -# - yum install -y environment-modules -# # python -# - wget https://www.python.org/ftp/python/3.12.8/Python-3.12.8.tgz -# - tar xzf Python-3.12.8.tgz -# - cd Python-3.12.8 -# - ./configure --with-system-ffi --with-computed-gotos --enable-loadable-sqlite-extensions -# - make -j ${nproc} -# - make altinstall -# - alternatives --install /usr/bin/python3 python3 /usr/local/bin/python3.12 1 -# - alternatives --set python3 /usr/local/bin/python3.12 -# - mv /usr/local/lib/libpython3.12.a libpython3.12.a.bak -# - python3 -V -# - cd .. -# - pwd -# - ls -# # additional -# - yum install -y git -# - export OMPI_ALLOW_RUN_AS_ROOT=1 -# - export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 -# - export OMPI_MCA_pml=ob1 -# - export OMPI_MCA_btl=tcp,self -# - export PATH="/usr/lib64/openmpi/bin:$PATH" -# # gvec -# - yum install -y g++ cmake which flexiblas-devel -# - yum install -y epel-release -# - yum install -y netcdf-devel netcdf-fortran-devel -# - export FC=`which gfortran` -# - export CC=`which gcc` -# - export CXX=`which g++` -# # virtual env -# - python3 -m pip list -# - python3 -m venv env -# - source env/bin/activate - # # --- scripts --- .scripts: @@ -524,609 +281,3 @@ test_pitagora05: - buildah push gitlab-registry.mpcdf.mpg.de/struphy/struphy/opensuse-latest - buildah push gitlab-registry.mpcdf.mpg.de/struphy/struphy/ubuntu-latest-with-struphy - buildah push gitlab-registry.mpcdf.mpg.de/struphy/struphy/ubuntu-latest - -# # --- artifacts --- - -# .artifacts_scheduled: -# artifacts: -# name: 'python-env-installed-${LANGUAGE}-${OMP}' -# paths: -# - env_${CI_PIPELINE_ID}_${LANGUAGE}_${OMP} -# expire_in: 1 day - -# ##################### -# ### STARTUP STAGE ### -# ##################### - -# update_images_buildah: -# tags: [podman] #necessary to run on the correct runner -# stage: build_images -# image: quay.io/buildah/stable #stable buildah image should suffice -# extends: -# - .rules_update_images -# variables: -# BUILDAH_FORMAT: docker #necessary -# BUILDAH_ISOLATION: chroot #necessary -# before_script: -# - !reference [.scripts, inspect_directory] -# - buildah login -u $CI_REGISTRY_USER --password $CI_REGISTRY_PASSWORD $CI_REGISTRY -# script: -# - !reference [.scripts, build_images] -# - !reference [.scripts, push_images] - -# # Print predefined variables: https://gitlab.mpcdf.mpg.de/help/ci/variables/predefined_variables.md -# ci_vars: -# stage: startup -# extends: -# - .rules_startup -# - .image_gitlab_mpcdf_gcc_openmp -# - .before_script_load_modules -# script: -# - echo "All environment variables:" -# - env - -# inspect_repo: -# stage: startup -# extends: -# - .rules_startup -# script: -# - curl https://raw.githubusercontent.com/AlDanial/cloc/master/cloc > cloc -# - chmod +x cloc -# - ./cloc --version -# - ./cloc $(git ls-files) - -# ##################### -# ### PUSH PIPELINE ### -# ##################### - -# install_tests: -# stage: test -# extends: -# - .rules_mr_to_devel -# - .image_gitlab_mpcdf_struphy -# - .before_script_load_modules -# - .parallel_matrix_install -# script: -# - !reference [.scripts, install_on_push] - -# unit_tests: -# stage: test -# extends: -# - .rules_mr_to_devel -# - .image_gitlab_mpcdf_struphy -# - .before_script_load_modules -# - .variables_push -# script: -# - !reference [.scripts, install_on_push] -# - !reference [.scripts, unit_tests] - -# tests_pyccel_devel: -# stage: test -# extends: -# - .rules_scheduled -# - .image_gitlab_mpcdf_struphy -# - .before_script_load_modules -# - .variables_push -# script: -# - !reference [.scripts, install_pyccel_devel] -# - !reference [.scripts, unit_tests] -# - !reference [.scripts, model_tests] - -# model_tests: -# stage: test -# extends: -# - .rules_mr_to_devel -# - .image_gitlab_mpcdf_struphy -# - .before_script_load_modules -# - .variables_push -# script: -# - !reference [.scripts, install_on_push] -# - !reference [.scripts, model_tests] - -# quickstart_tests: -# stage: test -# extends: -# - .rules_mr_to_devel -# - .image_gitlab_mpcdf_struphy -# - .before_script_load_modules -# - .variables_push -# script: -# - !reference [.scripts, install_on_push] -# - !reference [.scripts, quickstart_tests] - -# pages_tests: -# stage: test -# extends: -# - .rules_mr_to_devel -# - .image_gitlab_mpcdf_struphy -# - .before_script_load_modules -# - .variables_push -# script: -# - !reference [.scripts, install_on_push] -# - !reference [.scripts, inspect_struphy] -# - module load pandoc -# - module list -# - pip install -e .[doc] -# - | -# struphy compile -y || ( -# echo "Initial compile failed. Removing compiled kernels and trying again..." && -# struphy compile -d -y && -# struphy compile -y -# ) -# - cd doc ; make html; cd .. - -# ubuntu_latest: -# stage: test -# extends: -# - .rules_mr_to_devel -# - .image_ubuntu_struphy -# - .variables_push -# script: -# - !reference [.scripts, inspect_directory] -# - !reference [.scripts, install_on_push] -# - !reference [.scripts, quickstart_tests] -# - !reference [.scripts, model_tests] - -# macos_nmpp: -# stage: test -# tags: [macos, nmpp, private] -# needs: [] -# variables: -# # Job ID changes for each job, so reusing the directory becomes impossible, -# # therefore, the fetch strategy also becomes unusable -# GIT_CLONE_PATH: $CI_BUILDS_DIR/$CI_PROJECT_NAME/$CI_JOB_ID/$CI_CONCURRENT_PROJECT_ID -# _JOB_PATH: $CI_BUILDS_DIR/$CI_PROJECT_NAME/$CI_JOB_ID -# extends: -# - .rules_mr_to_devel -# - .variables_push -# before_script: -# # - brew install cmake -# - brew link --overwrite cmake -# - cmake --version -# - make -v -# - printenv -# - system_profiler SPHardwareDataType -# - export FC=`which gfortran` # for gvec -# - export CC=`which gcc` # for gvec -# - export CXX=`which g++` # for gvec -# script: -# - !reference [.scripts, create_venv] -# - !reference [.scripts, install] -# - !reference [.scripts, compile] -# - !reference [.scripts, quickstart_tests] -# - !reference [.scripts, model_tests] -# after_script: -# - pwd -# - ls -a -# - echo $_JOB_PATH -# - ls -a $_JOB_PATH -# - rm -rf $_JOB_PATH - -# cleanup_macos: -# stage: test -# tags: [macos, nmpp, private] -# needs: [] -# extends: -# - .rules_cleanup_macos -# before_script: -# - make -v -# - printenv -# - system_profiler SPHardwareDataType -# script: -# - _STRUPHY_PATH=$CI_BUILDS_DIR/$CI_PROJECT_NAME -# - pwd -# - ls -a -# - echo $_STRUPHY_PATH -# - ls -a $_STRUPHY_PATH -# - rm -rf $_STRUPHY_PATH/* -# - ls -a $_STRUPHY_PATH - -# ########################## -# ### SCHEDULED PIPELINE ### -# ########################## - -# install_scheduled: -# stage: install -# needs: [] -# extends: -# - .rules_scheduled -# - .image_gitlab_mpcdf_gcc_openmp -# - .before_script_load_modules -# - .parallel_matrix_scheduled -# - .artifacts_scheduled -# script: -# - !reference [.scripts, create_venv_matrix] -# - !reference [.scripts, install_no_binary] -# - !reference [.scripts, compile_matrix] - -# compile_timing: -# stage: install -# needs: [] -# extends: -# - .image_gitlab_mpcdf_gcc_openmp -# - .before_script_load_modules -# - .rules_scheduled -# script: -# - !reference [.scripts, create_venv] -# - !reference [.scripts, install_no_binary] -# - pyccel --version -# - struphy compile --status -# - struphy compile --delete -# - struphy compile --time-execution -y --language c > timings_c.txt -# - struphy compile --delete -# - struphy compile --time-execution -y --language fortran > timings_fortran.txt -# - | -# cat << 'EOF' > check_pyccel.sh -# #!/bin/bash -# grep 'pyccel --libdir' timings_c.txt | sed -E 's|.*/site-packages/||' | while read filepath; do -# echo "========================================" -# echo "Kernel: $filepath" -# echo "-------------------- Timers -------------------------" -# echo "# Language: C" -# grep -A 11 "pyccel.*${filepath}" timings_c.txt | tail -n +3 -# echo "# Language: Fortran" -# grep -A 13 "pyccel.*${filepath}" timings_fortran.txt | tail -n +3 -# done -# EOF -# - cat check_pyccel.sh -# - chmod +x check_pyccel.sh -# - ./check_pyccel.sh - -# test_build: -# stage: test -# extends: -# - .image_gitlab_mpcdf_gcc_openmp -# - .rules_scheduled -# - .before_script_load_modules -# needs: [] -# script: -# - !reference [.scripts, create_venv] -# - !reference [.scripts, build] -# - pip install $(echo dist/struphy*.whl) --no-cache-dir --no-binary mpi4py - -# unit_tests_scheduled: -# stage: test -# extends: -# - .image_gitlab_mpcdf_gcc_openmp -# - .rules_scheduled -# - .parallel_matrix_scheduled -# - .before_script_load_modules -# script: -# - !reference [.scripts, source_env_matrix] -# - !reference [.scripts, unit_tests] - -# model_tests_scheduled: -# stage: test -# extends: -# - .image_gitlab_mpcdf_gcc_openmp -# - .rules_scheduled -# - .parallel_matrix_scheduled -# - .before_script_load_modules -# script: -# - !reference [.scripts, source_env_matrix] -# - !reference [.scripts, model_tests] - -# check_release_dependencies: -# stage: test -# extends: -# - .image_gitlab_mpcdf_gcc_openmp -# - .before_script_load_modules -# - .rules_mr_to_master -# script: -# - !reference [.scripts, inspect_directory] -# - !reference [.scripts, create_venv] -# - !reference [.scripts, install_all] -# - python src/struphy/utils/set_release_dependencies.py -# - | -# if git diff --exit-code pyproject.toml; then -# echo "pyproject.toml has not changed, dependencies are set correctly!" -# else -# echo "pyproject.toml has changed, dependencies are set incorrectly!" -# echo "Run python src/struphy/utils/set_release_dependencies.py to update pyproject.toml" -# exit 1 -# fi - -# Ubuntu-latest-from-registry: -# stage: test -# needs: [] -# extends: -# - .image_ubuntu_latest -# - .rules_scheduled -# - .parallel_matrix_scheduled -# before_script: -# - !reference [.scripts, inspect_directory] -# - !reference [.scripts, create_venv] -# script: -# - !reference [.scripts, install] -# - !reference [.scripts, compile_matrix] -# - !reference [.scripts, model_tests] - -# Fedora-latest: -# stage: test -# needs: [] -# extends: -# - .rules_scheduled -# - .parallel_matrix_scheduled -# - .requirements_fedora -# image: fedora:latest -# script: -# - !reference [.scripts, install] -# - !reference [.scripts, compile_matrix] -# - !reference [.scripts, model_tests] - -# Fedora-latest-from-registry: -# stage: test -# needs: [] -# extends: -# - .rules_scheduled -# - .parallel_matrix_scheduled -# image: gitlab-registry.mpcdf.mpg.de/struphy/struphy/fedora-latest -# before_script: -# - !reference [.scripts, inspect_directory] -# - !reference [.scripts, create_venv] -# - . /etc/profile.d/modules.sh -# - module load mpi/openmpi-$(arch) -# - module list -# script: -# - !reference [.scripts, install] -# - !reference [.scripts, compile_matrix] -# - !reference [.scripts, model_tests] - -# Fedora-latest-from-registry-base: -# stage: test -# needs: [] -# extends: -# - .rules_scheduled -# - .parallel_matrix_scheduled -# image: gitlab-registry.mpcdf.mpg.de/struphy/struphy/fedora-latest -# before_script: -# - !reference [.scripts, inspect_directory] -# - !reference [.scripts, create_venv] -# - . /etc/profile.d/modules.sh -# - module load mpi/openmpi-$(arch) -# - module list -# script: -# - !reference [.scripts, install_base] -# - !reference [.scripts, compile_matrix] -# - !reference [.scripts, model_tests] - -# OpenSuse-latest: -# stage: test -# needs: [] -# extends: -# - .rules_scheduled -# - .parallel_matrix_scheduled -# - .requirements_opensuse -# image: opensuse/tumbleweed:latest -# script: -# - !reference [.scripts, install] -# - !reference [.scripts, compile_matrix] -# - !reference [.scripts, model_tests] - -# Opensuse-latest-from-registry: -# stage: test -# needs: [] -# extends: -# - .rules_scheduled -# - .parallel_matrix_scheduled -# image: gitlab-registry.mpcdf.mpg.de/struphy/struphy/opensuse-latest -# before_script: -# - !reference [.scripts, inspect_directory] -# - set -x -# - !reference [.scripts, create_venv] -# script: -# - !reference [.scripts, install] -# - !reference [.scripts, compile_matrix] -# - !reference [.scripts, model_tests] - -# Opensuse-latest-from-registry-base: -# stage: test -# needs: [] -# extends: -# - .rules_scheduled -# - .parallel_matrix_scheduled -# image: gitlab-registry.mpcdf.mpg.de/struphy/struphy/opensuse-latest -# before_script: -# - !reference [.scripts, inspect_directory] -# - set -x -# - !reference [.scripts, create_venv] -# script: -# - !reference [.scripts, install_base] -# - !reference [.scripts, compile_matrix] -# - !reference [.scripts, model_tests] - -# AlmaLinux-latest: -# stage: test -# needs: [] -# extends: -# - .rules_scheduled -# - .parallel_matrix_scheduled -# - .requirements_almalinux -# image: almalinux:latest -# script: -# - !reference [.scripts, install_no_binary] -# - !reference [.scripts, compile_matrix] -# - !reference [.scripts, model_tests] - -# Almalinux-latest-from-registry: -# stage: test -# needs: [] -# extends: -# - .rules_scheduled -# - .parallel_matrix_scheduled -# image: gitlab-registry.mpcdf.mpg.de/struphy/struphy/almalinux-latest -# before_script: -# - !reference [.scripts, inspect_directory] -# - python3 -m ensurepip --upgrade -# - python3 -m pip install --upgrade pip -# - !reference [.scripts, create_venv] -# script: -# - !reference [.scripts, install_no_binary] -# - !reference [.scripts, compile_matrix] -# - !reference [.scripts, model_tests] - -# Almalinux-latest-from-registry-base: -# stage: test -# needs: [] -# extends: -# - .rules_scheduled -# - .parallel_matrix_scheduled -# image: gitlab-registry.mpcdf.mpg.de/struphy/struphy/almalinux-latest -# before_script: -# - !reference [.scripts, inspect_directory] -# - python3 -m ensurepip --upgrade -# - python3 -m pip install --upgrade pip -# - !reference [.scripts, create_venv] -# script: -# - !reference [.scripts, install_base_no_binary] -# - !reference [.scripts, compile_matrix] -# - !reference [.scripts, model_tests] - -# ################## -# ### LINT STAGE ### -# ################## - -# # Run in weekly CI pipeline only -# lint_full_repo_report: -# stage: lint -# needs: [] -# extends: -# - .rules_scheduled -# - .image_ubuntu_latest -# script: -# - !reference [.scripts, inspect_directory] -# - !reference [.scripts, create_venv] -# - pip install -e .[dev] # We have to install struphy in editable mode for struphy lint to work -# - !reference [.scripts, inspect_struphy] -# - struphy lint all --output-format report -# - mkdir -p lint_reports -# - mv *.html lint_reports/ 2>/dev/null || echo "No HTML files found." -# allow_failure: true -# artifacts: -# paths: -# - lint_reports/ -# expire_in: 1 month - -# # Lint struphy with `struphy lint` -# lint_repo: -# stage: lint -# needs: [] -# extends: -# - .rules_mr_to_devel -# - .image_ubuntu_latest -# script: -# - !reference [.scripts, inspect_directory] -# - !reference [.scripts, create_venv] -# - pip install -e .[dev] # We have to install struphy in editable mode for struphy lint to work -# - !reference [.scripts, inspect_struphy] -# # Lint all files in current branch, FAIL the CI pipeline if any files are incorrectly formatted -# - struphy lint all --output-format plain --verbose - -# # Lint struphy with `struphy lint` -# lint_branch_report: -# stage: lint -# needs: [] -# extends: -# - .rules_mr_to_devel -# - .image_ubuntu_latest -# script: -# - !reference [.scripts, inspect_directory] -# - !reference [.scripts, create_venv] -# - pip install -e .[dev] # We have to install struphy in editable mode for struphy lint to work -# - !reference [.scripts, inspect_struphy] -# - struphy lint branch --output-format report -# - mkdir -p lint_reports -# - mv *.html lint_reports/ 2>/dev/null || echo "No HTML files found." -# allow_failure: true -# artifacts: -# paths: -# - lint_reports/ -# expire_in: 1 month - -# ################### -# ### PAGES STAGE ### -# ################### -# pages: -# stage: pages -# extends: -# - .image_ubuntu_latest -# - .rules_pages_release -# before_script: -# - !reference [.scripts, inspect_directory] -# - !reference [.scripts, create_venv] -# - apt show pandoc -# - apt show graphviz -# - echo $CI_COMMIT_MESSAGE -# - pip install .[doc] -# script: -# - !reference [.scripts, inspect_struphy] -# - !reference [.scripts, compile] -# - cd doc ; make html -# - mv _build/html/ ../public/ -# artifacts: -# name: 'pages' -# paths: -# - public/ - -# ##################### -# ### RELEASE STAGE ### -# ##################### - -# deploy: -# stage: release -# extends: -# - .rules_pypi_release -# - .image_gitlab_mpcdf_gcc_openmp -# - .before_script_load_modules -# script: -# - !reference [.scripts, create_venv] -# - !reference [.scripts, build] -# - pip install -U twine -# - twine upload dist/* - -# vars: -# stage: release -# extends: -# - .rules_gitlab_release -# before_script: -# - !reference [.scripts, inspect_directory] -# script: -# - var=$(> vars.env -# artifacts: -# name: 'vars' -# reports: -# dotenv: vars.env -# expire_in: 1 day - -# release_job: -# stage: release -# image: registry.gitlab.com/gitlab-org/release-cli:latest -# extends: -# - .rules_gitlab_release -# needs: ['vars'] -# before_script: -# - !reference [.scripts, inspect_directory] -# script: -# - cat /etc/*-release -# - echo $VERSION -# release: # See https://docs.gitlab.com/ee/ci/yaml/#release for available properties -# tag_name: 'v$VERSION' # The version is incremented per pipeline. -# name: 'v$VERSION' -# ref: '$CI_COMMIT_SHA' # The tag is created from the pipeline SHA. -# description: 'CHANGELOG.md' -# assets: -# links: -# - name: 'Documentation' -# url: 'https://struphy.pages.mpcdf.de/struphy/index.html' -# - name: 'PyPI' -# url: 'https://pypi.org/project/struphy/' From 112fdd01fe78fcf7ea943fc948dfdbe002d624ca Mon Sep 17 00:00:00 2001 From: Max Date: Sun, 14 Sep 2025 14:55:44 +0200 Subject: [PATCH 23/35] Install with .[dev] --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a1247ad4d..b0c688375 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -115,7 +115,7 @@ test_pitagora05: - source env/bin/activate - python -m pip install --upgrade pip - pip install --no-binary=mpi4py mpi4py - - pip install . + - pip install ".[dev]" - python -c "from mpi4py import MPI" - struphy -h - struphy --refresh-models From b26a39723a678e5af547200939d29bb9f97cc6f2 Mon Sep 17 00:00:00 2001 From: Max Date: Sun, 14 Sep 2025 14:57:25 +0200 Subject: [PATCH 24/35] print(MPI) --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b0c688375..72262b47b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -116,7 +116,7 @@ test_pitagora05: - python -m pip install --upgrade pip - pip install --no-binary=mpi4py mpi4py - pip install ".[dev]" - - python -c "from mpi4py import MPI" + - python -c "from mpi4py import MPI; print(MPI)" - struphy -h - struphy --refresh-models - struphy compile From 005fda8509325f683c981530586c02a0dfb8c4a1 Mon Sep 17 00:00:00 2001 From: Max Lindqvist Date: Wed, 8 Oct 2025 14:46:27 +0200 Subject: [PATCH 25/35] Install gvec with pip install gvec --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 72262b47b..b77c29713 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -115,6 +115,7 @@ test_pitagora05: - source env/bin/activate - python -m pip install --upgrade pip - pip install --no-binary=mpi4py mpi4py + - pip install gvec - pip install ".[dev]" - python -c "from mpi4py import MPI; print(MPI)" - struphy -h From d9a64665f8641e4eb8b086b135cafe71ca5856da Mon Sep 17 00:00:00 2001 From: Max Lindqvist Date: Wed, 8 Oct 2025 15:39:17 +0200 Subject: [PATCH 26/35] Added cmake module --- .gitlab-ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b77c29713..a0a2fbf3f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -53,7 +53,7 @@ test_toki01: - module purge - module list - module avail - - module load gcc/14 openmpi/5.0 python-waterboa/2025.06 + - module load gcc/14 openmpi/5.0 python-waterboa/2025.06 cmake - module list - !reference [.scripts, install_struphy_clusters] @@ -70,7 +70,7 @@ test_viper01: - module purge - module list - module avail - - module load gcc/14 openmpi/5.0 python-waterboa/2025.06 + - module load gcc/14 openmpi/5.0 python-waterboa/2025.06 cmake - module list - !reference [.scripts, install_struphy_clusters] @@ -86,7 +86,7 @@ test_raven01: - module purge - module list - module avail - - module load gcc/14 openmpi/5.0 python-waterboa/2025.06 + - module load gcc/14 openmpi/5.0 python-waterboa/2025.06 cmake - module list - !reference [.scripts, install_struphy_clusters] @@ -102,7 +102,7 @@ test_pitagora05: - module purge - module list - module avail - - module load gcc/12.3.0 openmpi/4.1.6--gcc--12.3.0 python/3.11.7 + - module load gcc/12.3.0 openmpi/4.1.6--gcc--12.3.0 python/3.11.7 cmake - module list - !reference [.scripts, install_struphy_clusters] From e84b54165e961799f2b8ba22f5a944d0a076694a Mon Sep 17 00:00:00 2001 From: Max Lindqvist Date: Thu, 9 Oct 2025 16:18:40 +0200 Subject: [PATCH 27/35] Install with phys and load netcdf module --- .gitlab-ci.yml | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a0a2fbf3f..dd78f3d79 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -53,8 +53,11 @@ test_toki01: - module purge - module list - module avail - - module load gcc/14 openmpi/5.0 python-waterboa/2025.06 cmake + - module load gcc/14 openmpi/5.0 python-waterboa/2025.06 cmake netcdf-serial mkl hdf5-serial - module list + - export FC=`which gfortran` + - export CC=`which gcc` + - export CXX=`which g++` - !reference [.scripts, install_struphy_clusters] @@ -70,8 +73,11 @@ test_viper01: - module purge - module list - module avail - - module load gcc/14 openmpi/5.0 python-waterboa/2025.06 cmake + - module load gcc/14 openmpi/5.0 python-waterboa/2025.06 cmake netcdf-serial mkl hdf5-serial - module list + - export FC=`which gfortran` + - export CC=`which gcc` + - export CXX=`which g++` - !reference [.scripts, install_struphy_clusters] test_raven01: @@ -86,8 +92,11 @@ test_raven01: - module purge - module list - module avail - - module load gcc/14 openmpi/5.0 python-waterboa/2025.06 cmake + - module load gcc/14 openmpi/5.0 python-waterboa/2025.06 cmake netcdf-serial mkl hdf5-serial - module list + - export FC=`which gfortran` + - export CC=`which gcc` + - export CXX=`which g++` - !reference [.scripts, install_struphy_clusters] test_pitagora05: @@ -102,8 +111,11 @@ test_pitagora05: - module purge - module list - module avail - - module load gcc/12.3.0 openmpi/4.1.6--gcc--12.3.0 python/3.11.7 cmake + - module load gcc/12.3.0 openmpi/4.1.6--gcc--12.3.0 python/3.11.7 cmake netcdf-serial mkl hdf5-serial - module list + - export FC=`which gfortran` + - export CC=`which gcc` + - export CXX=`which g++` - !reference [.scripts, install_struphy_clusters] @@ -116,7 +128,7 @@ test_pitagora05: - python -m pip install --upgrade pip - pip install --no-binary=mpi4py mpi4py - pip install gvec - - pip install ".[dev]" + - pip install ".[phys]" - python -c "from mpi4py import MPI; print(MPI)" - struphy -h - struphy --refresh-models From bc0770bc4b86605d3c57abc28baf5113167d2fae Mon Sep 17 00:00:00 2001 From: Max Lindqvist Date: Thu, 9 Oct 2025 16:20:48 +0200 Subject: [PATCH 28/35] Updated pitagora modules --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index dd78f3d79..97b3e937f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -111,7 +111,8 @@ test_pitagora05: - module purge - module list - module avail - - module load gcc/12.3.0 openmpi/4.1.6--gcc--12.3.0 python/3.11.7 cmake netcdf-serial mkl hdf5-serial + - module load gcc/12.3.0 openmpi/4.1.6--gcc--12.3.0 python/3.11.7 cmake + - module load netcdf-fortran/4.6.1--openmpi--4.1.6--gcc--12.3.0 intel-oneapi-mkl/2024.0.0--intel-oneapi-mpi--2021.12.1 hdf5/1.14.3--gcc--12.3.0 - module list - export FC=`which gfortran` - export CC=`which gcc` From 3bcee9c059aed334b99838e1520c262b45d6e985 Mon Sep 17 00:00:00 2001 From: Max Lindqvist Date: Thu, 9 Oct 2025 16:22:47 +0200 Subject: [PATCH 29/35] Updated pitagora modules --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 97b3e937f..7d217a23d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -112,7 +112,7 @@ test_pitagora05: - module list - module avail - module load gcc/12.3.0 openmpi/4.1.6--gcc--12.3.0 python/3.11.7 cmake - - module load netcdf-fortran/4.6.1--openmpi--4.1.6--gcc--12.3.0 intel-oneapi-mkl/2024.0.0--intel-oneapi-mpi--2021.12.1 hdf5/1.14.3--gcc--12.3.0 + - module load netcdf-fortran/4.6.1--openmpi--4.1.6--gcc--12.3.0 intel-oneapi-mkl/2024.0.0--intel-oneapi-mpi--2021.12.1 - module list - export FC=`which gfortran` - export CC=`which gcc` From 0a8c213de5b688c5f891751fe87fbff75c5d772b Mon Sep 17 00:00:00 2001 From: Max Lindqvist Date: Fri, 10 Oct 2025 08:15:51 +0000 Subject: [PATCH 30/35] Update .gitlab-ci.yml file --- .gitlab-ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7d217a23d..1a2895c8b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -78,6 +78,8 @@ test_viper01: - export FC=`which gfortran` - export CC=`which gcc` - export CXX=`which g++` + - unset I_MPI_HYDRA_BOOTSTRAP + - unset I_MPI_PMI_LIBRARY - !reference [.scripts, install_struphy_clusters] test_raven01: @@ -97,6 +99,8 @@ test_raven01: - export FC=`which gfortran` - export CC=`which gcc` - export CXX=`which g++` + - unset I_MPI_HYDRA_BOOTSTRAP + - unset I_MPI_PMI_LIBRARY - !reference [.scripts, install_struphy_clusters] test_pitagora05: @@ -117,6 +121,8 @@ test_pitagora05: - export FC=`which gfortran` - export CC=`which gcc` - export CXX=`which g++` + - unset I_MPI_HYDRA_BOOTSTRAP + - unset I_MPI_PMI_LIBRARY - !reference [.scripts, install_struphy_clusters] From e5636360b0552120ffb4b32be69b4ff455880f8a Mon Sep 17 00:00:00 2001 From: Max Lindqvist Date: Fri, 10 Oct 2025 09:44:00 +0000 Subject: [PATCH 31/35] Update .gitlab-ci.yml file --- .gitlab-ci.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1a2895c8b..bc9eef299 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -78,8 +78,6 @@ test_viper01: - export FC=`which gfortran` - export CC=`which gcc` - export CXX=`which g++` - - unset I_MPI_HYDRA_BOOTSTRAP - - unset I_MPI_PMI_LIBRARY - !reference [.scripts, install_struphy_clusters] test_raven01: @@ -99,8 +97,7 @@ test_raven01: - export FC=`which gfortran` - export CC=`which gcc` - export CXX=`which g++` - - unset I_MPI_HYDRA_BOOTSTRAP - - unset I_MPI_PMI_LIBRARY + - export JAX_DISABLE_JIT=1 - !reference [.scripts, install_struphy_clusters] test_pitagora05: @@ -121,8 +118,7 @@ test_pitagora05: - export FC=`which gfortran` - export CC=`which gcc` - export CXX=`which g++` - - unset I_MPI_HYDRA_BOOTSTRAP - - unset I_MPI_PMI_LIBRARY + - export JAX_DISABLE_JIT=1 - !reference [.scripts, install_struphy_clusters] From e7ef8c1561d16274ab6abf08bf64e4f78f6dd80b Mon Sep 17 00:00:00 2001 From: Max Date: Mon, 13 Oct 2025 19:10:11 +0200 Subject: [PATCH 32/35] reverted gitlab ci --- .gitlab-ci.yml | 1013 ++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 942 insertions(+), 71 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bc9eef299..e06efe4c4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -33,6 +33,7 @@ variables: stages: - startup + - clusters - build_images - install - test @@ -40,89 +41,250 @@ stages: - pages - release +# --- images --- -test_toki01: - stage: startup - tags: - - maxlin - - private - - toki01 - script: - - echo "Connected to toki01" - - !reference [.scripts, inspect_directory] - - module purge - - module list - - module avail - - module load gcc/14 openmpi/5.0 python-waterboa/2025.06 cmake netcdf-serial mkl hdf5-serial - - module list - - export FC=`which gfortran` - - export CC=`which gcc` - - export CXX=`which g++` - - !reference [.scripts, install_struphy_clusters] +.image_gitlab_mpcdf_struphy: + image: gitlab-registry.mpcdf.mpg.de/struphy/struphy/mpcdf-gcc-openmpi-with-struphy +.image_ubuntu_struphy: + image: gitlab-registry.mpcdf.mpg.de/struphy/struphy/ubuntu-latest-with-struphy -test_viper01: - stage: startup - tags: - - maxlin - - private - - viper01 - script: - - echo "Connected to viper01" - - !reference [.scripts, inspect_directory] - - module purge +.image_gitlab_mpcdf_gcc_openmp: + image: gitlab-registry.mpcdf.mpg.de/mpcdf/ci-module-image/gcc_14-openmpi_5_0:2025 + +.image_ubuntu_latest: + image: gitlab-registry.mpcdf.mpg.de/struphy/struphy/ubuntu-latest + +# --- job variables --- + +.variables_push: + variables: + LANGUAGE: 'c' + OMP: '' + +# --- rules --- + +# Options for CI_PIPELINE_SOURCE: push, merge_request_event, web, schedule, api +.rules_startup: + rules: + - if: $CI_COMMIT_TAG + when: never + - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_REF_PROTECTED == "false" + when: never + - when: always # this is a fix from https://forum.gitlab.com/t/push-to-gitlab-triggers-one-or-two-pipelines/61070/2 + +.rules_common: + skip_pages_and_scheduled: + - if: $TEST_SCHEDULED == "true" + when: never + - if: $MAKE_PAGES == "true" + when: never + - if: $CI_PIPELINE_SOURCE == "schedule" + when: never + - if: $CI_COMMIT_TAG + when: never + - if: $CI_PIPELINE_SOURCE == "push" + when: never + release: + - if: $TEST_SCHEDULED == "true" + when: never + - if: $CI_PIPELINE_SOURCE == "schedule" + when: never + - if: $CI_COMMIT_TAG + when: never + - if: $CI_COMMIT_REF_NAME == "master" + +.rules_mr_to_devel: + rules: + - !reference [.rules_common, skip_pages_and_scheduled] + - if: $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "devel" && $CI_PIPELINE_SOURCE == "merge_request_event" + +.rules_mr_to_master: + rules: + - !reference [.rules_common, skip_pages_and_scheduled] + - if: $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "master" + when: always + +.rules_scheduled: + rules: + - if: $TEST_SCHEDULED == "true" + - if: $CI_PIPELINE_SOURCE == "schedule" + +.rules_update_images: + rules: + - if: $UPDATE_IMAGES == "true" + +.rules_pypi_release: + rules: + - !reference [.rules_common, release] + +.rules_gitlab_release: + rules: + - !reference [.rules_common, release] + - if: $MAKE_GITLAB_RELEASE == "true" + +.rules_pages_release: + rules: + - !reference [.rules_common, release] + - if: $MAKE_PAGES == "true" + +.rules_cleanup_macos: + rules: + - if: $CLEANUP_MACOS_NMPP == "true" + +# --- parallel matrices --- + +.parallel_matrix_install: + parallel: + matrix: + - LANGUAGE: "fortran" + OMP: ["", "--omp-pic"] + # - LANGUAGE: "c" + # OMP: [""] + +.parallel_matrix_scheduled: + parallel: + matrix: + - LANGUAGE: "fortran" + OMP: [""] #, "--omp-pic"] # TODO: Enable when unit tests are working with --omp-pic! + - LANGUAGE: "c" + OMP: [""] + +# --- before scripts --- + +.before_script_load_modules: + before_script: + - date + - pwd + - ls -1a + - uname -a - module list - module avail - - module load gcc/14 openmpi/5.0 python-waterboa/2025.06 cmake netcdf-serial mkl hdf5-serial + # struphy modules + - module load gcc/14 openmpi/5.0 python-waterboa/2024.06 git graphviz/8 + # gvec modules and variables + - module load cmake netcdf-serial mkl hdf5-serial + - export FC=`which gfortran` + - export CC=`which gcc` + - export CXX=`which g++` + # verify - module list - - export FC=`which gfortran` - - export CC=`which gcc` + - echo "Fortran compiler is ${FC}" + - make -v + - python3 --version + +.requirements_fedora: + before_script: + - !reference [.scripts, inspect_directory] + # basic + - dnf install -y wget yum-utils make openssl-devel bzip2-devel libffi-devel zlib-devel + - dnf update -y + # compilers and mpi + - dnf install -y gcc + - dnf install -y gfortran + - dnf install -y blas-devel lapack-devel + - dnf install -y openmpi openmpi-devel + # python + - dnf install -y python3-devel + - dnf install -y python3-mpi4py-openmpi + # gvec + - dnf install -y g++ cmake netcdf netcdf-devel netcdf-fortran netcdf-fortran-devel pkgconf + - export FC=`which gfortran` + - export CC=`which gcc` - export CXX=`which g++` - - !reference [.scripts, install_struphy_clusters] + # additional + - dnf install -y git + - dnf install -y pandoc + - dnf update -y + - export OMPI_ALLOW_RUN_AS_ROOT=1 + - export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 + # virtual env + - python3 -m venv env + - source env/bin/activate + - . /etc/profile.d/modules.sh + - module load mpi/openmpi-$(arch) + - module list -test_raven01: - stage: startup - tags: - - maxlin - - private - - raven01 - script: - - echo "Connected to raven01" +.requirements_opensuse: + before_script: - !reference [.scripts, inspect_directory] - - module purge - - module list - - module avail - - module load gcc/14 openmpi/5.0 python-waterboa/2025.06 cmake netcdf-serial mkl hdf5-serial - - module list - - export FC=`which gfortran` - - export CC=`which gcc` + # basic + - zypper refresh + # compilers and mpi + - zypper install -y gcc-fortran gcc + - zypper install -y blas-devel lapack-devel + - zypper install -y openmpi openmpi-devel openmpi4-devel + - zypper install -y libgomp1 + # python + - zypper install -y python3 python3-devel + - zypper install -y python3-pip python3-virtualenv python3-pkgconfig + # gvec + - zypper install -y gcc-c++ cmake netcdf + - zypper addrepo -G https://download.opensuse.org/repositories/science/openSUSE_Tumbleweed/science.repo + - zypper install -y netcdf-fortran-devel + - export FC=`which gfortran` + - export CC=`which gcc` - export CXX=`which g++` - - export JAX_DISABLE_JIT=1 - - !reference [.scripts, install_struphy_clusters] + # additional + - zypper install -y git + - zypper install -y pandoc + - zypper install -y vim + - zypper install -y make + - export OMPI_ALLOW_RUN_AS_ROOT=1 + - export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 + - export OMPI_MCA_pml=ob1 + - export OMPI_MCA_btl=tcp,self + - export PATH="/usr/lib64/mpi/gcc/openmpi4/bin/:$PATH" + - export LD_LIBRARY_PATH="/usr/lib64/mpi/gcc/openmpi4/lib64:$LD_LIBRARY_PATH" + - !reference [.scripts, create_venv] -test_pitagora05: - stage: startup - tags: - - maxlin - - private - - pitagora05 - script: - - echo "Connected to pitagora05" +.requirements_almalinux: + before_script: - !reference [.scripts, inspect_directory] - - module purge - - module list - - module avail - - module load gcc/12.3.0 openmpi/4.1.6--gcc--12.3.0 python/3.11.7 cmake - - module load netcdf-fortran/4.6.1--openmpi--4.1.6--gcc--12.3.0 intel-oneapi-mkl/2024.0.0--intel-oneapi-mpi--2021.12.1 - - module list - - export FC=`which gfortran` - - export CC=`which gcc` + # basic + - yum install -y wget yum-utils make openssl-devel bzip2-devel libffi-devel zlib-devel + - yum update -y + - yum clean all + # compilers and mpi + - yum install -y gcc + - yum install -y gfortran + - yum install -y openmpi openmpi-devel + - yum install -y libgomp + - yum install -y environment-modules + # python + - wget https://www.python.org/ftp/python/3.12.8/Python-3.12.8.tgz + - tar xzf Python-3.12.8.tgz + - cd Python-3.12.8 + - ./configure --with-system-ffi --with-computed-gotos --enable-loadable-sqlite-extensions + - make -j ${nproc} + - make altinstall + - alternatives --install /usr/bin/python3 python3 /usr/local/bin/python3.12 1 + - alternatives --set python3 /usr/local/bin/python3.12 + - mv /usr/local/lib/libpython3.12.a libpython3.12.a.bak + - python3 -V + - cd .. + - pwd + - ls + # additional + - yum install -y git + - export OMPI_ALLOW_RUN_AS_ROOT=1 + - export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 + - export OMPI_MCA_pml=ob1 + - export OMPI_MCA_btl=tcp,self + - export PATH="/usr/lib64/openmpi/bin:$PATH" + # gvec + - yum install -y g++ cmake which flexiblas-devel + - yum install -y epel-release + - yum install -y netcdf-devel netcdf-fortran-devel + - export FC=`which gfortran` + - export CC=`which gcc` - export CXX=`which g++` - - export JAX_DISABLE_JIT=1 - - !reference [.scripts, install_struphy_clusters] - + # virtual env + - python3 -m pip list + - python3 -m venv env + - source env/bin/activate -# # --- scripts --- +# --- scripts --- .scripts: install_struphy_clusters: @@ -138,8 +300,6 @@ test_pitagora05: - struphy compile - struphy test unit inspect_directory: - - hostname -f - - whoami - date - pwd - ls -1a @@ -298,3 +458,714 @@ test_pitagora05: - buildah push gitlab-registry.mpcdf.mpg.de/struphy/struphy/opensuse-latest - buildah push gitlab-registry.mpcdf.mpg.de/struphy/struphy/ubuntu-latest-with-struphy - buildah push gitlab-registry.mpcdf.mpg.de/struphy/struphy/ubuntu-latest + +# --- artifacts --- + +.artifacts_scheduled: + artifacts: + name: 'python-env-installed-${LANGUAGE}-${OMP}' + paths: + - env_${CI_PIPELINE_ID}_${LANGUAGE}_${OMP} + expire_in: 1 day + +##################### +### STARTUP STAGE ### +##################### + +update_images_buildah: + tags: [podman] #necessary to run on the correct runner + stage: build_images + image: quay.io/buildah/stable #stable buildah image should suffice + extends: + - .rules_update_images + variables: + BUILDAH_FORMAT: docker #necessary + BUILDAH_ISOLATION: chroot #necessary + before_script: + - !reference [.scripts, inspect_directory] + - buildah login -u $CI_REGISTRY_USER --password $CI_REGISTRY_PASSWORD $CI_REGISTRY + script: + - !reference [.scripts, build_images] + - !reference [.scripts, push_images] + +# Print predefined variables: https://gitlab.mpcdf.mpg.de/help/ci/variables/predefined_variables.md +ci_vars: + stage: startup + extends: + - .rules_startup + - .image_gitlab_mpcdf_gcc_openmp + - .before_script_load_modules + script: + - echo "All environment variables:" + - env + +inspect_repo: + stage: startup + extends: + - .rules_startup + script: + - curl https://raw.githubusercontent.com/AlDanial/cloc/master/cloc > cloc + - chmod +x cloc + - ./cloc --version + - ./cloc $(git ls-files) + +##################### +### PUSH PIPELINE ### +##################### + +# check_struphy_simulation_params: +# stage: test +# extends: +# - .rules_mr_to_devel +# - .image_gitlab_mpcdf_struphy +# - .before_script_load_modules +# - .variables_push +# script: +# - !reference [.scripts, install_on_push] +# - git clone https://gitlab.mpcdf.mpg.de/struphy/struphy-simulations.git +# - | +# for file in struphy-simulations/**/*.yml; do +# echo "Checking $file" +# # TODO: How to deduce the model? +# struphy params LinearVlasovAmpereOneSpecies --check-file $file +# done + +install_tests: + stage: test + extends: + - .rules_mr_to_devel + - .image_gitlab_mpcdf_struphy + - .before_script_load_modules + - .parallel_matrix_install + script: + - !reference [.scripts, install_on_push] + +unit_tests: + stage: test + extends: + - .rules_mr_to_devel + - .image_gitlab_mpcdf_struphy + - .before_script_load_modules + - .variables_push + script: + - !reference [.scripts, install_on_push] + - !reference [.scripts, unit_tests] + +tests_pyccel_devel: + stage: test + extends: + - .rules_scheduled + - .image_gitlab_mpcdf_struphy + - .before_script_load_modules + - .variables_push + script: + - !reference [.scripts, install_pyccel_devel] + - !reference [.scripts, unit_tests] + - !reference [.scripts, model_tests] + +model_tests: + stage: test + extends: + - .rules_mr_to_devel + - .image_gitlab_mpcdf_struphy + - .before_script_load_modules + - .variables_push + script: + - !reference [.scripts, install_on_push] + - !reference [.scripts, model_tests] + +quickstart_tests: + stage: test + extends: + - .rules_mr_to_devel + - .image_gitlab_mpcdf_struphy + - .before_script_load_modules + - .variables_push + script: + - !reference [.scripts, install_on_push] + - !reference [.scripts, quickstart_tests] + +pages_tests: + stage: test + extends: + - .rules_mr_to_devel + - .image_gitlab_mpcdf_struphy + - .before_script_load_modules + - .variables_push + script: + - !reference [.scripts, install_on_push] + - !reference [.scripts, inspect_struphy] + - module load pandoc + - module list + - pip install .[doc] + - | + struphy compile -y || ( + echo "Initial compile failed. Removing compiled kernels and trying again..." && + struphy compile -d -y && + struphy compile -y + ) + - cd doc ; make html + - mv _build/html/ $CI_PROJECT_DIR/documentation/ + artifacts: + expose_as: 'Documentation' + paths: + - documentation/ + +ubuntu_latest: + stage: test + extends: + - .rules_mr_to_devel + - .image_ubuntu_struphy + - .variables_push + script: + - !reference [.scripts, inspect_directory] + - !reference [.scripts, install_on_push] + - !reference [.scripts, quickstart_tests] + - !reference [.scripts, model_tests] + +macos_nmpp: + stage: test + tags: [macos, nmpp, private] + needs: [] + variables: + # Job ID changes for each job, so reusing the directory becomes impossible, + # therefore, the fetch strategy also becomes unusable + GIT_CLONE_PATH: $CI_BUILDS_DIR/$CI_PROJECT_NAME/$CI_JOB_ID/$CI_CONCURRENT_PROJECT_ID + _JOB_PATH: $CI_BUILDS_DIR/$CI_PROJECT_NAME/$CI_JOB_ID + extends: + - .rules_mr_to_devel + - .variables_push + before_script: + # - brew install cmake + - brew link --overwrite cmake + - cmake --version + - make -v + - printenv + - system_profiler SPHardwareDataType + - export FC=`which gfortran` # for gvec + - export CC=`which gcc` # for gvec + - export CXX=`which g++` # for gvec + script: + - !reference [.scripts, create_venv] + - !reference [.scripts, install] + - !reference [.scripts, compile] + - !reference [.scripts, quickstart_tests] + - !reference [.scripts, model_tests] + after_script: + - pwd + - ls -a + - echo $_JOB_PATH + - ls -a $_JOB_PATH + - rm -rf $_JOB_PATH + +cleanup_macos: + stage: test + tags: [macos, nmpp, private] + needs: [] + extends: + - .rules_cleanup_macos + before_script: + - make -v + - printenv + - system_profiler SPHardwareDataType + script: + - _STRUPHY_PATH=$CI_BUILDS_DIR/$CI_PROJECT_NAME + - pwd + - ls -a + - echo $_STRUPHY_PATH + - ls -a $_STRUPHY_PATH + - rm -rf $_STRUPHY_PATH/* + - ls -a $_STRUPHY_PATH + +########################## +### SCHEDULED PIPELINE ### +########################## + + +install_scheduled: + stage: install + needs: [] + extends: + - .rules_scheduled + - .image_gitlab_mpcdf_gcc_openmp + - .before_script_load_modules + - .parallel_matrix_scheduled + - .artifacts_scheduled + script: + - !reference [.scripts, create_venv_matrix] + - !reference [.scripts, install_no_binary] + - !reference [.scripts, compile_matrix] + +compile_timing: + stage: install + needs: [] + extends: + - .image_gitlab_mpcdf_gcc_openmp + - .before_script_load_modules + - .rules_scheduled + script: + - !reference [.scripts, create_venv] + - !reference [.scripts, install_no_binary] + - pyccel --version + - struphy compile --status + - struphy compile --delete + - struphy compile --time-execution -y --language c > timings_c.txt + - struphy compile --delete + - struphy compile --time-execution -y --language fortran > timings_fortran.txt + - | + cat << 'EOF' > check_pyccel.sh + #!/bin/bash + grep 'pyccel --libdir' timings_c.txt | sed -E 's|.*/site-packages/||' | while read filepath; do + echo "========================================" + echo "Kernel: $filepath" + echo "-------------------- Timers -------------------------" + echo "# Language: C" + grep -A 11 "pyccel.*${filepath}" timings_c.txt | tail -n +3 + echo "# Language: Fortran" + grep -A 13 "pyccel.*${filepath}" timings_fortran.txt | tail -n +3 + done + EOF + - cat check_pyccel.sh + - chmod +x check_pyccel.sh + - ./check_pyccel.sh + +test_build: + stage: test + extends: + - .image_gitlab_mpcdf_gcc_openmp + - .rules_scheduled + - .before_script_load_modules + needs: [] + script: + - !reference [.scripts, create_venv] + - !reference [.scripts, build] + - pip install $(echo dist/struphy*.whl) --no-cache-dir --no-binary mpi4py + +unit_tests_scheduled: + stage: test + extends: + - .image_gitlab_mpcdf_gcc_openmp + - .rules_scheduled + - .parallel_matrix_scheduled + - .before_script_load_modules + script: + - !reference [.scripts, source_env_matrix] + - !reference [.scripts, unit_tests] + +model_tests_scheduled: + stage: test + extends: + - .image_gitlab_mpcdf_gcc_openmp + - .rules_scheduled + - .parallel_matrix_scheduled + - .before_script_load_modules + script: + - !reference [.scripts, source_env_matrix] + - !reference [.scripts, model_tests] + +check_release_dependencies: + stage: test + extends: + - .image_gitlab_mpcdf_gcc_openmp + - .before_script_load_modules + - .rules_mr_to_master + script: + - !reference [.scripts, inspect_directory] + - !reference [.scripts, create_venv] + - !reference [.scripts, install_all] + - python src/struphy/utils/set_release_dependencies.py + - | + if git diff --exit-code pyproject.toml; then + echo "pyproject.toml has not changed, dependencies are set correctly!" + else + echo "pyproject.toml has changed, dependencies are set incorrectly!" + echo "Run python src/struphy/utils/set_release_dependencies.py to update pyproject.toml" + exit 1 + fi + +Ubuntu-latest-from-registry: + stage: test + needs: [] + extends: + - .image_ubuntu_latest + - .rules_scheduled + - .parallel_matrix_scheduled + before_script: + - !reference [.scripts, inspect_directory] + - !reference [.scripts, create_venv] + script: + - !reference [.scripts, install] + - !reference [.scripts, compile_matrix] + - !reference [.scripts, model_tests] + +Fedora-latest: + stage: test + needs: [] + extends: + - .rules_scheduled + - .parallel_matrix_scheduled + - .requirements_fedora + image: fedora:latest + script: + - !reference [.scripts, install] + - !reference [.scripts, compile_matrix] + - !reference [.scripts, model_tests] + +Fedora-latest-from-registry: + stage: test + needs: [] + extends: + - .rules_scheduled + - .parallel_matrix_scheduled + image: gitlab-registry.mpcdf.mpg.de/struphy/struphy/fedora-latest + before_script: + - !reference [.scripts, inspect_directory] + - !reference [.scripts, create_venv] + - . /etc/profile.d/modules.sh + - module load mpi/openmpi-$(arch) + - module list + script: + - !reference [.scripts, install] + - !reference [.scripts, compile_matrix] + - !reference [.scripts, model_tests] + +Fedora-latest-from-registry-base: + stage: test + needs: [] + extends: + - .rules_scheduled + - .parallel_matrix_scheduled + image: gitlab-registry.mpcdf.mpg.de/struphy/struphy/fedora-latest + before_script: + - !reference [.scripts, inspect_directory] + - !reference [.scripts, create_venv] + - . /etc/profile.d/modules.sh + - module load mpi/openmpi-$(arch) + - module list + script: + - !reference [.scripts, install_base] + - !reference [.scripts, compile_matrix] + - !reference [.scripts, model_tests] + +OpenSuse-latest: + stage: test + needs: [] + extends: + - .rules_scheduled + - .parallel_matrix_scheduled + - .requirements_opensuse + image: opensuse/tumbleweed:latest + script: + - !reference [.scripts, install] + - !reference [.scripts, compile_matrix] + - !reference [.scripts, model_tests] + +Opensuse-latest-from-registry: + stage: test + needs: [] + extends: + - .rules_scheduled + - .parallel_matrix_scheduled + image: gitlab-registry.mpcdf.mpg.de/struphy/struphy/opensuse-latest + before_script: + - !reference [.scripts, inspect_directory] + - set -x + - !reference [.scripts, create_venv] + script: + - !reference [.scripts, install] + - !reference [.scripts, compile_matrix] + - !reference [.scripts, model_tests] + +Opensuse-latest-from-registry-base: + stage: test + needs: [] + extends: + - .rules_scheduled + - .parallel_matrix_scheduled + image: gitlab-registry.mpcdf.mpg.de/struphy/struphy/opensuse-latest + before_script: + - !reference [.scripts, inspect_directory] + - set -x + - !reference [.scripts, create_venv] + script: + - !reference [.scripts, install_base] + - !reference [.scripts, compile_matrix] + - !reference [.scripts, model_tests] + +AlmaLinux-latest: + stage: test + needs: [] + extends: + - .rules_scheduled + - .parallel_matrix_scheduled + - .requirements_almalinux + image: almalinux:latest + script: + - !reference [.scripts, install_no_binary] + - !reference [.scripts, compile_matrix] + - !reference [.scripts, model_tests] + +Almalinux-latest-from-registry: + stage: test + needs: [] + extends: + - .rules_scheduled + - .parallel_matrix_scheduled + image: gitlab-registry.mpcdf.mpg.de/struphy/struphy/almalinux-latest + before_script: + - !reference [.scripts, inspect_directory] + - python3 -m ensurepip --upgrade + - python3 -m pip install --upgrade pip + - !reference [.scripts, create_venv] + script: + - !reference [.scripts, install_no_binary] + - !reference [.scripts, compile_matrix] + - !reference [.scripts, model_tests] + +Almalinux-latest-from-registry-base: + stage: test + needs: [] + extends: + - .rules_scheduled + - .parallel_matrix_scheduled + image: gitlab-registry.mpcdf.mpg.de/struphy/struphy/almalinux-latest + before_script: + - !reference [.scripts, inspect_directory] + - python3 -m ensurepip --upgrade + - python3 -m pip install --upgrade pip + - !reference [.scripts, create_venv] + script: + - !reference [.scripts, install_base_no_binary] + - !reference [.scripts, compile_matrix] + - !reference [.scripts, model_tests] + +################## +### LINT STAGE ### +################## + +# Run in weekly CI pipeline only +lint_full_repo_report: + stage: lint + needs: [] + extends: + - .rules_scheduled + - .image_ubuntu_latest + script: + - !reference [.scripts, inspect_directory] + - !reference [.scripts, create_venv] + - pip install -e .[dev] # We have to install struphy in editable mode for struphy lint to work + - !reference [.scripts, inspect_struphy] + - struphy lint all --output-format report + allow_failure: true + artifacts: + expose_as: 'Branch lint report' + paths: + - code_analysis_report.html + expire_in: 1 month + +# Lint struphy with `struphy lint` +lint_repo: + stage: lint + needs: [] + extends: + - .rules_mr_to_devel + - .image_ubuntu_latest + script: + - !reference [.scripts, inspect_directory] + - !reference [.scripts, create_venv] + - pip install -e .[dev] # We have to install struphy in editable mode for struphy lint to work + - !reference [.scripts, inspect_struphy] + # Lint all files in current branch, FAIL the CI pipeline if any files are incorrectly formatted + - struphy lint all --output-format plain --verbose + +# Lint struphy with `struphy lint` +lint_branch_report: + stage: lint + needs: [] + extends: + - .rules_mr_to_devel + - .image_ubuntu_latest + script: + - !reference [.scripts, inspect_directory] + - !reference [.scripts, create_venv] + - pip install -e .[dev] # We have to install struphy in editable mode for struphy lint to work + - !reference [.scripts, inspect_struphy] + - struphy lint branch --output-format report + allow_failure: true + artifacts: + expose_as: 'Branch lint report' + paths: + - code_analysis_report.html + expire_in: 1 month + +################### +### PAGES STAGE ### +################### +pages: + stage: pages + extends: + - .image_ubuntu_latest + - .rules_pages_release + before_script: + - !reference [.scripts, inspect_directory] + - !reference [.scripts, create_venv] + - apt show pandoc + - apt show graphviz + - echo $CI_COMMIT_MESSAGE + - pip install .[doc] + script: + - !reference [.scripts, inspect_struphy] + - !reference [.scripts, compile] + - cd doc ; make html + - mv _build/html/ $CI_PROJECT_DIR/public/ + artifacts: + name: 'pages' + paths: + - public/ + +##################### +### Clusters runners ### +##################### + +test_toki01: + stage: clusters + tags: + - maxlin + - private + - toki01 + script: + - echo "Connected to toki01" + - !reference [.scripts, inspect_directory] + - module purge + - module list + - module avail + - module load gcc/14 openmpi/5.0 python-waterboa/2025.06 cmake netcdf-serial mkl hdf5-serial + - module list + - export FC=`which gfortran` + - export CC=`which gcc` + - export CXX=`which g++` + - !reference [.scripts, install_struphy_clusters] + + +test_viper01: + stage: clusters + tags: + - maxlin + - private + - viper01 + script: + - echo "Connected to viper01" + - !reference [.scripts, inspect_directory] + - module purge + - module list + - module avail + - module load gcc/14 openmpi/5.0 python-waterboa/2025.06 cmake netcdf-serial mkl hdf5-serial + - module list + - export FC=`which gfortran` + - export CC=`which gcc` + - export CXX=`which g++` + - !reference [.scripts, install_struphy_clusters] + +test_raven01: + stage: clusters + tags: + - maxlin + - private + - raven01 + script: + - echo "Connected to raven01" + - !reference [.scripts, inspect_directory] + - module purge + - module list + - module avail + - module load gcc/14 openmpi/5.0 python-waterboa/2025.06 cmake netcdf-serial mkl hdf5-serial + - module list + - export FC=`which gfortran` + - export CC=`which gcc` + - export CXX=`which g++` + - export JAX_DISABLE_JIT=1 + - !reference [.scripts, install_struphy_clusters] + +test_pitagora05: + stage: clusters + tags: + - maxlin + - private + - pitagora05 + script: + - echo "Connected to pitagora05" + - !reference [.scripts, inspect_directory] + - module purge + - module list + - module avail + - module load gcc/12.3.0 openmpi/4.1.6--gcc--12.3.0 python/3.11.7 cmake + - module load netcdf-fortran/4.6.1--openmpi--4.1.6--gcc--12.3.0 intel-oneapi-mkl/2024.0.0--intel-oneapi-mpi--2021.12.1 + - module list + - export FC=`which gfortran` + - export CC=`which gcc` + - export CXX=`which g++` + - export JAX_DISABLE_JIT=1 + - !reference [.scripts, install_struphy_clusters] + +##################### +### RELEASE STAGE ### +##################### + +deploy: + stage: release + extends: + - .rules_pypi_release + - .image_gitlab_mpcdf_gcc_openmp + - .before_script_load_modules + script: + - !reference [.scripts, create_venv] + - !reference [.scripts, build] + - pip install -U twine + - twine upload dist/* + +vars: + stage: release + extends: + - .rules_gitlab_release + before_script: + - !reference [.scripts, inspect_directory] + script: + - var=$(> vars.env + artifacts: + name: 'vars' + reports: + dotenv: vars.env + expire_in: 1 day + +release_job: + stage: release + image: registry.gitlab.com/gitlab-org/release-cli:latest + extends: + - .rules_gitlab_release + needs: ['vars'] + before_script: + - !reference [.scripts, inspect_directory] + script: + - cat /etc/*-release + - echo $VERSION + release: # See https://docs.gitlab.com/ee/ci/yaml/#release for available properties + tag_name: 'v$VERSION' # The version is incremented per pipeline. + name: 'v$VERSION' + ref: '$CI_COMMIT_SHA' # The tag is created from the pipeline SHA. + description: 'CHANGELOG.md' + assets: + links: + - name: 'Documentation' + url: 'https://struphy.pages.mpcdf.de/struphy/index.html' + - name: 'PyPI' + url: 'https://pypi.org/project/struphy/' From ba1d6741f0cf48a4416d45b4f691c0e003237bab Mon Sep 17 00:00:00 2001 From: Max Date: Mon, 13 Oct 2025 19:14:31 +0200 Subject: [PATCH 33/35] Added rules --- .gitlab-ci.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e06efe4c4..96a04a51f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1029,6 +1029,8 @@ pages: test_toki01: stage: clusters + extends: + - .rules_mr_to_devel tags: - maxlin - private @@ -1049,6 +1051,8 @@ test_toki01: test_viper01: stage: clusters + extends: + - .rules_mr_to_devel tags: - maxlin - private @@ -1068,6 +1072,8 @@ test_viper01: test_raven01: stage: clusters + extends: + - .rules_mr_to_devel tags: - maxlin - private @@ -1088,6 +1094,8 @@ test_raven01: test_pitagora05: stage: clusters + extends: + - .rules_mr_to_devel tags: - maxlin - private From fa0cd63703c5219bc7283d02aaeaa1453e4acef9 Mon Sep 17 00:00:00 2001 From: Max Date: Mon, 13 Oct 2025 19:21:38 +0200 Subject: [PATCH 34/35] Removed rules --- .gitlab-ci.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 96a04a51f..66fb054e0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1029,8 +1029,8 @@ pages: test_toki01: stage: clusters - extends: - - .rules_mr_to_devel + # extends: + # - .rules_mr_to_devel tags: - maxlin - private @@ -1051,8 +1051,8 @@ test_toki01: test_viper01: stage: clusters - extends: - - .rules_mr_to_devel + # extends: + # - .rules_mr_to_devel tags: - maxlin - private @@ -1072,8 +1072,8 @@ test_viper01: test_raven01: stage: clusters - extends: - - .rules_mr_to_devel + # extends: + # - .rules_mr_to_devel tags: - maxlin - private @@ -1094,8 +1094,8 @@ test_raven01: test_pitagora05: stage: clusters - extends: - - .rules_mr_to_devel + # extends: + # - .rules_mr_to_devel tags: - maxlin - private From d27c7bf0f159c8fef4956089a0cd77de1ce560ed Mon Sep 17 00:00:00 2001 From: Max Date: Mon, 13 Oct 2025 19:24:36 +0200 Subject: [PATCH 35/35] Commented out old ci --- .gitlab-ci.yml | 1094 ++++++++++++++++++++++++------------------------ 1 file changed, 547 insertions(+), 547 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 66fb054e0..e75f5e5f2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -489,25 +489,25 @@ update_images_buildah: - !reference [.scripts, push_images] # Print predefined variables: https://gitlab.mpcdf.mpg.de/help/ci/variables/predefined_variables.md -ci_vars: - stage: startup - extends: - - .rules_startup - - .image_gitlab_mpcdf_gcc_openmp - - .before_script_load_modules - script: - - echo "All environment variables:" - - env +# ci_vars: +# stage: startup +# extends: +# - .rules_startup +# - .image_gitlab_mpcdf_gcc_openmp +# - .before_script_load_modules +# script: +# - echo "All environment variables:" +# - env -inspect_repo: - stage: startup - extends: - - .rules_startup - script: - - curl https://raw.githubusercontent.com/AlDanial/cloc/master/cloc > cloc - - chmod +x cloc - - ./cloc --version - - ./cloc $(git ls-files) +# inspect_repo: +# stage: startup +# extends: +# - .rules_startup +# script: +# - curl https://raw.githubusercontent.com/AlDanial/cloc/master/cloc > cloc +# - chmod +x cloc +# - ./cloc --version +# - ./cloc $(git ls-files) ##################### ### PUSH PIPELINE ### @@ -530,498 +530,498 @@ inspect_repo: # struphy params LinearVlasovAmpereOneSpecies --check-file $file # done -install_tests: - stage: test - extends: - - .rules_mr_to_devel - - .image_gitlab_mpcdf_struphy - - .before_script_load_modules - - .parallel_matrix_install - script: - - !reference [.scripts, install_on_push] - -unit_tests: - stage: test - extends: - - .rules_mr_to_devel - - .image_gitlab_mpcdf_struphy - - .before_script_load_modules - - .variables_push - script: - - !reference [.scripts, install_on_push] - - !reference [.scripts, unit_tests] - -tests_pyccel_devel: - stage: test - extends: - - .rules_scheduled - - .image_gitlab_mpcdf_struphy - - .before_script_load_modules - - .variables_push - script: - - !reference [.scripts, install_pyccel_devel] - - !reference [.scripts, unit_tests] - - !reference [.scripts, model_tests] - -model_tests: - stage: test - extends: - - .rules_mr_to_devel - - .image_gitlab_mpcdf_struphy - - .before_script_load_modules - - .variables_push - script: - - !reference [.scripts, install_on_push] - - !reference [.scripts, model_tests] - -quickstart_tests: - stage: test - extends: - - .rules_mr_to_devel - - .image_gitlab_mpcdf_struphy - - .before_script_load_modules - - .variables_push - script: - - !reference [.scripts, install_on_push] - - !reference [.scripts, quickstart_tests] - -pages_tests: - stage: test - extends: - - .rules_mr_to_devel - - .image_gitlab_mpcdf_struphy - - .before_script_load_modules - - .variables_push - script: - - !reference [.scripts, install_on_push] - - !reference [.scripts, inspect_struphy] - - module load pandoc - - module list - - pip install .[doc] - - | - struphy compile -y || ( - echo "Initial compile failed. Removing compiled kernels and trying again..." && - struphy compile -d -y && - struphy compile -y - ) - - cd doc ; make html - - mv _build/html/ $CI_PROJECT_DIR/documentation/ - artifacts: - expose_as: 'Documentation' - paths: - - documentation/ +# install_tests: +# stage: test +# extends: +# - .rules_mr_to_devel +# - .image_gitlab_mpcdf_struphy +# - .before_script_load_modules +# - .parallel_matrix_install +# script: +# - !reference [.scripts, install_on_push] -ubuntu_latest: - stage: test - extends: - - .rules_mr_to_devel - - .image_ubuntu_struphy - - .variables_push - script: - - !reference [.scripts, inspect_directory] - - !reference [.scripts, install_on_push] - - !reference [.scripts, quickstart_tests] - - !reference [.scripts, model_tests] - -macos_nmpp: - stage: test - tags: [macos, nmpp, private] - needs: [] - variables: - # Job ID changes for each job, so reusing the directory becomes impossible, - # therefore, the fetch strategy also becomes unusable - GIT_CLONE_PATH: $CI_BUILDS_DIR/$CI_PROJECT_NAME/$CI_JOB_ID/$CI_CONCURRENT_PROJECT_ID - _JOB_PATH: $CI_BUILDS_DIR/$CI_PROJECT_NAME/$CI_JOB_ID - extends: - - .rules_mr_to_devel - - .variables_push - before_script: - # - brew install cmake - - brew link --overwrite cmake - - cmake --version - - make -v - - printenv - - system_profiler SPHardwareDataType - - export FC=`which gfortran` # for gvec - - export CC=`which gcc` # for gvec - - export CXX=`which g++` # for gvec - script: - - !reference [.scripts, create_venv] - - !reference [.scripts, install] - - !reference [.scripts, compile] - - !reference [.scripts, quickstart_tests] - - !reference [.scripts, model_tests] - after_script: - - pwd - - ls -a - - echo $_JOB_PATH - - ls -a $_JOB_PATH - - rm -rf $_JOB_PATH - -cleanup_macos: - stage: test - tags: [macos, nmpp, private] - needs: [] - extends: - - .rules_cleanup_macos - before_script: - - make -v - - printenv - - system_profiler SPHardwareDataType - script: - - _STRUPHY_PATH=$CI_BUILDS_DIR/$CI_PROJECT_NAME - - pwd - - ls -a - - echo $_STRUPHY_PATH - - ls -a $_STRUPHY_PATH - - rm -rf $_STRUPHY_PATH/* - - ls -a $_STRUPHY_PATH +# unit_tests: +# stage: test +# extends: +# - .rules_mr_to_devel +# - .image_gitlab_mpcdf_struphy +# - .before_script_load_modules +# - .variables_push +# script: +# - !reference [.scripts, install_on_push] +# - !reference [.scripts, unit_tests] -########################## -### SCHEDULED PIPELINE ### -########################## +# tests_pyccel_devel: +# stage: test +# extends: +# - .rules_scheduled +# - .image_gitlab_mpcdf_struphy +# - .before_script_load_modules +# - .variables_push +# script: +# - !reference [.scripts, install_pyccel_devel] +# - !reference [.scripts, unit_tests] +# - !reference [.scripts, model_tests] +# model_tests: +# stage: test +# extends: +# - .rules_mr_to_devel +# - .image_gitlab_mpcdf_struphy +# - .before_script_load_modules +# - .variables_push +# script: +# - !reference [.scripts, install_on_push] +# - !reference [.scripts, model_tests] -install_scheduled: - stage: install - needs: [] - extends: - - .rules_scheduled - - .image_gitlab_mpcdf_gcc_openmp - - .before_script_load_modules - - .parallel_matrix_scheduled - - .artifacts_scheduled - script: - - !reference [.scripts, create_venv_matrix] - - !reference [.scripts, install_no_binary] - - !reference [.scripts, compile_matrix] +# quickstart_tests: +# stage: test +# extends: +# - .rules_mr_to_devel +# - .image_gitlab_mpcdf_struphy +# - .before_script_load_modules +# - .variables_push +# script: +# - !reference [.scripts, install_on_push] +# - !reference [.scripts, quickstart_tests] -compile_timing: - stage: install - needs: [] - extends: - - .image_gitlab_mpcdf_gcc_openmp - - .before_script_load_modules - - .rules_scheduled - script: - - !reference [.scripts, create_venv] - - !reference [.scripts, install_no_binary] - - pyccel --version - - struphy compile --status - - struphy compile --delete - - struphy compile --time-execution -y --language c > timings_c.txt - - struphy compile --delete - - struphy compile --time-execution -y --language fortran > timings_fortran.txt - - | - cat << 'EOF' > check_pyccel.sh - #!/bin/bash - grep 'pyccel --libdir' timings_c.txt | sed -E 's|.*/site-packages/||' | while read filepath; do - echo "========================================" - echo "Kernel: $filepath" - echo "-------------------- Timers -------------------------" - echo "# Language: C" - grep -A 11 "pyccel.*${filepath}" timings_c.txt | tail -n +3 - echo "# Language: Fortran" - grep -A 13 "pyccel.*${filepath}" timings_fortran.txt | tail -n +3 - done - EOF - - cat check_pyccel.sh - - chmod +x check_pyccel.sh - - ./check_pyccel.sh - -test_build: - stage: test - extends: - - .image_gitlab_mpcdf_gcc_openmp - - .rules_scheduled - - .before_script_load_modules - needs: [] - script: - - !reference [.scripts, create_venv] - - !reference [.scripts, build] - - pip install $(echo dist/struphy*.whl) --no-cache-dir --no-binary mpi4py +# pages_tests: +# stage: test +# extends: +# - .rules_mr_to_devel +# - .image_gitlab_mpcdf_struphy +# - .before_script_load_modules +# - .variables_push +# script: +# - !reference [.scripts, install_on_push] +# - !reference [.scripts, inspect_struphy] +# - module load pandoc +# - module list +# - pip install .[doc] +# - | +# struphy compile -y || ( +# echo "Initial compile failed. Removing compiled kernels and trying again..." && +# struphy compile -d -y && +# struphy compile -y +# ) +# - cd doc ; make html +# - mv _build/html/ $CI_PROJECT_DIR/documentation/ +# artifacts: +# expose_as: 'Documentation' +# paths: +# - documentation/ + +# ubuntu_latest: +# stage: test +# extends: +# - .rules_mr_to_devel +# - .image_ubuntu_struphy +# - .variables_push +# script: +# - !reference [.scripts, inspect_directory] +# - !reference [.scripts, install_on_push] +# - !reference [.scripts, quickstart_tests] +# - !reference [.scripts, model_tests] -unit_tests_scheduled: - stage: test - extends: - - .image_gitlab_mpcdf_gcc_openmp - - .rules_scheduled - - .parallel_matrix_scheduled - - .before_script_load_modules - script: - - !reference [.scripts, source_env_matrix] - - !reference [.scripts, unit_tests] +# macos_nmpp: +# stage: test +# tags: [macos, nmpp, private] +# needs: [] +# variables: +# # Job ID changes for each job, so reusing the directory becomes impossible, +# # therefore, the fetch strategy also becomes unusable +# GIT_CLONE_PATH: $CI_BUILDS_DIR/$CI_PROJECT_NAME/$CI_JOB_ID/$CI_CONCURRENT_PROJECT_ID +# _JOB_PATH: $CI_BUILDS_DIR/$CI_PROJECT_NAME/$CI_JOB_ID +# extends: +# - .rules_mr_to_devel +# - .variables_push +# before_script: +# # - brew install cmake +# - brew link --overwrite cmake +# - cmake --version +# - make -v +# - printenv +# - system_profiler SPHardwareDataType +# - export FC=`which gfortran` # for gvec +# - export CC=`which gcc` # for gvec +# - export CXX=`which g++` # for gvec +# script: +# - !reference [.scripts, create_venv] +# - !reference [.scripts, install] +# - !reference [.scripts, compile] +# - !reference [.scripts, quickstart_tests] +# - !reference [.scripts, model_tests] +# after_script: +# - pwd +# - ls -a +# - echo $_JOB_PATH +# - ls -a $_JOB_PATH +# - rm -rf $_JOB_PATH + +# cleanup_macos: +# stage: test +# tags: [macos, nmpp, private] +# needs: [] +# extends: +# - .rules_cleanup_macos +# before_script: +# - make -v +# - printenv +# - system_profiler SPHardwareDataType +# script: +# - _STRUPHY_PATH=$CI_BUILDS_DIR/$CI_PROJECT_NAME +# - pwd +# - ls -a +# - echo $_STRUPHY_PATH +# - ls -a $_STRUPHY_PATH +# - rm -rf $_STRUPHY_PATH/* +# - ls -a $_STRUPHY_PATH + +# ########################## +# ### SCHEDULED PIPELINE ### +# ########################## + + +# install_scheduled: +# stage: install +# needs: [] +# extends: +# - .rules_scheduled +# - .image_gitlab_mpcdf_gcc_openmp +# - .before_script_load_modules +# - .parallel_matrix_scheduled +# - .artifacts_scheduled +# script: +# - !reference [.scripts, create_venv_matrix] +# - !reference [.scripts, install_no_binary] +# - !reference [.scripts, compile_matrix] -model_tests_scheduled: - stage: test - extends: - - .image_gitlab_mpcdf_gcc_openmp - - .rules_scheduled - - .parallel_matrix_scheduled - - .before_script_load_modules - script: - - !reference [.scripts, source_env_matrix] - - !reference [.scripts, model_tests] +# compile_timing: +# stage: install +# needs: [] +# extends: +# - .image_gitlab_mpcdf_gcc_openmp +# - .before_script_load_modules +# - .rules_scheduled +# script: +# - !reference [.scripts, create_venv] +# - !reference [.scripts, install_no_binary] +# - pyccel --version +# - struphy compile --status +# - struphy compile --delete +# - struphy compile --time-execution -y --language c > timings_c.txt +# - struphy compile --delete +# - struphy compile --time-execution -y --language fortran > timings_fortran.txt +# - | +# cat << 'EOF' > check_pyccel.sh +# #!/bin/bash +# grep 'pyccel --libdir' timings_c.txt | sed -E 's|.*/site-packages/||' | while read filepath; do +# echo "========================================" +# echo "Kernel: $filepath" +# echo "-------------------- Timers -------------------------" +# echo "# Language: C" +# grep -A 11 "pyccel.*${filepath}" timings_c.txt | tail -n +3 +# echo "# Language: Fortran" +# grep -A 13 "pyccel.*${filepath}" timings_fortran.txt | tail -n +3 +# done +# EOF +# - cat check_pyccel.sh +# - chmod +x check_pyccel.sh +# - ./check_pyccel.sh -check_release_dependencies: - stage: test - extends: - - .image_gitlab_mpcdf_gcc_openmp - - .before_script_load_modules - - .rules_mr_to_master - script: - - !reference [.scripts, inspect_directory] - - !reference [.scripts, create_venv] - - !reference [.scripts, install_all] - - python src/struphy/utils/set_release_dependencies.py - - | - if git diff --exit-code pyproject.toml; then - echo "pyproject.toml has not changed, dependencies are set correctly!" - else - echo "pyproject.toml has changed, dependencies are set incorrectly!" - echo "Run python src/struphy/utils/set_release_dependencies.py to update pyproject.toml" - exit 1 - fi - -Ubuntu-latest-from-registry: - stage: test - needs: [] - extends: - - .image_ubuntu_latest - - .rules_scheduled - - .parallel_matrix_scheduled - before_script: - - !reference [.scripts, inspect_directory] - - !reference [.scripts, create_venv] - script: - - !reference [.scripts, install] - - !reference [.scripts, compile_matrix] - - !reference [.scripts, model_tests] +# test_build: +# stage: test +# extends: +# - .image_gitlab_mpcdf_gcc_openmp +# - .rules_scheduled +# - .before_script_load_modules +# needs: [] +# script: +# - !reference [.scripts, create_venv] +# - !reference [.scripts, build] +# - pip install $(echo dist/struphy*.whl) --no-cache-dir --no-binary mpi4py -Fedora-latest: - stage: test - needs: [] - extends: - - .rules_scheduled - - .parallel_matrix_scheduled - - .requirements_fedora - image: fedora:latest - script: - - !reference [.scripts, install] - - !reference [.scripts, compile_matrix] - - !reference [.scripts, model_tests] +# unit_tests_scheduled: +# stage: test +# extends: +# - .image_gitlab_mpcdf_gcc_openmp +# - .rules_scheduled +# - .parallel_matrix_scheduled +# - .before_script_load_modules +# script: +# - !reference [.scripts, source_env_matrix] +# - !reference [.scripts, unit_tests] -Fedora-latest-from-registry: - stage: test - needs: [] - extends: - - .rules_scheduled - - .parallel_matrix_scheduled - image: gitlab-registry.mpcdf.mpg.de/struphy/struphy/fedora-latest - before_script: - - !reference [.scripts, inspect_directory] - - !reference [.scripts, create_venv] - - . /etc/profile.d/modules.sh - - module load mpi/openmpi-$(arch) - - module list - script: - - !reference [.scripts, install] - - !reference [.scripts, compile_matrix] - - !reference [.scripts, model_tests] +# model_tests_scheduled: +# stage: test +# extends: +# - .image_gitlab_mpcdf_gcc_openmp +# - .rules_scheduled +# - .parallel_matrix_scheduled +# - .before_script_load_modules +# script: +# - !reference [.scripts, source_env_matrix] +# - !reference [.scripts, model_tests] -Fedora-latest-from-registry-base: - stage: test - needs: [] - extends: - - .rules_scheduled - - .parallel_matrix_scheduled - image: gitlab-registry.mpcdf.mpg.de/struphy/struphy/fedora-latest - before_script: - - !reference [.scripts, inspect_directory] - - !reference [.scripts, create_venv] - - . /etc/profile.d/modules.sh - - module load mpi/openmpi-$(arch) - - module list - script: - - !reference [.scripts, install_base] - - !reference [.scripts, compile_matrix] - - !reference [.scripts, model_tests] +# check_release_dependencies: +# stage: test +# extends: +# - .image_gitlab_mpcdf_gcc_openmp +# - .before_script_load_modules +# - .rules_mr_to_master +# script: +# - !reference [.scripts, inspect_directory] +# - !reference [.scripts, create_venv] +# - !reference [.scripts, install_all] +# - python src/struphy/utils/set_release_dependencies.py +# - | +# if git diff --exit-code pyproject.toml; then +# echo "pyproject.toml has not changed, dependencies are set correctly!" +# else +# echo "pyproject.toml has changed, dependencies are set incorrectly!" +# echo "Run python src/struphy/utils/set_release_dependencies.py to update pyproject.toml" +# exit 1 +# fi + +# Ubuntu-latest-from-registry: +# stage: test +# needs: [] +# extends: +# - .image_ubuntu_latest +# - .rules_scheduled +# - .parallel_matrix_scheduled +# before_script: +# - !reference [.scripts, inspect_directory] +# - !reference [.scripts, create_venv] +# script: +# - !reference [.scripts, install] +# - !reference [.scripts, compile_matrix] +# - !reference [.scripts, model_tests] -OpenSuse-latest: - stage: test - needs: [] - extends: - - .rules_scheduled - - .parallel_matrix_scheduled - - .requirements_opensuse - image: opensuse/tumbleweed:latest - script: - - !reference [.scripts, install] - - !reference [.scripts, compile_matrix] - - !reference [.scripts, model_tests] +# Fedora-latest: +# stage: test +# needs: [] +# extends: +# - .rules_scheduled +# - .parallel_matrix_scheduled +# - .requirements_fedora +# image: fedora:latest +# script: +# - !reference [.scripts, install] +# - !reference [.scripts, compile_matrix] +# - !reference [.scripts, model_tests] -Opensuse-latest-from-registry: - stage: test - needs: [] - extends: - - .rules_scheduled - - .parallel_matrix_scheduled - image: gitlab-registry.mpcdf.mpg.de/struphy/struphy/opensuse-latest - before_script: - - !reference [.scripts, inspect_directory] - - set -x - - !reference [.scripts, create_venv] - script: - - !reference [.scripts, install] - - !reference [.scripts, compile_matrix] - - !reference [.scripts, model_tests] +# Fedora-latest-from-registry: +# stage: test +# needs: [] +# extends: +# - .rules_scheduled +# - .parallel_matrix_scheduled +# image: gitlab-registry.mpcdf.mpg.de/struphy/struphy/fedora-latest +# before_script: +# - !reference [.scripts, inspect_directory] +# - !reference [.scripts, create_venv] +# - . /etc/profile.d/modules.sh +# - module load mpi/openmpi-$(arch) +# - module list +# script: +# - !reference [.scripts, install] +# - !reference [.scripts, compile_matrix] +# - !reference [.scripts, model_tests] -Opensuse-latest-from-registry-base: - stage: test - needs: [] - extends: - - .rules_scheduled - - .parallel_matrix_scheduled - image: gitlab-registry.mpcdf.mpg.de/struphy/struphy/opensuse-latest - before_script: - - !reference [.scripts, inspect_directory] - - set -x - - !reference [.scripts, create_venv] - script: - - !reference [.scripts, install_base] - - !reference [.scripts, compile_matrix] - - !reference [.scripts, model_tests] +# Fedora-latest-from-registry-base: +# stage: test +# needs: [] +# extends: +# - .rules_scheduled +# - .parallel_matrix_scheduled +# image: gitlab-registry.mpcdf.mpg.de/struphy/struphy/fedora-latest +# before_script: +# - !reference [.scripts, inspect_directory] +# - !reference [.scripts, create_venv] +# - . /etc/profile.d/modules.sh +# - module load mpi/openmpi-$(arch) +# - module list +# script: +# - !reference [.scripts, install_base] +# - !reference [.scripts, compile_matrix] +# - !reference [.scripts, model_tests] -AlmaLinux-latest: - stage: test - needs: [] - extends: - - .rules_scheduled - - .parallel_matrix_scheduled - - .requirements_almalinux - image: almalinux:latest - script: - - !reference [.scripts, install_no_binary] - - !reference [.scripts, compile_matrix] - - !reference [.scripts, model_tests] +# OpenSuse-latest: +# stage: test +# needs: [] +# extends: +# - .rules_scheduled +# - .parallel_matrix_scheduled +# - .requirements_opensuse +# image: opensuse/tumbleweed:latest +# script: +# - !reference [.scripts, install] +# - !reference [.scripts, compile_matrix] +# - !reference [.scripts, model_tests] -Almalinux-latest-from-registry: - stage: test - needs: [] - extends: - - .rules_scheduled - - .parallel_matrix_scheduled - image: gitlab-registry.mpcdf.mpg.de/struphy/struphy/almalinux-latest - before_script: - - !reference [.scripts, inspect_directory] - - python3 -m ensurepip --upgrade - - python3 -m pip install --upgrade pip - - !reference [.scripts, create_venv] - script: - - !reference [.scripts, install_no_binary] - - !reference [.scripts, compile_matrix] - - !reference [.scripts, model_tests] +# Opensuse-latest-from-registry: +# stage: test +# needs: [] +# extends: +# - .rules_scheduled +# - .parallel_matrix_scheduled +# image: gitlab-registry.mpcdf.mpg.de/struphy/struphy/opensuse-latest +# before_script: +# - !reference [.scripts, inspect_directory] +# - set -x +# - !reference [.scripts, create_venv] +# script: +# - !reference [.scripts, install] +# - !reference [.scripts, compile_matrix] +# - !reference [.scripts, model_tests] -Almalinux-latest-from-registry-base: - stage: test - needs: [] - extends: - - .rules_scheduled - - .parallel_matrix_scheduled - image: gitlab-registry.mpcdf.mpg.de/struphy/struphy/almalinux-latest - before_script: - - !reference [.scripts, inspect_directory] - - python3 -m ensurepip --upgrade - - python3 -m pip install --upgrade pip - - !reference [.scripts, create_venv] - script: - - !reference [.scripts, install_base_no_binary] - - !reference [.scripts, compile_matrix] - - !reference [.scripts, model_tests] +# Opensuse-latest-from-registry-base: +# stage: test +# needs: [] +# extends: +# - .rules_scheduled +# - .parallel_matrix_scheduled +# image: gitlab-registry.mpcdf.mpg.de/struphy/struphy/opensuse-latest +# before_script: +# - !reference [.scripts, inspect_directory] +# - set -x +# - !reference [.scripts, create_venv] +# script: +# - !reference [.scripts, install_base] +# - !reference [.scripts, compile_matrix] +# - !reference [.scripts, model_tests] -################## -### LINT STAGE ### -################## +# AlmaLinux-latest: +# stage: test +# needs: [] +# extends: +# - .rules_scheduled +# - .parallel_matrix_scheduled +# - .requirements_almalinux +# image: almalinux:latest +# script: +# - !reference [.scripts, install_no_binary] +# - !reference [.scripts, compile_matrix] +# - !reference [.scripts, model_tests] -# Run in weekly CI pipeline only -lint_full_repo_report: - stage: lint - needs: [] - extends: - - .rules_scheduled - - .image_ubuntu_latest - script: - - !reference [.scripts, inspect_directory] - - !reference [.scripts, create_venv] - - pip install -e .[dev] # We have to install struphy in editable mode for struphy lint to work - - !reference [.scripts, inspect_struphy] - - struphy lint all --output-format report - allow_failure: true - artifacts: - expose_as: 'Branch lint report' - paths: - - code_analysis_report.html - expire_in: 1 month +# Almalinux-latest-from-registry: +# stage: test +# needs: [] +# extends: +# - .rules_scheduled +# - .parallel_matrix_scheduled +# image: gitlab-registry.mpcdf.mpg.de/struphy/struphy/almalinux-latest +# before_script: +# - !reference [.scripts, inspect_directory] +# - python3 -m ensurepip --upgrade +# - python3 -m pip install --upgrade pip +# - !reference [.scripts, create_venv] +# script: +# - !reference [.scripts, install_no_binary] +# - !reference [.scripts, compile_matrix] +# - !reference [.scripts, model_tests] -# Lint struphy with `struphy lint` -lint_repo: - stage: lint - needs: [] - extends: - - .rules_mr_to_devel - - .image_ubuntu_latest - script: - - !reference [.scripts, inspect_directory] - - !reference [.scripts, create_venv] - - pip install -e .[dev] # We have to install struphy in editable mode for struphy lint to work - - !reference [.scripts, inspect_struphy] - # Lint all files in current branch, FAIL the CI pipeline if any files are incorrectly formatted - - struphy lint all --output-format plain --verbose - -# Lint struphy with `struphy lint` -lint_branch_report: - stage: lint - needs: [] - extends: - - .rules_mr_to_devel - - .image_ubuntu_latest - script: - - !reference [.scripts, inspect_directory] - - !reference [.scripts, create_venv] - - pip install -e .[dev] # We have to install struphy in editable mode for struphy lint to work - - !reference [.scripts, inspect_struphy] - - struphy lint branch --output-format report - allow_failure: true - artifacts: - expose_as: 'Branch lint report' - paths: - - code_analysis_report.html - expire_in: 1 month - -################### -### PAGES STAGE ### -################### -pages: - stage: pages - extends: - - .image_ubuntu_latest - - .rules_pages_release - before_script: - - !reference [.scripts, inspect_directory] - - !reference [.scripts, create_venv] - - apt show pandoc - - apt show graphviz - - echo $CI_COMMIT_MESSAGE - - pip install .[doc] - script: - - !reference [.scripts, inspect_struphy] - - !reference [.scripts, compile] - - cd doc ; make html - - mv _build/html/ $CI_PROJECT_DIR/public/ - artifacts: - name: 'pages' - paths: - - public/ +# Almalinux-latest-from-registry-base: +# stage: test +# needs: [] +# extends: +# - .rules_scheduled +# - .parallel_matrix_scheduled +# image: gitlab-registry.mpcdf.mpg.de/struphy/struphy/almalinux-latest +# before_script: +# - !reference [.scripts, inspect_directory] +# - python3 -m ensurepip --upgrade +# - python3 -m pip install --upgrade pip +# - !reference [.scripts, create_venv] +# script: +# - !reference [.scripts, install_base_no_binary] +# - !reference [.scripts, compile_matrix] +# - !reference [.scripts, model_tests] + +# ################## +# ### LINT STAGE ### +# ################## + +# # Run in weekly CI pipeline only +# lint_full_repo_report: +# stage: lint +# needs: [] +# extends: +# - .rules_scheduled +# - .image_ubuntu_latest +# script: +# - !reference [.scripts, inspect_directory] +# - !reference [.scripts, create_venv] +# - pip install -e .[dev] # We have to install struphy in editable mode for struphy lint to work +# - !reference [.scripts, inspect_struphy] +# - struphy lint all --output-format report +# allow_failure: true +# artifacts: +# expose_as: 'Branch lint report' +# paths: +# - code_analysis_report.html +# expire_in: 1 month + +# # Lint struphy with `struphy lint` +# lint_repo: +# stage: lint +# needs: [] +# extends: +# - .rules_mr_to_devel +# - .image_ubuntu_latest +# script: +# - !reference [.scripts, inspect_directory] +# - !reference [.scripts, create_venv] +# - pip install -e .[dev] # We have to install struphy in editable mode for struphy lint to work +# - !reference [.scripts, inspect_struphy] +# # Lint all files in current branch, FAIL the CI pipeline if any files are incorrectly formatted +# - struphy lint all --output-format plain --verbose + +# # Lint struphy with `struphy lint` +# lint_branch_report: +# stage: lint +# needs: [] +# extends: +# - .rules_mr_to_devel +# - .image_ubuntu_latest +# script: +# - !reference [.scripts, inspect_directory] +# - !reference [.scripts, create_venv] +# - pip install -e .[dev] # We have to install struphy in editable mode for struphy lint to work +# - !reference [.scripts, inspect_struphy] +# - struphy lint branch --output-format report +# allow_failure: true +# artifacts: +# expose_as: 'Branch lint report' +# paths: +# - code_analysis_report.html +# expire_in: 1 month + +# ################### +# ### PAGES STAGE ### +# ################### +# pages: +# stage: pages +# extends: +# - .image_ubuntu_latest +# - .rules_pages_release +# before_script: +# - !reference [.scripts, inspect_directory] +# - !reference [.scripts, create_venv] +# - apt show pandoc +# - apt show graphviz +# - echo $CI_COMMIT_MESSAGE +# - pip install .[doc] +# script: +# - !reference [.scripts, inspect_struphy] +# - !reference [.scripts, compile] +# - cd doc ; make html +# - mv _build/html/ $CI_PROJECT_DIR/public/ +# artifacts: +# name: 'pages' +# paths: +# - public/ ##################### ### Clusters runners ### @@ -1119,61 +1119,61 @@ test_pitagora05: ### RELEASE STAGE ### ##################### -deploy: - stage: release - extends: - - .rules_pypi_release - - .image_gitlab_mpcdf_gcc_openmp - - .before_script_load_modules - script: - - !reference [.scripts, create_venv] - - !reference [.scripts, build] - - pip install -U twine - - twine upload dist/* - -vars: - stage: release - extends: - - .rules_gitlab_release - before_script: - - !reference [.scripts, inspect_directory] - script: - - var=$(> vars.env - artifacts: - name: 'vars' - reports: - dotenv: vars.env - expire_in: 1 day +# deploy: +# stage: release +# extends: +# - .rules_pypi_release +# - .image_gitlab_mpcdf_gcc_openmp +# - .before_script_load_modules +# script: +# - !reference [.scripts, create_venv] +# - !reference [.scripts, build] +# - pip install -U twine +# - twine upload dist/* -release_job: - stage: release - image: registry.gitlab.com/gitlab-org/release-cli:latest - extends: - - .rules_gitlab_release - needs: ['vars'] - before_script: - - !reference [.scripts, inspect_directory] - script: - - cat /etc/*-release - - echo $VERSION - release: # See https://docs.gitlab.com/ee/ci/yaml/#release for available properties - tag_name: 'v$VERSION' # The version is incremented per pipeline. - name: 'v$VERSION' - ref: '$CI_COMMIT_SHA' # The tag is created from the pipeline SHA. - description: 'CHANGELOG.md' - assets: - links: - - name: 'Documentation' - url: 'https://struphy.pages.mpcdf.de/struphy/index.html' - - name: 'PyPI' - url: 'https://pypi.org/project/struphy/' +# vars: +# stage: release +# extends: +# - .rules_gitlab_release +# before_script: +# - !reference [.scripts, inspect_directory] +# script: +# - var=$(> vars.env +# artifacts: +# name: 'vars' +# reports: +# dotenv: vars.env +# expire_in: 1 day + +# release_job: +# stage: release +# image: registry.gitlab.com/gitlab-org/release-cli:latest +# extends: +# - .rules_gitlab_release +# needs: ['vars'] +# before_script: +# - !reference [.scripts, inspect_directory] +# script: +# - cat /etc/*-release +# - echo $VERSION +# release: # See https://docs.gitlab.com/ee/ci/yaml/#release for available properties +# tag_name: 'v$VERSION' # The version is incremented per pipeline. +# name: 'v$VERSION' +# ref: '$CI_COMMIT_SHA' # The tag is created from the pipeline SHA. +# description: 'CHANGELOG.md' +# assets: +# links: +# - name: 'Documentation' +# url: 'https://struphy.pages.mpcdf.de/struphy/index.html' +# - name: 'PyPI' +# url: 'https://pypi.org/project/struphy/'