diff --git a/.codecov.yml b/.codecov.yaml similarity index 100% rename from .codecov.yml rename to .codecov.yaml diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS deleted file mode 100644 index 5155aa3..0000000 --- a/.github/CODEOWNERS +++ /dev/null @@ -1 +0,0 @@ -* @mdolab/infrastructure-maintainers diff --git a/.github/dependabot.yml b/.github/dependabot.yaml similarity index 100% rename from .github/dependabot.yml rename to .github/dependabot.yaml diff --git a/.github/stale.yml b/.github/stale.yaml similarity index 100% rename from .github/stale.yml rename to .github/stale.yaml diff --git a/.github/workflows/README.md b/.github/workflows/README.md new file mode 100644 index 0000000..8b65332 --- /dev/null +++ b/.github/workflows/README.md @@ -0,0 +1,234 @@ +# GitHub Actions Workflows + +Reusable GitHub Actions workflows for Supercritical repositories. These workflows are called from individual repositories using the `workflow_call` trigger. + +## Available Workflows + +| Workflow | Description | +| :------- | :---------- | +| `build.yaml` | Build and test code in Docker container | +| `tapenade.yaml` | Tapenade automatic differentiation checks | +| `clang_format.yaml` | C/C++ formatting checks | +| `fprettify.yaml` | Fortran 90 formatting checks | +| `ruff.yaml` | Python formatting and linting with Ruff | +| `mypy.yaml` | Python type checking with MyPy | +| `branch-name-check.yaml` | Enforce branch naming conventions | + +--- + +## Workflow Options + +Configuration inheritance/overrides are documented in each workflow section below. + +### build.yaml + +Docker-based build and test workflow using the `scritical/private-dev` image. +Runs GCC and/or Intel jobs based on `GCC` and `INTEL` input flags. + +| Name | Type | Default | Description | +| :--- | :--- | :------ | :---------- | +| `TIMEOUT` | number | `120` | Runtime allowed for the job, in minutes | +| `GCC_CONFIG` | string | `""` | Path to GCC configuration file (from repository root) | +| `INTEL_CONFIG` | string | `""` | Path to Intel configuration file (from repository root) | +| `INTEL` | boolean | `false` | Whether to run Intel-specific build and test steps | +| `GCC` | boolean | `true` | Whether to run GCC-specific build and test steps | +| `BUILD_SCRIPT` | string | `.github/build_real.sh` | Path to build script. Empty string skips this step | +| `TEST_SCRIPT` | string | `.github/test_real.sh` | Path to test script. Empty string skips this step | +| `TEST` | boolean | `true` | Whether to run the test step | + +**Required Secrets:** +| Name | Description | +| :--- | :---------- | +| `DOCKER_USER` | Docker registry username | +| `DOCKER_OAT` | Docker registry Organization Access Token | + +If these secrets are configured at the organization level, callers can use `secrets: inherit` instead of listing each secret. + +--- + +### ruff.yaml + +Python formatting and linting using Ruff. +The workflow checks out the org-wide Ruff configuration from `scritical/.github` and uses it for format, lint, and isort checks. + +| Name | Type | Default | Description | +| :--- | :--- | :------ | :---------- | +| `MCCABE` | boolean | `false` | Enable McCabe complexity check (pass/fail, max complexity = 10) | +| `ISORT` | boolean | `false` | Enable import sorting check (pass/fail) | + +**Configuration Override:** Create a `ruff.toml` in your repo with: +```toml +extend = "~/.config/ruff/ruff.toml" + +# Local overrides here +[lint] +ignore = ["N802"] +``` + +--- + +### mypy.yaml + +Runs MyPy type checking in the GCC OpenMPI Docker image. + +| Name | Type | Default | Description | +| :--- | :--- | :------ | :---------- | +| `TIMEOUT` | number | `30` | Runtime allowed for the job, in minutes | + +**Required Secrets:** +| Name | Description | +| :--- | :---------- | +| `DOCKER_USER` | Docker registry username | +| `DOCKER_OAT` | Docker registry Organization Access Token | + +If these secrets are configured at the organization level, callers can use `secrets: inherit` instead of listing each secret. + +--- + +### tapenade.yaml + +Run Tapenade automatic differentiation and check for uncommitted changes. + +| Name | Type | Default | Description | +| :--- | :--- | :------ | :---------- | +| `TIMEOUT` | number | `10` | Runtime allowed for the job, in minutes | +| `TAPENADE_SCRIPT` | string | `.github/build_tapenade.sh` | Path to Tapenade build script | + +Uses Tapenade version 3.16 from tapenade_3.16-v2-723-ge8da61555.tar. Using a different version will create a diff. + +Here is the link to our tapenade version: https://gitlab.inria.fr/tapenade/tapenade/-/package_files/112870/download + +--- + +### clang_format.yaml + +C/C++ code formatting checks using clang-format. + +| Name | Type | Default | Description | +| :--- | :--- | :------ | :---------- | +| `TIMEOUT` | number | `10` | Runtime allowed for the job, in minutes | + +**Configuration Override:** Create a `.clang-format` file in your repo root. + +--- + +### fprettify.yaml + +Fortran 90 code formatting checks using fprettify. + +| Name | Type | Default | Description | +| :--- | :--- | :------ | :---------- | +| `TIMEOUT` | number | `10` | Runtime allowed for the job, in minutes | + +**Configuration Override:** Create a `.fprettify.rc` file in your repo root. + +--- + +### branch-name-check.yaml + +Enforces branch naming conventions for pull requests: + +- For PRs targeting `main`, source branches must start with `feature-`, `bugfix-`, or `hotfix-`. +- For PRs targeting `client-*`, source branches must start with `feature-`, `bugfix-`, or `hotfix-`. + +## Setting Up Workflows + +### Step 1: Create Workflow File + +Create `.github/workflows/ci.yaml` in your repository: + +```yaml +name: CI + +on: + push: + branches: [main] + pull_request: + branches: [main] + +jobs: + build: + uses: scritical/.github/.github/workflows/build.yaml@main + with: + GCC_CONFIG: config/defaults/config.LINUX_GFORTRAN.mk # If necessary + secrets: + DOCKER_USER: ${{ secrets.DOCKER_USER }} + DOCKER_OAT: ${{ secrets.DOCKER_OAT }} + + ruff: + uses: scritical/.github/.github/workflows/ruff.yaml@main + + clang-format: + uses: scritical/.github/.github/workflows/clang_format.yaml@main + + fprettify: + uses: scritical/.github/.github/workflows/fprettify.yaml@main +``` + +### Step 2: Write Build Script + +Create `.github/build_real.sh` in your repository: + +```bash +#!/bin/bash +set -e +cp $CONFIG_FILE config/config.mk +make +pip install . +``` + +### Step 3: Write Test Script + +Create `.github/test_real.sh` in your repository: + +```bash +#!/bin/bash +set -e +./input_files/get-input-files.sh +testflo -v . -n 1 +``` + +### Step 4: Add Secrets + +In your orginaization settings, add the required secrets: +- `DOCKER_USER` - Docker organization name +- `DOCKER_OAT` - Docker Organization Access Token for pulling private images + +--- + +## Complex Builds + +For repositories requiring both real and complex builds: + +```yaml +jobs: + build-real: + uses: scritical/.github/.github/workflows/build.yaml@main + with: + GCC_CONFIG: config/defaults/config.LINUX_GFORTRAN.mk + BUILD_SCRIPT: .github/build_real.sh + TEST_SCRIPT: .github/test_real.sh + secrets: + DOCKER_USER: ${{ secrets.DOCKER_USER }} + DOCKER_OAT: ${{ secrets.DOCKER_OAT }} + + build-complex: + uses: scritical/.github/.github/workflows/build.yaml@main + with: + GCC_CONFIG: config/defaults/config.LINUX_GFORTRAN.mk + BUILD_SCRIPT: .github/build_complex.sh + TEST_SCRIPT: .github/test_complex.sh + secrets: + DOCKER_USER: ${{ secrets.DOCKER_USER }} + DOCKER_OAT: ${{ secrets.DOCKER_OAT }} +``` + +--- + +## Adding Status Badge + +Add a status badge to your README: + +```markdown +![CI](https://github.com/scritical/REPO_NAME/actions/workflows/ci.yaml/badge.svg) +``` diff --git a/.github/workflows/branch-name-check.yml b/.github/workflows/branch-name-check.yaml similarity index 88% rename from .github/workflows/branch-name-check.yml rename to .github/workflows/branch-name-check.yaml index f02ea69..cd70185 100644 --- a/.github/workflows/branch-name-check.yml +++ b/.github/workflows/branch-name-check.yaml @@ -1,15 +1,9 @@ -name: Branch Name Check - on: - pull_request: - types: [opened, synchronize, reopened, edited] - branches: - - main - - 'client-*' + workflow_call: jobs: check-branch-name: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Check branch name for main if: github.base_ref == 'main' diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml new file mode 100644 index 0000000..49497d1 --- /dev/null +++ b/.github/workflows/build.yaml @@ -0,0 +1,113 @@ +on: + workflow_call: + inputs: + TIMEOUT: + type: number + default: 120 + description: "Timeout for the job in minutes" + GCC_CONFIG: + type: string + default: "" + description: "Path to GCC configuration file" + INTEL_CONFIG: + type: string + default: "" + description: "Path to Intel configuration file" + INTEL: + type: boolean + default: false + description: "Whether to run Intel-specific build and test steps (if false, runs GCC steps)" + GCC: + type: boolean + default: true + description: "Whether to run GCC-specific build and test steps (if false, runs Intel steps)" + BUILD_SCRIPT: + type: string + default: ".github/build_real.sh" + description: "Path to build script" + TEST_SCRIPT: + type: string + default: ".github/test_real.sh" + description: "Path to test script" + TEST: + type: boolean + default: true + description: "Whether to run the test step" + secrets: + DOCKER_USER: + required: true + description: "Docker registry username" + DOCKER_OAT: + required: true + description: "Docker registry Organization Access Token" + +env: + SCRITICAL_HOMEDIR: /home/scriticaluser + BASHRC: /home/scriticaluser/.bashrc_scritical + +jobs: + build-gcc: + if: inputs.GCC + runs-on: ubuntu-24.04 + timeout-minutes: ${{ inputs.TIMEOUT }} + env: + DOCKER_TAG: u24-gcc-ompi-latest + CONFIG_FILE: ${{ inputs.GCC_CONFIG }} + steps: &build_steps + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Set repo environment variables + run: | + repo_name="${{ github.event.repository.name }}" + echo "REPO_NAME=$repo_name" >> "$GITHUB_ENV" + echo "DOCKER_WORKING_DIR=${{ env.SCRITICAL_HOMEDIR }}/repos/$repo_name" >> "$GITHUB_ENV" + echo "DOCKER_MOUNT_DIR=${{ env.SCRITICAL_HOMEDIR }}/mount/$repo_name" >> "$GITHUB_ENV" + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_USER }} + password: ${{ secrets.DOCKER_OAT }} + + - name: Start Docker container + run: | + docker run -t -d --name app \ + --mount "type=bind,src=${{ github.workspace }},target=${{ env.DOCKER_MOUNT_DIR }}" \ + scritical/private-dev:${{ env.DOCKER_TAG }} /bin/bash + + # Copy repo to working directory + docker exec app /bin/bash -c "rm -rf $DOCKER_WORKING_DIR && mkdir -p $DOCKER_WORKING_DIR && cp -a $DOCKER_MOUNT_DIR/. $DOCKER_WORKING_DIR/" + + - name: Build + if: inputs.BUILD_SCRIPT != '' + run: | + docker exec \ + -e CONFIG_FILE="${{ env.CONFIG_FILE }}" \ + app /bin/bash -c ". $BASHRC && cd $DOCKER_WORKING_DIR && /bin/bash ${{ inputs.BUILD_SCRIPT }}" + + - name: Test + if: inputs.TEST_SCRIPT != '' && inputs.TEST + run: | + docker exec \ + -e AGENT_NAME="${{ runner.name }}" \ + -e BUILD_REASON="${{ github.event_name }}" \ + app /bin/bash -c ". $BASHRC && cd $DOCKER_WORKING_DIR && source ~/MPIOversubscribe.sh && /bin/bash ${{ inputs.TEST_SCRIPT }}" + + - name: Cleanup + if: always() + run: | + # Stop and remove the container only if it exists to avoid noisy errors. + if docker ps -a --format '{{.Names}}' | grep -qx app; then + docker stop app && docker rm app || true + fi + + build-intel: + if: inputs.INTEL + runs-on: ubuntu-24.04 + timeout-minutes: ${{ inputs.TIMEOUT }} + env: + DOCKER_TAG: u24-intel-impi-latest + CONFIG_FILE: ${{ inputs.INTEL_CONFIG }} + steps: *build_steps + diff --git a/.github/workflows/clang_format.yaml b/.github/workflows/clang_format.yaml index 03aeea9..b76cdd8 100644 --- a/.github/workflows/clang_format.yaml +++ b/.github/workflows/clang_format.yaml @@ -1,13 +1,32 @@ on: workflow_call: + inputs: + TIMEOUT: + type: number + default: 10 + description: "Timeout for the job in minutes" jobs: clang_format: runs-on: ubuntu-24.04 + timeout-minutes: ${{ inputs.TIMEOUT }} steps: - - name: Install dependencies + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Install clang-format + run: sudo apt-get install clang-format-20 -y + + - name: Get clang-format config run: | - sudo apt-get install clang-format-10 -y - - name: Run clang_format + LOCAL_CONFIG_FILE=".clang-format" + GLOBAL_CONFIG_URL="https://raw.githubusercontent.com/scritical/.github/main/.clang-format" + + if [[ ! -f "$LOCAL_CONFIG_FILE" ]]; then + echo "Downloading global clang-format config..." + wget "$GLOBAL_CONFIG_URL" -O "$LOCAL_CONFIG_FILE" || echo "No global clang-format config found; proceeding without a config file. The subsequent clang-format step may fail because -style=file requires a .clang-format configuration." + fi + + - name: Run clang-format run: | - bash ./.github/clang_format.sh + find . -iname '*.h' -o -iname '*.hpp' -o -iname '*.c' -o -iname '*.cpp' | xargs clang-format -style=file --dry-run --Werror diff --git a/.github/workflows/format-and-lint.yaml b/.github/workflows/format-and-lint.yaml deleted file mode 100644 index 4ff965c..0000000 --- a/.github/workflows/format-and-lint.yaml +++ /dev/null @@ -1,31 +0,0 @@ -on: - workflow_call: - -jobs: - format-and-lint: - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v5 - - name: Set up Python - uses: actions/setup-python@v6 - with: - python-version: 3.12 - - name: Install dependencies - run: | - wget https://raw.githubusercontent.com/mdolab/.github/main/.pre-commit-config.yaml - pip install pre-commit - - # Set the ruff config in the .github repo as the global config, then the local config should extend it by including the line `extend = "~/.config/ruff/ruff.toml"` - mkdir -p ~/.config/ruff - url=https://raw.githubusercontent.com/mdolab/.github/main/ruff.toml - wget $url -O ~/.config/ruff/ruff.toml - - echo "Ruff config:" - cat ~/.config/ruff/ruff.toml - if [[ -f "ruff.toml" ]]; then - cat ruff.toml - fi - - - name: Run pre-commit checks - run: | - pre-commit run --all-files --show-diff-on-failure diff --git a/.github/workflows/fprettify.yaml b/.github/workflows/fprettify.yaml new file mode 100644 index 0000000..9f65560 --- /dev/null +++ b/.github/workflows/fprettify.yaml @@ -0,0 +1,52 @@ +on: + workflow_call: + inputs: + TIMEOUT: + type: number + default: 10 + description: "Timeout for the job in minutes" + +jobs: + fprettify: + runs-on: ubuntu-24.04 + timeout-minutes: ${{ inputs.TIMEOUT }} + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.11" + + - name: Install fprettify + run: pip install fprettify==0.3.7 + + - name: Get fprettify config + run: | + LOCAL_CONFIG_FILE=".fprettify.rc" + GLOBAL_CONFIG_URL="https://raw.githubusercontent.com/scritical/.github/main/.fprettify.rc" + + if [[ ! -f "$LOCAL_CONFIG_FILE" ]]; then + echo "Downloading global fprettify config..." + wget "$GLOBAL_CONFIG_URL" -O "$LOCAL_CONFIG_FILE" || echo "No global config found; fprettify will use its built-in defaults" + fi + + - name: Run fprettify + run: | + CONFIG_FILE=".fprettify.rc" + CONFIG_ARG="" + if [[ -f "$CONFIG_FILE" ]]; then + CONFIG_ARG="--config-file $CONFIG_FILE" + fi + + # Run fprettify on non-differentiated Fortran 90 source files + find . -type f -iname '*.f90' ! -iname '*_b.f90' ! -iname '*_d.f90' -print0 | while read -d $'\0' fileName + do + echo "Checking $fileName" + fprettify $CONFIG_ARG "$fileName" + done + + - name: Check for changes + run: | + git diff --summary --exit-code || (echo "fprettify produced changes. Please format and commit." && exit 1) diff --git a/.github/workflows/isort.yaml b/.github/workflows/isort.yaml deleted file mode 100644 index 37c8e44..0000000 --- a/.github/workflows/isort.yaml +++ /dev/null @@ -1,19 +0,0 @@ -on: - workflow_call: - -jobs: - isort: - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v5 - - name: Set up Python - uses: actions/setup-python@v6 - with: - python-version: 3.9 - - name: Get isort config file - run: | - # copy over the isort config file - if [[ ! -f ".isort.cfg" ]]; then - wget https://raw.githubusercontent.com/mdolab/.github/main/.isort.cfg - fi - - uses: isort/isort-action@master diff --git a/.github/workflows/mypy.yaml b/.github/workflows/mypy.yaml index 97c4221..eb31ad0 100644 --- a/.github/workflows/mypy.yaml +++ b/.github/workflows/mypy.yaml @@ -1,23 +1,65 @@ on: workflow_call: + inputs: + TIMEOUT: + type: number + default: 30 + description: "Timeout for the job in minutes" + secrets: + DOCKER_USER: + required: true + description: "Docker registry username" + DOCKER_OAT: + required: true + description: "Docker registry Organization Access Token" + +env: + SCRITICAL_HOMEDIR: /home/scriticaluser + BASHRC: /home/scriticaluser/.bashrc_scritical jobs: mypy: runs-on: ubuntu-24.04 - strategy: - matrix: - python-version: [3.9, 3.11] + timeout-minutes: ${{ inputs.TIMEOUT }} + env: + DOCKER_TAG: u24-gcc-ompi-latest steps: - - uses: actions/checkout@v5 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v6 + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Set repo environment variables + run: | + repo_name="${{ github.event.repository.name }}" + echo "REPO_NAME=$repo_name" >> "$GITHUB_ENV" + echo "DOCKER_WORKING_DIR=${{ env.SCRITICAL_HOMEDIR }}/repos/$repo_name" >> "$GITHUB_ENV" + echo "DOCKER_MOUNT_DIR=${{ env.SCRITICAL_HOMEDIR }}/mount/$repo_name" >> "$GITHUB_ENV" + + - name: Login to Docker Hub + uses: docker/login-action@v3 with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies + username: ${{ secrets.DOCKER_USER }} + password: ${{ secrets.DOCKER_OAT }} + + - name: Start Docker container run: | - pip install wheel - pip install mypy - - name: Mypy check + docker run -t -d --name app \ + --mount "type=bind,src=${{ github.workspace }},target=${{ env.DOCKER_MOUNT_DIR }}" \ + scritical/private-dev:${{ env.DOCKER_TAG }} /bin/bash + + # Copy repo to working directory + docker exec app /bin/bash -c "rm -rf $DOCKER_WORKING_DIR && mkdir -p $DOCKER_WORKING_DIR && cp -a $DOCKER_MOUNT_DIR/. $DOCKER_WORKING_DIR/" + + - name: MyPy type checking + run: | + docker exec \ + -e AGENT_NAME="${{ runner.name }}" \ + -e BUILD_REASON="${{ github.event_name }}" \ + app /bin/bash -c ". $BASHRC && cd $DOCKER_WORKING_DIR && mypy ." + + - name: Cleanup + if: always() run: | - mypy --any-exprs-report mypy-output . - cat mypy-output/any-exprs.txt + # Stop and remove the container only if it exists to avoid noisy errors. + if docker ps -a --format '{{.Names}}' | grep -qx app; then + docker stop app && docker rm app || true + fi diff --git a/.github/workflows/pylint.yaml b/.github/workflows/pylint.yaml deleted file mode 100644 index 01921c6..0000000 --- a/.github/workflows/pylint.yaml +++ /dev/null @@ -1,21 +0,0 @@ -on: - workflow_call: - -jobs: - pylint: - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v5 - - name: Set up Python - uses: actions/setup-python@v6 - with: - python-version: 3.9 - - name: Install dependencies - run: | - pip install wheel - pip install pylint - - name: Run pylint - run: | - # copy over the pylint config file - wget https://raw.githubusercontent.com/mdolab/.github/main/.pylintrc - find . -type f -name "*.py" -not -path "*/doc/*" | xargs pylint diff --git a/.github/workflows/pypi.yaml b/.github/workflows/pypi.yaml deleted file mode 100644 index 1ecfb57..0000000 --- a/.github/workflows/pypi.yaml +++ /dev/null @@ -1,26 +0,0 @@ -on: - workflow_call: - secrets: - PYPI_API_TOKEN: - required: true - -jobs: - pypi: - if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v5 - - uses: actions/setup-python@v6 - with: - python-version: 3.9 - - name: Build package - run: | - python --version - python -m pip install -U pip - pip install setuptools wheel - python setup.py bdist_wheel - - name: Publish package - uses: pypa/gh-action-pypi-publish@release/v1 - with: - user: __token__ - password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/.github/workflows/ruff.yaml b/.github/workflows/ruff.yaml new file mode 100644 index 0000000..d697baf --- /dev/null +++ b/.github/workflows/ruff.yaml @@ -0,0 +1,78 @@ +on: + workflow_call: + inputs: + MCCABE: + type: boolean + default: false + description: "Enable McCabe complexity check (pass/fail)" + ISORT: + type: boolean + default: false + description: "Enable isort import sorting check (pass/fail)" + +jobs: + ruff: + runs-on: ubuntu-24.04 + defaults: + run: + working-directory: project + steps: + - name: Checkout repository + uses: actions/checkout@v6 + with: + path: project + + - name: Checkout org-wide configs + uses: actions/checkout@v6 + with: + repository: scritical/.github + path: .org-dotgithub + fetch-depth: 1 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.11" + + - name: Setup Ruff Configuration + run: | + # Get the org-wide ruff config and copy it to ~/.config/ruff/ruff.toml + mkdir -p ~/.config/ruff + cp ../.org-dotgithub/ruff.toml ~/.config/ruff/ruff.toml + + if [[ -f ruff.toml ]]; then + echo "Found repo-specific ruff.toml" + echo "RUFF_CONFIG=ruff.toml" >> "$GITHUB_ENV" + cat ruff.toml + else + echo "No repo-specific ruff.toml found, using org-wide config" + echo "RUFF_CONFIG=~/.config/ruff/ruff.toml" >> "$GITHUB_ENV" + fi + + - name: Install Ruff + run: | + python -m pip install --upgrade pip + pip install ruff + + - name: Ruff formatting + run: | + echo "=== Ruff Formatting Check ===" + ruff format --check --exit-non-zero-on-fix --config $RUFF_CONFIG . + + - name: Ruff linting + run: | + echo "=== Ruff Linting Check ===" + ruff check --exit-non-zero-on-fix --config $RUFF_CONFIG . + + - name: isort check + if: inputs.ISORT + run: | + echo "=== isort Import Sorting Check ===" + ruff check --exit-non-zero-on-fix --select=I --config $RUFF_CONFIG . + + - name: McCabe complexity check + if: inputs.MCCABE + run: | + echo "=== McCabe Complexity Report ===" + echo "Checking for functions with complexity > 10..." + ruff check --select=C90 --config $RUFF_CONFIG . \ No newline at end of file diff --git a/.github/workflows/tapenade.yaml b/.github/workflows/tapenade.yaml new file mode 100644 index 0000000..3bc90f7 --- /dev/null +++ b/.github/workflows/tapenade.yaml @@ -0,0 +1,41 @@ +on: + workflow_call: + inputs: + TIMEOUT: + type: number + default: 10 + description: "Timeout for the job in minutes" + TAPENADE_SCRIPT: + type: string + default: ".github/build_tapenade.sh" + description: "Path to Tapenade build script" + +jobs: + tapenade: + runs-on: ubuntu-24.04 + timeout-minutes: ${{ inputs.TIMEOUT }} + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Set up Java + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: 8 + + - name: Install and run Tapenade + run: | + # Download Tapenade 3.16 (tapenade_3.16-v2-723-ge8da61555.tar) + TAPENADE_URL="https://gitlab.inria.fr/tapenade/tapenade/-/package_files/112870/download" + wget "$TAPENADE_URL" -O tapenade_3.16.tar + tar -xzf tapenade_3.16.tar + export PATH=$(pwd)/tapenade_3.16/bin:$PATH + + # Run Tapenade + touch config.mk # empty config file to make "make" happy + bash ${{ inputs.TAPENADE_SCRIPT }} + + - name: Check for changes + run: | + git diff --exit-code || (echo "Tapenade produced changes. Please regenerate and commit." && exit 1) diff --git a/.isort.cfg b/.isort.cfg deleted file mode 100644 index d6b8b9d..0000000 --- a/.isort.cfg +++ /dev/null @@ -1,9 +0,0 @@ -[isort] -profile=black -line_length=120 -force_sort_within_sections=true -import_heading_stdlib=Standard Python modules -import_heading_thirdparty=External modules -import_heading_firstparty=First party modules -import_heading_localfolder=Local modules -skip_glob=**__init__.py,**setup.py diff --git a/README.md b/README.md index 6b306c3..1672029 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,18 @@ # .github This repo stores the following shared repository settings/configurations/templates: - Issue and pull request templates on GitHub -- Shared configurations for - - `pre-commit` - - `ruff` - - `isort` - - `pylint` - - `codecov` - - `mypy` -- Shared configurations for Azure Pipelines - Shared configurations for GitHub Actions + - `ruff` + - formatting + - linting + - isort + - McCabe Complexity + - `clang_format` + - `build` + - Docker build + - Integration testing + - `mypy` + - `fprettify` + - `tapenade` + - `branch-name-check` - Issue/PR labels shared across the organization -- Configuration script for the build bot diff --git a/azure/README.md b/azure/README.md deleted file mode 100644 index 32c5aed..0000000 --- a/azure/README.md +++ /dev/null @@ -1,170 +0,0 @@ -# Azure -The Supercritical Azure Pipelines page is found on the [Azure Website](https://dev.azure.com/scritical/). -The process outlined here is split into Public and Private projects, used accordingly for public and private repositories. -However, at the moment we only support Private projects, but the processes support public repositories. -The pipelines for each repository can be found by selecting its parent project and then the pipelines button. -Each pipeline is set up using the templates located in this `.github` repository so only configuration files are needed in each repository. - -The templates are organized into the following files: -- `azure_build.yaml` which handles the build and test jobs for the code -- `azure_pypi.yaml` which handles the PyPI deployment if applicable -- `azure_style.yaml` which handles style checks on the code -- `azure_template.yaml` which handles the job itself, calling the necessary sub-templates. - -## Template Options -| Name | Type | Default | Description. | -| :--------------------- | :------ | :-------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------ | -| `REPO_NAME` | string | | Name of repository | -| `COMPLEX` | boolean | `false` | Flag for triggering complex build and tests | -| `TIMEOUT_BUILD` | number | `120` | Runtime allowed for each build and test job, in minutes | -| `GCC_CONFIG` | string | `None` | Path to GCC configuration file (from repository root) | -| `INTEL_CONFIG` | string | `None` | Path to Intel configuration file (from repository root) | -| `BUILD_REAL` | string | `.github/build_real.sh` | Path to Bash script with commands to build real code. Using `None` will skip this step. | -| `TEST_REAL` | string | `.github/text_real.sh` | Path to Bash script to run real tests. Using `None` will skip this step. | -| `BUILD_COMPLEX` | string | `.github/build_complex.sh` | Path to Bash script with commands to build complex code. Using `None` will skip this step. | -| `TEST_COMPLEX` | string | `.github/text_complex.sh` | Path to Bash script with commands to run complex tests. Using `None` will skip this step. | -| `IMAGE` | string | `public` | Select Docker image. Can be `public`, `private`, or `auto`. `auto` uses the private image on trusted builds and the public image otherwise. | -| `COVERAGE` | boolean | `false` | Flag to report test coverage to `codecov` | -| `TIMEOUT_STYLE` | number | `10` | Runtime allowed for each style check, in minutes | -| `IGNORE_STYLE` | boolean | `false` | Flag to allow `formatting and linting` checks to fail without failing the pipeline | -| `BASE_FORMAT_AND_LINT` | boolean | `true` | Flag to trigger the `base_format_and_lint` check | -| `ISORT` | boolean | `false` | Flag to trigger the `isort` check | -| `PYLINT` | boolean | `false` | Flag to trigger the `pylint` check | -| `CLANG_FORMAT` | boolean | `false` | Flag to trigger the `clang-format` check | -| `FPRETTIFY` | boolean | `false` | Flag to trigger the `fprettify` check | -| `MYPY` | boolean | `false` | Flag to trigger the `mypy` check | -| `TAPENADE` | boolean | `false` | Flag to trigger the Tapenade check | -| `TIMEOUT_TAPENADE` | number | `10` | Runtime allowed for the Tapenade check, in minutes | -| `TAPENADE_SCRIPT` | string | `.github/build_tapenade.sh` | Path to Bash script with commands to run Tapenade | -| `TAPENADE_VERSION` | string | `3.10` | Version of Tapenade to use | - -## Setting up a pipeline -### Step 1: Setup Azure Pipelines YAML File: - -1. Create `azure_template.yaml` in the `.github/` directory of your repository -2. Add triggers (see example below) - - Only set triggers for the `main` branch and pull requests to `main` -3. Add resources (see example below) - - This resource pulls the `azure_template` from the `scritical/.github` repository -4. Add parameters (see example below and the options table above) - -``` -trigger: -- main - -pr: -- main - -resources: - repositories: - - repository: azure_template - type: github - name: scritical/.github - endpoint: scritical - -extends: - template: azure/azure_template.yaml@azure_template - parameters: - REPO_NAME: adflow - IGNORE_STYLE: true - COMPLEX: true - GCC_CONFIG: config/defaults/config.LINUX_GFORTRAN.mk - INTEL_CONFIG: config/defaults/config.LINUX_INTEL.mk -``` - -### Step 2: Write Build Bash Script: - -1. Create `build_real.sh` in the `.github/` directory of your repository -2. Add `#!/bin/bash` followed by `set -e` as the first two lines -3. Add configuration file copy if needed (see example below). Note that the environment variable `$CONFIG_FILE` is provided by the Azure template and is automatically set depending on the compiler used for each build. -4. Add make command if needed (see example below) -5. Add python install command (see example below) -6. Repeat with `build_complex.sh` if needed - -``` -#!/bin/bash -set -e -cp $CONFIG_FILE config/config.mk -make -pip install . -``` - -### Step 3: Write Test Bash Script: - -1. Create `test_real.sh` in `.github/` directory of your repository -2. Add `#!/bin/bash` followed by `set -e` as the first two lines -3. Add input file download if needed (see example below) -4. Add `testflo` command -5. Repeat with `test_complex.sh` if needed - -``` -#!/bin/bash -set -e -./input_files/get-input-files.sh -testflo -v . -n 1 -``` - -### Step 4: Push branch to `scritical/` repository and create pull request - -1. Push branch directly to `scritical` repository, not your personal fork - -### Step 5: Create new Pipeline on Microsoft Azure - -1. Go to [Azure Pipelines](https://dev.azure.com/scritical/) and select public / private as needed by current repository -2. Select "Pipelines" and "New pipeline" -3. Select "Github" and authorize the Azure Pipeline app if you have not already -4. Select correct scritical repository (again, not personal fork) -5. Select "Existing Azure Pipelines YAML file" -6. In pop-up menu, set "Branch" to your new Azure-transition branch and "Path" to `.github/azure-pipelines.yaml` -7. Click "continue" - -### Step 6: Add Badge to README - -1. Go to the pipeline on Azure -2. Click on the three dots next to "Run pipeline" -3. Click on "status badge" -4. Set "Branch" to "main" -5. Copy the text for "Sample markdown" and paste it into README - -### Step 7: Create PR - -1. Create pull request to `main` -2. Ensure the Azure jobs start and appear in the pull request - -## PyPI with Azure -If using PyPI for a repository, the yaml file does not follow this same structure, but imports components as stages instead of as an extend. -An example yaml file using PyPI is: - -``` -trigger: - branches: - include: - - main - tags: - include: - - v*.*.* - -pr: -- main - -resources: - repositories: - - repository: azure_template - type: github - name: scritical/.github - endpoint: scritical - -stages: -- template: azure/azure_template.yaml@azure_template - parameters: - REPO_NAME: REPOSITORY NAME - -- stage: - dependsOn: - - Test_Real - - Style - displayName: PyPI - condition: and(succeeded(), contains(variables['build.sourceBranch'], 'tags')) - jobs: - - template: azure/azure_pypi.yaml@azure_template -``` diff --git a/azure/azure_build.yaml b/azure/azure_build.yaml deleted file mode 100644 index effbc18..0000000 --- a/azure/azure_build.yaml +++ /dev/null @@ -1,157 +0,0 @@ -parameters: - - name: REPO_NAME - type: string - - name: TIMEOUT - type: number - - name: GCC_CONFIG - type: string - - name: INTEL_CONFIG - type: string - - name: BUILD - type: string - - name: TEST - type: string - - name: IMAGE - type: string - - name: COVERAGE - type: boolean - - name: C_COVERAGE - type: boolean - -jobs: - - job: - pool: - vmImage: "ubuntu-24.04" - timeoutInMinutes: ${{ parameters.TIMEOUT }} - strategy: - matrix: - "u24-gcc-ompi-latest": - DOCKER_TAG: u24-gcc-ompi-latest - CONFIG_FILE: ${{ parameters.GCC_CONFIG }} - # "u22-gcc-ompi-stable": - # DOCKER_TAG: u22-gcc-ompi-stable - # CONFIG_FILE: ${{ parameters.GCC_CONFIG }} - variables: - - group: Docker - - group: Codecov - - name: SCRITICAL_HOMEDIR - value: /home/scriticaluser - - name: DOCKER_WORKING_DIR - value: ${{ variables.SCRITICAL_HOMEDIR }}/repos/${{ parameters.REPO_NAME }} - - name: DOCKER_MOUNT_DIR - value: ${{ variables.SCRITICAL_HOMEDIR }}/azure/${{ parameters.REPO_NAME }} - - name: BASHRC - value: ${{ variables.SCRITICAL_HOMEDIR }}/.bashrc_scritical - steps: - - checkout: self - - checkout: azure_template - - script: | - # This is a trusted build if DOCKER_USERNAME is defined - if [[ ! -z $(DOCKER_USERNAME) ]] ; then - echo $(DOCKER_PASSWORD) | docker login -u $(DOCKER_USERNAME) --password-stdin; - fi - # "auto" pulls the private image for trusted builds - if [[ "${{ parameters.IMAGE }}" == "private" ]] || [[ "${{ parameters.IMAGE }}" == "auto" && ! -z $(DOCKER_USERNAME) ]] ; then - export DOCKER_IMAGE=private-dev; - else - export DOCKER_IMAGE=public-dev; - fi - docker pull scritical/$DOCKER_IMAGE:$(DOCKER_TAG); - docker run -t -d --name app --mount "type=bind,src=$(pwd)/${{ parameters.REPO_NAME }},target=${{ variables.DOCKER_MOUNT_DIR }}" scritical/$DOCKER_IMAGE:$(DOCKER_TAG) /bin/bash; - docker exec app /bin/bash -c "rm -rf ${{ variables.DOCKER_WORKING_DIR }} && cp -r ${{ variables.DOCKER_MOUNT_DIR }} ${{ variables.DOCKER_WORKING_DIR }}"; - displayName: Prepare Repository - - script: docker exec -e CONFIG_FILE=$(CONFIG_FILE) app /bin/bash -c ". ${{ variables.BASHRC }} && cd ${{ variables.DOCKER_WORKING_DIR }} && /bin/bash ${{ parameters.BUILD }}" - displayName: Build - condition: and(succeeded(), ne('${{ parameters.BUILD }}', 'None')) - - script: | - set -e - docker exec -e AGENT_NAME="$AGENT_NAME" -e BUILD_REASON=$(Build.Reason) app /bin/bash -c ". ${{ variables.BASHRC }} && cd ${{ variables.DOCKER_WORKING_DIR }} && source ~/MPIOversubscribe.sh && /bin/bash ${{ parameters.TEST }}" - displayName: Run Tests - condition: and(succeeded(), ne('${{ parameters.TEST }}', 'None')) - - script: | - cd ${{ parameters.REPO_NAME }} - - # validate the uploader - curl https://keybase.io/codecovsecurity/pgp_keys.asc | gpg --no-default-keyring --keyring trustedkeys.gpg --import - curl -Os https://uploader.codecov.io/latest/linux/codecov - curl -Os https://uploader.codecov.io/latest/linux/codecov.SHA256SUM - curl -Os https://uploader.codecov.io/latest/linux/codecov.SHA256SUM.sig - gpgv codecov.SHA256SUM.sig codecov.SHA256SUM - shasum -a 256 -c codecov.SHA256SUM - - # exit if checksum fails - if [[ $? -ne 0 ]] ; then - echo "checksum failed" - exit 1 - fi - - # clean validation files - rm codecov.SHA256SUM* - - # grab the token - if [[ ! -z $(${{ parameters.REPO_NAME }}_CODECOV_TOKEN) ]] ; then - echo "codecov token found!" - export CODECOV_TOKEN=$(${{ parameters.REPO_NAME }}_CODECOV_TOKEN) - export CODECOV_ARGS="-t $CODECOV_TOKEN" - else - export CODECOV_ARGS="" - fi - - # convert coverage file to xml inside Docker - sudo docker exec app /bin/bash -c ". ${{ variables.BASHRC }} && cd ${{ variables.DOCKER_WORKING_DIR }} && \ - mv \$(find . -name ".coverage" -type f) . && coverage xml" - - # copy coverage file out of Docker - docker cp app:${{ variables.DOCKER_WORKING_DIR }}/coverage.xml $(pwd) - - # run codecov - chmod +x codecov - ./codecov $CODECOV_ARGS - condition: and(succeeded(), ${{ parameters.COVERAGE }}) - displayName: Coverage - - script: | - cd ${{ parameters.REPO_NAME }} - - # validate the uploader - curl https://keybase.io/codecovsecurity/pgp_keys.asc | gpg --no-default-keyring --keyring trustedkeys.gpg --import - curl -Os https://uploader.codecov.io/latest/linux/codecov - curl -Os https://uploader.codecov.io/latest/linux/codecov.SHA256SUM - curl -Os https://uploader.codecov.io/latest/linux/codecov.SHA256SUM.sig - gpgv codecov.SHA256SUM.sig codecov.SHA256SUM - shasum -a 256 -c codecov.SHA256SUM - - # exit if checksum fails - if [[ $? -ne 0 ]] ; then - echo "checksum failed" - exit 1 - fi - - # clean validation files - rm codecov.SHA256SUM* - - # grab the token - if [[ ! -z $(${{ parameters.REPO_NAME }}_CODECOV_TOKEN) ]] ; then - echo "codecov token found!" - export CODECOV_TOKEN=$(${{ parameters.REPO_NAME }}_CODECOV_TOKEN) - export CODECOV_ARGS="-t $CODECOV_TOKEN" - else - export CODECOV_ARGS="" - fi - - # convert coverage file to xml inside Docker - sudo docker exec app /bin/bash -c ". ${{ variables.BASHRC }} && cd ${{ variables.DOCKER_WORKING_DIR }} && \ - lcov --capture --directory . --output-file coverage.info && \ - lcov --remove coverage.info '/usr/*' --output-file coverage.info && \ - lcov --remove coverage.info '*/numpy/*' --output-file coverage.info && \ - lcov --remove coverage.info '*/.pyenv/versions/*' --output-file coverage.info && \ - lcov --remove coverage.info '*mpi4py*' --output-file coverage.info" && \ - lcov --list coverage.info - - # copy coverage file out of Docker - docker cp app:${{ variables.DOCKER_WORKING_DIR }}/coverage.info $(pwd) - - # run codecov - chmod +x codecov - ./codecov $CODECOV_ARGS - condition: ${{ parameters.C_COVERAGE }} - displayName: C Coverage diff --git a/azure/azure_pypi.yaml b/azure/azure_pypi.yaml deleted file mode 100644 index 7ed4c46..0000000 --- a/azure/azure_pypi.yaml +++ /dev/null @@ -1,29 +0,0 @@ -parameters: - - name: SERVICE_CONNECTION - type: string - default: pypi - - name: TIMEOUT - type: number - default: 10 - -jobs: - - job: PyPI_Deploy - pool: - vmImage: "ubuntu-22.04" - timeoutInMinutes: ${{ parameters.TIMEOUT }} - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: "3.8" - - script: | - python3 --version - python3 -m pip install -U pip - pip3 install setuptools wheel twine - python3 setup.py bdist_wheel - displayName: "Install Packages" - - task: TwineAuthenticate@1 - displayName: "Twine Authenticate" - inputs: - pythonUploadServiceConnection: ${{ parameters.SERVICE_CONNECTION }} - - script: python3 -m twine upload --config-file $(PYPIRC_PATH) dist/*.whl - displayName: "Twine Upload" diff --git a/azure/azure_style.yaml b/azure/azure_style.yaml deleted file mode 100644 index 7bbd391..0000000 --- a/azure/azure_style.yaml +++ /dev/null @@ -1,208 +0,0 @@ -parameters: - - name: REPO_NAME - type: string - - name: IMAGE - type: string - # The following parameters have defaults because this template could be used by repos directly - - name: TIMEOUT - type: number - default: 10 - - name: IGNORE_STYLE - type: boolean - default: false - - name: BASE_FORMAT_AND_LINT - type: boolean - default: true - - name: ISORT - type: boolean - default: false - - name: PYLINT - type: boolean - default: false - - name: CLANG_FORMAT - type: boolean - default: false - - name: FPRETTIFY - type: boolean - default: false - - name: MYPY - type: boolean - default: false -jobs: - - job: base_format_and_lint - pool: - vmImage: "ubuntu-22.04" - timeoutInMinutes: ${{ parameters.TIMEOUT }} - continueOnError: ${{ parameters.IGNORE_STYLE }} - condition: ${{ parameters.BASE_FORMAT_AND_LINT }} - steps: - - checkout: self - - checkout: azure_template - - task: UsePythonVersion@0 - inputs: - versionSpec: "3.11" - - script: | - cd ${{ parameters.REPO_NAME }} - cp ../.github/.pre-commit-config.yaml . - pip install pre-commit - - # Set the ruff config in the .github repo as the global config, then the local config should extend it by including the line `extend = "~/.config/ruff/ruff.toml"` - mkdir -p ~/.config/ruff - cp ../.github/ruff.toml ~/.config/ruff/ruff.toml - - echo "Ruff config:" - cat ~/.config/ruff/ruff.toml - if [[ -f "ruff.toml" ]]; then - cat ruff.toml - fi - - # Run the pre-commit checks - pre-commit run --all-files --show-diff-on-failure - - - job: isort - pool: - vmImage: "ubuntu-22.04" - timeoutInMinutes: ${{ parameters.TIMEOUT }} - continueOnError: ${{ parameters.IGNORE_STYLE }} - condition: ${{ parameters.ISORT }} - steps: - - checkout: self - - checkout: azure_template - - task: UsePythonVersion@0 - inputs: - versionSpec: "3.11" - - script: | - cd ${{ parameters.REPO_NAME }} - - # copy over the isort config file - if [[ ! -f ".isort.cfg" ]]; then - cp ../.github/.isort.cfg . - fi - - pip install wheel - pip install isort - isort . -c - - - job: pylint - pool: - vmImage: "ubuntu-22.04" - timeoutInMinutes: ${{ parameters.TIMEOUT }} - continueOnError: ${{ parameters.IGNORE_STYLE }} - condition: ${{ parameters.PYLINT }} - steps: - - checkout: self - - checkout: azure_template - - task: UsePythonVersion@0 - inputs: - versionSpec: "3.11" - - script: | - cd ${{ parameters.REPO_NAME }} - - # copy over the pylint config file - cp ../.github/.pylintrc . - - pip install pylint - find . -type f -name "*.py" -not -path "*/doc/*" | xargs pylint - - - job: clang_format - pool: - vmImage: "ubuntu-22.04" - timeoutInMinutes: ${{ parameters.TIMEOUT }} - continueOnError: ${{ parameters.IGNORE_STYLE }} - condition: ${{ parameters.CLANG_FORMAT }} - steps: - - checkout: self - - checkout: azure_template - - script: | - # Install prerequisites - sudo apt-get install clang-format-10 -y - - cd ${{ parameters.REPO_NAME }} - - # Check if we can access script, if not exit - if [[ ! -f ../.github/azure/clang-format.sh ]]; then - echo "clang-format.sh not found. Exiting." - exit 1 - fi - - # Run the formatting - bash ../.github/azure/clang-format.sh --dry-run || exit $? - - - job: fprettify - pool: - vmImage: "ubuntu-22.04" - timeoutInMinutes: ${{ parameters.TIMEOUT }} - continueOnError: ${{ parameters.IGNORE_STYLE }} - condition: ${{ parameters.FPRETTIFY }} - steps: - - checkout: self - - checkout: azure_template - - task: UsePythonVersion@0 - inputs: - versionSpec: "3.11" - - script: | - cd ${{ parameters.REPO_NAME }} - - # Check if we can access script, if not exit - if [[ ! -f ../.github/azure/fprettify.sh ]]; then - echo "fprettify.sh not found. Exiting." - exit 1 - fi - - # Install fprettify - pip install fprettify==0.3.7 - - # Run the formatting - bash ../.github/azure/fprettify.sh || exit $? - - # Exit with an error if any of the tracked files changed - git diff --summary --exit-code - - - job: mypy - pool: - vmImage: "ubuntu-22.04" - timeoutInMinutes: ${{ parameters.TIMEOUT }} - continueOnError: ${{ parameters.IGNORE_STYLE }} - condition: ${{ parameters.MYPY }} - strategy: - matrix: - "u24-gcc-ompi-latest": - DOCKER_TAG: u24-gcc-ompi-latest - variables: - - group: Docker - - name: SCRITICAL_HOMEDIR - value: /home/scriticaluser - - name: DOCKER_WORKING_DIR - value: ${{ variables.SCRITICAL_HOMEDIR }}/repos/${{ parameters.REPO_NAME }} - - name: DOCKER_MOUNT_DIR - value: ${{ variables.SCRITICAL_HOMEDIR }}/azure/${{ parameters.REPO_NAME }} - - name: BASHRC - value: ${{ variables.SCRITICAL_HOMEDIR }}/.bashrc_scritical - steps: - - checkout: self - - checkout: azure_template - - task: UsePythonVersion@0 - inputs: - versionSpec: "3.11" - - script: | - # This is a trusted build if DOCKER_USERNAME is defined - if [[ ! -z $(DOCKER_USERNAME) ]] ; then - echo $(DOCKER_PASSWORD) | docker login -u $(DOCKER_USERNAME) --password-stdin; - fi - # "auto" pulls the private image for trusted builds - if [[ "${{ parameters.IMAGE }}" == "private" ]] || [[ "${{ parameters.IMAGE }}" == "auto" && ! -z $(DOCKER_USERNAME) ]] ; then - export DOCKER_IMAGE=private-dev; - else - export DOCKER_IMAGE=public-dev; - fi - docker pull scritical/$DOCKER_IMAGE:$(DOCKER_TAG); - docker run -t -d --name app --mount "type=bind,src=$(pwd)/${{ parameters.REPO_NAME }},target=${{ variables.DOCKER_MOUNT_DIR }}" scritical/$DOCKER_IMAGE:$(DOCKER_TAG) /bin/bash; - docker exec app /bin/bash -c "rm -rf ${{ variables.DOCKER_WORKING_DIR }} && cp -r ${{ variables.DOCKER_MOUNT_DIR }} ${{ variables.DOCKER_WORKING_DIR }}"; - displayName: Prepare Repository - - script: | - set -e - docker exec app /bin/bash -c ". ${{ variables.BASHRC }} && cd ${{ variables.DOCKER_WORKING_DIR }} && mypy" - displayName: Run mypy - condition: succeeded() - - diff --git a/azure/azure_tapenade.yaml b/azure/azure_tapenade.yaml deleted file mode 100644 index a6209e4..0000000 --- a/azure/azure_tapenade.yaml +++ /dev/null @@ -1,45 +0,0 @@ -parameters: - - name: REPO_NAME - type: string - - name: TIMEOUT - type: number - - name: TAPENADE_SCRIPT - type: string - - name: TAPENADE_VERSION - type: string - -jobs: - - job: tapenade - pool: - vmImage: "ubuntu-22.04" - timeoutInMinutes: ${{ parameters.TIMEOUT }} - steps: - - task: JavaToolInstaller@0 - inputs: - versionSpec: "8" - jdkArchitectureOption: "x64" - jdkSourceOption: "PreInstalled" - - script: | - if [[ "${{ parameters.TAPENADE_VERSION }}" == "3.10" ]]; then - export TAPENADE_URL=https://websites.umich.edu/~mdolaboratory/misc/tapenade3.10.tar - export TAPENADE_HOME=$(pwd)/tapenade${{ parameters.TAPENADE_VERSION }} - elif [[ "${{ parameters.TAPENADE_VERSION }}" == "3.16" ]]; then - # The following link corresponds to tapenade_3.16-v2-723-ge8da61555.tar - export TAPENADE_URL="https://gitlab.inria.fr/tapenade/tapenade/-/package_files/112870/download -O tapenade_3.16.tar" - export TAPENADE_HOME=$(pwd)/tapenade_${{ parameters.TAPENADE_VERSION }} - fi - - # Install Tapenade - wget $TAPENADE_URL - tar -xzf $TAPENADE_HOME.tar - export PATH=$TAPENADE_HOME/bin:$PATH - - # Run Tapenade - touch config.mk # empty config file to make "make" happy - bash ${{ parameters.TAPENADE_SCRIPT }} - if [ $? -ne 0 ]; then - echo "Tapenade runscript failed" - exit 1 - fi - - git diff --exit-code diff --git a/azure/azure_template.yaml b/azure/azure_template.yaml deleted file mode 100644 index 783c31e..0000000 --- a/azure/azure_template.yaml +++ /dev/null @@ -1,139 +0,0 @@ -parameters: - - name: REPO_NAME - type: string - # Build and test - - name: COMPLEX - type: boolean - default: false - - name: TIMEOUT_BUILD - type: number - default: 120 - - name: GCC_CONFIG - type: string - default: None - - name: INTEL_CONFIG - type: string - default: None - - name: BUILD_REAL - type: string - default: .github/build_real.sh - - name: TEST_REAL - type: string - default: .github/test_real.sh - - name: BUILD_COMPLEX - type: string - default: .github/build_complex.sh - - name: TEST_COMPLEX - type: string - default: .github/test_complex.sh - - name: IMAGE - type: string - default: private - - name: COVERAGE - type: boolean - default: false - - name: C_COVERAGE - type: boolean - default: false - # Style checks - - name: TIMEOUT_STYLE - type: number - default: 10 - - name: IGNORE_STYLE - type: boolean - default: false - - name: BASE_FORMAT_AND_LINT - type: boolean - default: true - - name: ISORT - type: boolean - default: false - - name: PYLINT - type: boolean - default: false - - name: CLANG_FORMAT - type: boolean - default: false - - name: FPRETTIFY - type: boolean - default: false - - name: MYPY - type: boolean - default: false - # Tapenade - - name: TAPENADE - type: boolean - default: false - - name: TIMEOUT_TAPENADE - type: number - default: 10 - - name: TAPENADE_SCRIPT - type: string - default: .github/build_tapenade.sh - - name: TAPENADE_VERSION - type: string - default: 3.10 - - -stages: - - stage: Test_Real - dependsOn: [] - displayName: Real Test - jobs: - - template: azure_build.yaml - parameters: - REPO_NAME: ${{ parameters.REPO_NAME }} - TIMEOUT: ${{ parameters.TIMEOUT_BUILD }} - GCC_CONFIG: ${{ parameters.GCC_CONFIG }} - INTEL_CONFIG: ${{ parameters.INTEL_CONFIG }} - BUILD: ${{ parameters.BUILD_REAL }} - TEST: ${{ parameters.TEST_REAL }} - IMAGE: ${{ parameters.IMAGE }} - COVERAGE: ${{ parameters.COVERAGE }} - C_COVERAGE: ${{ parameters.C_COVERAGE }} - - - stage: Test_Complex - dependsOn: [] - displayName: Complex Test - condition: ${{ parameters.COMPLEX }} - jobs: - - template: azure_build.yaml - parameters: - REPO_NAME: ${{ parameters.REPO_NAME }} - TIMEOUT: ${{ parameters.TIMEOUT_BUILD }} - GCC_CONFIG: ${{ parameters.GCC_CONFIG }} - INTEL_CONFIG: ${{ parameters.INTEL_CONFIG }} - BUILD: ${{ parameters.BUILD_COMPLEX }} - TEST: ${{ parameters.TEST_COMPLEX }} - IMAGE: ${{ parameters.IMAGE }} - COVERAGE: ${{ parameters.COVERAGE }} - C_COVERAGE: ${{ parameters.C_COVERAGE }} - - - stage: Style - dependsOn: [] - displayName: Style Checks - jobs: - - template: azure_style.yaml - parameters: - REPO_NAME: ${{ parameters.REPO_NAME }} - IMAGE: ${{ parameters.IMAGE }} - TIMEOUT: ${{ parameters.TIMEOUT_STYLE }} - IGNORE_STYLE: ${{ parameters.IGNORE_STYLE }} - BASE_FORMAT_AND_LINT: ${{ parameters.BASE_FORMAT_AND_LINT }} - ISORT: ${{ parameters.ISORT }} - PYLINT: ${{ parameters.PYLINT }} - CLANG_FORMAT: ${{ parameters.CLANG_FORMAT }} - FPRETTIFY: ${{ parameters.FPRETTIFY }} - MYPY: ${{ parameters.MYPY }} - - - stage: Tapenade - dependsOn: [] - displayName: Tapenade Checks - condition: ${{ parameters.TAPENADE }} - jobs: - - template: azure_tapenade.yaml - parameters: - REPO_NAME: ${{ parameters.REPO_NAME }} - TIMEOUT: ${{ parameters.TIMEOUT_TAPENADE }} - TAPENADE_SCRIPT: ${{ parameters.TAPENADE_SCRIPT }} - TAPENADE_VERSION: ${{ parameters.TAPENADE_VERSION }} diff --git a/azure/clang-format.sh b/azure/clang-format.sh deleted file mode 100644 index ac6ded6..0000000 --- a/azure/clang-format.sh +++ /dev/null @@ -1,70 +0,0 @@ -#!/bin/bash - -# clang-format expects a .clang-format file in the root (local dir) where the command -# is run. Need to check for existing files first. If none copy over the global file -# and remove once done to make sure its not added to git. - -# -------------- Command line input -------------- -USAGE=" -usage: [-d] - -Argument description: - -d|--dry-run Do not make changes, only simulate formatting changes -" -DRY_RUN="" -while [[ "$#" -gt 0 ]]; do - case $1 in - -d|--dry-run) DRY_RUN="--dry-run"; shift ;; - -h|--help) echo "$USAGE"; exit 1 ;; - *) echo "Unknown parameter passed: $1"; echo "$USAGE"; exit 1 ;; - esac - shift -done - -# Set some constants -CLANGFORMAT_CONFIG_FILE="" -LOCAL_CONFIG_FILE=".clang-format" -GLOBAL_CONFIG_FILE="../.github/.clang-format" - -# Initialize variables -global=0 - -if [[ -f "$LOCAL_CONFIG_FILE" ]]; then - # Use local config file - echo "Using local config file: $LOCAL_CONFIG_FILE" - CLANGFORMAT_CONFIG_FILE="$LOCAL_CONFIG_FILE" -elif [[ -f "$GLOBAL_CONFIG_FILE" ]]; then - # Use global config file - echo "Using .github shared config file: $GLOBAL_CONFIG_FILE" - CLANGFORMAT_CONFIG_FILE="$GLOBAL_CONFIG_FILE" - cp "$GLOBAL_CONFIG_FILE" . - global=1 -else - echo "No clang-format config was found. Exiting" - exit 2 -fi - -# Setting common values -clang_format_args="-style=file --verbose --Werror" -if [[ -z "$DRY_RUN" ]]; then - # Apply the formatting inplace - clang_format_args="$clang_format_args -i" -else - # Only do a dry run - clang_format_args="$clang_format_args --dry-run" -fi - -echo "Running clang-format with args: $clang_format_args" - -# Run the formatting -find . -iname '*.h' -o -iname '*.hpp' -o -iname '*.c' -o -iname '*.cpp' | xargs clang-format $clang_format_args - -# Store the exit code in case we are running dry-run and find -clang_exit_code=$? - -if [[ 1 == "$global" ]]; then - # Remove only of we copied over - rm "$LOCAL_CONFIG_FILE" -fi - -exit $clang_exit_code \ No newline at end of file diff --git a/azure/combine-config.py b/azure/combine-config.py deleted file mode 100644 index 9de9d6a..0000000 --- a/azure/combine-config.py +++ /dev/null @@ -1,28 +0,0 @@ -""" -This script will read in two files via command line arguments -It will then merge them, and write it out to the appropriate output file name -""" -import argparse -import configparser - -parser = argparse.ArgumentParser() -parser.add_argument("default", type=str, help="file name for the default config file") -parser.add_argument("repo", type=str, help="file name for the repo-specific config file") -parser.add_argument("output", type=str, help="output file name") -args = parser.parse_args() - -# default config -config_def = configparser.ConfigParser() -config_def.read(args.default) -# project config -config_repo = configparser.ConfigParser() -config_repo.read(args.repo) - -# merge them together, with existing keys overwritten by project config -for config in config_def.keys(): - for key in config_repo[config].keys(): - config_def[config][key] = config_repo[config][key] - -# write out the file -with open(args.output, "w") as configfile: - config_def.write(configfile) diff --git a/combine-config.py b/combine-config.py new file mode 100644 index 0000000..9b8c282 --- /dev/null +++ b/combine-config.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +"""Merge two INI-style configuration files. + +This script merges a *default* INI config with an optional *repo-specific* INI +config, writing the merged result to an output path. If the repo-specific +config file does not exist, the default config is written as-is. + +Notes +----- +The merge behavior is: + +- Sections present only in the default config are preserved. +- Sections present in the repo config are added if missing. +- For keys present in both configs, the repo config value overrides the + default. + +Examples +-------- +From the command line:: + + python combine-config.py default.cfg repo.cfg merged.cfg + +""" +import argparse +import configparser +import sys +from pathlib import Path + + +def merge_configs(default_path: str, repo_path: str, output_path: str) -> None: + """Merge a default config with an optional repo config. + + Parameters + ---------- + default_path : str + Path to the default INI-style configuration file. + repo_path : str + Path to the repo-specific INI-style configuration file. If this file + does not exist, the default config is used without modification. + output_path : str + Path to write the merged INI configuration. + + Returns + ------- + None + This function writes the merged configuration to ``output_path``. + """ + # Read default config + config = configparser.ConfigParser() + config.read(default_path) + + # Read and merge repo config if it exists + repo_file = Path(repo_path) + if repo_file.exists(): + config_repo = configparser.ConfigParser() + config_repo.read(repo_path) + + # Merge: repo config overrides default + for section in config_repo.sections(): + if not config.has_section(section): + config.add_section(section) + for key, value in config_repo.items(section): + config.set(section, key, value) + + # Write merged config + with open(output_path, "w") as f: + config.write(f) + + +def main() -> int: + """CLI entry point. + + Parses command-line arguments and writes a merged configuration file. + + Returns + ------- + int + Process exit code. Returns ``0`` on successful completion. + """ + parser = argparse.ArgumentParser( + description="Merge two INI-style configuration files" + ) + parser.add_argument("default", type=str, help="Path to default config file") + parser.add_argument("repo", type=str, help="Path to repo-specific config file") + parser.add_argument("output", type=str, help="Path for merged output file") + args = parser.parse_args() + + merge_configs(args.default, args.repo, args.output) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/azure/fprettify.sh b/fprettify.sh similarity index 96% rename from azure/fprettify.sh rename to fprettify.sh index 91333e8..ea941b8 100644 --- a/azure/fprettify.sh +++ b/fprettify.sh @@ -2,7 +2,7 @@ FPRETTIFY_CONFIG_FILE="" LOCAL_CONFIG_FILE=".fprettify.rc" -GLOBAL_CONFIG_FILE="../.github/.fprettify.rc" +GLOBAL_CONFIG_FILE="./.fprettify.rc" if [[ ! -z "$1" ]]; then # Offer the option of supplying config file through an argument diff --git a/ruff.toml b/ruff.toml index d59433f..c2d4df6 100644 --- a/ruff.toml +++ b/ruff.toml @@ -1,5 +1,5 @@ # Minimum Python syntax support -target-version = "py39" +target-version = "py311" line-length = 120 @@ -18,7 +18,7 @@ ignore = [ "E501", # line length, exceeded by some docstrings "W605", # TODO (to be fixed), invalid escape sequence, necessary for sphinx directives in docstrings but should switch to raw string "D301", # Same reason as W605 - "D10", # D100-107, missing docstring in public function, class etc (yes we should have docstrings for all public methods, but that is a pipe dream) + "D10", # D100-107, missing docstring in public function, class etc (yes we should have docstrings for all public methods, but that is a pipe dream) "D400", # First line of docstring should end with a period (assumes the summary fits on one line, which is not always the case) "D205", # 1 blank line required between summary line and description (Makes same assumption as D400) "D200", # One-line docstring should fit on one line, don't like this because it enforces inconsistent formatting between one-line and multi-line docstrings @@ -27,17 +27,17 @@ ignore = [ "D404", # First word of the docstring should not be "This" (Probably a good rule to follow going forward but it occurs in so many places in our code and it's not a big enough issue to warrant fixing) ] -select = [ - "B", - "D", - "E", - "F", - "W", -] +select = ["B", "D", "E", "F", "W"] [lint.pydocstyle] convention = "numpy" +[lint.mccabe] +max-complexity = 10 + +[lint.isort] +force-sort-within-sections = true + [format] quote-style = "double" indent-style = "space"