diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index df3d2afa..cdeb5272 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -27,13 +27,16 @@ jobs: - uses: "actions/setup-python@v5" with: python-version: "3.11" + - name: "Install python dependencies" + working-directory: "plugin_template" + run: | + echo ::group::PYDEPS + pip install pre-commit + echo ::endgroup:: - name: "Lint plugin_template" working-directory: "plugin_template" run: | - pip3 install black==24.3.0 flake8 - black --version - black --check --diff plugin-template utils.py - flake8 plugin-template utils.py + pre-commit run --all-files -v - name: "Bootstrap catdog plugin" working-directory: "plugin_template" run: | @@ -41,48 +44,19 @@ jobs: # Below this line we include the steps of the ci workflow of the generated plugin - - name: "Install python dependencies" - run: | - echo ::group::PYDEPS - pip install -r lint_requirements.txt - echo ::endgroup:: - - - name: "Lint workflow files" - run: | - yamllint -s -d '{extends: relaxed, rules: {line-length: disable}}' .github/workflows - - - name: "Verify bump version config" - run: | - bump-my-version bump --dry-run release - bump-my-version show-bump - - # run black separately from flake8 to get a diff - - name: "Run black" - run: | - black --version - black --check --diff . - - # Lint code. - - name: "Run flake8" - run: | - flake8 - - - name: "Run extra lint checks" - run: | - [ ! -x .ci/scripts/extra_linting.sh ] || .ci/scripts/extra_linting.sh - - - name: "Check for any files unintentionally left out of MANIFEST.in" - run: | - check-manifest - - - name: "Verify requirements files" - run: | - python .ci/scripts/check_requirements.py - - - name: "Check for pulpcore imports outside of pulpcore.plugin" - run: | - sh .ci/scripts/check_pulpcore_imports.sh - - - name: "Check for common gettext problems" - run: | - sh .ci/scripts/check_gettext.sh + # cache example from: https://pre-commit.com/#github-actions-example + - name: set PY + run: echo "PY=$(python -VV | sha256sum | cut -d' ' -f1)" >> $GITHUB_ENV + - uses: actions/cache@v5 + with: + path: ~/.cache/pre-commit + key: pre-commit|${{ env.PY }}|${{ hashFiles('.pre-commit-config.yaml') }} + - name: Run pre-commit + shell: "bash" + env: + PY_COLORS: "1" + FORCE_COLOR: "1" + PRE_COMMIT_COLOR: "always" + TERM: "xterm-256color" + run: | + pre-commit run --all-files -v diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..ebdb1a38 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,29 @@ +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +repos: + - repo: 'https://github.com/pre-commit/pre-commit-hooks' + rev: v6.0.0 + hooks: + - id: check-yaml + - id: check-added-large-files + + - repo: 'https://github.com/pycqa/flake8' + rev: 7.3.0 + hooks: + - id: flake8 + args: ["--config", ".flake8"] + language_version: '3.11' + + - repo: 'https://github.com/psf/black' + rev: 24.3.0 + hooks: + - id: black + language_version: "3.11" + args: ["--extend-exclude", ".*venv.*", "."] + additional_dependencies: ["flake8-black"] + + - repo: 'https://github.com/adrienverge/yamllint' + rev: v1.37.1 + hooks: + - id: yamllint + args: ["--strict"] diff --git a/.yamllint.yaml b/.yamllint.yaml new file mode 100644 index 00000000..b83e80a1 --- /dev/null +++ b/.yamllint.yaml @@ -0,0 +1,3 @@ +extends: relaxed +rules: + line-length: disable diff --git a/lint_requirements.txt b/lint_requirements.txt new file mode 100644 index 00000000..b131606e --- /dev/null +++ b/lint_requirements.txt @@ -0,0 +1,4 @@ +# CI linting uses pre-commit +# keeping here for compatibility +black==24.3.0 +flake8==7.3.0 diff --git a/plugin-template b/plugin-template index 41d275a1..4cd2340a 100755 --- a/plugin-template +++ b/plugin-template @@ -5,7 +5,6 @@ import os import pprint import shlex import shutil -import subprocess import sys import textwrap from pathlib import Path @@ -184,6 +183,14 @@ DEPRECATED_FILES = { ], } +PRECOMMIT_VERSIONS = { + "pre-commit-hooks": "v3.2.0", + "flake8": "7.3.0", + "black": "24.3.0", + "yamllint": "v1.37.1", + "check-manifest": "0.51", +} + def main(): parser = argparse.ArgumentParser( @@ -374,22 +381,37 @@ def main(): if "github" in sections: migrate_dummy(plugin_root_dir) + migrate_checkmanifest_ignores(plugin_root_dir) if plugin_root_dir: print("\nDeprecation check:") check_for_deprecated_files(plugin_root_dir, sections) - if config["black"]: - try: - subprocess.run(["black", "--quiet", "."], cwd=plugin_root_dir) - except FileNotFoundError: - pass - -def migrate_dummy(plugin_root_dir): +def migrate_dummy(plugin_root_dir: str): pass +def migrate_checkmanifest_ignores(plugin_root_dir: str): + # appends new check-manifest.ignore files to pyproject unmanaged section + # remove after plugins are up-to-date + import tomlkit + import contextlib + + pyproject_path = Path(plugin_root_dir) / "pyproject.toml" + if not pyproject_path.exists(): + # handle manually + return + pyproject_data = tomlkit.loads(pyproject_path.read_text()) + ignore_files = [".yamllint.yaml", ".pre-commit-config.yaml"] + for ignore_file in ignore_files: + with contextlib.suppress(KeyError): + if ignore_file in pyproject_data["tool"]["check-manifest"]["ignore"]: + continue + pyproject_data["tool"]["check-manifest"]["ignore"].append(ignore_file) + pyproject_path.write_text(tomlkit.dumps(pyproject_data)) + + def write_template_section(config, name, plugin_root_dir, verbose=False): """ Template or copy all files for the section. @@ -414,6 +436,8 @@ def write_template_section(config, name, plugin_root_dir, verbose=False): env.filters["to_yaml"] = utils.to_nice_yaml env.filters["from_yaml"] = utils.from_yaml env.filters["shquote"] = shlex.quote + env.filters["normalize_rev"] = utils.normalize_rev + env.filters["quote"] = lambda o: repr(o) files_templated = 0 files_copied = 0 @@ -428,7 +452,7 @@ def write_template_section(config, name, plugin_root_dir, verbose=False): "current_version": utils.current_version(plugin_root_path), "pulpdocs_branch": PULPDOCS_BRANCH, "is_pulpdocs_member": config["plugin_name"] in utils.get_pulpdocs_members(PULPDOCS_BRANCH), - "black_version": utils.black_version(), + "precommit_versions": PRECOMMIT_VERSIONS, **config, } diff --git a/requirements.txt b/requirements.txt index fb4fc085..087b9192 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,4 @@ -# This needs to match the version in templates/github/lint_requirement.txt.j2 -black==24.3.0 +pre-commit jinja2 pyyaml requests~=2.32.3 diff --git a/scripts/update_ci.sh b/scripts/update_ci.sh index 637917ca..d9685e75 100755 --- a/scripts/update_ci.sh +++ b/scripts/update_ci.sh @@ -42,8 +42,7 @@ fi if [[ "${use_black}" == "True" ]] then - pip install -r lint_requirements.txt - black . + pre-commit run -a black || true # always zero exit-code if [[ "$(git status --porcelain)" ]] then diff --git a/templates/bootstrap/pyproject.toml.j2 b/templates/bootstrap/pyproject.toml.j2 index 28526468..2749308f 100644 --- a/templates/bootstrap/pyproject.toml.j2 +++ b/templates/bootstrap/pyproject.toml.j2 @@ -111,6 +111,8 @@ ignore = [ "lint_requirements.txt", "docs/**", ".flake8", + ".yamllint.yaml", + ".pre-commit-config.yaml", "template_config.yml", ".coveragerc", ".dependabot/config.yml", diff --git a/templates/github/.ci/ansible/build_container.yaml b/templates/github/.ci/ansible/build_container.yaml index c380b430..93ce3ebb 100644 --- a/templates/github/.ci/ansible/build_container.yaml +++ b/templates/github/.ci/ansible/build_container.yaml @@ -23,5 +23,5 @@ - name: "Clean image cache" docker_prune: - images : true + images: true ... diff --git a/templates/github/.ci/scripts/check_bump.sh.j2 b/templates/github/.ci/scripts/check_bump.sh.j2 new file mode 100755 index 00000000..16288522 --- /dev/null +++ b/templates/github/.ci/scripts/check_bump.sh.j2 @@ -0,0 +1,7 @@ +#!/bin/bash + +{% include 'header.j2' %} + +set -eux +bump-my-version bump release --dry-run --allow-dirty +bump-my-version show-bump diff --git a/templates/github/.ci/scripts/check_gettext.sh.j2 b/templates/github/.ci/scripts/check_gettext.sh.j2 index 5492a1d1..ca8eb0fb 100755 --- a/templates/github/.ci/scripts/check_gettext.sh.j2 +++ b/templates/github/.ci/scripts/check_gettext.sh.j2 @@ -1,16 +1,21 @@ -#!/bin/bash +#!/bin/sh {% include 'header.j2' %} # make sure this script runs at the repo root cd "$(dirname "$(realpath -e "$0")")"/../.. -set -uv +set -eu -MATCHES=$(grep -n -r --include \*.py "_(f") +if [ $# -eq 0 ]; then + echo "Usage: $0 [filename ...]" + exit 1 +fi -if [ $? -ne 1 ]; then - printf "\nERROR: Detected mix of f-strings and gettext:\n" - echo "$MATCHES" +# grep returns 1 if it doesn't find a match +PATTERN="_(f[\"\']" +if RESULT=$(grep -n "$PATTERN" "$@"); then + echo "ERROR: Detected mix of f-strings and gettext:" + echo "$RESULT" exit 1 fi diff --git a/templates/github/.ci/scripts/check_pulpcore_imports.sh.j2 b/templates/github/.ci/scripts/check_pulpcore_imports.sh.j2 index d1b03be4..931f5b72 100755 --- a/templates/github/.ci/scripts/check_pulpcore_imports.sh.j2 +++ b/templates/github/.ci/scripts/check_pulpcore_imports.sh.j2 @@ -5,14 +5,20 @@ # make sure this script runs at the repo root cd "$(dirname "$(realpath -e "$0")")"/../.. -set -uv +set -eu -# check for imports not from pulpcore.plugin. exclude tests -MATCHES=$(grep -n -r --include \*.py "from pulpcore.*import" . | grep -v "tests\|plugin" {%- for allow in core_import_allowed -%}| grep -v "{{allow}}" {%- endfor -%}) +if [ $# -eq 0 ]; then + echo "Usage: $0 [filename ...]" + exit 1 +fi -if [ $? -ne 1 ]; then - printf "\nERROR: Detected bad imports from pulpcore:\n" +# check for imports not from pulpcore.plugin. exclude tests +if MATCHES=$( + grep -n "from pulpcore.*import" "$@" | + grep -v "tests\|plugin" {%- for allow in core_import_allowed %} | + grep -v "{{allow}}" {%- endfor %} +); then + echo "ERROR: Detected bad imports. Plugins should import from 'pulpcore.plugin':" echo "$MATCHES" - printf "\nPlugins should import from pulpcore.plugin." exit 1 fi diff --git a/templates/github/.github/workflows/lint.yml.j2 b/templates/github/.github/workflows/lint.yml.j2 index d8e238ff..d314ab79 100644 --- a/templates/github/.github/workflows/lint.yml.j2 +++ b/templates/github/.github/workflows/lint.yml.j2 @@ -22,56 +22,23 @@ jobs: {{ setup_python() | indent(6) }} - {{ install_python_deps(["-r", "lint_requirements.txt"]) | indent(6) }} - - - name: "Lint workflow files" - run: | - yamllint -s -d '{extends: relaxed, rules: {line-length: disable}}' .github/workflows - - - name: "Verify bump version config" - run: | - bump-my-version bump --dry-run release - bump-my-version show-bump - {%- if black %} - - # run black separately from flake8 to get a diff - - name: "Run black" - run: | - black --version - black --check --diff . - {%- endif %} - {%- if flake8 %} - - # Lint code. - - name: "Run flake8" - run: | - flake8 - {%- endif %} - - - name: "Run extra lint checks" - run: | - [ ! -x .ci/scripts/extra_linting.sh ] || .ci/scripts/extra_linting.sh - {%- if check_manifest %} - - - name: "Check for any files unintentionally left out of MANIFEST.in" - run: | - check-manifest - {%- endif %} - {%- if lint_requirements %} - - - name: "Verify requirements files" - run: | - python .ci/scripts/check_requirements.py - {%- endif %} - {%- if check_stray_pulpcore_imports %} - - - name: "Check for pulpcore imports outside of pulpcore.plugin" - run: | - sh .ci/scripts/check_pulpcore_imports.sh - {%- endif %} - {%- if check_gettext %} - - - name: "Check for common gettext problems" - run: | - sh .ci/scripts/check_gettext.sh - {%- endif %} + {{ install_python_deps(["pre-commit"]) | indent(6) }} + + # cache example from: https://pre-commit.com/#github-actions-example + - name: set PY + run: echo "PY=$(python -VV | sha256sum | cut -d' ' -f1)" >> $GITHUB_ENV + - uses: actions/cache@v5 + with: + path: ~/.cache/pre-commit + {%- raw %} + key: pre-commit|${{ env.PY }}|${{ hashFiles('.pre-commit-config.yaml') }} + {%- endraw %} + - name: Run pre-commit + shell: "bash" + env: + PY_COLORS: "1" + FORCE_COLOR: "1" + PRE_COMMIT_COLOR: "always" + TERM: "xterm-256color" + run: | + pre-commit run --all-files -v diff --git a/templates/github/.pre-commit-config.yaml.j2 b/templates/github/.pre-commit-config.yaml.j2 new file mode 100644 index 00000000..35b63996 --- /dev/null +++ b/templates/github/.pre-commit-config.yaml.j2 @@ -0,0 +1,95 @@ +{% include 'header.j2' %} + +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +repos: + - repo: 'https://github.com/pre-commit/pre-commit-hooks' + rev: {{ precommit_versions["pre-commit-hooks"] | quote }} + hooks: + - id: check-yaml + - id: check-added-large-files + + - repo: 'https://github.com/adrienverge/yamllint' + rev: {{ precommit_versions["yamllint"] | quote }} + hooks: + - id: yamllint + name: Lint workflow files + args: ["--strict"] + {%- if black %} + + - repo: local + hooks: + - id: check-bump + name: Verify bump version config + language: python + entry: .ci/scripts/check_bump.sh + pass_filenames: false + additional_dependencies: ["bump-my-version"] + + - repo: 'https://github.com/psf/black' + rev: {{ precommit_versions["black"] | quote }} + hooks: + - id: black + name: Run black + language_version: "3.11" + pass_filenames: false + args: ["--extend-exclude", ".*venv.*", "."] + {%- if flake8 %} + additional_dependencies: ["flake8-black"] + {%- endif %} + {%- endif %} + {%- if flake8 %} + + - repo: 'https://github.com/pycqa/flake8' + rev: {{ precommit_versions["flake8"] | quote }} + hooks: + - id: flake8 + name: Run flake8 + args: ["--config", ".flake8"] + language_version: '3.11' + {%- endif %} + + - repo: local + hooks: + - id: extra-lint + name: Run extra lint checks + language: unsupported + types: [python] + entry: sh -c "[ ! -x .ci/scripts/extra_linting.sh ] || .ci/scripts/extra_linting.sh" + + {%- if check_manifest %} + + - repo: 'https://github.com/mgedmin/check-manifest' + rev: {{ precommit_versions["check-manifest"] | quote }} + hooks: + - id: check-manifest + name: Check files unintentionally left out of MANIFEST.in + {%- endif %} + + # Custom checks + - repo: local + hooks: + {%- if lint_requirements %} + - id: check-requirements + name: Verify requirements files + language: python + entry: python .ci/scripts/check_requirements.py + pass_filenames: false + additional_dependencies: ["packaging"] + {%- endif %} + {%- if check_stray_pulpcore_imports %} + + - id: check-pulpcore-imports + name: Check pulpcore imports outside 'pulpcore.plugin' + language: unsupported_script + types: [python] + entry: .ci/scripts/check_pulpcore_imports.sh + {%- endif %} + {%- if check_gettext %} + + - id: check-gettext + name: Check for common gettext problems + language: unsupported_script + types: [python] + entry: .ci/scripts/check_gettext.sh + {%- endif %} diff --git a/templates/github/.yamllint.yaml.j2 b/templates/github/.yamllint.yaml.j2 new file mode 100644 index 00000000..746254f1 --- /dev/null +++ b/templates/github/.yamllint.yaml.j2 @@ -0,0 +1,5 @@ +{% include 'header.j2' %} + +extends: relaxed +rules: + line-length: disable diff --git a/templates/github/lint_requirements.txt.j2 b/templates/github/lint_requirements.txt.j2 index 8cccca14..ce6e20af 100644 --- a/templates/github/lint_requirements.txt.j2 +++ b/templates/github/lint_requirements.txt.j2 @@ -1,13 +1,16 @@ {% include 'header.j2' %} +# CI linting uses pre-commit +# keeping here for compatibility {% if black -%} -black=={{ black_version }} +black=={{ precommit_versions["black"] | normalize_rev }} {% endif -%} -bump-my-version -check-manifest -flake8 +check-manifest=={{ precommit_versions["check-manifest"] | normalize_rev }} +flake8=={{ precommit_versions["flake8"] | normalize_rev }} +yamllint=={{ precommit_versions["yamllint"] | normalize_rev }} + {% if black -%} flake8-black {% endif -%} +bump-my-version packaging -yamllint diff --git a/utils.py b/utils.py index 64b60b65..e42badbc 100644 --- a/utils.py +++ b/utils.py @@ -1,5 +1,4 @@ from datetime import timedelta -from pathlib import Path import re import requests_cache import stat @@ -51,9 +50,16 @@ def to_snake(name): return name.replace("-", "_") +def normalize_rev(rev: str): + """ + Normalize common revision patterns. E.g.: 'v1.2.3' -> '1.2.3'. + """ + return re.sub("^[a-zA-Z]+|[a-zA-Z]+$", "", rev) + + def to_nice_yaml(data): """Implement a filter for Jinja 2 templates to render human readable YAML.""" - return yaml.dump(data, indent=2, allow_unicode=True, default_flow_style=False) + return yaml.dump(data, indent=2, allow_unicode=True, default_flow_style=False).strip() def from_yaml(data): @@ -94,16 +100,6 @@ def current_version(plugin_root_path): return current_version -def black_version(): - PATTERN = re.compile(r"^black==(?P.*)$") - requirements_file = Path(__file__).parent / "requirements.txt" - for line in requirements_file.read_text().splitlines(): - if match := PATTERN.fullmatch(line): - return match.group("version") - - raise ValueError("'black' not found in 'requirements.txt'") - - def get_pulpdocs_members(pulpdocs_branch="main") -> list[str]: """ Get repositories which are members of the Pulp managed documentation.