[PULP-1105] Add pre-commit in github templates

.github/workflows/lint.yml

@@ -27,62 +27,36 @@ jobs:
       - uses: "actions/setup-python@v5"
         with:
           python-version: "3.11"
+      - name: "Install python dependencies"
+        working-directory: "plugin_template"
+        run: |
+          echo ::group::PYDEPS
+          pip install pre-commit
+          echo ::endgroup::
       - name: "Lint plugin_template"
         working-directory: "plugin_template"
         run: |
-          pip3 install black==24.3.0 flake8
-          black --version
-          black --check --diff plugin-template utils.py
-          flake8 plugin-template utils.py
+          pre-commit run --all-files -v
       - name: "Bootstrap catdog plugin"
         working-directory: "plugin_template"
         run: |
           .ci/bootstrap_catdog.sh
 
       # Below this line we include the steps of the ci workflow of the generated plugin
 
-      - name: "Install python dependencies"
-        run: |
-          echo ::group::PYDEPS
-          pip install -r lint_requirements.txt
-          echo ::endgroup::
-
-      - name: "Lint workflow files"
-        run: |
-          yamllint -s -d '{extends: relaxed, rules: {line-length: disable}}' .github/workflows
-
-      - name: "Verify bump version config"
-        run: |
-          bump-my-version bump --dry-run release
-          bump-my-version show-bump
-
-      # run black separately from flake8 to get a diff
-      - name: "Run black"
-        run: |
-          black --version
-          black --check --diff .
-
-      # Lint code.
-      - name: "Run flake8"
-        run: |
-          flake8
-
-      - name: "Run extra lint checks"
-        run: |
-          [ ! -x .ci/scripts/extra_linting.sh ] || .ci/scripts/extra_linting.sh
-
-      - name: "Check for any files unintentionally left out of MANIFEST.in"
-        run: |
-          check-manifest
-
-      - name: "Verify requirements files"
-        run: |
-          python .ci/scripts/check_requirements.py
-
-      - name: "Check for pulpcore imports outside of pulpcore.plugin"
-        run: |
-          sh .ci/scripts/check_pulpcore_imports.sh
-
-      - name: "Check for common gettext problems"
-        run: |
-          sh .ci/scripts/check_gettext.sh
+      # cache example from: https://pre-commit.com/#github-actions-example
+      - name: set PY
+        run: echo "PY=$(python -VV | sha256sum | cut -d' ' -f1)" >> $GITHUB_ENV
+      - uses: actions/cache@v5
+        with:
+          path: ~/.cache/pre-commit
+          key: pre-commit|${{ env.PY }}|${{ hashFiles('.pre-commit-config.yaml') }}
+      - name: Run pre-commit
+        shell: "bash"
+        env:
+          PY_COLORS: "1"
+          FORCE_COLOR: "1"
+          PRE_COMMIT_COLOR: "always"
+          TERM: "xterm-256color"
+        run: |
+          pre-commit run --all-files -v

.pre-commit-config.yaml

@@ -0,0 +1,29 @@
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+repos:
+  - repo: 'https://github.com/pre-commit/pre-commit-hooks'
+    rev: v6.0.0
+    hooks:
+      - id: check-yaml
+      - id: check-added-large-files
+
+  - repo: 'https://github.com/pycqa/flake8'
+    rev: 7.3.0
+    hooks:
+      - id: flake8
+        args: ["--config", ".flake8"]
+        language_version: '3.11'
+
+  - repo: 'https://github.com/psf/black'
+    rev: 24.3.0
+    hooks:
+      - id: black
+        language_version: "3.11"
+        args: ["--extend-exclude", ".*venv.*", "."]
+        additional_dependencies: ["flake8-black"]
+
+  - repo: 'https://github.com/adrienverge/yamllint'
+    rev: v1.37.1
+    hooks:
+      - id: yamllint
+        args: ["--strict"]

.yamllint.yaml

@@ -0,0 +1,3 @@
+extends: relaxed
+rules:
+  line-length: disable

lint_requirements.txt

@@ -0,0 +1,4 @@
+# CI linting uses pre-commit
+# keeping here for compatibility
+black==24.3.0
+flake8==7.3.0

plugin-template

@@ -5,7 +5,6 @@ import os
 import pprint
 import shlex
 import shutil
-import subprocess
 import sys
 import textwrap
 from pathlib import Path
@@ -184,6 +183,14 @@ DEPRECATED_FILES = {
     ],
 }
 
+PRECOMMIT_VERSIONS = {
+    "pre-commit-hooks": "v3.2.0",
+    "flake8": "7.3.0",
+    "black": "24.3.0",
+    "yamllint": "v1.37.1",
+    "check-manifest": "0.51",
+}
+
 
 def main():
     parser = argparse.ArgumentParser(
@@ -374,22 +381,37 @@ def main():
 
     if "github" in sections:
         migrate_dummy(plugin_root_dir)
+        migrate_checkmanifest_ignores(plugin_root_dir)
 
     if plugin_root_dir:
         print("\nDeprecation check:")
         check_for_deprecated_files(plugin_root_dir, sections)
 
-    if config["black"]:
-        try:
-            subprocess.run(["black", "--quiet", "."], cwd=plugin_root_dir)
-        except FileNotFoundError:
-            pass
-
 
-def migrate_dummy(plugin_root_dir):
+def migrate_dummy(plugin_root_dir: str):
     pass
 
 
+def migrate_checkmanifest_ignores(plugin_root_dir: str):
+    # appends new check-manifest.ignore files to pyproject unmanaged section
+    # remove after plugins are up-to-date
+    import tomlkit
+    import contextlib
+
+    pyproject_path = Path(plugin_root_dir) / "pyproject.toml"
+    if not pyproject_path.exists():
+        # handle manually
+        return
+    pyproject_data = tomlkit.loads(pyproject_path.read_text())
+    ignore_files = [".yamllint.yaml", ".pre-commit-config.yaml"]
+    for ignore_file in ignore_files:
+        with contextlib.suppress(KeyError):
+            if ignore_file in pyproject_data["tool"]["check-manifest"]["ignore"]:
+                continue
+            pyproject_data["tool"]["check-manifest"]["ignore"].append(ignore_file)
+    pyproject_path.write_text(tomlkit.dumps(pyproject_data))
+
+
 def write_template_section(config, name, plugin_root_dir, verbose=False):
     """
     Template or copy all files for the section.
@@ -414,6 +436,8 @@ def write_template_section(config, name, plugin_root_dir, verbose=False):
     env.filters["to_yaml"] = utils.to_nice_yaml
     env.filters["from_yaml"] = utils.from_yaml
     env.filters["shquote"] = shlex.quote
+    env.filters["normalize_rev"] = utils.normalize_rev
+    env.filters["quote"] = lambda o: repr(o)
 
     files_templated = 0
     files_copied = 0
@@ -428,7 +452,7 @@ def write_template_section(config, name, plugin_root_dir, verbose=False):
         "current_version": utils.current_version(plugin_root_path),
         "pulpdocs_branch": PULPDOCS_BRANCH,
         "is_pulpdocs_member": config["plugin_name"] in utils.get_pulpdocs_members(PULPDOCS_BRANCH),
-        "black_version": utils.black_version(),
+        "precommit_versions": PRECOMMIT_VERSIONS,
         **config,
     }
 

requirements.txt

@@ -1,5 +1,4 @@
-# This needs to match the version in templates/github/lint_requirement.txt.j2
-black==24.3.0
+pre-commit
 jinja2
 pyyaml
 requests~=2.32.3

scripts/update_ci.sh

@@ -42,8 +42,7 @@ fi
 
 if [[ "${use_black}" == "True" ]]
 then
-  pip install -r lint_requirements.txt
-  black .
+  pre-commit run -a black || true  # always zero exit-code
 
   if [[ "$(git status --porcelain)" ]]
   then

templates/bootstrap/pyproject.toml.j2

@@ -111,6 +111,8 @@ ignore = [
     "lint_requirements.txt",
     "docs/**",
     ".flake8",
+    ".yamllint.yaml",
+    ".pre-commit-config.yaml",
     "template_config.yml",
     ".coveragerc",
     ".dependabot/config.yml",

templates/github/.ci/ansible/build_container.yaml

@@ -23,5 +23,5 @@
 
     - name: "Clean image cache"
       docker_prune:
-        images : true
+        images: true
 ...

templates/github/.ci/scripts/check_bump.sh.j2

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+{% include 'header.j2' %}
+
+set -eux
+bump-my-version bump release --dry-run --allow-dirty
+bump-my-version show-bump

templates/github/.ci/scripts/check_gettext.sh.j2

@@ -1,16 +1,21 @@
-#!/bin/bash
+#!/bin/sh
 
 {% include 'header.j2' %}
 
 # make sure this script runs at the repo root
 cd "$(dirname "$(realpath -e "$0")")"/../..
 
-set -uv
+set -eu
 
-MATCHES=$(grep -n -r --include \*.py "_(f")
+if [ $# -eq 0 ]; then
+  echo "Usage: $0 <filename> [filename ...]"
+  exit 1
+fi
 
-if [ $? -ne 1 ]; then
-  printf "\nERROR: Detected mix of f-strings and gettext:\n"
-  echo "$MATCHES"
+# grep returns 1 if it doesn't find a match
+PATTERN="_(f[\"\']"
+if RESULT=$(grep -n "$PATTERN" "$@"); then
+  echo "ERROR: Detected mix of f-strings and gettext:"
+  echo "$RESULT"
   exit 1
 fi

templates/github/.ci/scripts/check_pulpcore_imports.sh.j2

@@ -5,14 +5,20 @@
 # make sure this script runs at the repo root
 cd "$(dirname "$(realpath -e "$0")")"/../..
 
-set -uv
+set -eu
 
-# check for imports not from pulpcore.plugin. exclude tests
-MATCHES=$(grep -n -r --include \*.py "from pulpcore.*import" . | grep -v "tests\|plugin" {%- for allow in core_import_allowed -%}| grep -v "{{allow}}" {%- endfor -%})
+if [ $# -eq 0 ]; then
+  echo "Usage: $0 <filename> [filename ...]"
+  exit 1
+fi
 
-if [ $? -ne 1 ]; then
-  printf "\nERROR: Detected bad imports from pulpcore:\n"
+# check for imports not from pulpcore.plugin. exclude tests
+if MATCHES=$(
+  grep -n "from pulpcore.*import" "$@" |
+  grep -v "tests\|plugin" {%- for allow in core_import_allowed %} |
+  grep -v "{{allow}}" {%- endfor %}
+); then
+  echo "ERROR: Detected bad imports. Plugins should import from 'pulpcore.plugin':"
   echo "$MATCHES"
-  printf "\nPlugins should import from pulpcore.plugin."
   exit 1
 fi

templates/github/.github/workflows/lint.yml.j2

@@ -22,56 +22,23 @@ jobs:
 
       {{ setup_python() | indent(6) }}
 
-      {{ install_python_deps(["-r", "lint_requirements.txt"]) | indent(6) }}
-
-      - name: "Lint workflow files"
-        run: |
-          yamllint -s -d '{extends: relaxed, rules: {line-length: disable}}' .github/workflows
-
-      - name: "Verify bump version config"
-        run: |
-          bump-my-version bump --dry-run release
-          bump-my-version show-bump
-      {%- if black %}
-
-      # run black separately from flake8 to get a diff
-      - name: "Run black"
-        run: |
-          black --version
-          black --check --diff .
-      {%- endif %}
-      {%- if flake8 %}
-
-      # Lint code.
-      - name: "Run flake8"
-        run: |
-          flake8
-      {%- endif %}
-
-      - name: "Run extra lint checks"
-        run: |
-          [ ! -x .ci/scripts/extra_linting.sh ] || .ci/scripts/extra_linting.sh
-      {%- if check_manifest %}
-
-      - name: "Check for any files unintentionally left out of MANIFEST.in"
-        run: |
-          check-manifest
-      {%- endif %}
-      {%- if lint_requirements %}
-
-      - name: "Verify requirements files"
-        run: |
-          python .ci/scripts/check_requirements.py
-      {%- endif %}
-      {%- if check_stray_pulpcore_imports %}
-
-      - name: "Check for pulpcore imports outside of pulpcore.plugin"
-        run: |
-          sh .ci/scripts/check_pulpcore_imports.sh
-      {%- endif %}
-      {%- if check_gettext %}
-
-      - name: "Check for common gettext problems"
-        run: |
-          sh .ci/scripts/check_gettext.sh
-      {%- endif %}
+      {{ install_python_deps(["pre-commit"]) | indent(6) }}
+
+      # cache example from: https://pre-commit.com/#github-actions-example
+      - name: set PY
+        run: echo "PY=$(python -VV | sha256sum | cut -d' ' -f1)" >> $GITHUB_ENV
+      - uses: actions/cache@v5
+        with:
+          path: ~/.cache/pre-commit
+          {%- raw %}
+          key: pre-commit|${{ env.PY }}|${{ hashFiles('.pre-commit-config.yaml') }}
+          {%- endraw %}
+      - name: Run pre-commit
+        shell: "bash"
+        env:
+          PY_COLORS: "1"
+          FORCE_COLOR: "1"
+          PRE_COMMIT_COLOR: "always"
+          TERM: "xterm-256color"
+        run: |
+          pre-commit run --all-files -v