Catch catalog drift vs generated hooks

Add generated hook comments to help contributors, instructing them to
modify the catalog.py + a GHA job to check for catalog discrepancies.

Contributors easily modify `.pre-commit-hooks.yaml` without modifying
the catalog file, which puts us in the state that changes can easily
be accidentally reverted. The job enforces that if regenerating the
hooks produces a change, the PR fails in CI, and the comments help
contributors avoid failing CI.

Author: sirosen

.github/workflows/build.yaml

@@ -99,6 +99,38 @@ jobs:
       # collate and report
       - run: python ./scripts/aggregate-pytest-reports.py artifacts/*/pytest.*.xml
 
+  # this job ensures that any changes to the hook config are reflected properly
+  # in hook config generation
+  # if it fails, please see the CONTRIBUTING.md for the repo and read the section titled
+  # "Adding new hooks"
+  ensure-catalog-matches-hook-config:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: get date for caching
+        run: /bin/date -u "+%U" > cachedate.txt
+        shell: bash
+
+      - uses: actions/setup-python@v4
+        id: setup-python
+        with:
+          python-version: "3.x"
+          cache: "pip"
+          cache-dependency-path: |
+            .github/workflows/build.yaml
+            setup.cfg
+            tox.ini
+            cachedate.txt
+
+      - run: python -m pip install tox
+
+      - name: regenerate hook config
+        run: tox run -e generate-hooks-config
+
+      - name: fail if .pre-commit-hooks.yaml changed
+        run: git diff --exit-code -- .pre-commit-hooks.yaml
+
   self-check:
     name: "Self-Check"
     runs-on: ubuntu-latest

.pre-commit-hooks.yaml

@@ -18,6 +18,9 @@
 
 # --AUTOGEN_HOOKS_START-- #
 
+# this hook is autogenerated from a script
+# to modify this hook, update `src/check_jsonschema/catalog.py`
+# and run `make generate-hooks` or `tox run -e generate-hooks-config`
 - id: check-azure-pipelines
   name: Validate Azure Pipelines
   description: 'Validate Azure Pipelines config against the schema provided by Microsoft'
@@ -26,6 +29,9 @@
   files: ^(\.)?azure-pipelines.(yml|yaml)$
   types: [yaml]
 
+# this hook is autogenerated from a script
+# to modify this hook, update `src/check_jsonschema/catalog.py`
+# and run `make generate-hooks` or `tox run -e generate-hooks-config`
 - id: check-bamboo-spec
   name: Validate Bamboo Specs
   description: 'Validate Bamboo Specs against the schema provided by SchemaStore'
@@ -34,6 +40,9 @@
   files: ^bamboo-specs/.*\.(yml|yaml)$
   types: [yaml]
 
+# this hook is autogenerated from a script
+# to modify this hook, update `src/check_jsonschema/catalog.py`
+# and run `make generate-hooks` or `tox run -e generate-hooks-config`
 - id: check-buildkite
   name: Validate Buildkite Pipelines
   description: 'Validate Buildkite Pipelines against the schema provided by Buildkite'
@@ -48,6 +57,9 @@
     )$
   types_or: [json,yaml]
 
+# this hook is autogenerated from a script
+# to modify this hook, update `src/check_jsonschema/catalog.py`
+# and run `make generate-hooks` or `tox run -e generate-hooks-config`
 - id: check-dependabot
   name: Validate Dependabot Config (v2)
   description: 'Validate Dependabot Config (v2) against the schema provided by SchemaStore'
@@ -56,6 +68,9 @@
   files: ^\.github/dependabot.(yml|yaml)$
   types: [yaml]
 
+# this hook is autogenerated from a script
+# to modify this hook, update `src/check_jsonschema/catalog.py`
+# and run `make generate-hooks` or `tox run -e generate-hooks-config`
 - id: check-github-actions
   name: Validate GitHub Actions
   description: 'Validate GitHub Actions against the schema provided by SchemaStore'
@@ -68,6 +83,9 @@
     )$
   types: [yaml]
 
+# this hook is autogenerated from a script
+# to modify this hook, update `src/check_jsonschema/catalog.py`
+# and run `make generate-hooks` or `tox run -e generate-hooks-config`
 - id: check-github-workflows
   name: Validate GitHub Workflows
   description: 'Validate GitHub Workflows against the schema provided by SchemaStore'
@@ -76,6 +94,9 @@
   files: ^\.github/workflows/[^/]+$
   types: [yaml]
 
+# this hook is autogenerated from a script
+# to modify this hook, update `src/check_jsonschema/catalog.py`
+# and run `make generate-hooks` or `tox run -e generate-hooks-config`
 - id: check-gitlab-ci
   name: Validate GitLab CI config
   description: 'Validate GitLab CI config against the schema provided by SchemaStore'
@@ -84,6 +105,9 @@
   files: ^.*\.gitlab-ci.yml$
   types: [yaml]
 
+# this hook is autogenerated from a script
+# to modify this hook, update `src/check_jsonschema/catalog.py`
+# and run `make generate-hooks` or `tox run -e generate-hooks-config`
 - id: check-readthedocs
   name: Validate ReadTheDocs Config
   description: 'Validate ReadTheDocs config against the schema provided by ReadTheDocs'
@@ -92,6 +116,9 @@
   files: ^\.readthedocs.(yml|yaml)$
   types: [yaml]
 
+# this hook is autogenerated from a script
+# to modify this hook, update `src/check_jsonschema/catalog.py`
+# and run `make generate-hooks` or `tox run -e generate-hooks-config`
 - id: check-renovate
   name: Validate Renovate Config
   description: 'Validate Renovate config against the schema provided by Renovate (does not support renovate config in package.json)'
@@ -104,6 +131,9 @@
       \.renovaterc(\.json)?
     )$
 
+# this hook is autogenerated from a script
+# to modify this hook, update `src/check_jsonschema/catalog.py`
+# and run `make generate-hooks` or `tox run -e generate-hooks-config`
 - id: check-travis
   name: Validate Travis Config
   description: 'Validate Travis Config against the schema provided by SchemaStore'

scripts/generate-hooks-config.py

@@ -42,6 +42,9 @@ def update_hook_config(new_config: str) -> None:
 
 
 def generate_hook_lines(config) -> t.Iterator[str]:
+    yield "# this hook is autogenerated from a script"
+    yield "# to modify this hook, update `src/check_jsonschema/catalog.py`"
+    yield "# and run `make generate-hooks` or `tox run -e generate-hooks-config`"
     yield f"- id: {config['id']}"
     yield f"  name: {config['name']}"
     yield f"  description: '{config['description']}'"