add qmd doctester

.github/workflows/runtests.yml

@@ -11,6 +11,7 @@ on:
     - dev
     paths:
       - 'pyproject.toml'
+      - 'docs/**/*.qmd'
       - '**.py'
       - '.github/workflows/*.yml'
       - '.github/actions/**/*.yml'
@@ -72,6 +73,11 @@ jobs:
           cache-number: ${{ env.CACHE_NUMBER }}
           prepare-test-data: "true"
 
+      - name: generate qmd docs tests
+        id: generate_qmd_tests
+        shell: bash -el {0}
+        run: python scripts/generate_doc_code_tests.py
+
       # Runs test suite and calculates coverage
       - name: run test suite
         id: run_test_suite
@@ -111,6 +117,6 @@ jobs:
           token: ${{ secrets.CODECOV_TOKEN }}
 
       - name: fail job after debug session if tests failed
-        if: steps.run_test_suite.outcome == 'failure' || steps.run_docstrings.outcome == 'failure'
+        if: steps.generate_qmd_tests.outcome == 'failure' || steps.run_test_suite.outcome == 'failure' || steps.run_docstrings.outcome == 'failure'
         shell: bash
         run: exit 1

.gitignore

@@ -89,6 +89,7 @@ docs/**/*.ipynb
 scratch/**
 .ruff_cache
 uv.lock
+tests/test_autogenerated_doccode/
 
 docs/index_files
 docs/index.quarto_ipynb

docs/contributing/testing.qmd

@@ -45,6 +45,20 @@ To run the docstring tests use the following:
 pytest dascore --doctest-modules
 ```
 
+To validate executable examples in the hand-written documentation without building the full site, generate the mirrored tests with:
+
+```bash
+python scripts/generate_doc_code_tests.py
+```
+
+Then run the generated tests:
+
+```bash
+pytest tests/test_autogenerated_doccode
+```
+
+The `tests/test_autogenerated_doccode` directory is intentionally gitignored and should be regenerated locally rather than committed.
+
 ## Writing Tests
 
 Tests should go into the `tests/` folder, which mirrors the structure of the main package.

environment.yml

@@ -3,6 +3,7 @@ channels:
   - conda-forge
 dependencies:
   - pytest
+  - pytest-timeout
   - numpy>=1.24
   - pydantic>=2.1
   - pip

pyproject.toml

@@ -226,6 +226,7 @@ markers = [
     "network: tests that require network-style filesystem access",
     "slow: tests skipped by default unless explicitly selected with -m slow",
     "benchmark: performance benchmark tests",
+    "docs_examples: autogenerated tests for executable qmd examples",
 ]
 
 [tool.ruff.format]