Add documentation validation

.github/workflows/docs-validation-unit-tests.yml

@@ -0,0 +1,46 @@
+name: Documentation Unit Tests
+
+on:
+  push:
+    paths:
+      - "ci/**"
+  pull_request:
+    paths:
+      - "ci/**"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: docs-unit-tests-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  unit-tests:
+    name: Pytest (selector & executor)
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.12"]
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install test dependencies
+        run: |
+          python -m pip install pytest requests-mock anyio
+
+      - name: Run pytest
+        env:
+          PYTHONUNBUFFERED: "1"
+        run: |
+          pytest -q --maxfail=1

.github/workflows/docs-validation.yml

@@ -0,0 +1,145 @@
+name: Documentation Functional Testing
+
+on:
+  push:
+    paths:
+      - "tutorial/**/*.task.sh"
+      - "how-to/**/*.task.sh"
+  pull_request:
+    paths:
+      - "tutorial/**/*.task.sh"
+      - "how-to/**/*.task.sh"
+  # Manual trigger
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: docs-validation-${{ github.ref }}
+  cancel-in-progress: true
+
+defaults:
+  run:
+    shell: bash
+
+env:
+  PYTHONUNBUFFERED: "1"
+  PIP_DISABLE_PIP_VERSION_CHECK: "1"
+  # Dry-run switch. Set to "1" to print without executing.
+  DOCS_DRY_RUN: "1"
+
+jobs:
+  lint-snippets:
+    name: Lint snippet shell files
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Lint shell snippets
+        run: make lint
+
+  plan-and-run:
+    name: Plan & run docs tests
+    needs: lint-snippets
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Determine changed snippet files
+        id: changed
+        uses: tj-actions/changed-files@v45
+        with:
+          files: |
+            tutorial/**/*.task.sh
+            how-to/**/*.task.sh
+
+      - name: Write changed list to file
+        id: write-changed
+        if: steps.changed.outputs.all_changed_files != ''
+        run: |
+          # No quotes around the variable allows the shell to perform word splitting
+          printf '%s\n' ${{ steps.changed.outputs.all_changed_files }} > changed_snippets.txt
+
+          COUNT=$(wc -l < changed_snippets.txt | tr -d ' ')
+          echo "count=${COUNT}" >> "$GITHUB_OUTPUT"
+
+          {
+            echo "### Changed snippets"
+            echo
+            echo "**${COUNT} file(s) changed:**"
+            echo
+            sed 's/^/- /' changed_snippets.txt
+          } >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Short-circuit if nothing to run
+        if: steps.write-changed.outputs.count == null || steps.write-changed.outputs.count == '0'
+        run: echo "No .task.sh files changed. Skipping subsequent steps."
+
+      - name: Set up Python (executor/selector)
+        if: steps.write-changed.outputs.count > 0
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Build and display execution plan
+        if: steps.write-changed.outputs.count > 0
+        id: plan
+        run: |
+          python ci/select-doc-pages.py --changed-file-list changed_snippets.txt --out plan.txt
+          PLAN_COUNT=$(wc -l < plan.txt | tr -d ' ')
+          echo "count=${PLAN_COUNT}" >> "$GITHUB_OUTPUT"
+
+          echo "--- Execution Plan (${PLAN_COUNT} steps) ---"
+          nl -ba plan.txt || true
+          echo "-----------------------------------"
+
+          {
+            echo "### Execution plan"
+            if [[ "$PLAN_COUNT" -gt 0 ]]; then
+              echo "Selector produced a plan with **${PLAN_COUNT}** item(s)."
+              echo '```'
+              cat plan.txt
+              echo '```'
+            else
+              echo "_Selector produced an empty plan._"
+            fi
+          } >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Fail if plan is unexpectedly empty
+        if: steps.write-changed.outputs.count > 0 && steps.plan.outputs.count == '0'
+        run: |
+          echo "::error::Selector produced an empty plan, but there were changed snippets."
+          exit 1
+
+      - name: Dry-run plan validation
+        if: env.DOCS_DRY_RUN == '1' && steps.plan.outputs.count > 0
+        run: |
+          echo "Executing dry-run validation of the plan..."
+          python ci/run-doc-pages.py --plan plan.txt --dry-run
+
+      - name: Execute plan
+        if: env.DOCS_DRY_RUN != '1' && steps.plan.outputs.count > 0
+        run: |
+          python ci/run-doc-pages.py --plan plan.txt 2>&1 | tee run.log
+
+      - name: Upload artifacts (plan & logs)
+        if: always() && steps.write-changed.outputs.count > 0
+        uses: actions/upload-artifact@v4
+        with:
+          name: docs-validation-artifacts-${{ github.run_id }}
+          path: |
+            plan.txt
+            changed_snippets.txt
+            run.log
+          if-no-files-found: ignore

Makefile

@@ -17,6 +17,7 @@ help:
         "* clean built doc files:                     make clean-doc \n" \
         "* clean full environment:                    make clean \n" \
         "* check links:                               make linkcheck \n" \
+        "* lint shell snippets:                       make lint \n" \
         "* check spelling:                            make spelling \n" \
         "* check spelling (without building again):   make spellcheck \n" \
         "* check inclusive language:                  make woke \n" \

Makefile.sp

@@ -138,6 +138,12 @@ sp-pdf: sp-pdf-prep
 	@rm -r $(BUILDDIR)/latex
 	@echo "\nOutput can be found in ./$(BUILDDIR)\n"
 
+sp-lint: sp-install
+	@echo "Linting shell snippets with bashate..."
+	@. $(VENV); pip show bashate > /dev/null || (echo "--> Installing bashate..."; pip install bashate)
+	@. $(VENV); find tutorial how-to -type f -name '*.task.sh' -print0 2>/dev/null | xargs -0 -r bashate -i E006
+	@echo "Linting finished successfully."
+
 # Catch-all target: route all unknown targets to Sphinx using the new
 # "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
 %: Makefile.sp

ci/README.md

@@ -0,0 +1,135 @@
+# Documentation Testing Framework
+
+## Purpose
+
+The goal is to ensure that every command shown in our documentation matches the software.
+Each tutorial or how-to snippet contains **both display and execution blocks**:
+
+- `[docs-view:<name>]` — what appears in the rendered documentation (human-readable example)
+- `[docs-exec:<name>]` — what CI executes (machine-runnable equivalent)
+
+Both appear in the same `.task.sh` snippet file, so you only maintain one source of truth.
+
+---
+
+## Block Types
+
+### `[docs-view:<name>]` — for rendered docs
+Used with the Sphinx `literalinclude` directive.
+Contains clean, readable shell commands.
+
+Example:
+```bash
+# [docs-view:enable-secrets]
+sunbeam enable secrets
+# [docs-view:enable-secrets-end]
+```
+
+### `[docs-exec:<name>]`
+Used for **execution** in CI or local validation runs.
+It can include wrappers, environment variables, or anything needed to actually make it work.
+
+Example:
+```bash
+# [docs-exec:enable-secrets]
+sg snap_daemon 'sunbeam enable secrets'
+# [docs-exec:enable-secrets-end]
+```
+
+> The `docs-view` block is what appears in the docs.
+> The `docs-exec` block is what actually runs — but they should mirror each other closely.
+
+## Creating or Editing a Snippet
+
+Create a new file like `how-to/snippets/secrets.task.sh`:
+
+
+```bash
+# [docs-view:enable-secrets]
+sunbeam enable secrets
+# [docs-view:enable-secrets-end]
+
+# [docs-exec:enable-secrets]
+sg snap_daemon 'sunbeam enable secrets'
+# [docs-exec:enable-secrets-end]
+
+# [docs-view:disable-secrets]
+sunbeam disable secrets
+# [docs-view:disable-secrets-end]
+
+# [docs-exec:disable-secrets]
+sg snap_daemon 'sunbeam disable secrets'
+# [docs-exec:disable-secrets-end]
+```
+
+**Rules**
+- Use `# [docs-exec:<name>]` … `# [docs-exec:<name>-end]` for every runnable block.
+- Blocks must not overlap or nest.
+- Keep commands non-interactive and idempotent where possible.
+
+---
+
+## Including `[docs-view]` Blocks in RST Files
+
+To include a `docs-view` block in the rendered documentation, use `literalinclude` with `start-after` and `end-before` markers:
+
+```rst
+.. literalinclude:: ../snippets/ldap.task.sh
+   :language: bash
+   :start-after: [docs-view:enable-ldap]
+   :end-before:  [docs-view:enable-ldap-end]
+```
+
+**Tips**
+- Always point to your `*.task.sh` file in the appropriate `snippets/` directory.
+- Make sure your markers (`[docs-view:NAME]` and `[docs-view:NAME-end]`) match exactly.
+- Sphinx will automatically include only the lines between those markers.
+
+---
+
+## Declaring Dependencies
+
+If one snippet must run after another (e.g., a feature enable depends on sunbeam being deployed), add a `# @depends:` line near the top pointing to the prerequisite `*.task.sh` script:
+
+```bash
+# @depends: tutorial/snippets/get-started-with-openstack.task.sh
+```
+
+**Notes**
+- Colon after `@depends` is **required**.
+- Paths must be **repo-relative**.
+- Multiple dependencies are allowed (one per line).
+- The selector resolves them such that dependencies always appear **before** dependents in the plan.
+
+---
+
+
+## CI Workflows
+
+There are two workflows using this system:
+
+### 1. **Documentation Functional Testing**
+- Triggers when `*.task.sh` files change.
+- Uses the selector to build `plan.txt`.
+- Runs the runner in dry-run mode to verify parsing and order.
+- Artifacts: `plan.txt`, `changed_snippets.txt`, `run.log`.
+
+### 2. **Documentation Unit Tests**
+- Triggers when `ci/**` changes.
+- Runs `pytest` on the selector/runner themselves.
+
+---
+
+## Adding Certificates, DNS, or Other Setup
+
+If a how-to requires additional setup (e.g., certificates, DNS records), wrap those commands in their own `[docs-exec:*]` blocks inside the same `.task.sh` file:
+
+```bash
+# [docs-exec:generate-cert]
+openssl req -x509 -nodes -newkey rsa:2048 \
+  -keyout /tmp/demo.key -out /tmp/demo.crt \
+  -subj "/CN=demo.internal" -days 365
+# [docs-exec:generate-cert-end]
+```
+
+These commands will appear in the printed (or executed) plan just like any other snippet.