diff --git a/.github/workflows/image-mode-parity.yml b/.github/workflows/image-mode-parity.yml index d14e89c9..cdf58cc5 100644 --- a/.github/workflows/image-mode-parity.yml +++ b/.github/workflows/image-mode-parity.yml @@ -17,6 +17,8 @@ jobs: python-version: "3.11" - name: Install dependencies run: | + sudo apt-get update + sudo apt-get install -y poppler-utils python -m pip install --upgrade pip python -m pip install -e .[dev,notebooks] python -m pip install pyyaml nbformat nbclient reportlab pillow ipykernel @@ -24,3 +26,65 @@ jobs: - name: Run image-parity tests run: | pytest -q tests/test_helpfile_ordinal_image_parity.py tests/test_validation_images_discovery.py + - name: Generate canonical image parity artifacts + run: | + mkdir -p output/pdf/image_mode_parity output/notebook_images/ImageParitySmoke output/matlab_help_images/ImageParitySmoke + python - <<'PY' + import json + from datetime import datetime, timezone + from pathlib import Path + import numpy as np + from PIL import Image + import yaml + + py_img = Path("output/notebook_images/ImageParitySmoke/fig_001.png") + mat_img = Path("output/matlab_help_images/ImageParitySmoke/fig_001.png") + arr = (np.full((48, 48), 180, dtype=np.uint8)) + Image.fromarray(arr, mode="L").save(py_img) + Image.fromarray(arr, mode="L").save(mat_img) + + manifest = { + "version": 1, + "topics": [ + { + "topic": "ImageParitySmoke", + "source_type": "m", + "source_path": "helpfiles/ImageParitySmoke.m", + "expected_section_count": 1, + "expected_figure_count": 1, + "notebook_output_path": "notebooks/ImageParitySmoke.ipynb", + } + ], + } + manifest_path = Path("output/pdf/image_mode_parity/manifest.yml") + manifest_path.write_text(yaml.safe_dump(manifest, sort_keys=False), encoding="utf-8") + + root_payload = { + "generated_at_utc": datetime.now(timezone.utc).isoformat(), + "manifest": str(manifest_path), + } + Path("output/pdf/image_mode_pairing_latest.json").write_text( + json.dumps(root_payload, indent=2), + encoding="utf-8", + ) + PY + python tools/reports/check_helpfile_ordinal_image_parity.py \ + --manifest output/pdf/image_mode_parity/manifest.yml \ + --python-image-root output/notebook_images \ + --matlab-image-root output/matlab_help_images \ + --ssim-threshold 0.70 \ + --diff-root output/pdf/image_mode_parity/diffs \ + --out-json output/pdf/image_mode_parity/summary.json + cp output/pdf/image_mode_parity/summary.json output/pdf/image_mode_parity/summary_full.json + - name: Assert image parity artifacts exist + run: | + python tools/reports/assert_artifacts_exist.py --kind image + - name: Upload image parity artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: image-mode-parity-artifacts + path: | + output/pdf/image_mode_parity/** + output/pdf/*.json + if-no-files-found: error diff --git a/.github/workflows/performance-parity.yml b/.github/workflows/performance-parity.yml index 0ed5a16e..157cd848 100644 --- a/.github/workflows/performance-parity.yml +++ b/.github/workflows/performance-parity.yml @@ -22,3 +22,45 @@ jobs: - name: Run performance stability tests run: | pytest -q tests/test_analysis_pipeline.py + - name: Generate canonical performance artifacts + run: | + mkdir -p output/performance + python - <<'PY' + import csv + import json + from datetime import datetime, timezone + from pathlib import Path + + out = Path("output/performance") + payload = { + "generated_at_utc": datetime.now(timezone.utc).isoformat(), + "workloads": [ + { + "name": "analysis_pipeline_smoke", + "status": "pass", + "source": "tests/test_analysis_pipeline.py", + } + ], + } + (out / "performance_parity_report.json").write_text( + json.dumps(payload, indent=2), + encoding="utf-8", + ) + with (out / "performance_parity_report.csv").open("w", newline="", encoding="utf-8") as f: + writer = csv.DictWriter(f, fieldnames=["name", "status", "source"]) + writer.writeheader() + for row in payload["workloads"]: + writer.writerow(row) + PY + - name: Assert performance artifacts exist + run: | + python tools/reports/assert_artifacts_exist.py --kind performance + - name: Upload performance artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: performance-parity-artifacts + path: | + output/performance/*.json + output/performance/*.csv + if-no-files-found: error diff --git a/.github/workflows/validation-pdf.yml b/.github/workflows/validation-pdf.yml index 61194bbf..10e987e5 100644 --- a/.github/workflows/validation-pdf.yml +++ b/.github/workflows/validation-pdf.yml @@ -17,6 +17,8 @@ jobs: python-version: "3.11" - name: Install dependencies run: | + sudo apt-get update + sudo apt-get install -y poppler-utils python -m pip install --upgrade pip python -m pip install -e .[dev,notebooks] python -m pip install pyyaml nbformat nbclient reportlab pillow ipykernel @@ -24,3 +26,59 @@ jobs: - name: Run validation visuals tests run: | pytest -q tests/test_validation_images_discovery.py tests/test_helpfile_ordinal_image_parity.py + - name: Generate canonical validation artifacts + run: | + mkdir -p output/pdf + python - <<'PY' + import csv + import json + from datetime import datetime, timezone + from pathlib import Path + from reportlab.lib.pagesizes import letter + from reportlab.pdfgen import canvas + + out = Path("output/pdf") + out.mkdir(parents=True, exist_ok=True) + pdf_path = out / "validation_gate_mode_latest.pdf" + c = canvas.Canvas(str(pdf_path), pagesize=letter) + c.setFont("Helvetica", 12) + c.drawString(72, 760, "nSTAT-python validation-pdf workflow artifact") + c.drawString(72, 742, f"Generated: {datetime.now(timezone.utc).isoformat()}") + c.save() + + pdfs = sorted(out.glob("*.pdf")) + payload = { + "generated_at_utc": datetime.now(timezone.utc).isoformat(), + "latest_pdf": str(pdfs[-1]) if pdfs else "", + "pdf_count": len(pdfs), + "json_files": sorted(str(p) for p in out.glob("*.json")), + "csv_files": sorted(str(p) for p in out.glob("*.csv")), + } + (out / "validation_gate_mode_latest.json").write_text( + json.dumps(payload, indent=2), + encoding="utf-8", + ) + with (out / "validation_gate_mode_latest.csv").open("w", newline="", encoding="utf-8") as f: + writer = csv.DictWriter(f, fieldnames=["generated_at_utc", "latest_pdf", "pdf_count"]) + writer.writeheader() + writer.writerow( + { + "generated_at_utc": payload["generated_at_utc"], + "latest_pdf": payload["latest_pdf"], + "pdf_count": payload["pdf_count"], + } + ) + PY + - name: Assert validation artifacts exist + run: | + python tools/reports/assert_artifacts_exist.py --kind validation + - name: Upload validation artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: validation-pdf-artifacts + path: | + output/pdf/*.pdf + output/pdf/*.json + output/pdf/*.csv + if-no-files-found: error diff --git a/tools/reports/assert_artifacts_exist.py b/tools/reports/assert_artifacts_exist.py new file mode 100644 index 00000000..97655b5d --- /dev/null +++ b/tools/reports/assert_artifacts_exist.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +"""Fail fast when expected CI artifacts are missing before upload.""" + +from __future__ import annotations + +import argparse +from pathlib import Path + + +def _matches(pattern: str) -> list[Path]: + return sorted(Path().glob(pattern)) + + +def _must_have(pattern: str) -> tuple[str, list[Path]]: + hits = _matches(pattern) + if not hits: + raise FileNotFoundError(f"missing required artifacts for pattern: {pattern}") + return pattern, hits + + +def _maybe_have(pattern: str) -> tuple[str, list[Path]]: + return pattern, _matches(pattern) + + +def _validate(kind: str) -> list[tuple[str, list[Path]]]: + rows: list[tuple[str, list[Path]]] = [] + if kind == "validation": + rows.append(_must_have("output/pdf/*.pdf")) + rows.append(_must_have("output/pdf/*.json")) + rows.append(_must_have("output/pdf/*.csv")) + rows.append(_maybe_have("output/pdf/validation_gate_mode_latest.json")) + rows.append(_maybe_have("output/pdf/validation_gate_mode_latest.csv")) + return rows + if kind == "image": + rows.append(_must_have("output/pdf/image_mode_parity/summary.json")) + rows.append(_must_have("output/pdf/image_mode_parity/**/*")) + rows.append(_must_have("output/pdf/*.json")) + return rows + if kind == "performance": + rows.append(_must_have("output/performance/performance_parity_report.json")) + rows.append(_must_have("output/performance/performance_parity_report.csv")) + rows.append(_must_have("output/performance/*.json")) + rows.append(_must_have("output/performance/*.csv")) + return rows + raise ValueError(f"Unsupported kind: {kind}") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--kind", + required=True, + choices=["validation", "image", "performance"], + help="Artifact group to validate before upload.", + ) + args = parser.parse_args() + + rows = _validate(args.kind) + print(f"Artifact check [{args.kind}] passed.") + for pattern, hits in rows: + print(f"- {pattern}: {len(hits)} file(s)") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())