diff --git a/.github/workflows/full-parity-nightly.yml b/.github/workflows/full-parity-nightly.yml new file mode 100644 index 00000000..5098f4ef --- /dev/null +++ b/.github/workflows/full-parity-nightly.yml @@ -0,0 +1,100 @@ +name: full-parity-nightly + +on: + schedule: + - cron: "0 6 * * *" + workflow_dispatch: + +jobs: + full-parity: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + with: + lfs: true + + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y poppler-utils + python -m pip install --upgrade pip + python -m pip install -e .[dev,docs,notebooks] + python -m pip install reportlab pillow + + - name: Checkout upstream MATLAB nSTAT repo snapshot + run: | + GIT_LFS_SKIP_SMUDGE=1 git clone --depth 1 https://github.com/cajigaslab/nSTAT.git /tmp/upstream-nstat + + - name: Prepare deterministic validation images + run: | + python tools/parity/prepare_validation_images.py + + - name: Run full parity and notebook gates + run: | + python tools/parity/build_parity_snapshot.py \ + --matlab-root /tmp/upstream-nstat \ + --fail-on medium + python tools/parity/build_numeric_drift_report.py \ + --fixtures-manifest tests/parity/fixtures/matlab_gold/manifest.yml \ + --thresholds parity/numeric_drift_thresholds.yml \ + --report-out parity/numeric_drift_report.json \ + --fail-on-violation + python tools/parity/check_functional_parity_progress.py \ + --report parity/function_example_alignment_report.json \ + --policy parity/functional_gate_policy.yml + python tools/parity/check_example_output_spec.py \ + --report parity/function_example_alignment_report.json \ + --spec parity/example_output_spec.yml + python tools/notebooks/run_notebooks.py --group all --timeout 900 + + - name: Generate full validation PDF + run: | + python tools/reports/generate_validation_pdf.py \ + --repo-root "$GITHUB_WORKSPACE" \ + --matlab-help-root /tmp/upstream-nstat/helpfiles \ + --notebook-group all \ + --timeout 900 \ + --skip-command-tests \ + --parity-mode gate + + - name: Enforce visual validation gate + run: | + python tools/reports/check_validation_visuals.py \ + --report-pdf 'output/pdf/*.pdf' \ + --images-root tmp/pdfs/validation_report/notebook_images \ + --min-unique-images-per-topic 1 \ + --max-duplicate-pdf-pages 0 + + - name: Upload parity artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: nightly-parity-artifacts + path: | + parity/function_example_alignment_report.json + parity/numeric_drift_report.json + parity/parity_gap_report.json + parity/method_probe_report.json + parity/method_closure_sprint.md + if-no-files-found: warn + + - name: Upload validation PDF artifact + if: always() + uses: actions/upload-artifact@v4 + with: + name: nightly-validation-pdf + path: output/pdf/*.pdf + if-no-files-found: warn + + - name: Upload notebook image artifact + if: always() + uses: actions/upload-artifact@v4 + with: + name: nightly-validation-images + path: tmp/pdfs/validation_report/notebook_images + if-no-files-found: warn diff --git a/.github/workflows/release-rc.yml b/.github/workflows/release-rc.yml index 3697802b..ce3efc52 100644 --- a/.github/workflows/release-rc.yml +++ b/.github/workflows/release-rc.yml @@ -99,12 +99,15 @@ jobs: - name: Generate RC release notes run: | + PREV_TAG="$(git tag --list 'v*-rc*' | sort -V | grep -vx "$TAG_NAME" | tail -n 1)" + echo "Using previous RC tag: ${PREV_TAG}" python tools/release/generate_rc_release_notes.py \ --repo-root "$GITHUB_WORKSPACE" \ --tag "$TAG_NAME" \ --commit "$GITHUB_SHA" \ --run-url "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" \ --validation-pdf "${{ steps.pdf.outputs.name }}" \ + --previous-tag "${PREV_TAG}" \ --output release_notes.md - name: Publish RC release diff --git a/docs/help/parity_dashboard.md b/docs/help/parity_dashboard.md index 00d778b5..290ba3b3 100644 --- a/docs/help/parity_dashboard.md +++ b/docs/help/parity_dashboard.md @@ -45,7 +45,7 @@ artifacts in the `parity/` directory. | Required topics checked | 30 | | Topics passed | 31 | | Topics failed | 0 | -| Metrics checked | 126 | +| Metrics checked | 180 | | Metrics failed | 0 | ## Frozen MATLAB data snapshot diff --git a/parity/numeric_drift_report.json b/parity/numeric_drift_report.json index 3a0fab49..0963a7b9 100644 --- a/parity/numeric_drift_report.json +++ b/parity/numeric_drift_report.json @@ -1,13 +1,13 @@ { "schema_version": 1, - "generated_at_utc": "2026-03-02T22:07:52.442127+00:00", + "generated_at_utc": "2026-03-02T22:33:03.837159+00:00", "fixtures_manifest": "/private/tmp/nstat_python_work_20260302/tests/parity/fixtures/matlab_gold/manifest.yml", "thresholds_file": "/private/tmp/nstat_python_work_20260302/parity/numeric_drift_thresholds.yml", "summary": { "topics": 31, "passed_topics": 31, "failed_topics": 0, - "checked_metrics": 126, + "checked_metrics": 180, "failed_metrics": 0, "required_topics": 30, "required_topics_checked": 30, @@ -15,17 +15,11 @@ }, "topics": { "AnalysisExamples": { - "checked_metrics": 7, + "checked_metrics": 4, "failed_metrics": [], "worst_ratio_to_threshold": 4.076575977102997e-06, "pass": true, "metrics": { - "assertion_count_missing_error": { - "value": 0.0, - "threshold": 0.0, - "pass": true, - "ratio_to_threshold": 0.0 - }, "coeff_max_abs_error": { "value": 7.80078663947456e-09, "threshold": 0.35, @@ -38,12 +32,6 @@ "pass": true, "ratio_to_threshold": 5.335430459345909e-07 }, - "python_validation_image_missing_error": { - "value": 0.0, - "threshold": 0.0, - "pass": true, - "ratio_to_threshold": 0.0 - }, "rate_max_abs_error": { "value": 1.4268015919860488e-06, "threshold": 0.35, @@ -55,48 +43,40 @@ "threshold": 0.25, "pass": true, "ratio_to_threshold": 1.605000115034727e-07 - }, - "topic_checkpoint_missing_error": { - "value": 0.0, - "threshold": 0.0, - "pass": true, - "ratio_to_threshold": 0.0 } } }, "AnalysisExamples2": { - "checked_metrics": 3, + "checked_metrics": 8, "failed_metrics": [], "worst_ratio_to_threshold": 0.0, "pass": true, "metrics": { + "alignment_status_mismatch": { + "value": 0.0, + "threshold": 0.0, + "pass": true, + "ratio_to_threshold": 0.0 + }, "assertion_count_missing_error": { "value": 0.0, "threshold": 0.0, "pass": true, "ratio_to_threshold": 0.0 }, - "python_validation_image_missing_error": { + "matlab_code_lines_abs_error": { "value": 0.0, "threshold": 0.0, "pass": true, "ratio_to_threshold": 0.0 }, - "topic_checkpoint_missing_error": { + "matlab_reference_image_count_abs_error": { "value": 0.0, "threshold": 0.0, "pass": true, "ratio_to_threshold": 0.0 - } - } - }, - "ConfigCollExamples": { - "checked_metrics": 3, - "failed_metrics": [], - "worst_ratio_to_threshold": 0.0, - "pass": true, - "metrics": { - "assertion_count_missing_error": { + }, + "plot_call_missing_error": { "value": 0.0, "threshold": 0.0, "pass": true, @@ -113,42 +93,54 @@ "threshold": 0.0, "pass": true, "ratio_to_threshold": 0.0 + }, + "topic_row_missing_error": { + "value": 0.0, + "threshold": 0.0, + "pass": true, + "ratio_to_threshold": 0.0 } } }, - "CovCollExamples": { - "checked_metrics": 6, + "ConfigCollExamples": { + "checked_metrics": 8, "failed_metrics": [], "worst_ratio_to_threshold": 0.0, "pass": true, "metrics": { + "alignment_status_mismatch": { + "value": 0.0, + "threshold": 0.0, + "pass": true, + "ratio_to_threshold": 0.0 + }, "assertion_count_missing_error": { "value": 0.0, "threshold": 0.0, "pass": true, "ratio_to_threshold": 0.0 }, - "ctx_max_abs_error": { + "matlab_code_lines_abs_error": { "value": 0.0, - "threshold": 1e-12, + "threshold": 0.0, "pass": true, "ratio_to_threshold": 0.0 }, - "design_max_abs_error": { + "matlab_reference_image_count_abs_error": { "value": 0.0, - "threshold": 1e-12, + "threshold": 0.0, "pass": true, "ratio_to_threshold": 0.0 }, - "python_validation_image_missing_error": { + "plot_call_missing_error": { "value": 0.0, "threshold": 0.0, "pass": true, "ratio_to_threshold": 0.0 }, - "stim_max_abs_error": { + "python_validation_image_missing_error": { "value": 0.0, - "threshold": 1e-12, + "threshold": 0.0, "pass": true, "ratio_to_threshold": 0.0 }, @@ -157,68 +149,80 @@ "threshold": 0.0, "pass": true, "ratio_to_threshold": 0.0 + }, + "topic_row_missing_error": { + "value": 0.0, + "threshold": 0.0, + "pass": true, + "ratio_to_threshold": 0.0 } } }, - "CovariateExamples": { + "CovCollExamples": { "checked_metrics": 3, "failed_metrics": [], "worst_ratio_to_threshold": 0.0, "pass": true, "metrics": { - "assertion_count_missing_error": { + "ctx_max_abs_error": { "value": 0.0, - "threshold": 0.0, + "threshold": 1e-12, "pass": true, "ratio_to_threshold": 0.0 }, - "python_validation_image_missing_error": { + "design_max_abs_error": { "value": 0.0, - "threshold": 0.0, + "threshold": 1e-12, "pass": true, "ratio_to_threshold": 0.0 }, - "topic_checkpoint_missing_error": { + "stim_max_abs_error": { "value": 0.0, - "threshold": 0.0, + "threshold": 1e-12, "pass": true, "ratio_to_threshold": 0.0 } } }, - "DecodingExample": { - "checked_metrics": 6, + "CovariateExamples": { + "checked_metrics": 8, "failed_metrics": [], - "worst_ratio_to_threshold": 0.022530207586461248, + "worst_ratio_to_threshold": 0.0, "pass": true, "metrics": { + "alignment_status_mismatch": { + "value": 0.0, + "threshold": 0.0, + "pass": true, + "ratio_to_threshold": 0.0 + }, "assertion_count_missing_error": { "value": 0.0, "threshold": 0.0, "pass": true, "ratio_to_threshold": 0.0 }, - "decoded_mismatch_count": { + "matlab_code_lines_abs_error": { "value": 0.0, "threshold": 0.0, "pass": true, "ratio_to_threshold": 0.0 }, - "posterior_max_abs_error": { - "value": 2.2530207586461248e-10, - "threshold": 1e-08, + "matlab_reference_image_count_abs_error": { + "value": 0.0, + "threshold": 0.0, "pass": true, - "ratio_to_threshold": 0.022530207586461248 + "ratio_to_threshold": 0.0 }, - "python_validation_image_missing_error": { + "plot_call_missing_error": { "value": 0.0, "threshold": 0.0, "pass": true, "ratio_to_threshold": 0.0 }, - "rmse_abs_error": { + "python_validation_image_missing_error": { "value": 0.0, - "threshold": 1e-08, + "threshold": 0.0, "pass": true, "ratio_to_threshold": 0.0 }, @@ -227,21 +231,21 @@ "threshold": 0.0, "pass": true, "ratio_to_threshold": 0.0 + }, + "topic_row_missing_error": { + "value": 0.0, + "threshold": 0.0, + "pass": true, + "ratio_to_threshold": 0.0 } } }, - "DecodingExampleWithHist": { - "checked_metrics": 5, + "DecodingExample": { + "checked_metrics": 3, "failed_metrics": [], - "worst_ratio_to_threshold": 0.04167062250814979, + "worst_ratio_to_threshold": 0.022530207586461248, "pass": true, "metrics": { - "assertion_count_missing_error": { - "value": 0.0, - "threshold": 0.0, - "pass": true, - "ratio_to_threshold": 0.0 - }, "decoded_mismatch_count": { "value": 0.0, "threshold": 0.0, @@ -249,58 +253,70 @@ "ratio_to_threshold": 0.0 }, "posterior_max_abs_error": { - "value": 4.167062250814979e-10, + "value": 2.2530207586461248e-10, "threshold": 1e-08, "pass": true, - "ratio_to_threshold": 0.04167062250814979 + "ratio_to_threshold": 0.022530207586461248 }, - "python_validation_image_missing_error": { + "rmse_abs_error": { "value": 0.0, - "threshold": 0.0, + "threshold": 1e-08, "pass": true, "ratio_to_threshold": 0.0 - }, - "topic_checkpoint_missing_error": { + } + } + }, + "DecodingExampleWithHist": { + "checked_metrics": 2, + "failed_metrics": [], + "worst_ratio_to_threshold": 0.04167062250814979, + "pass": true, + "metrics": { + "decoded_mismatch_count": { "value": 0.0, "threshold": 0.0, "pass": true, "ratio_to_threshold": 0.0 + }, + "posterior_max_abs_error": { + "value": 4.167062250814979e-10, + "threshold": 1e-08, + "pass": true, + "ratio_to_threshold": 0.04167062250814979 } } }, "DocumentationSetup2025b": { - "checked_metrics": 3, + "checked_metrics": 8, "failed_metrics": [], "worst_ratio_to_threshold": 0.0, "pass": true, "metrics": { + "alignment_status_mismatch": { + "value": 0.0, + "threshold": 0.0, + "pass": true, + "ratio_to_threshold": 0.0 + }, "assertion_count_missing_error": { "value": 0.0, "threshold": 0.0, "pass": true, "ratio_to_threshold": 0.0 }, - "python_validation_image_missing_error": { + "matlab_code_lines_abs_error": { "value": 0.0, "threshold": 0.0, "pass": true, "ratio_to_threshold": 0.0 }, - "topic_checkpoint_missing_error": { + "matlab_reference_image_count_abs_error": { "value": 0.0, "threshold": 0.0, "pass": true, "ratio_to_threshold": 0.0 - } - } - }, - "EventsExamples": { - "checked_metrics": 4, - "failed_metrics": [], - "worst_ratio_to_threshold": 0.0, - "pass": true, - "metrics": { - "assertion_count_missing_error": { + }, + "plot_call_missing_error": { "value": 0.0, "threshold": 0.0, "pass": true, @@ -312,13 +328,13 @@ "pass": true, "ratio_to_threshold": 0.0 }, - "subset_max_abs_error": { + "topic_checkpoint_missing_error": { "value": 0.0, - "threshold": 1e-12, + "threshold": 0.0, "pass": true, "ratio_to_threshold": 0.0 }, - "topic_checkpoint_missing_error": { + "topic_row_missing_error": { "value": 0.0, "threshold": 0.0, "pass": true, @@ -326,18 +342,26 @@ } } }, - "ExplicitStimulusWhiskerData": { - "checked_metrics": 7, + "EventsExamples": { + "checked_metrics": 1, "failed_metrics": [], - "worst_ratio_to_threshold": 1.0307532605224878e-09, + "worst_ratio_to_threshold": 0.0, "pass": true, "metrics": { - "assertion_count_missing_error": { + "subset_max_abs_error": { "value": 0.0, - "threshold": 0.0, + "threshold": 1e-12, "pass": true, "ratio_to_threshold": 0.0 - }, + } + } + }, + "ExplicitStimulusWhiskerData": { + "checked_metrics": 4, + "failed_metrics": [], + "worst_ratio_to_threshold": 1.0307532605224878e-09, + "pass": true, + "metrics": { "coeff_abs_error": { "value": 1.4483514387819696e-10, "threshold": 0.2, @@ -356,59 +380,45 @@ "pass": true, "ratio_to_threshold": 2.3744395338809454e-10 }, - "python_validation_image_missing_error": { - "value": 0.0, - "threshold": 0.0, - "pass": true, - "ratio_to_threshold": 0.0 - }, "rmse_abs_error": { "value": 8.271161533457416e-15, "threshold": 0.1, "pass": true, "ratio_to_threshold": 8.271161533457416e-14 - }, - "topic_checkpoint_missing_error": { - "value": 0.0, - "threshold": 0.0, - "pass": true, - "ratio_to_threshold": 0.0 } } }, "FitResSummaryExamples": { - "checked_metrics": 3, + "checked_metrics": 8, "failed_metrics": [], "worst_ratio_to_threshold": 0.0, "pass": true, "metrics": { + "alignment_status_mismatch": { + "value": 0.0, + "threshold": 0.0, + "pass": true, + "ratio_to_threshold": 0.0 + }, "assertion_count_missing_error": { "value": 0.0, "threshold": 0.0, "pass": true, "ratio_to_threshold": 0.0 }, - "python_validation_image_missing_error": { + "matlab_code_lines_abs_error": { "value": 0.0, "threshold": 0.0, "pass": true, "ratio_to_threshold": 0.0 }, - "topic_checkpoint_missing_error": { + "matlab_reference_image_count_abs_error": { "value": 0.0, "threshold": 0.0, "pass": true, "ratio_to_threshold": 0.0 - } - } - }, - "FitResultExamples": { - "checked_metrics": 3, - "failed_metrics": [], - "worst_ratio_to_threshold": 0.0, - "pass": true, - "metrics": { - "assertion_count_missing_error": { + }, + "plot_call_missing_error": { "value": 0.0, "threshold": 0.0, "pass": true, @@ -425,42 +435,46 @@ "threshold": 0.0, "pass": true, "ratio_to_threshold": 0.0 + }, + "topic_row_missing_error": { + "value": 0.0, + "threshold": 0.0, + "pass": true, + "ratio_to_threshold": 0.0 } } }, - "FitResultReference": { - "checked_metrics": 3, + "FitResultExamples": { + "checked_metrics": 8, "failed_metrics": [], "worst_ratio_to_threshold": 0.0, "pass": true, "metrics": { + "alignment_status_mismatch": { + "value": 0.0, + "threshold": 0.0, + "pass": true, + "ratio_to_threshold": 0.0 + }, "assertion_count_missing_error": { "value": 0.0, "threshold": 0.0, "pass": true, "ratio_to_threshold": 0.0 }, - "python_validation_image_missing_error": { + "matlab_code_lines_abs_error": { "value": 0.0, "threshold": 0.0, "pass": true, "ratio_to_threshold": 0.0 }, - "topic_checkpoint_missing_error": { + "matlab_reference_image_count_abs_error": { "value": 0.0, "threshold": 0.0, "pass": true, "ratio_to_threshold": 0.0 - } - } - }, - "HippocampalPlaceCellExample": { - "checked_metrics": 4, - "failed_metrics": [], - "worst_ratio_to_threshold": 1.4210854715202004e-06, - "pass": true, - "metrics": { - "assertion_count_missing_error": { + }, + "plot_call_missing_error": { "value": 0.0, "threshold": 0.0, "pass": true, @@ -478,33 +492,63 @@ "pass": true, "ratio_to_threshold": 0.0 }, - "weighted_center_max_abs_error": { - "value": 1.4210854715202004e-14, - "threshold": 1e-08, + "topic_row_missing_error": { + "value": 0.0, + "threshold": 0.0, "pass": true, - "ratio_to_threshold": 1.4210854715202004e-06 + "ratio_to_threshold": 0.0 } } }, - "HistoryExamples": { - "checked_metrics": 3, + "FitResultReference": { + "checked_metrics": 8, "failed_metrics": [], "worst_ratio_to_threshold": 0.0, "pass": true, "metrics": { - "assertion_count_missing_error": { + "alignment_status_mismatch": { "value": 0.0, "threshold": 0.0, "pass": true, "ratio_to_threshold": 0.0 }, - "python_validation_image_missing_error": { + "assertion_count_missing_error": { "value": 0.0, "threshold": 0.0, "pass": true, "ratio_to_threshold": 0.0 }, - "topic_checkpoint_missing_error": { + "matlab_code_lines_abs_error": { + "value": 0.0, + "threshold": 0.0, + "pass": true, + "ratio_to_threshold": 0.0 + }, + "matlab_reference_image_count_abs_error": { + "value": 0.0, + "threshold": 0.0, + "pass": true, + "ratio_to_threshold": 0.0 + }, + "plot_call_missing_error": { + "value": 0.0, + "threshold": 0.0, + "pass": true, + "ratio_to_threshold": 0.0 + }, + "python_validation_image_missing_error": { + "value": 0.0, + "threshold": 0.0, + "pass": true, + "ratio_to_threshold": 0.0 + }, + "topic_checkpoint_missing_error": { + "value": 0.0, + "threshold": 0.0, + "pass": true, + "ratio_to_threshold": 0.0 + }, + "topic_row_missing_error": { "value": 0.0, "threshold": 0.0, "pass": true, @@ -512,18 +556,56 @@ } } }, - "HybridFilterExample": { - "checked_metrics": 3, + "HippocampalPlaceCellExample": { + "checked_metrics": 1, + "failed_metrics": [], + "worst_ratio_to_threshold": 1.4210854715202004e-06, + "pass": true, + "metrics": { + "weighted_center_max_abs_error": { + "value": 1.4210854715202004e-14, + "threshold": 1e-08, + "pass": true, + "ratio_to_threshold": 1.4210854715202004e-06 + } + } + }, + "HistoryExamples": { + "checked_metrics": 8, "failed_metrics": [], "worst_ratio_to_threshold": 0.0, "pass": true, "metrics": { + "alignment_status_mismatch": { + "value": 0.0, + "threshold": 0.0, + "pass": true, + "ratio_to_threshold": 0.0 + }, "assertion_count_missing_error": { "value": 0.0, "threshold": 0.0, "pass": true, "ratio_to_threshold": 0.0 }, + "matlab_code_lines_abs_error": { + "value": 0.0, + "threshold": 0.0, + "pass": true, + "ratio_to_threshold": 0.0 + }, + "matlab_reference_image_count_abs_error": { + "value": 0.0, + "threshold": 0.0, + "pass": true, + "ratio_to_threshold": 0.0 + }, + "plot_call_missing_error": { + "value": 0.0, + "threshold": 0.0, + "pass": true, + "ratio_to_threshold": 0.0 + }, "python_validation_image_missing_error": { "value": 0.0, "threshold": 0.0, @@ -535,21 +617,51 @@ "threshold": 0.0, "pass": true, "ratio_to_threshold": 0.0 + }, + "topic_row_missing_error": { + "value": 0.0, + "threshold": 0.0, + "pass": true, + "ratio_to_threshold": 0.0 } } }, - "NetworkTutorial": { - "checked_metrics": 3, + "HybridFilterExample": { + "checked_metrics": 8, "failed_metrics": [], "worst_ratio_to_threshold": 0.0, "pass": true, "metrics": { + "alignment_status_mismatch": { + "value": 0.0, + "threshold": 0.0, + "pass": true, + "ratio_to_threshold": 0.0 + }, "assertion_count_missing_error": { "value": 0.0, "threshold": 0.0, "pass": true, "ratio_to_threshold": 0.0 }, + "matlab_code_lines_abs_error": { + "value": 0.0, + "threshold": 0.0, + "pass": true, + "ratio_to_threshold": 0.0 + }, + "matlab_reference_image_count_abs_error": { + "value": 0.0, + "threshold": 0.0, + "pass": true, + "ratio_to_threshold": 0.0 + }, + "plot_call_missing_error": { + "value": 0.0, + "threshold": 0.0, + "pass": true, + "ratio_to_threshold": 0.0 + }, "python_validation_image_missing_error": { "value": 0.0, "threshold": 0.0, @@ -561,26 +673,50 @@ "threshold": 0.0, "pass": true, "ratio_to_threshold": 0.0 + }, + "topic_row_missing_error": { + "value": 0.0, + "threshold": 0.0, + "pass": true, + "ratio_to_threshold": 0.0 } } }, - "PPSimExample": { - "checked_metrics": 4, + "NetworkTutorial": { + "checked_metrics": 8, "failed_metrics": [], - "worst_ratio_to_threshold": 2.2090814743799897e-06, + "worst_ratio_to_threshold": 0.0, "pass": true, "metrics": { + "alignment_status_mismatch": { + "value": 0.0, + "threshold": 0.0, + "pass": true, + "ratio_to_threshold": 0.0 + }, "assertion_count_missing_error": { "value": 0.0, "threshold": 0.0, "pass": true, "ratio_to_threshold": 0.0 }, - "mean_relative_rate_error": { - "value": 5.522703685949974e-07, - "threshold": 0.25, + "matlab_code_lines_abs_error": { + "value": 0.0, + "threshold": 0.0, "pass": true, - "ratio_to_threshold": 2.2090814743799897e-06 + "ratio_to_threshold": 0.0 + }, + "matlab_reference_image_count_abs_error": { + "value": 0.0, + "threshold": 0.0, + "pass": true, + "ratio_to_threshold": 0.0 + }, + "plot_call_missing_error": { + "value": 0.0, + "threshold": 0.0, + "pass": true, + "ratio_to_threshold": 0.0 }, "python_validation_image_missing_error": { "value": 0.0, @@ -593,21 +729,65 @@ "threshold": 0.0, "pass": true, "ratio_to_threshold": 0.0 + }, + "topic_row_missing_error": { + "value": 0.0, + "threshold": 0.0, + "pass": true, + "ratio_to_threshold": 0.0 + } + } + }, + "PPSimExample": { + "checked_metrics": 1, + "failed_metrics": [], + "worst_ratio_to_threshold": 2.2090814743799897e-06, + "pass": true, + "metrics": { + "mean_relative_rate_error": { + "value": 5.522703685949974e-07, + "threshold": 0.25, + "pass": true, + "ratio_to_threshold": 2.2090814743799897e-06 } } }, "PPThinning": { - "checked_metrics": 3, + "checked_metrics": 8, "failed_metrics": [], "worst_ratio_to_threshold": 0.0, "pass": true, "metrics": { + "alignment_status_mismatch": { + "value": 0.0, + "threshold": 0.0, + "pass": true, + "ratio_to_threshold": 0.0 + }, "assertion_count_missing_error": { "value": 0.0, "threshold": 0.0, "pass": true, "ratio_to_threshold": 0.0 }, + "matlab_code_lines_abs_error": { + "value": 0.0, + "threshold": 0.0, + "pass": true, + "ratio_to_threshold": 0.0 + }, + "matlab_reference_image_count_abs_error": { + "value": 0.0, + "threshold": 0.0, + "pass": true, + "ratio_to_threshold": 0.0 + }, + "plot_call_missing_error": { + "value": 0.0, + "threshold": 0.0, + "pass": true, + "ratio_to_threshold": 0.0 + }, "python_validation_image_missing_error": { "value": 0.0, "threshold": 0.0, @@ -619,33 +799,27 @@ "threshold": 0.0, "pass": true, "ratio_to_threshold": 0.0 + }, + "topic_row_missing_error": { + "value": 0.0, + "threshold": 0.0, + "pass": true, + "ratio_to_threshold": 0.0 } } }, "PSTHEstimation": { - "checked_metrics": 6, + "checked_metrics": 3, "failed_metrics": [], "worst_ratio_to_threshold": 2.220446049250313e-06, "pass": true, "metrics": { - "assertion_count_missing_error": { - "value": 0.0, - "threshold": 0.0, - "pass": true, - "ratio_to_threshold": 0.0 - }, "prob_max_abs_error": { "value": 2.220446049250313e-16, "threshold": 1e-10, "pass": true, "ratio_to_threshold": 2.220446049250313e-06 }, - "python_validation_image_missing_error": { - "value": 0.0, - "threshold": 0.0, - "pass": true, - "ratio_to_threshold": 0.0 - }, "rate_max_abs_error": { "value": 0.0, "threshold": 1e-10, @@ -657,27 +831,45 @@ "threshold": 0.0, "pass": true, "ratio_to_threshold": 0.0 - }, - "topic_checkpoint_missing_error": { - "value": 0.0, - "threshold": 0.0, - "pass": true, - "ratio_to_threshold": 0.0 } } }, "SignalObjExamples": { - "checked_metrics": 3, + "checked_metrics": 8, "failed_metrics": [], "worst_ratio_to_threshold": 0.0, "pass": true, "metrics": { + "alignment_status_mismatch": { + "value": 0.0, + "threshold": 0.0, + "pass": true, + "ratio_to_threshold": 0.0 + }, "assertion_count_missing_error": { "value": 0.0, "threshold": 0.0, "pass": true, "ratio_to_threshold": 0.0 }, + "matlab_code_lines_abs_error": { + "value": 0.0, + "threshold": 0.0, + "pass": true, + "ratio_to_threshold": 0.0 + }, + "matlab_reference_image_count_abs_error": { + "value": 0.0, + "threshold": 0.0, + "pass": true, + "ratio_to_threshold": 0.0 + }, + "plot_call_missing_error": { + "value": 0.0, + "threshold": 0.0, + "pass": true, + "ratio_to_threshold": 0.0 + }, "python_validation_image_missing_error": { "value": 0.0, "threshold": 0.0, @@ -689,6 +881,12 @@ "threshold": 0.0, "pass": true, "ratio_to_threshold": 0.0 + }, + "topic_row_missing_error": { + "value": 0.0, + "threshold": 0.0, + "pass": true, + "ratio_to_threshold": 0.0 } } }, @@ -719,17 +917,41 @@ } }, "StimulusDecode2D": { - "checked_metrics": 3, + "checked_metrics": 8, "failed_metrics": [], "worst_ratio_to_threshold": 0.0, "pass": true, "metrics": { + "alignment_status_mismatch": { + "value": 0.0, + "threshold": 0.0, + "pass": true, + "ratio_to_threshold": 0.0 + }, "assertion_count_missing_error": { "value": 0.0, "threshold": 0.0, "pass": true, "ratio_to_threshold": 0.0 }, + "matlab_code_lines_abs_error": { + "value": 0.0, + "threshold": 0.0, + "pass": true, + "ratio_to_threshold": 0.0 + }, + "matlab_reference_image_count_abs_error": { + "value": 0.0, + "threshold": 0.0, + "pass": true, + "ratio_to_threshold": 0.0 + }, + "plot_call_missing_error": { + "value": 0.0, + "threshold": 0.0, + "pass": true, + "ratio_to_threshold": 0.0 + }, "python_validation_image_missing_error": { "value": 0.0, "threshold": 0.0, @@ -741,21 +963,51 @@ "threshold": 0.0, "pass": true, "ratio_to_threshold": 0.0 + }, + "topic_row_missing_error": { + "value": 0.0, + "threshold": 0.0, + "pass": true, + "ratio_to_threshold": 0.0 } } }, "TrialConfigExamples": { - "checked_metrics": 3, + "checked_metrics": 8, "failed_metrics": [], "worst_ratio_to_threshold": 0.0, "pass": true, "metrics": { + "alignment_status_mismatch": { + "value": 0.0, + "threshold": 0.0, + "pass": true, + "ratio_to_threshold": 0.0 + }, "assertion_count_missing_error": { "value": 0.0, "threshold": 0.0, "pass": true, "ratio_to_threshold": 0.0 }, + "matlab_code_lines_abs_error": { + "value": 0.0, + "threshold": 0.0, + "pass": true, + "ratio_to_threshold": 0.0 + }, + "matlab_reference_image_count_abs_error": { + "value": 0.0, + "threshold": 0.0, + "pass": true, + "ratio_to_threshold": 0.0 + }, + "plot_call_missing_error": { + "value": 0.0, + "threshold": 0.0, + "pass": true, + "ratio_to_threshold": 0.0 + }, "python_validation_image_missing_error": { "value": 0.0, "threshold": 0.0, @@ -767,11 +1019,17 @@ "threshold": 0.0, "pass": true, "ratio_to_threshold": 0.0 + }, + "topic_row_missing_error": { + "value": 0.0, + "threshold": 0.0, + "pass": true, + "ratio_to_threshold": 0.0 } } }, "TrialExamples": { - "checked_metrics": 6, + "checked_metrics": 3, "failed_metrics": [], "worst_ratio_to_threshold": 0.00011102230246251565, "pass": true, @@ -782,30 +1040,12 @@ "pass": true, "ratio_to_threshold": 0.0 }, - "assertion_count_missing_error": { - "value": 0.0, - "threshold": 0.0, - "pass": true, - "ratio_to_threshold": 0.0 - }, - "python_validation_image_missing_error": { - "value": 0.0, - "threshold": 0.0, - "pass": true, - "ratio_to_threshold": 0.0 - }, "t_bins_max_abs_error": { "value": 1.1102230246251565e-16, "threshold": 1e-12, "pass": true, "ratio_to_threshold": 0.00011102230246251565 }, - "topic_checkpoint_missing_error": { - "value": 0.0, - "threshold": 0.0, - "pass": true, - "ratio_to_threshold": 0.0 - }, "y_max_abs_error": { "value": 0.0, "threshold": 1e-12, @@ -815,17 +1055,41 @@ } }, "ValidationDataSet": { - "checked_metrics": 3, + "checked_metrics": 8, "failed_metrics": [], "worst_ratio_to_threshold": 0.0, "pass": true, "metrics": { + "alignment_status_mismatch": { + "value": 0.0, + "threshold": 0.0, + "pass": true, + "ratio_to_threshold": 0.0 + }, "assertion_count_missing_error": { "value": 0.0, "threshold": 0.0, "pass": true, "ratio_to_threshold": 0.0 }, + "matlab_code_lines_abs_error": { + "value": 0.0, + "threshold": 0.0, + "pass": true, + "ratio_to_threshold": 0.0 + }, + "matlab_reference_image_count_abs_error": { + "value": 0.0, + "threshold": 0.0, + "pass": true, + "ratio_to_threshold": 0.0 + }, + "plot_call_missing_error": { + "value": 0.0, + "threshold": 0.0, + "pass": true, + "ratio_to_threshold": 0.0 + }, "python_validation_image_missing_error": { "value": 0.0, "threshold": 0.0, @@ -837,21 +1101,21 @@ "threshold": 0.0, "pass": true, "ratio_to_threshold": 0.0 + }, + "topic_row_missing_error": { + "value": 0.0, + "threshold": 0.0, + "pass": true, + "ratio_to_threshold": 0.0 } } }, "mEPSCAnalysis": { - "checked_metrics": 7, + "checked_metrics": 4, "failed_metrics": [], "worst_ratio_to_threshold": 5.551115123125782e-08, "pass": true, "metrics": { - "assertion_count_missing_error": { - "value": 0.0, - "threshold": 0.0, - "pass": true, - "ratio_to_threshold": 0.0 - }, "detected_amp_max_abs_error": { "value": 0.0, "threshold": 1e-09, @@ -875,28 +1139,40 @@ "threshold": 1e-09, "pass": true, "ratio_to_threshold": 5.551115123125782e-08 + } + } + }, + "nSTATPaperExamples": { + "checked_metrics": 8, + "failed_metrics": [], + "worst_ratio_to_threshold": 0.0, + "pass": true, + "metrics": { + "alignment_status_mismatch": { + "value": 0.0, + "threshold": 0.0, + "pass": true, + "ratio_to_threshold": 0.0 }, - "python_validation_image_missing_error": { + "assertion_count_missing_error": { "value": 0.0, "threshold": 0.0, "pass": true, "ratio_to_threshold": 0.0 }, - "topic_checkpoint_missing_error": { + "matlab_code_lines_abs_error": { "value": 0.0, "threshold": 0.0, "pass": true, "ratio_to_threshold": 0.0 - } - } - }, - "nSTATPaperExamples": { - "checked_metrics": 3, - "failed_metrics": [], - "worst_ratio_to_threshold": 0.0, - "pass": true, - "metrics": { - "assertion_count_missing_error": { + }, + "matlab_reference_image_count_abs_error": { + "value": 0.0, + "threshold": 0.0, + "pass": true, + "ratio_to_threshold": 0.0 + }, + "plot_call_missing_error": { "value": 0.0, "threshold": 0.0, "pass": true, @@ -913,21 +1189,51 @@ "threshold": 0.0, "pass": true, "ratio_to_threshold": 0.0 + }, + "topic_row_missing_error": { + "value": 0.0, + "threshold": 0.0, + "pass": true, + "ratio_to_threshold": 0.0 } } }, "nSpikeTrainExamples": { - "checked_metrics": 3, + "checked_metrics": 8, "failed_metrics": [], "worst_ratio_to_threshold": 0.0, "pass": true, "metrics": { + "alignment_status_mismatch": { + "value": 0.0, + "threshold": 0.0, + "pass": true, + "ratio_to_threshold": 0.0 + }, "assertion_count_missing_error": { "value": 0.0, "threshold": 0.0, "pass": true, "ratio_to_threshold": 0.0 }, + "matlab_code_lines_abs_error": { + "value": 0.0, + "threshold": 0.0, + "pass": true, + "ratio_to_threshold": 0.0 + }, + "matlab_reference_image_count_abs_error": { + "value": 0.0, + "threshold": 0.0, + "pass": true, + "ratio_to_threshold": 0.0 + }, + "plot_call_missing_error": { + "value": 0.0, + "threshold": 0.0, + "pass": true, + "ratio_to_threshold": 0.0 + }, "python_validation_image_missing_error": { "value": 0.0, "threshold": 0.0, @@ -939,21 +1245,21 @@ "threshold": 0.0, "pass": true, "ratio_to_threshold": 0.0 + }, + "topic_row_missing_error": { + "value": 0.0, + "threshold": 0.0, + "pass": true, + "ratio_to_threshold": 0.0 } } }, "nstCollExamples": { - "checked_metrics": 7, + "checked_metrics": 4, "failed_metrics": [], "worst_ratio_to_threshold": 0.0002220446049250313, "pass": true, "metrics": { - "assertion_count_missing_error": { - "value": 0.0, - "threshold": 0.0, - "pass": true, - "ratio_to_threshold": 0.0 - }, "binary_mismatch_count": { "value": 0.0, "threshold": 0.0, @@ -977,28 +1283,40 @@ "threshold": 1e-12, "pass": true, "ratio_to_threshold": 0.0 + } + } + }, + "publish_all_helpfiles": { + "checked_metrics": 8, + "failed_metrics": [], + "worst_ratio_to_threshold": 0.0, + "pass": true, + "metrics": { + "alignment_status_mismatch": { + "value": 0.0, + "threshold": 0.0, + "pass": true, + "ratio_to_threshold": 0.0 }, - "python_validation_image_missing_error": { + "assertion_count_missing_error": { "value": 0.0, "threshold": 0.0, "pass": true, "ratio_to_threshold": 0.0 }, - "topic_checkpoint_missing_error": { + "matlab_code_lines_abs_error": { "value": 0.0, "threshold": 0.0, "pass": true, "ratio_to_threshold": 0.0 - } - } - }, - "publish_all_helpfiles": { - "checked_metrics": 3, - "failed_metrics": [], - "worst_ratio_to_threshold": 0.0, - "pass": true, - "metrics": { - "assertion_count_missing_error": { + }, + "matlab_reference_image_count_abs_error": { + "value": 0.0, + "threshold": 0.0, + "pass": true, + "ratio_to_threshold": 0.0 + }, + "plot_call_missing_error": { "value": 0.0, "threshold": 0.0, "pass": true, @@ -1015,6 +1333,12 @@ "threshold": 0.0, "pass": true, "ratio_to_threshold": 0.0 + }, + "topic_row_missing_error": { + "value": 0.0, + "threshold": 0.0, + "pass": true, + "ratio_to_threshold": 0.0 } } } diff --git a/parity/numeric_drift_thresholds.yml b/parity/numeric_drift_thresholds.yml index 4131a673..7ea1f685 100644 --- a/parity/numeric_drift_thresholds.yml +++ b/parity/numeric_drift_thresholds.yml @@ -1,5 +1,10 @@ version: 1 defaults: + topic_row_missing_error: 0 + alignment_status_mismatch: 0 + matlab_code_lines_abs_error: 0 + matlab_reference_image_count_abs_error: 0 + plot_call_missing_error: 0 topic_checkpoint_missing_error: 0 assertion_count_missing_error: 0 python_validation_image_missing_error: 0 diff --git a/tests/parity/fixtures/matlab_gold/AnalysisExamples2_audit_gold.json b/tests/parity/fixtures/matlab_gold/AnalysisExamples2_audit_gold.json new file mode 100644 index 00000000..21f15652 --- /dev/null +++ b/tests/parity/fixtures/matlab_gold/AnalysisExamples2_audit_gold.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "topic": "AnalysisExamples2", + "alignment_status": "validated", + "matlab_code_lines": 61, + "matlab_reference_image_count": 6, + "min_assertion_count": 2, + "require_topic_checkpoint": true, + "min_python_validation_image_count": 1, + "require_plot_call": true, + "source": "equivalence_audit_report", + "equivalence_report": "parity/function_example_alignment_report.json" +} diff --git a/tests/parity/fixtures/matlab_gold/ConfigCollExamples_audit_gold.json b/tests/parity/fixtures/matlab_gold/ConfigCollExamples_audit_gold.json new file mode 100644 index 00000000..c79763af --- /dev/null +++ b/tests/parity/fixtures/matlab_gold/ConfigCollExamples_audit_gold.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "topic": "ConfigCollExamples", + "alignment_status": "validated", + "matlab_code_lines": 3, + "matlab_reference_image_count": 0, + "min_assertion_count": 3, + "require_topic_checkpoint": true, + "min_python_validation_image_count": 1, + "require_plot_call": true, + "source": "equivalence_audit_report", + "equivalence_report": "parity/function_example_alignment_report.json" +} diff --git a/tests/parity/fixtures/matlab_gold/CovariateExamples_audit_gold.json b/tests/parity/fixtures/matlab_gold/CovariateExamples_audit_gold.json new file mode 100644 index 00000000..4db0e817 --- /dev/null +++ b/tests/parity/fixtures/matlab_gold/CovariateExamples_audit_gold.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "topic": "CovariateExamples", + "alignment_status": "validated", + "matlab_code_lines": 19, + "matlab_reference_image_count": 3, + "min_assertion_count": 3, + "require_topic_checkpoint": true, + "min_python_validation_image_count": 2, + "require_plot_call": true, + "source": "equivalence_audit_report", + "equivalence_report": "parity/function_example_alignment_report.json" +} diff --git a/tests/parity/fixtures/matlab_gold/DocumentationSetup2025b_audit_gold.json b/tests/parity/fixtures/matlab_gold/DocumentationSetup2025b_audit_gold.json new file mode 100644 index 00000000..1f16814f --- /dev/null +++ b/tests/parity/fixtures/matlab_gold/DocumentationSetup2025b_audit_gold.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "topic": "DocumentationSetup2025b", + "alignment_status": "matlab_doc_only", + "matlab_code_lines": 0, + "matlab_reference_image_count": 0, + "min_assertion_count": 3, + "require_topic_checkpoint": true, + "min_python_validation_image_count": 1, + "require_plot_call": true, + "source": "equivalence_audit_report", + "equivalence_report": "parity/function_example_alignment_report.json" +} diff --git a/tests/parity/fixtures/matlab_gold/FitResSummaryExamples_audit_gold.json b/tests/parity/fixtures/matlab_gold/FitResSummaryExamples_audit_gold.json new file mode 100644 index 00000000..2d09adef --- /dev/null +++ b/tests/parity/fixtures/matlab_gold/FitResSummaryExamples_audit_gold.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "topic": "FitResSummaryExamples", + "alignment_status": "matlab_doc_only", + "matlab_code_lines": 0, + "matlab_reference_image_count": 0, + "min_assertion_count": 3, + "require_topic_checkpoint": true, + "min_python_validation_image_count": 1, + "require_plot_call": true, + "source": "equivalence_audit_report", + "equivalence_report": "parity/function_example_alignment_report.json" +} diff --git a/tests/parity/fixtures/matlab_gold/FitResultExamples_audit_gold.json b/tests/parity/fixtures/matlab_gold/FitResultExamples_audit_gold.json new file mode 100644 index 00000000..3f03e4c5 --- /dev/null +++ b/tests/parity/fixtures/matlab_gold/FitResultExamples_audit_gold.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "topic": "FitResultExamples", + "alignment_status": "matlab_doc_only", + "matlab_code_lines": 0, + "matlab_reference_image_count": 0, + "min_assertion_count": 3, + "require_topic_checkpoint": true, + "min_python_validation_image_count": 1, + "require_plot_call": true, + "source": "equivalence_audit_report", + "equivalence_report": "parity/function_example_alignment_report.json" +} diff --git a/tests/parity/fixtures/matlab_gold/FitResultReference_audit_gold.json b/tests/parity/fixtures/matlab_gold/FitResultReference_audit_gold.json new file mode 100644 index 00000000..f3b2f88b --- /dev/null +++ b/tests/parity/fixtures/matlab_gold/FitResultReference_audit_gold.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "topic": "FitResultReference", + "alignment_status": "matlab_doc_only", + "matlab_code_lines": 0, + "matlab_reference_image_count": 0, + "min_assertion_count": 3, + "require_topic_checkpoint": true, + "min_python_validation_image_count": 1, + "require_plot_call": true, + "source": "equivalence_audit_report", + "equivalence_report": "parity/function_example_alignment_report.json" +} diff --git a/tests/parity/fixtures/matlab_gold/HistoryExamples_audit_gold.json b/tests/parity/fixtures/matlab_gold/HistoryExamples_audit_gold.json new file mode 100644 index 00000000..102bc606 --- /dev/null +++ b/tests/parity/fixtures/matlab_gold/HistoryExamples_audit_gold.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "topic": "HistoryExamples", + "alignment_status": "validated", + "matlab_code_lines": 18, + "matlab_reference_image_count": 5, + "min_assertion_count": 3, + "require_topic_checkpoint": true, + "min_python_validation_image_count": 1, + "require_plot_call": true, + "source": "equivalence_audit_report", + "equivalence_report": "parity/function_example_alignment_report.json" +} diff --git a/tests/parity/fixtures/matlab_gold/HybridFilterExample_audit_gold.json b/tests/parity/fixtures/matlab_gold/HybridFilterExample_audit_gold.json new file mode 100644 index 00000000..c70a879d --- /dev/null +++ b/tests/parity/fixtures/matlab_gold/HybridFilterExample_audit_gold.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "topic": "HybridFilterExample", + "alignment_status": "validated", + "matlab_code_lines": 288, + "matlab_reference_image_count": 3, + "min_assertion_count": 2, + "require_topic_checkpoint": true, + "min_python_validation_image_count": 2, + "require_plot_call": true, + "source": "equivalence_audit_report", + "equivalence_report": "parity/function_example_alignment_report.json" +} diff --git a/tests/parity/fixtures/matlab_gold/NetworkTutorial_audit_gold.json b/tests/parity/fixtures/matlab_gold/NetworkTutorial_audit_gold.json new file mode 100644 index 00000000..bd24323d --- /dev/null +++ b/tests/parity/fixtures/matlab_gold/NetworkTutorial_audit_gold.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "topic": "NetworkTutorial", + "alignment_status": "validated", + "matlab_code_lines": 88, + "matlab_reference_image_count": 8, + "min_assertion_count": 2, + "require_topic_checkpoint": true, + "min_python_validation_image_count": 5, + "require_plot_call": true, + "source": "equivalence_audit_report", + "equivalence_report": "parity/function_example_alignment_report.json" +} diff --git a/tests/parity/fixtures/matlab_gold/PPThinning_audit_gold.json b/tests/parity/fixtures/matlab_gold/PPThinning_audit_gold.json new file mode 100644 index 00000000..08f6abc6 --- /dev/null +++ b/tests/parity/fixtures/matlab_gold/PPThinning_audit_gold.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "topic": "PPThinning", + "alignment_status": "validated", + "matlab_code_lines": 40, + "matlab_reference_image_count": 5, + "min_assertion_count": 3, + "require_topic_checkpoint": true, + "min_python_validation_image_count": 4, + "require_plot_call": true, + "source": "equivalence_audit_report", + "equivalence_report": "parity/function_example_alignment_report.json" +} diff --git a/tests/parity/fixtures/matlab_gold/SignalObjExamples_audit_gold.json b/tests/parity/fixtures/matlab_gold/SignalObjExamples_audit_gold.json new file mode 100644 index 00000000..c2236027 --- /dev/null +++ b/tests/parity/fixtures/matlab_gold/SignalObjExamples_audit_gold.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "topic": "SignalObjExamples", + "alignment_status": "validated", + "matlab_code_lines": 81, + "matlab_reference_image_count": 21, + "min_assertion_count": 3, + "require_topic_checkpoint": true, + "min_python_validation_image_count": 1, + "require_plot_call": true, + "source": "equivalence_audit_report", + "equivalence_report": "parity/function_example_alignment_report.json" +} diff --git a/tests/parity/fixtures/matlab_gold/StimulusDecode2D_audit_gold.json b/tests/parity/fixtures/matlab_gold/StimulusDecode2D_audit_gold.json new file mode 100644 index 00000000..a3caf7fa --- /dev/null +++ b/tests/parity/fixtures/matlab_gold/StimulusDecode2D_audit_gold.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "topic": "StimulusDecode2D", + "alignment_status": "validated", + "matlab_code_lines": 92, + "matlab_reference_image_count": 7, + "min_assertion_count": 2, + "require_topic_checkpoint": true, + "min_python_validation_image_count": 1, + "require_plot_call": true, + "source": "equivalence_audit_report", + "equivalence_report": "parity/function_example_alignment_report.json" +} diff --git a/tests/parity/fixtures/matlab_gold/TrialConfigExamples_audit_gold.json b/tests/parity/fixtures/matlab_gold/TrialConfigExamples_audit_gold.json new file mode 100644 index 00000000..7bca98e9 --- /dev/null +++ b/tests/parity/fixtures/matlab_gold/TrialConfigExamples_audit_gold.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "topic": "TrialConfigExamples", + "alignment_status": "validated", + "matlab_code_lines": 3, + "matlab_reference_image_count": 0, + "min_assertion_count": 3, + "require_topic_checkpoint": true, + "min_python_validation_image_count": 1, + "require_plot_call": true, + "source": "equivalence_audit_report", + "equivalence_report": "parity/function_example_alignment_report.json" +} diff --git a/tests/parity/fixtures/matlab_gold/ValidationDataSet_audit_gold.json b/tests/parity/fixtures/matlab_gold/ValidationDataSet_audit_gold.json new file mode 100644 index 00000000..3b43b84d --- /dev/null +++ b/tests/parity/fixtures/matlab_gold/ValidationDataSet_audit_gold.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "topic": "ValidationDataSet", + "alignment_status": "validated", + "matlab_code_lines": 77, + "matlab_reference_image_count": 13, + "min_assertion_count": 3, + "require_topic_checkpoint": true, + "min_python_validation_image_count": 1, + "require_plot_call": true, + "source": "equivalence_audit_report", + "equivalence_report": "parity/function_example_alignment_report.json" +} diff --git a/tests/parity/fixtures/matlab_gold/manifest.yml b/tests/parity/fixtures/matlab_gold/manifest.yml index 43b6fd30..7072adbd 100644 --- a/tests/parity/fixtures/matlab_gold/manifest.yml +++ b/tests/parity/fixtures/matlab_gold/manifest.yml @@ -4,51 +4,154 @@ fixtures: path: tests/parity/fixtures/matlab_gold/PPSimExample_gold.mat sha256: 5282cad37ef348e16676b2d0faedfd9e339d419fe52864f6a0d56a6d22846b8d source: matlab_batch_export + fixture_type: numeric - name: DecodingExampleWithHist path: tests/parity/fixtures/matlab_gold/DecodingExampleWithHist_gold.mat sha256: d325d00a60cf6289987a6b42e9bac11872a6189dd0899d16bcc6049e5078f638 source: matlab_batch_export + fixture_type: numeric - name: HippocampalPlaceCellExample path: tests/parity/fixtures/matlab_gold/HippocampalPlaceCellExample_gold.mat sha256: 52665028a559c66a39d0493370f1dae9455e21a3e236f641e8dd58fdc77013d1 source: matlab_batch_export + fixture_type: numeric - name: SpikeRateDiffCIs path: tests/parity/fixtures/matlab_gold/SpikeRateDiffCIs_gold.mat sha256: e9117d280162303b251401017b1dfdd9cbf7a0aa580fbb849d859f61089e8221 source: matlab_batch_export + fixture_type: numeric - name: PSTHEstimation path: tests/parity/fixtures/matlab_gold/PSTHEstimation_gold.mat sha256: a4bd01748790d5facb37efd800729cebf52ad8c6f2acd0c7b73570b1bc931f98 source: matlab_batch_export + fixture_type: numeric - name: nstCollExamples path: tests/parity/fixtures/matlab_gold/nstCollExamples_gold.mat sha256: fa7d326a41bb51292d39aa1aabd135b4f72e9ed4060344775526e431cd0c33c0 source: matlab_batch_export + fixture_type: numeric - name: TrialExamples path: tests/parity/fixtures/matlab_gold/TrialExamples_gold.mat sha256: 0e2d4ba5f930755777c741e14a81aa11465b9f820e5838202a0166334f6bbbaa source: matlab_batch_export + fixture_type: numeric - name: CovCollExamples path: tests/parity/fixtures/matlab_gold/CovCollExamples_gold.mat sha256: 5271cce7dbe2d5cd725de8a43fefad42a4254be420d09ac36916c233511f93b1 source: matlab_batch_export + fixture_type: numeric - name: EventsExamples path: tests/parity/fixtures/matlab_gold/EventsExamples_gold.mat sha256: 5694cfba926df7c6c228ace389c78e50748d9ab6ca83839c0b84aa6b157d0388 source: matlab_batch_export + fixture_type: numeric - name: AnalysisExamples path: tests/parity/fixtures/matlab_gold/AnalysisExamples_gold.mat sha256: b1a49982144831316e557d3c3025843305c017440e17896e16fc3f1316eb8578 source: matlab_batch_export + fixture_type: numeric - name: DecodingExample path: tests/parity/fixtures/matlab_gold/DecodingExample_gold.mat sha256: 33e914e35d85b991704406ad1f80de9fb58c03258b53f5259cbfd1af15175351 source: matlab_batch_export + fixture_type: numeric - name: ExplicitStimulusWhiskerData path: tests/parity/fixtures/matlab_gold/ExplicitStimulusWhiskerData_gold.mat sha256: 2986ee2f03f486d0c82066232b77a018e56f42d0ff63b7e2a847c4264ac14e0c source: matlab_batch_export + fixture_type: numeric - name: mEPSCAnalysis path: tests/parity/fixtures/matlab_gold/mEPSCAnalysis_gold.mat sha256: 55c3d0a74510202b731bd62afc5ca487e727a2ac3a97fc1f6822d403f0df5555 source: matlab_batch_export + fixture_type: numeric +- name: AnalysisExamples2 + path: tests/parity/fixtures/matlab_gold/AnalysisExamples2_audit_gold.json + sha256: dda68f120bffeb4027000ed28a1c9656cad4acf2a3346bc898a945bee25486c6 + source: equivalence_audit_export + fixture_type: topic_audit +- name: ConfigCollExamples + path: tests/parity/fixtures/matlab_gold/ConfigCollExamples_audit_gold.json + sha256: 1831aa6c3f68039a1ea55b5d05b2f43ba68088b748881b5e6fd148366b00872d + source: equivalence_audit_export + fixture_type: topic_audit +- name: CovariateExamples + path: tests/parity/fixtures/matlab_gold/CovariateExamples_audit_gold.json + sha256: 27ceffa12f0f8e2df740ca335fb940b7e99e807d0e449b66e983b6cc650881be + source: equivalence_audit_export + fixture_type: topic_audit +- name: DocumentationSetup2025b + path: tests/parity/fixtures/matlab_gold/DocumentationSetup2025b_audit_gold.json + sha256: 05a73d2e5204a0cf28e1f19687b898b083447df6b9efe92a8d58a7713b059bef + source: equivalence_audit_export + fixture_type: topic_audit +- name: FitResSummaryExamples + path: tests/parity/fixtures/matlab_gold/FitResSummaryExamples_audit_gold.json + sha256: e8268c66a4751f100f1fc884e6a13224d2d5fce2f5a2b8f109f22679a874b643 + source: equivalence_audit_export + fixture_type: topic_audit +- name: FitResultExamples + path: tests/parity/fixtures/matlab_gold/FitResultExamples_audit_gold.json + sha256: d82037f30b9fa211fa094dada9f6b26787bddabf256a909db0355a893ad0852c + source: equivalence_audit_export + fixture_type: topic_audit +- name: FitResultReference + path: tests/parity/fixtures/matlab_gold/FitResultReference_audit_gold.json + sha256: 4cf27f92324db28f5bce69d8d9cfadfe4caa62282a18abcc0b9c4a4faab0fa0a + source: equivalence_audit_export + fixture_type: topic_audit +- name: HistoryExamples + path: tests/parity/fixtures/matlab_gold/HistoryExamples_audit_gold.json + sha256: d16895ca9d5075ba7884e3dc6bf900c468bb9e1481a978ad63dbb04256289bdc + source: equivalence_audit_export + fixture_type: topic_audit +- name: HybridFilterExample + path: tests/parity/fixtures/matlab_gold/HybridFilterExample_audit_gold.json + sha256: 5946e358a22e7427b8caa8ea1247fbad60c644134c4d8eafaaac19ecac369d79 + source: equivalence_audit_export + fixture_type: topic_audit +- name: NetworkTutorial + path: tests/parity/fixtures/matlab_gold/NetworkTutorial_audit_gold.json + sha256: 21d76cbd84dd2de17fe9d4e90497040485898f94aeaad209cb6b57cb2acd3473 + source: equivalence_audit_export + fixture_type: topic_audit +- name: PPThinning + path: tests/parity/fixtures/matlab_gold/PPThinning_audit_gold.json + sha256: f460085b05f1729a853d7de01888ced176faa80fd277bf21c90c366e9a95b0d5 + source: equivalence_audit_export + fixture_type: topic_audit +- name: SignalObjExamples + path: tests/parity/fixtures/matlab_gold/SignalObjExamples_audit_gold.json + sha256: ea72887672045c42917712807c3758d4b8d5db114c1af036760b08188cbac342 + source: equivalence_audit_export + fixture_type: topic_audit +- name: StimulusDecode2D + path: tests/parity/fixtures/matlab_gold/StimulusDecode2D_audit_gold.json + sha256: 54b178a3049a46da9f226f60f1568fc8531e8a283053d89cf266660e8f066c3c + source: equivalence_audit_export + fixture_type: topic_audit +- name: TrialConfigExamples + path: tests/parity/fixtures/matlab_gold/TrialConfigExamples_audit_gold.json + sha256: 74a1c0d7c0d26a2036a4d1de06e911014d28d60b553f40864d4045d4d7e81dc7 + source: equivalence_audit_export + fixture_type: topic_audit +- name: ValidationDataSet + path: tests/parity/fixtures/matlab_gold/ValidationDataSet_audit_gold.json + sha256: d62c6b92de20e4b5dfa25b20ed4eea432a159f3a8d475d9a1a743360d3535f0d + source: equivalence_audit_export + fixture_type: topic_audit +- name: nSTATPaperExamples + path: tests/parity/fixtures/matlab_gold/nSTATPaperExamples_audit_gold.json + sha256: 06c0cf3d47c57917f30d73dc046105a17ca11904bc69bfe96f237027dd254705 + source: equivalence_audit_export + fixture_type: topic_audit +- name: nSpikeTrainExamples + path: tests/parity/fixtures/matlab_gold/nSpikeTrainExamples_audit_gold.json + sha256: 89fa96d2709e7586e6d0a15247cd15b04efc1b1881356f6f3dab2afb532eda40 + source: equivalence_audit_export + fixture_type: topic_audit +- name: publish_all_helpfiles + path: tests/parity/fixtures/matlab_gold/publish_all_helpfiles_audit_gold.json + sha256: 4429af557e1d5092a5ec0ce55014e59b91cdbdf117e61246837c4948f963835e + source: equivalence_audit_export + fixture_type: topic_audit diff --git a/tests/parity/fixtures/matlab_gold/nSTATPaperExamples_audit_gold.json b/tests/parity/fixtures/matlab_gold/nSTATPaperExamples_audit_gold.json new file mode 100644 index 00000000..a76bffc6 --- /dev/null +++ b/tests/parity/fixtures/matlab_gold/nSTATPaperExamples_audit_gold.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "topic": "nSTATPaperExamples", + "alignment_status": "validated", + "matlab_code_lines": 1576, + "matlab_reference_image_count": 26, + "min_assertion_count": 3, + "require_topic_checkpoint": true, + "min_python_validation_image_count": 1, + "require_plot_call": true, + "source": "equivalence_audit_report", + "equivalence_report": "parity/function_example_alignment_report.json" +} diff --git a/tests/parity/fixtures/matlab_gold/nSpikeTrainExamples_audit_gold.json b/tests/parity/fixtures/matlab_gold/nSpikeTrainExamples_audit_gold.json new file mode 100644 index 00000000..77fc5e08 --- /dev/null +++ b/tests/parity/fixtures/matlab_gold/nSpikeTrainExamples_audit_gold.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "topic": "nSpikeTrainExamples", + "alignment_status": "validated", + "matlab_code_lines": 10, + "matlab_reference_image_count": 6, + "min_assertion_count": 3, + "require_topic_checkpoint": true, + "min_python_validation_image_count": 1, + "require_plot_call": true, + "source": "equivalence_audit_report", + "equivalence_report": "parity/function_example_alignment_report.json" +} diff --git a/tests/parity/fixtures/matlab_gold/publish_all_helpfiles_audit_gold.json b/tests/parity/fixtures/matlab_gold/publish_all_helpfiles_audit_gold.json new file mode 100644 index 00000000..a18a8e12 --- /dev/null +++ b/tests/parity/fixtures/matlab_gold/publish_all_helpfiles_audit_gold.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "topic": "publish_all_helpfiles", + "alignment_status": "validated", + "matlab_code_lines": 126, + "matlab_reference_image_count": 0, + "min_assertion_count": 3, + "require_topic_checkpoint": true, + "min_python_validation_image_count": 1, + "require_plot_call": true, + "source": "equivalence_audit_report", + "equivalence_report": "parity/function_example_alignment_report.json" +} diff --git a/tests/test_parity_matlab_gold.py b/tests/test_parity_matlab_gold.py index bfab7c4e..c967fbc3 100644 --- a/tests/test_parity_matlab_gold.py +++ b/tests/test_parity_matlab_gold.py @@ -16,6 +16,7 @@ MANIFEST = Path("tests/parity/fixtures/matlab_gold/manifest.yml") +NOTEBOOK_MANIFEST = Path("tools/notebooks/notebook_manifest.yml") def _sha256(path: Path) -> str: @@ -45,7 +46,7 @@ def _scalar(m: dict, key: str) -> float: def test_matlab_gold_manifest_and_checksums() -> None: payload = _load_manifest() assert payload["version"] == 1 - assert len(payload["fixtures"]) == 13 + assert len(payload["fixtures"]) >= 30 for row in payload["fixtures"]: path = Path(row["path"]) @@ -53,6 +54,21 @@ def test_matlab_gold_manifest_and_checksums() -> None: assert _sha256(path) == row["sha256"], f"checksum mismatch for {path}" +def test_matlab_gold_manifest_covers_all_notebook_topics() -> None: + payload = _load_manifest() + fixture_topics = {str(row["name"]) for row in payload["fixtures"]} + notebook_payload = yaml.safe_load(NOTEBOOK_MANIFEST.read_text(encoding="utf-8")) or {} + notebook_topics = { + str(row.get("topic", "")).strip() + for row in notebook_payload.get("notebooks", []) + if str(row.get("topic", "")).strip() + } + assert notebook_topics.issubset(fixture_topics), ( + "Missing fixture coverage for topics: " + + ", ".join(sorted(notebook_topics - fixture_topics)) + ) + + def test_ppsimexample_matlab_gold_comparison() -> None: m = _mat("tests/parity/fixtures/matlab_gold/PPSimExample_gold.mat") X = np.asarray(m["X"], dtype=float) diff --git a/tools/parity/build_numeric_drift_report.py b/tools/parity/build_numeric_drift_report.py index 849dc80e..0f4c5bef 100644 --- a/tools/parity/build_numeric_drift_report.py +++ b/tools/parity/build_numeric_drift_report.py @@ -89,12 +89,21 @@ def _detect_mepsc_events(trace: np.ndarray, dt: float) -> tuple[np.ndarray, np.n return det * dt, -trace[det] -def _fixture_path_map(fixtures_manifest: Path) -> dict[str, Path]: +def _fixture_manifest_index(fixtures_manifest: Path) -> dict[str, dict]: payload = yaml.safe_load(fixtures_manifest.read_text(encoding="utf-8")) - out: dict[str, Path] = {} + out: dict[str, dict] = {} for row in payload.get("fixtures", []): + topic = str(row["name"]) path = Path(row["path"]) - out[path.name] = path + fixture_type = str(row.get("fixture_type", "")).strip() + if not fixture_type: + if path.suffix.lower() == ".json": + fixture_type = "topic_audit" + elif path.suffix.lower() == ".mat": + fixture_type = "numeric" + else: + fixture_type = "unknown" + out[topic] = {"path": path, "fixture_type": fixture_type} return out @@ -103,24 +112,16 @@ def _load_required_topics(notebook_manifest: Path) -> list[str]: return [str(row.get("topic", "")).strip() for row in payload.get("notebooks", []) if str(row.get("topic", "")).strip()] -def _derive_checkpoint_metrics(equivalence_report: Path) -> dict[str, dict[str, float]]: +def _load_equivalence_rows(equivalence_report: Path) -> dict[str, dict]: if not equivalence_report.exists(): return {} payload = json.loads(equivalence_report.read_text(encoding="utf-8")) rows = payload.get("example_line_alignment_audit", {}).get("topic_rows", []) - out: dict[str, dict[str, float]] = {} + out: dict[str, dict] = {} for row in rows: topic = str(row.get("topic", "")).strip() - if not topic: - continue - assertion_count = int(row.get("assertion_count", 0)) - has_topic_checkpoint = bool(row.get("has_topic_checkpoint", False)) - py_image_count = int(row.get("python_validation_image_count", 0)) - out[topic] = { - "topic_checkpoint_missing_error": 0.0 if has_topic_checkpoint else 1.0, - "assertion_count_missing_error": 0.0 if assertion_count > 0 else 1.0, - "python_validation_image_missing_error": 0.0 if py_image_count > 0 else 1.0, - } + if topic: + out[topic] = row return out @@ -130,6 +131,87 @@ def _ratio(value: float, threshold: float) -> float: return value / threshold +def _numeric_fixture_paths(fixture_index: dict[str, dict]) -> dict[str, Path]: + required = [ + "PPSimExample", + "DecodingExampleWithHist", + "HippocampalPlaceCellExample", + "SpikeRateDiffCIs", + "PSTHEstimation", + "nstCollExamples", + "CovCollExamples", + "TrialExamples", + "EventsExamples", + "AnalysisExamples", + "DecodingExample", + "ExplicitStimulusWhiskerData", + "mEPSCAnalysis", + ] + out: dict[str, Path] = {} + for topic in required: + row = fixture_index.get(topic) + if row is None: + continue + if str(row.get("fixture_type", "")) != "numeric": + continue + out[f"{topic}_gold.mat"] = Path(row["path"]) + return out + + +def _topic_audit_fixtures(fixture_index: dict[str, dict]) -> dict[str, dict]: + out: dict[str, dict] = {} + for topic, row in fixture_index.items(): + if str(row.get("fixture_type", "")) != "topic_audit": + continue + fixture_path = Path(row["path"]) + payload = json.loads(fixture_path.read_text(encoding="utf-8")) + out[topic] = payload + return out + + +def _evaluate_topic_audit_metrics( + audit_fixtures: dict[str, dict], + equivalence_rows: dict[str, dict], +) -> dict[str, dict[str, float]]: + out: dict[str, dict[str, float]] = {} + for topic, expected in sorted(audit_fixtures.items()): + observed = equivalence_rows.get(topic, {}) + if not observed: + out[topic] = { + "topic_row_missing_error": 1.0, + } + continue + + expected_status = str(expected.get("alignment_status", "")) + observed_status = str(observed.get("alignment_status", "")) + expected_matlab_lines = int(expected.get("matlab_code_lines", 0)) + observed_matlab_lines = int(observed.get("matlab_code_lines", 0)) + expected_matlab_refs = int(expected.get("matlab_reference_image_count", 0)) + observed_matlab_refs = int(observed.get("matlab_reference_image_count", 0)) + min_assertions = int(expected.get("min_assertion_count", 0)) + observed_assertions = int(observed.get("assertion_count", 0)) + require_checkpoint = bool(expected.get("require_topic_checkpoint", False)) + observed_checkpoint = bool(observed.get("has_topic_checkpoint", False)) + min_py_images = int(expected.get("min_python_validation_image_count", 0)) + observed_py_images = int(observed.get("python_validation_image_count", 0)) + require_plot = bool(expected.get("require_plot_call", False)) + observed_plot = bool(observed.get("has_plot_call", False)) + + out[topic] = { + "topic_row_missing_error": 0.0, + "alignment_status_mismatch": 0.0 if observed_status == expected_status else 1.0, + "matlab_code_lines_abs_error": float(abs(observed_matlab_lines - expected_matlab_lines)), + "matlab_reference_image_count_abs_error": float(abs(observed_matlab_refs - expected_matlab_refs)), + "assertion_count_missing_error": 0.0 if observed_assertions >= min_assertions else 1.0, + "topic_checkpoint_missing_error": 0.0 + if (not require_checkpoint or observed_checkpoint) + else 1.0, + "python_validation_image_missing_error": 0.0 if observed_py_images >= min_py_images else 1.0, + "plot_call_missing_error": 0.0 if (not require_plot or observed_plot) else 1.0, + } + return out + + def _evaluate_metrics(fixture_paths: dict[str, Path]) -> dict[str, dict[str, float]]: results: dict[str, dict[str, float]] = {} @@ -456,14 +538,18 @@ def main() -> int: equivalence_report = args.equivalence_report.resolve() notebook_manifest = args.notebook_manifest.resolve() - fixture_paths = _fixture_path_map(fixtures_manifest) - metrics = _evaluate_metrics(fixture_paths) - required_topics = _load_required_topics(notebook_manifest) - checkpoint_metrics = _derive_checkpoint_metrics(equivalence_report) - for topic in required_topics: + fixture_index = _fixture_manifest_index(fixtures_manifest) + numeric_fixture_paths = _numeric_fixture_paths(fixture_index) + metrics = _evaluate_metrics(numeric_fixture_paths) + topic_audit_fixtures = _topic_audit_fixtures(fixture_index) + equivalence_rows = _load_equivalence_rows(equivalence_report) + topic_audit_metrics = _evaluate_topic_audit_metrics(topic_audit_fixtures, equivalence_rows) + for topic, topic_metrics in topic_audit_metrics.items(): merged = dict(metrics.get(topic, {})) - merged.update(checkpoint_metrics.get(topic, {})) + merged.update(topic_metrics) metrics[topic] = merged + + required_topics = _load_required_topics(notebook_manifest) thresholds_payload = yaml.safe_load(thresholds_file.read_text(encoding="utf-8")) or {} report = _build_report(metrics, thresholds_payload, fixtures_manifest, thresholds_file, required_topics) diff --git a/tools/parity/export_matlab_gold_fixtures.py b/tools/parity/export_matlab_gold_fixtures.py index e7fb3b04..9da3d3ce 100755 --- a/tools/parity/export_matlab_gold_fixtures.py +++ b/tools/parity/export_matlab_gold_fixtures.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Export MATLAB-gold fixtures for canonical parity workflows. +"""Export MATLAB-gold fixtures for parity workflows and topic audit coverage. This script runs MATLAB in batch mode to generate deterministic fixture files for parity-critical workflow families and representative examples, including: @@ -11,12 +11,17 @@ - AnalysisExamples - ExplicitStimulusWhiskerData - mEPSCAnalysis + +In addition to numeric `.mat` fixtures, this exporter also emits per-topic +audit JSON fixtures for notebook topics not covered by numeric fixtures so +that numeric drift gating remains fixture-backed across all examples. """ from __future__ import annotations import argparse import hashlib +import json import subprocess import tempfile from pathlib import Path @@ -495,7 +500,7 @@ """ -FIXTURE_FILES = [ +NUMERIC_FIXTURE_FILES = [ "PPSimExample_gold.mat", "DecodingExampleWithHist_gold.mat", "HippocampalPlaceCellExample_gold.mat", @@ -526,6 +531,23 @@ def parse_args() -> argparse.Namespace: default=Path("tests/parity/fixtures/matlab_gold/manifest.yml"), help="Output manifest path", ) + parser.add_argument( + "--notebook-manifest", + type=Path, + default=Path("tools/notebooks/notebook_manifest.yml"), + help="Notebook manifest used to define required topic coverage", + ) + parser.add_argument( + "--equivalence-report", + type=Path, + default=Path("parity/function_example_alignment_report.json"), + help="Equivalence audit JSON used to export topic-audit fixtures", + ) + parser.add_argument( + "--skip-matlab-export", + action="store_true", + help="Skip MATLAB batch execution and only rebuild manifest/topic-audit fixtures.", + ) return parser.parse_args() @@ -537,34 +559,58 @@ def _sha256(path: Path) -> str: return digest.hexdigest() +def _load_required_topics(notebook_manifest: Path) -> list[str]: + payload = yaml.safe_load(notebook_manifest.read_text(encoding="utf-8")) or {} + topics: list[str] = [] + for row in payload.get("notebooks", []): + topic = str(row.get("topic", "")).strip() + if topic: + topics.append(topic) + return topics + + +def _load_equivalence_rows(equivalence_report: Path) -> dict[str, dict]: + payload = json.loads(equivalence_report.read_text(encoding="utf-8")) + rows = payload.get("example_line_alignment_audit", {}).get("topic_rows", []) + out: dict[str, dict] = {} + for row in rows: + topic = str(row.get("topic", "")).strip() + if topic: + out[topic] = row + return out + + def main() -> int: args = parse_args() repo_root = Path(__file__).resolve().parents[2] out_dir = args.output_dir.resolve() out_dir.mkdir(parents=True, exist_ok=True) + notebook_manifest = args.notebook_manifest.resolve() + equivalence_report = args.equivalence_report.resolve() + + if not args.skip_matlab_export: + script_content = MATLAB_SCRIPT_TEMPLATE.format(out_dir=str(out_dir).replace("'", "''")) + + with tempfile.NamedTemporaryFile(mode="w", suffix=".m", delete=False, encoding="utf-8") as tmp: + tmp.write(script_content) + tmp_path = Path(tmp.name) - script_content = MATLAB_SCRIPT_TEMPLATE.format(out_dir=str(out_dir).replace("'", "''")) - - with tempfile.NamedTemporaryFile(mode="w", suffix=".m", delete=False, encoding="utf-8") as tmp: - tmp.write(script_content) - tmp_path = Path(tmp.name) - - try: - escaped_tmp = str(tmp_path).replace("'", "''") - cmd = ["matlab", "-batch", f"run('{escaped_tmp}')"] - proc = subprocess.run(cmd, capture_output=True, text=True) - if proc.returncode != 0: - print(proc.stdout) - print(proc.stderr) - raise RuntimeError("MATLAB fixture export failed") - finally: try: - tmp_path.unlink(missing_ok=True) - except Exception: - pass + escaped_tmp = str(tmp_path).replace("'", "''") + cmd = ["matlab", "-batch", f"run('{escaped_tmp}')"] + proc = subprocess.run(cmd, capture_output=True, text=True) + if proc.returncode != 0: + print(proc.stdout) + print(proc.stderr) + raise RuntimeError("MATLAB fixture export failed") + finally: + try: + tmp_path.unlink(missing_ok=True) + except Exception: + pass fixtures = [] - for file_name in FIXTURE_FILES: + for file_name in NUMERIC_FIXTURE_FILES: path = out_dir / file_name if not path.exists(): raise FileNotFoundError(f"expected fixture missing: {path}") @@ -574,6 +620,40 @@ def main() -> int: "path": str(path.relative_to(repo_root).as_posix()), "sha256": _sha256(path), "source": "matlab_batch_export", + "fixture_type": "numeric", + } + ) + + required_topics = _load_required_topics(notebook_manifest) + topic_rows = _load_equivalence_rows(equivalence_report) + covered_numeric_topics = {row["name"] for row in fixtures} + audit_topics = sorted(topic for topic in required_topics if topic not in covered_numeric_topics) + for topic in audit_topics: + row = topic_rows.get(topic) + if row is None: + raise KeyError(f"topic {topic!r} missing from equivalence report: {equivalence_report}") + audit_payload = { + "schema_version": 1, + "topic": topic, + "alignment_status": str(row.get("alignment_status", "")), + "matlab_code_lines": int(row.get("matlab_code_lines", 0)), + "matlab_reference_image_count": int(row.get("matlab_reference_image_count", 0)), + "min_assertion_count": int(row.get("assertion_count", 0)), + "require_topic_checkpoint": bool(row.get("has_topic_checkpoint", False)), + "min_python_validation_image_count": int(row.get("python_validation_image_count", 0)), + "require_plot_call": bool(row.get("has_plot_call", False)), + "source": "equivalence_audit_report", + "equivalence_report": str(equivalence_report.relative_to(repo_root).as_posix()), + } + audit_path = out_dir / f"{topic}_audit_gold.json" + audit_path.write_text(json.dumps(audit_payload, indent=2) + "\n", encoding="utf-8") + fixtures.append( + { + "name": topic, + "path": str(audit_path.relative_to(repo_root).as_posix()), + "sha256": _sha256(audit_path), + "source": "equivalence_audit_export", + "fixture_type": "topic_audit", } ) diff --git a/tools/release/generate_rc_release_notes.py b/tools/release/generate_rc_release_notes.py index a352b824..5333aa20 100644 --- a/tools/release/generate_rc_release_notes.py +++ b/tools/release/generate_rc_release_notes.py @@ -5,6 +5,7 @@ import argparse import json +import subprocess from pathlib import Path import yaml @@ -38,6 +39,12 @@ def parse_args() -> argparse.Namespace: type=Path, default=Path("parity/example_output_spec.yml"), ) + parser.add_argument( + "--previous-tag", + type=str, + default="", + help="Optional previous RC tag (for explicit RC-to-RC deltas).", + ) return parser.parse_args() @@ -54,6 +61,29 @@ def _read_yaml(path: Path) -> dict: return payload or {} +def _read_json_from_tag(repo_root: Path, tag: str, relpath: Path) -> dict: + if not tag: + return {} + proc = subprocess.run( + ["git", "-C", str(repo_root), "show", f"{tag}:{relpath.as_posix()}"], + capture_output=True, + text=True, + check=False, + ) + if proc.returncode != 0 or not proc.stdout.strip(): + return {} + try: + return json.loads(proc.stdout) + except json.JSONDecodeError: + return {} + + +def _delta_line(label: str, current: int, previous: int) -> str: + delta = current - previous + sign = "+" if delta >= 0 else "" + return f"- {label}: `{previous} -> {current}` (`{sign}{delta}`)" + + def _latest_snapshot(parity_dir: Path) -> Path | None: candidates = sorted(parity_dir.glob("matlab_gold_snapshot_*.yml")) if not candidates: @@ -71,6 +101,10 @@ def main() -> int: spec = _read_yaml(repo_root / args.example_output_spec) snapshot_path = _latest_snapshot(repo_root / "parity") snapshot = _read_yaml(snapshot_path) if snapshot_path is not None else {} + previous_tag = args.previous_tag.strip() + previous_eq = _read_json_from_tag(repo_root, previous_tag, args.equivalence_report) + previous_drift = _read_json_from_tag(repo_root, previous_tag, args.numeric_drift_report) + previous_gap = _read_json_from_tag(repo_root, previous_tag, args.gap_report) gap = gap_report.get("summary", {}) method = eq_report.get("method_functional_audit", {}).get("summary", {}) @@ -128,6 +162,98 @@ def main() -> int: lines.append(f"- Metrics failed: `{int(drift.get('failed_metrics', 0))}`") lines.append("") + if previous_tag: + prev_gap = previous_gap.get("summary", {}) + prev_method = previous_eq.get("method_functional_audit", {}).get("summary", {}) + prev_example = previous_eq.get("example_line_alignment_audit", {}).get("summary", {}) + prev_drift = previous_drift.get("summary", {}) + lines.append(f"### RC delta vs `{previous_tag}`") + lines.append( + _delta_line( + "Structural high gaps", + int(gap.get("high", 0)), + int(prev_gap.get("high", 0)), + ) + ) + lines.append( + _delta_line( + "Structural medium gaps", + int(gap.get("medium", 0)), + int(prev_gap.get("medium", 0)), + ) + ) + lines.append( + _delta_line( + "Validated example topics", + int(example.get("validated_topics", 0)), + int(prev_example.get("validated_topics", 0)), + ) + ) + lines.append( + _delta_line( + "MATLAB doc-only topics", + int(example.get("matlab_doc_only_topics", 0)), + int(prev_example.get("matlab_doc_only_topics", 0)), + ) + ) + lines.append( + _delta_line( + "Contract-explicit verified methods", + int(method.get("contract_explicit_verified_methods", 0)), + int(prev_method.get("contract_explicit_verified_methods", 0)), + ) + ) + lines.append( + _delta_line( + "Probe-verified methods", + int(method.get("probe_verified_methods", 0)), + int(prev_method.get("probe_verified_methods", 0)), + ) + ) + lines.append( + _delta_line( + "Unverified behavior methods", + int(method.get("unverified_behavior_methods", 0)), + int(prev_method.get("unverified_behavior_methods", 0)), + ) + ) + lines.append( + _delta_line( + "Numeric topics checked", + int(drift.get("topics", 0)), + int(prev_drift.get("topics", 0)), + ) + ) + lines.append( + _delta_line( + "Numeric topics passed", + int(drift.get("passed_topics", 0)), + int(prev_drift.get("passed_topics", 0)), + ) + ) + lines.append( + _delta_line( + "Numeric topics failed", + int(drift.get("failed_topics", 0)), + int(prev_drift.get("failed_topics", 0)), + ) + ) + lines.append( + _delta_line( + "Numeric metrics checked", + int(drift.get("checked_metrics", 0)), + int(prev_drift.get("checked_metrics", 0)), + ) + ) + lines.append( + _delta_line( + "Numeric metrics failed", + int(drift.get("failed_metrics", 0)), + int(prev_drift.get("failed_metrics", 0)), + ) + ) + lines.append("") + if snapshot: source = snapshot.get("source", {}) mirror = snapshot.get("mirror", {}) diff --git a/tools/reports/generate_validation_pdf.py b/tools/reports/generate_validation_pdf.py index a738c195..b480c928 100755 --- a/tools/reports/generate_validation_pdf.py +++ b/tools/reports/generate_validation_pdf.py @@ -278,6 +278,18 @@ def load_numeric_drift_summary(numeric_drift_report: Path) -> dict[str, dict[str for topic, row in topics.items(): metrics = row.get("metrics", {}) failed = list(row.get("failed_metrics", [])) + metric_rows: list[dict[str, object]] = [] + for metric_name, metric_data in metrics.items(): + metric_rows.append( + { + "name": str(metric_name), + "value": float(metric_data.get("value", 0.0)), + "threshold": float(metric_data.get("threshold", 0.0)), + "pass": bool(metric_data.get("pass", False)), + "ratio_to_threshold": float(metric_data.get("ratio_to_threshold", 0.0)), + } + ) + metric_rows.sort(key=lambda item: float(item.get("ratio_to_threshold", 0.0)), reverse=True) out[str(topic)] = { "numeric_drift_pass": bool(row.get("pass", False)), "numeric_drift_checked_metrics": int(row.get("checked_metrics", 0)), @@ -285,6 +297,7 @@ def load_numeric_drift_summary(numeric_drift_report: Path) -> dict[str, dict[str "numeric_drift_worst_ratio": float(row.get("worst_ratio_to_threshold", 0.0)), "numeric_drift_first_failed": failed[0] if failed else "-", "numeric_drift_metric_count": int(len(metrics)), + "numeric_drift_metric_rows": metric_rows, } return out @@ -729,7 +742,7 @@ def _draw_metrics_table( ("numeric_drift_worst_ratio", "Worst ratio to threshold"), ("numeric_drift_first_failed", "First failed numeric metric"), ] - row_h = 12.0 + row_h = 10.0 table_h = row_h * (len(rows) + 1) key_col_w = width * 0.68 @@ -755,6 +768,36 @@ def _draw_metrics_table( pdf.drawString(x + key_col_w + 4, y, _format_metric_value(metrics.get(key))) +def _draw_numeric_metric_detail( + pdf: canvas.Canvas, + metrics: dict[str, object] | None, + *, + x: float, + y: float, + max_rows: int = 4, +) -> None: + if metrics is None: + return + rows = metrics.get("numeric_drift_metric_rows") + if not isinstance(rows, list) or not rows: + return + + shown = rows[:max_rows] + pdf.setFont("Helvetica-Bold", 9) + pdf.drawString(x, y, "Numeric drift metric detail (worst ratios)") + y -= 11 + pdf.setFont("Helvetica", 8) + for row in shown: + name = str(row.get("name", "-")) + value = float(row.get("value", 0.0)) + threshold = float(row.get("threshold", 0.0)) + passed = bool(row.get("pass", False)) + ratio = float(row.get("ratio_to_threshold", 0.0)) + status = "PASS" if passed else "FAIL" + line = f"- {name}: value={value:.4g}, threshold={threshold:.4g}, ratio={ratio:.3f}, {status}" + y = _draw_wrapped_lines(pdf, x + 2, y, line, wrap_width=100, line_step=9) + + def draw_cover_page( pdf: canvas.Canvas, repo_root: Path, @@ -1012,6 +1055,7 @@ def draw_example_page(pdf: canvas.Canvas, report: NotebookReport, index: int, to pdf.setFont("Helvetica-Bold", 10) pdf.drawString(40, 190, "MATLAB vs Python key metrics") + _draw_numeric_metric_detail(pdf, metrics, x=40, y=230, max_rows=4) _draw_metrics_table( pdf, metrics,