From 3fe5b7b88634ca11fe7368665411c9580138a6c2 Mon Sep 17 00:00:00 2001 From: Steve Van Hooser Date: Sat, 11 Apr 2026 13:09:47 -0400 Subject: [PATCH 1/5] Add cross-language symmetry test framework Mirrors the symmetry test framework introduced in NDI-matlab / NDI-python so that NDR-matlab and NDR-python can verify that they read and interpret example data identically. - Add +makeArtifacts and +readArtifacts sub-packages under tools/tests/+ndr/+symmetry with INSTRUCTIONS.md for each phase. - Add an example reader.readData pair that dumps channel metadata and a short sample of Intan RHD data to JSON and verifies it against artifacts produced by either the MATLAB or Python suites. --- .../+makeArtifacts/+reader/readData.m | 79 +++++++++++++++++++ .../+symmetry/+makeArtifacts/INSTRUCTIONS.md | 38 +++++++++ .../+readArtifacts/+reader/readData.m | 74 +++++++++++++++++ .../+symmetry/+readArtifacts/INSTRUCTIONS.md | 74 +++++++++++++++++ 4 files changed, 265 insertions(+) create mode 100644 tools/tests/+ndr/+symmetry/+makeArtifacts/+reader/readData.m create mode 100644 tools/tests/+ndr/+symmetry/+makeArtifacts/INSTRUCTIONS.md create mode 100644 tools/tests/+ndr/+symmetry/+readArtifacts/+reader/readData.m create mode 100644 tools/tests/+ndr/+symmetry/+readArtifacts/INSTRUCTIONS.md diff --git a/tools/tests/+ndr/+symmetry/+makeArtifacts/+reader/readData.m b/tools/tests/+ndr/+symmetry/+makeArtifacts/+reader/readData.m new file mode 100644 index 0000000..3c0d79f --- /dev/null +++ b/tools/tests/+ndr/+symmetry/+makeArtifacts/+reader/readData.m @@ -0,0 +1,79 @@ +classdef readData < matlab.unittest.TestCase + % READDATA - Generate symmetry artifacts for NDR reader tests. + % + % This test reads a small slice of the checked-in Intan RHD example + % data through ndr.reader.intan_rhd and dumps JSON artifacts that the + % Python symmetry test suite can re-read and verify. + + methods (Test) + function testReadDataArtifacts(testCase) + % Determine the artifact directory (must match NDR-python layout) + artifactDir = fullfile(tempdir(), 'NDR', 'symmetryTest', 'matlabArtifacts', ... + 'reader', 'readData', 'testReadDataArtifacts'); + + % Clear previous artifacts if they exist + if isfolder(artifactDir) + rmdir(artifactDir, 's'); + end + mkdir(artifactDir); + + % Locate the example RHD file checked into NDR-matlab + rhd_file = fullfile(ndr.fun.ndrpath(), 'example_data', 'example.rhd'); + testCase.assumeTrue(isfile(rhd_file), ... + 'Example RHD file not available; skipping makeArtifacts test.'); + + % Instantiate the reader and configure the epoch + reader = ndr.reader.intan_rhd(); + epochstreams = {rhd_file}; + epoch_select = 1; + channeltype = 'ai'; + channel = 1; + + % Collect metadata + channels = reader.getchannelsepoch(epochstreams, epoch_select); + sr = reader.samplerate(epochstreams, epoch_select, channeltype, channel); + t0t1 = reader.t0_t1(epochstreams, epoch_select); + ec = reader.epochclock(epochstreams, epoch_select); + + ecStrings = cell(1, numel(ec)); + for i = 1:numel(ec) + if ischar(ec{i}) || isstring(ec{i}) + ecStrings{i} = char(ec{i}); + else + try + ecStrings{i} = char(ec{i}.type); + catch + ecStrings{i} = class(ec{i}); + end + end + end + + metadata = struct(); + metadata.channels = channels; + metadata.samplerate = sr; + metadata.t0_t1 = t0t1{1}; + metadata.epochclock = ecStrings; + + metaJson = jsonencode(metadata, 'ConvertInfAndNaN', true, 'PrettyPrint', true); + fid = fopen(fullfile(artifactDir, 'metadata.json'), 'w'); + assert(fid > 0, 'Could not create metadata.json'); + fprintf(fid, '%s', metaJson); + fclose(fid); + + % Read a short, deterministic chunk of samples for numerical parity checks + s0 = 1; + s1 = 100; + data = reader.readchannels_epochsamples(channeltype, channel, ... + epochstreams, epoch_select, s0, s1); + + readStruct = struct(); + readStruct.ai_channel_1_samples_1_100 = data(:)'; + + readJson = jsonencode(readStruct, 'ConvertInfAndNaN', true, 'PrettyPrint', true); + fid = fopen(fullfile(artifactDir, 'readData.json'), 'w'); + assert(fid > 0, 'Could not create readData.json'); + fprintf(fid, '%s', readJson); + fclose(fid); + end + end +end diff --git a/tools/tests/+ndr/+symmetry/+makeArtifacts/INSTRUCTIONS.md b/tools/tests/+ndr/+symmetry/+makeArtifacts/INSTRUCTIONS.md new file mode 100644 index 0000000..897b9be --- /dev/null +++ b/tools/tests/+ndr/+symmetry/+makeArtifacts/INSTRUCTIONS.md @@ -0,0 +1,38 @@ +# NDR Symmetry Artifacts Instructions + +This folder contains MATLAB unit tests whose purpose is to generate standard NDR artifacts for symmetry testing with other NDR language ports (e.g., Python). + +## Rules for `makeArtifacts` tests: + +1. **Artifact Location**: Tests must store their generated artifacts in the system's temporary directory (`tempdir`). +2. **Directory Structure**: Inside the temporary directory, artifacts must be placed in a specific nested folder structure: + `NDR/symmetryTest/matlabArtifacts////` + + - ``: The last part of the MATLAB package namespace. For example, for a test located at `tools/tests/+ndr/+symmetry/+makeArtifacts/+reader`, the namespace is `reader`. + - ``: The name of the test class (e.g., `readData`). + - ``: The specific name of the test method being executed (e.g., `testReadDataArtifacts`). + +3. **Persistent Teardown**: The generated artifact files **MUST** persist in the temporary directory so that the Python test suite can read them. Do NOT delete the artifact directory in a test teardown method. + +4. **Artifact Contents**: Every `makeArtifacts` test should produce at minimum: + - A `metadata.json` file describing the channels, sample rates, `t0`/`t1` boundaries, + and epoch clock types returned by the reader. + - A `readData.json` file containing a small, reproducible sample of data read via + `readchannels_epochsamples(...)` (or the equivalent reader call) so the Python suite + can verify numerical parity. + +5. **Deterministic Input**: Tests should read from the checked-in `example_data/` files + in the NDR-matlab repository so that both language ports operate on byte-identical inputs. + +## Example: +For a test class `readData.m` in `tools/tests/+ndr/+symmetry/+makeArtifacts/+reader` with a test method `testReadDataArtifacts`, the artifacts should be saved to: +`[tempdir(), 'NDR/symmetryTest/matlabArtifacts/reader/readData/testReadDataArtifacts/']` + +## Running + +From MATLAB: + +```matlab +% Generate artifacts +results = runtests('ndr.symmetry.makeArtifacts', 'IncludeSubpackages', true); +``` diff --git a/tools/tests/+ndr/+symmetry/+readArtifacts/+reader/readData.m b/tools/tests/+ndr/+symmetry/+readArtifacts/+reader/readData.m new file mode 100644 index 0000000..16e3be6 --- /dev/null +++ b/tools/tests/+ndr/+symmetry/+readArtifacts/+reader/readData.m @@ -0,0 +1,74 @@ +classdef readData < matlab.unittest.TestCase + % READDATA - Verify symmetry artifacts for NDR reader tests. + % + % This test is parameterized over the two potential sources of + % artifacts ('matlabArtifacts', 'pythonArtifacts'). It loads the + % JSON dumps produced by the corresponding makeArtifacts suite and + % confirms that the MATLAB NDR reader returns matching values for + % the same example data file. + % + % When the artifact directory for a given SourceType does not exist, + % the test silently skips so that the suite can run on machines that + % only have one of the two language ports installed. + + properties (TestParameter) + SourceType = {'matlabArtifacts', 'pythonArtifacts'}; + end + + methods (Test) + function testReadDataArtifacts(testCase, SourceType) + artifactDir = fullfile(tempdir(), 'NDR', 'symmetryTest', SourceType, ... + 'reader', 'readData', 'testReadDataArtifacts'); + + if ~isfolder(artifactDir) + disp(['Artifact directory from ' SourceType ' does not exist. Skipping.']); + return; + end + + rhd_file = fullfile(ndr.fun.ndrpath(), 'example_data', 'example.rhd'); + testCase.assumeTrue(isfile(rhd_file), ... + 'Example RHD file not available; skipping readArtifacts test.'); + + reader = ndr.reader.intan_rhd(); + epochstreams = {rhd_file}; + epoch_select = 1; + channeltype = 'ai'; + channel = 1; + + % --- metadata parity --- + metaFile = fullfile(artifactDir, 'metadata.json'); + if isfile(metaFile) + fid = fopen(metaFile, 'r'); + expectedMeta = jsondecode(fread(fid, inf, '*char')'); + fclose(fid); + + actualSr = reader.samplerate(epochstreams, epoch_select, channeltype, channel); + testCase.verifyEqual(actualSr, expectedMeta.samplerate, 'AbsTol', 1e-9, ... + ['Sample rate mismatch against ' SourceType ' artifacts.']); + + actualT0T1 = reader.t0_t1(epochstreams, epoch_select); + testCase.verifyEqual(actualT0T1{1}(:)', expectedMeta.t0_t1(:)', 'AbsTol', 1e-6, ... + ['t0_t1 mismatch against ' SourceType ' artifacts.']); + else + disp(['metadata.json not found in ' SourceType ' artifact directory. Skipping metadata check.']); + end + + % --- raw data parity --- + readFile = fullfile(artifactDir, 'readData.json'); + if isfile(readFile) + fid = fopen(readFile, 'r'); + expectedRead = jsondecode(fread(fid, inf, '*char')'); + fclose(fid); + + expectedSamples = expectedRead.ai_channel_1_samples_1_100(:)'; + actualData = reader.readchannels_epochsamples(channeltype, channel, ... + epochstreams, epoch_select, 1, 100); + + testCase.verifyEqual(actualData(:)', expectedSamples, 'AbsTol', 1e-9, ... + ['Data mismatch for ai channel 1 samples 1-100 against ' SourceType ' artifacts.']); + else + disp(['readData.json not found in ' SourceType ' artifact directory. Skipping data check.']); + end + end + end +end diff --git a/tools/tests/+ndr/+symmetry/+readArtifacts/INSTRUCTIONS.md b/tools/tests/+ndr/+symmetry/+readArtifacts/INSTRUCTIONS.md new file mode 100644 index 0000000..1576b1c --- /dev/null +++ b/tools/tests/+ndr/+symmetry/+readArtifacts/INSTRUCTIONS.md @@ -0,0 +1,74 @@ +# NDR Read Artifacts Instructions + +This folder contains MATLAB unit tests whose purpose is to read standard NDR artifacts generated by both the Python NDR test suite and the MATLAB test suite, comparing them against expected values. + +## Process overview: + +1. **Artifact Location**: The test suites will place their generated artifacts in the system's temporary directory (`tempdir()`). +2. **Directory Structure**: Inside the temporary directory, artifacts can be found in a specific nested folder structure: + `NDR/symmetryTest/////` + + - ``: Either `matlabArtifacts` or `pythonArtifacts`. + - ``: The module/package location of the corresponding test (e.g., `reader`). + - ``: The name of the test class (e.g., `readData`). + - ``: The specific name of the test method that was executed (e.g., `testReadDataArtifacts`). + +3. **Testing Goals**: The MATLAB tests located in this `+readArtifacts` package should: + - Load the JSON representations created by the target suite (e.g., `metadata.json`, `readData.json`). + - Re-read the same example data files through the corresponding NDR reader in MATLAB. + - Assert that the numeric values (channel lists, sample rates, `t0`/`t1`, sample data) + match what the target suite stored. + - Run across both `pythonArtifacts` and `matlabArtifacts` using parameterized testing to + ensure parity in *both* directions. + +4. **Skipping**: When the artifact directory for a given `SourceType` does not exist, the + test must skip silently (not fail). This allows the suite to run on machines where only + one of the two languages has produced artifacts. + +## Example: + +Use MATLAB's `TestParameter` property block to dynamically pass the `SourceType` to your tests: + +```matlab +classdef readData < matlab.unittest.TestCase + properties (TestParameter) + % Define the two potential sources of artifacts + SourceType = {'matlabArtifacts', 'pythonArtifacts'}; + end + + methods (Test) + function testReadDataArtifacts(testCase, SourceType) + artifactDir = fullfile(tempdir, 'NDR', 'symmetryTest', SourceType, ... + 'reader', 'readData', 'testReadDataArtifacts'); + + if ~isfolder(artifactDir) + disp(['Artifact directory from ' SourceType ' does not exist. Skipping.']); + return; + end + + % 1. Load the ground-truth metadata JSON that the target suite dumped. + fid = fopen(fullfile(artifactDir, 'metadata.json'), 'r'); + expectedMeta = jsondecode(fread(fid, inf, '*char')'); + fclose(fid); + + % 2. Re-read the same example data in MATLAB and assert parity. + reader = ndr.reader.intan_rhd(); + rhd_file = fullfile(ndr.fun.ndrpath(), 'example_data', 'example.rhd'); + epochstreams = {rhd_file}; + sr = reader.samplerate(epochstreams, 1, 'ai', 1); + + testCase.verifyEqual(sr, expectedMeta.samplerate, 'AbsTol', 1e-9, ... + ['Sample rate mismatch against ' SourceType ' artifacts.']); + end + end +end +``` + +## Running + +From MATLAB: + +```matlab +% Verify artifacts +results = runtests('ndr.symmetry.readArtifacts', 'IncludeSubpackages', true); +``` From a6a320a2402f61924b708cfd112d2e93d5a5cd9d Mon Sep 17 00:00:00 2001 From: Steve Van Hooser Date: Sat, 11 Apr 2026 13:26:05 -0400 Subject: [PATCH 2/5] Add symmetry test CI workflow Runs the MATLAB +makeArtifacts and +readArtifacts test packages on pushes to main, pull requests targeting main, and manual dispatch. This workflow is separate from the regular run_tests.yml CI so that the cross-language symmetry suite can evolve independently. --- .github/workflows/test-symmetry.yml | 90 +++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 .github/workflows/test-symmetry.yml diff --git a/.github/workflows/test-symmetry.yml b/.github/workflows/test-symmetry.yml new file mode 100644 index 0000000..271b8f2 --- /dev/null +++ b/.github/workflows/test-symmetry.yml @@ -0,0 +1,90 @@ +name: Test NDR Symmetry + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + run_tests: + name: Run symmetry tests + runs-on: ubuntu-latest + steps: + - name: Check out repo + uses: actions/checkout@v4 + + - name: Start virtual display server + if: runner.os == 'Linux' + run: | + sudo apt-get install -y xvfb + Xvfb :99 & + echo "DISPLAY=:99" >> $GITHUB_ENV + + - name: Set up MATLAB + uses: matlab-actions/setup-matlab@v2 + with: + release: latest + cache: true + products: | + Signal_Processing_Toolbox + Statistics_and_Machine_Learning_Toolbox + + - name: Install MatBox + uses: ehennestad/matbox-actions/install-matbox@v1 + + - name: Run symmetry tests + uses: matlab-actions/run-command@v2 + with: + command: | + addpath(genpath("+ndr")); + addpath(genpath("tools")); + ndr_Init; + + import matlab.unittest.TestSuite + import matlab.unittest.TestRunner + + makeSuite = TestSuite.fromPackage("ndr.symmetry.makeArtifacts", "IncludingSubpackages", true); + readSuite = TestSuite.fromPackage("ndr.symmetry.readArtifacts", "IncludingSubpackages", true); + suite = [makeSuite, readSuite]; + + fprintf("\n=== Discovered %d test(s) ===\n", numel(suite)); + for k = 1:numel(suite) + fprintf(" %d. %s\n", k, suite(k).Name); + end + fprintf("\n"); + assert(~isempty(suite), "No symmetry tests were found. Check tools/tests/+ndr/+symmetry.") + + runner = TestRunner.withTextOutput("Verbosity", "Detailed"); + + nMake = numel(makeSuite); + nRead = numel(readSuite); + + makeResults = runner.run(makeSuite); + fprintf("\n=== makeArtifacts Results: %d passed, %d failed, %d incomplete out of %d ===\n", ... + nnz([makeResults.Passed]), nnz([makeResults.Failed]), nnz([makeResults.Incomplete]), nMake); + + readResults = runner.run(readSuite); + fprintf("\n=== readArtifacts Results: %d passed, %d failed, %d incomplete out of %d ===\n", ... + nnz([readResults.Passed]), nnz([readResults.Failed]), nnz([readResults.Incomplete]), nRead); + + results = [makeResults, readResults]; + fprintf("\n=== Overall Symmetry Test Results ===\n"); + fprintf(" %d passed, %d failed, %d incomplete out of %d total\n", ... + nnz([results.Passed]), nnz([results.Failed]), nnz([results.Incomplete]), numel(results)); + disp(table(results)); + + assert(all(~[results.Failed]), "Some symmetry tests failed") + + - name: Restore MATLAB Path + uses: matlab-actions/run-command@v2 + if: always() + with: + command: | + restoredefaultpath() + savepath() From baaa9292134a286b54b4b061d4c5582de3455b41 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 11 Apr 2026 21:10:47 +0000 Subject: [PATCH 3/5] Update GitHub badges --- .github/badges/tests.svg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/badges/tests.svg b/.github/badges/tests.svg index 76555dc..c431d40 100644 --- a/.github/badges/tests.svg +++ b/.github/badges/tests.svg @@ -1 +1 @@ -teststests123 passed123 passed \ No newline at end of file +teststests126 passed126 passed \ No newline at end of file From 4e626ad582a86ddb25a15e096102d6cd282948c2 Mon Sep 17 00:00:00 2001 From: Steve Van Hooser Date: Sat, 11 Apr 2026 17:31:14 -0400 Subject: [PATCH 4/5] Fix symmetry CI path setup Two issues were causing the workflow to fail before any tests ran: 1. `addpath(genpath("+ndr"))` is invalid because MATLAB namespace directories (anything starting with `+`) must not themselves appear on the path -- only their parent directories. Replace with `addpath(pwd)` plus `addpath(tools/tests)` so both the repo-root `+ndr` and the test `+ndr/+symmetry` packages are discoverable. 2. `ndr_Init` tried to `mkdir` a filecache path under `userpath` which isn't writable on the CI runner. The symmetry tests only need `ndr.fun.ndrpath()` (self-contained), so skip `ndr_Init` entirely. --- .github/workflows/test-symmetry.yml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-symmetry.yml b/.github/workflows/test-symmetry.yml index 271b8f2..9cb9dec 100644 --- a/.github/workflows/test-symmetry.yml +++ b/.github/workflows/test-symmetry.yml @@ -42,9 +42,16 @@ jobs: uses: matlab-actions/run-command@v2 with: command: | - addpath(genpath("+ndr")); - addpath(genpath("tools")); - ndr_Init; + % Add the repo root so MATLAB sees the top-level +ndr namespace, + % and add tools/tests so MATLAB sees the +ndr/+symmetry namespace. + % Note: MATLAB namespace (+foo) directories must NOT appear on the + % path themselves; only their parent directories should. + addpath(pwd); + addpath(fullfile(pwd, "tools", "tests")); + + % Skip ndr_Init -- it tries to mkdir filecache paths under + % userpath which are not writable on the CI runner, and the + % symmetry tests only use ndr.fun.ndrpath() which is self-contained. import matlab.unittest.TestSuite import matlab.unittest.TestRunner From 1a97cc6dadb1dad5d7dd2765714eb61531171b3e Mon Sep 17 00:00:00 2001 From: Steve Van Hooser Date: Sat, 11 Apr 2026 17:39:29 -0400 Subject: [PATCH 5/5] Install MATLAB dependencies via matbox in symmetry CI The readData make-artifact test calls ndr.reader.intan_rhd methods that depend on vlt.string.strcmp_substitution from vhlab-toolbox-matlab. That dependency is declared in tools/requirements.txt but nothing was installing it in the symmetry CI. Call matbox.installRequirements(tools) to fetch and add it to the path, matching how NDI-matlab's symmetry CI handles requirements. --- .github/workflows/test-symmetry.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/test-symmetry.yml b/.github/workflows/test-symmetry.yml index 9cb9dec..cee466a 100644 --- a/.github/workflows/test-symmetry.yml +++ b/.github/workflows/test-symmetry.yml @@ -49,6 +49,10 @@ jobs: addpath(pwd); addpath(fullfile(pwd, "tools", "tests")); + % Install MATLAB dependencies declared in tools/requirements.txt + % (e.g. vhlab-toolbox-matlab, which provides vlt.string.*). + matbox.installRequirements(fullfile(pwd, "tools")); + % Skip ndr_Init -- it tries to mkdir filecache paths under % userpath which are not writable on the CI runner, and the % symmetry tests only use ndr.fun.ndrpath() which is self-contained.