Add NDR symmetry testing framework for cross-language validation

.github/workflows/test-symmetry.yml

@@ -0,0 +1,101 @@
+name: Test NDR Symmetry
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  run_tests:
+    name: Run symmetry tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repo
+        uses: actions/checkout@v4
+
+      - name: Start virtual display server
+        if: runner.os == 'Linux'
+        run: |
+          sudo apt-get install -y xvfb
+          Xvfb :99 &
+          echo "DISPLAY=:99" >> $GITHUB_ENV
+
+      - name: Set up MATLAB
+        uses: matlab-actions/setup-matlab@v2
+        with:
+          release: latest
+          cache: true
+          products: |
+            Signal_Processing_Toolbox
+            Statistics_and_Machine_Learning_Toolbox
+
+      - name: Install MatBox
+        uses: ehennestad/matbox-actions/install-matbox@v1
+
+      - name: Run symmetry tests
+        uses: matlab-actions/run-command@v2
+        with:
+          command: |
+            % Add the repo root so MATLAB sees the top-level +ndr namespace,
+            % and add tools/tests so MATLAB sees the +ndr/+symmetry namespace.
+            % Note: MATLAB namespace (+foo) directories must NOT appear on the
+            % path themselves; only their parent directories should.
+            addpath(pwd);
+            addpath(fullfile(pwd, "tools", "tests"));
+
+            % Install MATLAB dependencies declared in tools/requirements.txt
+            % (e.g. vhlab-toolbox-matlab, which provides vlt.string.*).
+            matbox.installRequirements(fullfile(pwd, "tools"));
+
+            % Skip ndr_Init -- it tries to mkdir filecache paths under
+            % userpath which are not writable on the CI runner, and the
+            % symmetry tests only use ndr.fun.ndrpath() which is self-contained.
+
+            import matlab.unittest.TestSuite
+            import matlab.unittest.TestRunner
+
+            makeSuite = TestSuite.fromPackage("ndr.symmetry.makeArtifacts", "IncludingSubpackages", true);
+            readSuite = TestSuite.fromPackage("ndr.symmetry.readArtifacts", "IncludingSubpackages", true);
+            suite = [makeSuite, readSuite];
+
+            fprintf("\n=== Discovered %d test(s) ===\n", numel(suite));
+            for k = 1:numel(suite)
+                fprintf("  %d. %s\n", k, suite(k).Name);
+            end
+            fprintf("\n");
+            assert(~isempty(suite), "No symmetry tests were found. Check tools/tests/+ndr/+symmetry.")
+
+            runner = TestRunner.withTextOutput("Verbosity", "Detailed");
+
+            nMake = numel(makeSuite);
+            nRead = numel(readSuite);
+
+            makeResults = runner.run(makeSuite);
+            fprintf("\n=== makeArtifacts Results: %d passed, %d failed, %d incomplete out of %d ===\n", ...
+                nnz([makeResults.Passed]), nnz([makeResults.Failed]), nnz([makeResults.Incomplete]), nMake);
+
+            readResults = runner.run(readSuite);
+            fprintf("\n=== readArtifacts Results: %d passed, %d failed, %d incomplete out of %d ===\n", ...
+                nnz([readResults.Passed]), nnz([readResults.Failed]), nnz([readResults.Incomplete]), nRead);
+
+            results = [makeResults, readResults];
+            fprintf("\n=== Overall Symmetry Test Results ===\n");
+            fprintf("  %d passed, %d failed, %d incomplete out of %d total\n", ...
+                nnz([results.Passed]), nnz([results.Failed]), nnz([results.Incomplete]), numel(results));
+            disp(table(results));
+
+            assert(all(~[results.Failed]), "Some symmetry tests failed")
+
+      - name: Restore MATLAB Path
+        uses: matlab-actions/run-command@v2
+        if: always()
+        with:
+          command: |
+            restoredefaultpath()
+            savepath()

tools/tests/+ndr/+symmetry/+makeArtifacts/+reader/readData.m

@@ -0,0 +1,79 @@
+classdef readData < matlab.unittest.TestCase
+    % READDATA - Generate symmetry artifacts for NDR reader tests.
+    %
+    % This test reads a small slice of the checked-in Intan RHD example
+    % data through ndr.reader.intan_rhd and dumps JSON artifacts that the
+    % Python symmetry test suite can re-read and verify.
+
+    methods (Test)
+        function testReadDataArtifacts(testCase)
+            % Determine the artifact directory (must match NDR-python layout)
+            artifactDir = fullfile(tempdir(), 'NDR', 'symmetryTest', 'matlabArtifacts', ...
+                'reader', 'readData', 'testReadDataArtifacts');
+
+            % Clear previous artifacts if they exist
+            if isfolder(artifactDir)
+                rmdir(artifactDir, 's');
+            end
+            mkdir(artifactDir);
+
+            % Locate the example RHD file checked into NDR-matlab
+            rhd_file = fullfile(ndr.fun.ndrpath(), 'example_data', 'example.rhd');
+            testCase.assumeTrue(isfile(rhd_file), ...
+                'Example RHD file not available; skipping makeArtifacts test.');
+
+            % Instantiate the reader and configure the epoch
+            reader = ndr.reader.intan_rhd();
+            epochstreams = {rhd_file};
+            epoch_select = 1;
+            channeltype = 'ai';
+            channel = 1;
+
+            % Collect metadata
+            channels = reader.getchannelsepoch(epochstreams, epoch_select);
+            sr       = reader.samplerate(epochstreams, epoch_select, channeltype, channel);
+            t0t1     = reader.t0_t1(epochstreams, epoch_select);
+            ec       = reader.epochclock(epochstreams, epoch_select);
+
+            ecStrings = cell(1, numel(ec));
+            for i = 1:numel(ec)
+                if ischar(ec{i}) || isstring(ec{i})
+                    ecStrings{i} = char(ec{i});
+                else
+                    try
+                        ecStrings{i} = char(ec{i}.type);
+                    catch
+                        ecStrings{i} = class(ec{i});
+                    end
+                end
+            end
+
+            metadata = struct();
+            metadata.channels    = channels;
+            metadata.samplerate  = sr;
+            metadata.t0_t1       = t0t1{1};
+            metadata.epochclock  = ecStrings;
+
+            metaJson = jsonencode(metadata, 'ConvertInfAndNaN', true, 'PrettyPrint', true);
+            fid = fopen(fullfile(artifactDir, 'metadata.json'), 'w');
+            assert(fid > 0, 'Could not create metadata.json');
+            fprintf(fid, '%s', metaJson);
+            fclose(fid);
+
+            % Read a short, deterministic chunk of samples for numerical parity checks
+            s0 = 1;
+            s1 = 100;
+            data = reader.readchannels_epochsamples(channeltype, channel, ...
+                epochstreams, epoch_select, s0, s1);
+
+            readStruct = struct();
+            readStruct.ai_channel_1_samples_1_100 = data(:)';
+
+            readJson = jsonencode(readStruct, 'ConvertInfAndNaN', true, 'PrettyPrint', true);
+            fid = fopen(fullfile(artifactDir, 'readData.json'), 'w');
+            assert(fid > 0, 'Could not create readData.json');
+            fprintf(fid, '%s', readJson);
+            fclose(fid);
+        end
+    end
+end

tools/tests/+ndr/+symmetry/+makeArtifacts/INSTRUCTIONS.md

@@ -0,0 +1,38 @@
+# NDR Symmetry Artifacts Instructions
+
+This folder contains MATLAB unit tests whose purpose is to generate standard NDR artifacts for symmetry testing with other NDR language ports (e.g., Python).
+
+## Rules for `makeArtifacts` tests:
+
+1. **Artifact Location**: Tests must store their generated artifacts in the system's temporary directory (`tempdir`).
+2. **Directory Structure**: Inside the temporary directory, artifacts must be placed in a specific nested folder structure:
+   `NDR/symmetryTest/matlabArtifacts/<namespace>/<class_name>/<test_name>/`
+
+   - `<namespace>`: The last part of the MATLAB package namespace. For example, for a test located at `tools/tests/+ndr/+symmetry/+makeArtifacts/+reader`, the namespace is `reader`.
+   - `<class_name>`: The name of the test class (e.g., `readData`).
+   - `<test_name>`: The specific name of the test method being executed (e.g., `testReadDataArtifacts`).
+
+3. **Persistent Teardown**: The generated artifact files **MUST** persist in the temporary directory so that the Python test suite can read them. Do NOT delete the artifact directory in a test teardown method.
+
+4. **Artifact Contents**: Every `makeArtifacts` test should produce at minimum:
+   - A `metadata.json` file describing the channels, sample rates, `t0`/`t1` boundaries,
+     and epoch clock types returned by the reader.
+   - A `readData.json` file containing a small, reproducible sample of data read via
+     `readchannels_epochsamples(...)` (or the equivalent reader call) so the Python suite
+     can verify numerical parity.
+
+5. **Deterministic Input**: Tests should read from the checked-in `example_data/` files
+   in the NDR-matlab repository so that both language ports operate on byte-identical inputs.
+
+## Example:
+For a test class `readData.m` in `tools/tests/+ndr/+symmetry/+makeArtifacts/+reader` with a test method `testReadDataArtifacts`, the artifacts should be saved to:
+`[tempdir(), 'NDR/symmetryTest/matlabArtifacts/reader/readData/testReadDataArtifacts/']`
+
+## Running
+
+From MATLAB:
+
+```matlab
+% Generate artifacts
+results = runtests('ndr.symmetry.makeArtifacts', 'IncludeSubpackages', true);
+```

tools/tests/+ndr/+symmetry/+readArtifacts/+reader/readData.m

@@ -0,0 +1,74 @@
+classdef readData < matlab.unittest.TestCase
+    % READDATA - Verify symmetry artifacts for NDR reader tests.
+    %
+    % This test is parameterized over the two potential sources of
+    % artifacts ('matlabArtifacts', 'pythonArtifacts'). It loads the
+    % JSON dumps produced by the corresponding makeArtifacts suite and
+    % confirms that the MATLAB NDR reader returns matching values for
+    % the same example data file.
+    %
+    % When the artifact directory for a given SourceType does not exist,
+    % the test silently skips so that the suite can run on machines that
+    % only have one of the two language ports installed.
+
+    properties (TestParameter)
+        SourceType = {'matlabArtifacts', 'pythonArtifacts'};
+    end
+
+    methods (Test)
+        function testReadDataArtifacts(testCase, SourceType)
+            artifactDir = fullfile(tempdir(), 'NDR', 'symmetryTest', SourceType, ...
+                'reader', 'readData', 'testReadDataArtifacts');
+
+            if ~isfolder(artifactDir)
+                disp(['Artifact directory from ' SourceType ' does not exist. Skipping.']);
+                return;
+            end
+
+            rhd_file = fullfile(ndr.fun.ndrpath(), 'example_data', 'example.rhd');
+            testCase.assumeTrue(isfile(rhd_file), ...
+                'Example RHD file not available; skipping readArtifacts test.');
+
+            reader = ndr.reader.intan_rhd();
+            epochstreams = {rhd_file};
+            epoch_select = 1;
+            channeltype = 'ai';
+            channel = 1;
+
+            % --- metadata parity ---
+            metaFile = fullfile(artifactDir, 'metadata.json');
+            if isfile(metaFile)
+                fid = fopen(metaFile, 'r');
+                expectedMeta = jsondecode(fread(fid, inf, '*char')');
+                fclose(fid);
+
+                actualSr = reader.samplerate(epochstreams, epoch_select, channeltype, channel);
+                testCase.verifyEqual(actualSr, expectedMeta.samplerate, 'AbsTol', 1e-9, ...
+                    ['Sample rate mismatch against ' SourceType ' artifacts.']);
+
+                actualT0T1 = reader.t0_t1(epochstreams, epoch_select);
+                testCase.verifyEqual(actualT0T1{1}(:)', expectedMeta.t0_t1(:)', 'AbsTol', 1e-6, ...
+                    ['t0_t1 mismatch against ' SourceType ' artifacts.']);
+            else
+                disp(['metadata.json not found in ' SourceType ' artifact directory. Skipping metadata check.']);
+            end
+
+            % --- raw data parity ---
+            readFile = fullfile(artifactDir, 'readData.json');
+            if isfile(readFile)
+                fid = fopen(readFile, 'r');
+                expectedRead = jsondecode(fread(fid, inf, '*char')');
+                fclose(fid);
+
+                expectedSamples = expectedRead.ai_channel_1_samples_1_100(:)';
+                actualData = reader.readchannels_epochsamples(channeltype, channel, ...
+                    epochstreams, epoch_select, 1, 100);
+
+                testCase.verifyEqual(actualData(:)', expectedSamples, 'AbsTol', 1e-9, ...
+                    ['Data mismatch for ai channel 1 samples 1-100 against ' SourceType ' artifacts.']);
+            else
+                disp(['readData.json not found in ' SourceType ' artifact directory. Skipping data check.']);
+            end
+        end
+    end
+end

tools/tests/+ndr/+symmetry/+readArtifacts/INSTRUCTIONS.md

@@ -0,0 +1,74 @@
+# NDR Read Artifacts Instructions
+
+This folder contains MATLAB unit tests whose purpose is to read standard NDR artifacts generated by both the Python NDR test suite and the MATLAB test suite, comparing them against expected values.
+
+## Process overview:
+
+1. **Artifact Location**: The test suites will place their generated artifacts in the system's temporary directory (`tempdir()`).
+2. **Directory Structure**: Inside the temporary directory, artifacts can be found in a specific nested folder structure:
+   `NDR/symmetryTest/<SourceType>/<namespace>/<class_name>/<test_name>/`
+
+   - `<SourceType>`: Either `matlabArtifacts` or `pythonArtifacts`.
+   - `<namespace>`: The module/package location of the corresponding test (e.g., `reader`).
+   - `<class_name>`: The name of the test class (e.g., `readData`).
+   - `<test_name>`: The specific name of the test method that was executed (e.g., `testReadDataArtifacts`).
+
+3. **Testing Goals**: The MATLAB tests located in this `+readArtifacts` package should:
+   - Load the JSON representations created by the target suite (e.g., `metadata.json`, `readData.json`).
+   - Re-read the same example data files through the corresponding NDR reader in MATLAB.
+   - Assert that the numeric values (channel lists, sample rates, `t0`/`t1`, sample data)
+     match what the target suite stored.
+   - Run across both `pythonArtifacts` and `matlabArtifacts` using parameterized testing to
+     ensure parity in *both* directions.
+
+4. **Skipping**: When the artifact directory for a given `SourceType` does not exist, the
+   test must skip silently (not fail). This allows the suite to run on machines where only
+   one of the two languages has produced artifacts.
+
+## Example:
+
+Use MATLAB's `TestParameter` property block to dynamically pass the `SourceType` to your tests:
+
+```matlab
+classdef readData < matlab.unittest.TestCase
+    properties (TestParameter)
+        % Define the two potential sources of artifacts
+        SourceType = {'matlabArtifacts', 'pythonArtifacts'};
+    end
+
+    methods (Test)
+        function testReadDataArtifacts(testCase, SourceType)
+            artifactDir = fullfile(tempdir, 'NDR', 'symmetryTest', SourceType, ...
+                'reader', 'readData', 'testReadDataArtifacts');
+
+            if ~isfolder(artifactDir)
+                disp(['Artifact directory from ' SourceType ' does not exist. Skipping.']);
+                return;
+            end
+
+            % 1. Load the ground-truth metadata JSON that the target suite dumped.
+            fid = fopen(fullfile(artifactDir, 'metadata.json'), 'r');
+            expectedMeta = jsondecode(fread(fid, inf, '*char')');
+            fclose(fid);
+
+            % 2. Re-read the same example data in MATLAB and assert parity.
+            reader = ndr.reader.intan_rhd();
+            rhd_file = fullfile(ndr.fun.ndrpath(), 'example_data', 'example.rhd');
+            epochstreams = {rhd_file};
+            sr = reader.samplerate(epochstreams, 1, 'ai', 1);
+
+            testCase.verifyEqual(sr, expectedMeta.samplerate, 'AbsTol', 1e-9, ...
+                ['Sample rate mismatch against ' SourceType ' artifacts.']);
+        end
+    end
+end
+```
+
+## Running
+
+From MATLAB:
+
+```matlab
+% Verify artifacts
+results = runtests('ndr.symmetry.readArtifacts', 'IncludeSubpackages', true);
+```