Stabilize notebook parity gates and CI data checks

.github/workflows/ci.yml

@@ -16,7 +16,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
-          lfs: true
+          lfs: false
 
       - uses: actions/setup-python@v5
         with:
@@ -81,7 +81,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
-          lfs: true
+          lfs: false
 
       - uses: actions/setup-python@v5
         with:
@@ -111,7 +111,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
-          lfs: true
+          lfs: false
 
       - uses: actions/setup-python@v5
         with:

.github/workflows/data-mirror-refresh.yml

@@ -29,7 +29,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
-          lfs: true
+          lfs: false
 
       - uses: actions/setup-python@v5
         with:

.github/workflows/full-parity-nightly.yml

@@ -12,7 +12,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
-          lfs: true
+          lfs: false
 
       - uses: actions/setup-python@v5
         with:
@@ -60,7 +60,10 @@ jobs:
             --notebook-group all \
             --timeout 900 \
             --skip-command-tests \
-            --parity-mode gate
+            --parity-mode gate \
+            --enforce-unique-images \
+            --min-unique-images-per-topic 1 \
+            --max-cross-topic-reuse-ratio 1.0
 
       - name: Enforce visual validation gate
         run: |

.github/workflows/parity-gate.yml

@@ -13,7 +13,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
-          lfs: true
+          lfs: false
 
       - uses: actions/setup-python@v5
         with:

.github/workflows/release-rc.yml

@@ -25,7 +25,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
-          lfs: true
+          lfs: false
           fetch-depth: 0
 
       - uses: actions/setup-python@v5
@@ -77,7 +77,10 @@ jobs:
             --timeout 900 \
             --skip-command-tests \
             --parity-mode gate \
-            --example-output-spec parity/example_output_spec.yml
+            --example-output-spec parity/example_output_spec.yml \
+            --enforce-unique-images \
+            --min-unique-images-per-topic 1 \
+            --max-cross-topic-reuse-ratio 1.0
 
       - name: Resolve latest validation PDF
         id: pdf

.github/workflows/release-stable.yml

@@ -27,7 +27,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
-          lfs: true
+          lfs: false
           fetch-depth: 0
 
       - uses: actions/setup-python@v5
@@ -94,7 +94,10 @@ jobs:
             --timeout 900 \
             --skip-command-tests \
             --parity-mode gate \
-            --example-output-spec parity/example_output_spec.yml
+            --example-output-spec parity/example_output_spec.yml \
+            --enforce-unique-images \
+            --min-unique-images-per-topic 1 \
+            --max-cross-topic-reuse-ratio 1.0
 
       - name: Resolve latest validation PDF
         id: pdf

.github/workflows/validation-pdf.yml

@@ -12,7 +12,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
-          lfs: true
+          lfs: false
 
       - uses: actions/setup-python@v5
         with:
@@ -64,7 +64,10 @@ jobs:
             --notebook-group all \
             --timeout 900 \
             --skip-command-tests \
-            --parity-mode gate
+            --parity-mode gate \
+            --enforce-unique-images \
+            --min-unique-images-per-topic 1 \
+            --max-cross-topic-reuse-ratio 1.0
 
       - name: Enforce visual validation gate
         run: |

notebooks/ConfigCollExamples.ipynb

@@ -72,55 +72,42 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# Signal/History workflow: explore covariates, spikes, history design, and events.\n",
-    "time = np.linspace(0.0, 4.0, 4001)\n",
-    "s1 = np.sin(2.0 * np.pi * 1.2 * time)\n",
-    "s2 = 0.5 * np.cos(2.0 * np.pi * 0.45 * time + 0.4)\n",
-    "s3 = s1 + s2\n",
+    "# ConfigCollExamples: compose and edit configuration collections.\n",
+    "from nstat.compat.matlab import TrialConfig, ConfigColl\n",
     "\n",
-    "cov = Covariate(time=time, data=np.column_stack([s1, s2, s3]), name=\"signals\", labels=[\"s1\", \"s2\", \"s3\"])\n",
-    "base_prob = np.clip(0.005 + 0.03 * (s3 > np.percentile(s3, 65)), 0.0, 0.4)\n",
-    "spike_times = time[rng.random(time.size) < base_prob]\n",
-    "spikes = SpikeTrain(spike_times=spike_times, t_start=float(time[0]), t_end=float(time[-1]), name=\"unit_1\")\n",
+    "tc1 = TrialConfig(covariateLabels=[\"Force\", \"f_x\"], Fs=2000.0, fitType=\"poisson\", name=\"cfg_force\")\n",
+    "tc2 = TrialConfig(covariateLabels=[\"Position\", \"x\"], Fs=2000.0, fitType=\"poisson\", name=\"cfg_pos\")\n",
+    "tcc = ConfigColl([tc1, tc2])\n",
     "\n",
-    "history = HistoryBasis(np.array([0.0, 0.005, 0.010, 0.020, 0.050]))\n",
-    "sample_times = time[::20]\n",
-    "H = history.design_matrix(spikes.spike_times, sample_times)\n",
+    "replacement = TrialConfig(covariateLabels=[\"Position\", \"y\"], Fs=1000.0, fitType=\"poisson\", name=\"cfg_pos_y\")\n",
+    "tcc.setConfig(2, replacement)\n",
+    "subset = tcc.getSubsetConfigs([1, 2])\n",
     "\n",
-    "burst_events = Events(times=np.array([0.5, 1.6, 2.4, 3.2]), labels=[\"A\", \"B\", \"C\", \"D\"])\n",
-    "centers, counts = spikes.bin_counts(bin_size_s=0.02)\n",
+    "names = tcc.getConfigNames()\n",
+    "rates = np.array([cfg.getSampleRate() for cfg in tcc.getConfigs()], dtype=float)\n",
+    "n_cov = np.array([len(cfg.getCovariateLabels()) for cfg in tcc.getConfigs()], dtype=float)\n",
     "\n",
-    "fig, axes = plt.subplots(3, 1, figsize=(9, 7), sharex=False)\n",
-    "axes[0].plot(time, cov.data[:, 0], label=\"s1\", linewidth=1.0)\n",
-    "axes[0].plot(time, cov.data[:, 1], label=\"s2\", linewidth=1.0)\n",
-    "axes[0].plot(time, cov.data[:, 2], label=\"s3\", linewidth=1.0)\n",
-    "axes[0].set_title(f\"{TOPIC}: signal and covariates\")\n",
-    "axes[0].legend(loc=\"upper right\")\n",
-    "\n",
-    "axes[1].bar(centers, counts, width=0.018, color=\"tab:gray\")\n",
-    "axes[1].vlines(burst_events.times, ymin=0.0, ymax=max(counts.max(), 1.0), color=\"tab:red\", linewidth=1.0)\n",
-    "axes[1].set_title(\"Binned spikes with event markers\")\n",
-    "axes[1].set_ylabel(\"count/bin\")\n",
-    "\n",
-    "im = axes[2].imshow(H.T, aspect=\"auto\", origin=\"lower\", cmap=\"magma\")\n",
-    "axes[2].set_title(\"History basis design matrix\")\n",
-    "axes[2].set_xlabel(\"time index\")\n",
-    "axes[2].set_ylabel(\"history bin\")\n",
-    "fig.colorbar(im, ax=axes[2], fraction=0.035, pad=0.02)\n",
+    "fig, axes = plt.subplots(1, 2, figsize=(9.2, 3.8))\n",
+    "axes[0].bar(names, rates, color=\"tab:purple\")\n",
+    "axes[0].set_title(\"Config sample rates\")\n",
+    "axes[0].set_ylabel(\"Hz\")\n",
     "\n",
+    "axes[1].bar(names, n_cov, color=\"tab:green\")\n",
+    "axes[1].set_title(\"Covariates per config\")\n",
+    "axes[1].set_ylabel(\"count\")\n",
     "plt.tight_layout()\n",
     "plt.show()\n",
     "\n",
-    "assert H.ndim == 2 and H.shape[1] == history.n_bins\n",
-    "assert spikes.spike_times.size > 5, \"Not enough spikes generated\"\n",
+    "assert len(subset.getConfigs()) == 2\n",
+    "assert float(rates[1]) == 1000.0\n",
     "\n",
     "CHECKPOINT_METRICS = {\n",
-    "    \"history_rows\": float(H.shape[0]),\n",
-    "    \"spike_count\": float(spikes.spike_times.size),\n",
+    "    \"num_configs\": float(len(tcc.getConfigs())),\n",
+    "    \"mean_sample_rate\": float(np.mean(rates)),\n",
     "}\n",
     "CHECKPOINT_LIMITS = {\n",
-    "    \"history_rows\": (50.0, 5000.0),\n",
-    "    \"spike_count\": (6.0, 6000.0),\n",
+    "    \"num_configs\": (2.0, 2.0),\n",
+    "    \"mean_sample_rate\": (1400.0, 1800.0),\n",
     "}\n"
    ]
   },

notebooks/CovCollExamples.ipynb

@@ -72,55 +72,65 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# Signal/History workflow: explore covariates, spikes, history design, and events.\n",
-    "time = np.linspace(0.0, 4.0, 4001)\n",
-    "s1 = np.sin(2.0 * np.pi * 1.2 * time)\n",
-    "s2 = 0.5 * np.cos(2.0 * np.pi * 0.45 * time + 0.4)\n",
-    "s3 = s1 + s2\n",
+    "# CovCollExamples: covariate collection queries, masking, and resampling.\n",
+    "from nstat.compat.matlab import Covariate, CovColl, History, nspikeTrain\n",
     "\n",
-    "cov = Covariate(time=time, data=np.column_stack([s1, s2, s3]), name=\"signals\", labels=[\"s1\", \"s2\", \"s3\"])\n",
-    "base_prob = np.clip(0.005 + 0.03 * (s3 > np.percentile(s3, 65)), 0.0, 0.4)\n",
-    "spike_times = time[rng.random(time.size) < base_prob]\n",
-    "spikes = SpikeTrain(spike_times=spike_times, t_start=float(time[0]), t_end=float(time[-1]), name=\"unit_1\")\n",
+    "t = np.arange(0.0, 5.0 + 0.001, 0.001)\n",
+    "position = Covariate(\n",
+    "    time=t,\n",
+    "    data=np.column_stack([np.exp(-t), np.sin(2.0 * np.pi * t), np.sin(2.0 * np.pi * t) ** 3]),\n",
+    "    name=\"Position\",\n",
+    "    labels=[\"x\", \"y\", \"z\"],\n",
+    ")\n",
+    "force = Covariate(\n",
+    "    time=t,\n",
+    "    data=np.column_stack([np.abs(np.sin(2.0 * np.pi * t)), np.abs(np.sin(2.0 * np.pi * t)) ** 2]),\n",
+    "    name=\"Force\",\n",
+    "    labels=[\"f_x\", \"f_y\"],\n",
+    ")\n",
+    "cc = CovColl([position, force])\n",
     "\n",
-    "history = HistoryBasis(np.array([0.0, 0.005, 0.010, 0.020, 0.050]))\n",
-    "sample_times = time[::20]\n",
-    "H = history.design_matrix(spikes.spike_times, sample_times)\n",
-    "\n",
-    "burst_events = Events(times=np.array([0.5, 1.6, 2.4, 3.2]), labels=[\"A\", \"B\", \"C\", \"D\"])\n",
-    "centers, counts = spikes.bin_counts(bin_size_s=0.02)\n",
-    "\n",
-    "fig, axes = plt.subplots(3, 1, figsize=(9, 7), sharex=False)\n",
-    "axes[0].plot(time, cov.data[:, 0], label=\"s1\", linewidth=1.0)\n",
-    "axes[0].plot(time, cov.data[:, 1], label=\"s2\", linewidth=1.0)\n",
-    "axes[0].plot(time, cov.data[:, 2], label=\"s3\", linewidth=1.0)\n",
-    "axes[0].set_title(f\"{TOPIC}: signal and covariates\")\n",
-    "axes[0].legend(loc=\"upper right\")\n",
-    "\n",
-    "axes[1].bar(centers, counts, width=0.018, color=\"tab:gray\")\n",
-    "axes[1].vlines(burst_events.times, ymin=0.0, ymax=max(counts.max(), 1.0), color=\"tab:red\", linewidth=1.0)\n",
-    "axes[1].set_title(\"Binned spikes with event markers\")\n",
-    "axes[1].set_ylabel(\"count/bin\")\n",
+    "fig1 = plt.figure(figsize=(9.0, 4.2))\n",
+    "cc.plot()\n",
+    "plt.title(f\"{TOPIC}: all covariates\")\n",
+    "plt.xlabel(\"time [s]\")\n",
+    "plt.tight_layout()\n",
+    "plt.show()\n",
     "\n",
-    "im = axes[2].imshow(H.T, aspect=\"auto\", origin=\"lower\", cmap=\"magma\")\n",
-    "axes[2].set_title(\"History basis design matrix\")\n",
-    "axes[2].set_xlabel(\"time index\")\n",
-    "axes[2].set_ylabel(\"history bin\")\n",
-    "fig.colorbar(im, ax=axes[2], fraction=0.035, pad=0.02)\n",
+    "_pos = cc.getCov(\"Position\")\n",
+    "_force = cc.getCov(\"Force\")\n",
+    "cc.resample(200.0)\n",
+    "cc.setMask([\"Position\", \"Force\"])\n",
     "\n",
+    "fig2 = plt.figure(figsize=(9.0, 4.2))\n",
+    "cc.plot()\n",
+    "plt.title(\"Resampled/masked covariates\")\n",
+    "plt.xlabel(\"time [s]\")\n",
     "plt.tight_layout()\n",
     "plt.show()\n",
     "\n",
+    "X, labels = cc.dataToMatrix()\n",
+    "n_before_remove = cc.nActCovar()\n",
+    "cc.removeCovariate(\"Force\")\n",
+    "n_after_remove = cc.nActCovar()\n",
+    "\n",
+    "assert X.shape[1] >= 4\n",
+    "assert n_after_remove == max(1, n_before_remove - 1)\n",
+    "history = History(bin_edges_s=np.array([0.0, 0.01, 0.03], dtype=float))\n",
+    "spikes = nspikeTrain(spike_times=np.sort(rng.random(25) * 0.5), t_start=0.0, t_end=0.5, name=\"tmp\")\n",
+    "H = history.computeHistory(spikes.spike_times, np.arange(0.0, 0.5, 0.01))\n",
     "assert H.ndim == 2 and H.shape[1] == history.n_bins\n",
-    "assert spikes.spike_times.size > 5, \"Not enough spikes generated\"\n",
+    "assert spikes.spike_times.size > 5\n",
     "\n",
     "CHECKPOINT_METRICS = {\n",
-    "    \"history_rows\": float(H.shape[0]),\n",
-    "    \"spike_count\": float(spikes.spike_times.size),\n",
+    "    \"matrix_rows\": float(X.shape[0]),\n",
+    "    \"matrix_cols\": float(X.shape[1]),\n",
+    "    \"active_covariates_after_remove\": float(n_after_remove),\n",
     "}\n",
     "CHECKPOINT_LIMITS = {\n",
-    "    \"history_rows\": (50.0, 5000.0),\n",
-    "    \"spike_count\": (6.0, 6000.0),\n",
+    "    \"matrix_rows\": (200.0, 2000.0),\n",
+    "    \"matrix_cols\": (4.0, 8.0),\n",
+    "    \"active_covariates_after_remove\": (1.0, 3.0),\n",
     "}\n"
    ]
   },