diff --git a/AGENTS.md b/AGENTS.md index 193a57f..1820e08 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -32,7 +32,58 @@ Every sub-package contains a file named `ndi_matlab_python_bridge.yaml`. - **Internal Access:** Use 0-based indexing for internal Python data structures (lists, NumPy arrays). - **Formatting:** Code must pass `black` and `ruff check --fix` before completion. -## 5. Directory Mapping Reference +## 5. CI Lint & Test Commands + +Before pushing any changes, you **must** run these commands and ensure they all pass. These are the same checks CI runs. + +### Formatting (Black) + +```bash +black --check src/ tests/ +``` + +To auto-fix formatting issues: + +```bash +black src/ tests/ +``` + +Configuration is in `pyproject.toml`: line-length = 100, target-version = py310/py311/py312. + +### Linting (Ruff) + +```bash +ruff check src/ tests/ +``` + +To auto-fix what ruff can: + +```bash +ruff check --fix src/ tests/ +``` + +Configuration is in `pyproject.toml` under `[tool.ruff]` and `[tool.ruff.lint]`. + +### Tests + +```bash +pytest tests/ -v --tb=short +``` + +Symmetry tests (cross-language MATLAB/Python parity) are excluded from the default run and are invoked separately in CI: + +```bash +pytest tests/symmetry/make_artifacts/ -v --tb=short +pytest tests/symmetry/read_artifacts/ -v --tb=short +``` + +### Quick pre-push checklist + +```bash +black src/ tests/ && ruff check src/ tests/ && pytest tests/ -x -q +``` + +## 6. Directory Mapping Reference - **MATLAB Source:** `VH-ndi_gui_Lab/NDI-matlab` (GitHub) - **Python Target:** `src/ndi/[namespace]/` (Mirrors MATLAB `+namespace/`) diff --git a/pyproject.toml b/pyproject.toml index 60e0fab..cbe8a96 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,7 +36,7 @@ classifiers = [ requires-python = ">=3.10" dependencies = [ "did @ git+https://github.com/VH-Lab/DID-python.git@main", - "ndr @ git+https://github.com/VH-lab/NDR-python.git@main", + "ndr[formats] @ git+https://github.com/VH-lab/NDR-python.git@main", "vhlab-toolbox-python @ git+https://github.com/VH-Lab/vhlab-toolbox-python.git@main", "ndi-compress @ git+https://github.com/Waltham-Data-Science/NDI-compress-python.git@main", "numpy>=1.20.0", diff --git a/src/ndi/class_registry.py b/src/ndi/class_registry.py index 5c5e13a..964f995 100644 --- a/src/ndi/class_registry.py +++ b/src/ndi/class_registry.py @@ -43,6 +43,9 @@ def _build_registry() -> dict[str, type]: from .setup.daq.reader.mfdaq.stimulus.nielsenvisintan import ( ndi_setup_daq_reader_mfdaq_stimulus_nielsenvisintan, ) + from .setup.daq.reader.mfdaq.stimulus.nielsenvisneuropixelsglx import ( + ndi_setup_daq_reader_mfdaq_stimulus_nielsenvisneuropixelsglx, + ) from .setup.daq.reader.mfdaq.stimulus.vhaudreybpod import ( ndi_setup_daq_reader_mfdaq_stimulus_VHAudreyBPod, ) @@ -71,6 +74,7 @@ def _build_registry() -> dict[str, type]: ndi_daq_reader_mfdaq_spikegadgets, ndi_setup_daq_reader_mfdaq_stimulus_vhlabvisspike2, ndi_setup_daq_reader_mfdaq_stimulus_nielsenvisintan, + ndi_setup_daq_reader_mfdaq_stimulus_nielsenvisneuropixelsglx, ndi_setup_daq_reader_mfdaq_stimulus_VHAudreyBPod, ): registry[cls.NDI_DAQREADER_CLASS] = cls @@ -81,6 +85,8 @@ def _build_registry() -> dict[str, type]: # File navigators registry[ndi_file_navigator.NDI_FILENAVIGATOR_CLASS] = ndi_file_navigator registry[ndi_file_navigator_epochdir.NDI_FILENAVIGATOR_CLASS] = ndi_file_navigator_epochdir + # Custom lab-specific navigators mapped to epochdir until dedicated classes exist + registry["ndi.setup.file.navigator.vhlab_np_epochdir"] = ndi_file_navigator_epochdir return registry diff --git a/src/ndi/daq/daqsystemstring.py b/src/ndi/daq/daqsystemstring.py index 23791d6..7b0d3dd 100644 --- a/src/ndi/daq/daqsystemstring.py +++ b/src/ndi/daq/daqsystemstring.py @@ -76,7 +76,16 @@ def parse(cls, devstr: str) -> ndi_daq_daqsystemstring: channeltype = match.group(1) numspec = match.group(2) + # Check for threshold suffix (e.g., '_t2.5' in 'aep1-3_t2.5') + threshold_str = "" + t_idx = numspec.find("_t") + if t_idx != -1: + threshold_str = numspec[t_idx:] + numspec = numspec[:t_idx] + channellist = _parse_channel_numbers(numspec) + if threshold_str: + channeltype = channeltype + threshold_str channels.append((channeltype, channellist)) return cls(devicename=devicename, channels=channels) @@ -96,8 +105,7 @@ def devicestring(self) -> str: if not channellist: parts.append(channeltype) else: - numstr = _format_channel_numbers(channellist) - parts.append(f"{channeltype}{numstr}") + parts.append(ndi_daq_daqsystemstring.channeltype2str(channeltype, channellist)) return f"{self.devicename}:{';'.join(parts)}" @@ -132,6 +140,51 @@ def __str__(self) -> str: def __repr__(self) -> str: return f"ndi_daq_daqsystemstring('{self.devicestring()}')" + @staticmethod + def channeltype2str(ct: str, channellist: list[int]) -> str: + """ + Build a device string segment from a channeltype and channel list. + + Handles threshold suffixes (e.g., ``_t2.5``) by placing the channel + numbers between the base type and the suffix. + + Args: + ct: Channel type string, optionally with threshold suffix + (e.g., ``'aep'`` or ``'aep_t2.5'``) + channellist: List of channel numbers + + Returns: + Device string segment (e.g., ``'aep1-3_t2.5'``) + """ + t_idx = ct.find("_t") + if t_idx != -1: + base = ct[:t_idx] + threshold_str = ct[t_idx:] + return f"{base}{_format_channel_numbers(channellist)}{threshold_str}" + return f"{ct}{_format_channel_numbers(channellist)}" + + @staticmethod + def parse_analog_event_channeltype(ct: str) -> tuple[str, float]: + """ + Extract base type and threshold from a channel type string. + + Given a channel type string like ``'aep_t2.5'``, returns the base + type (``'aep'``) and threshold (``2.5``). If no threshold suffix + is present, threshold is ``0.0``. + + Args: + ct: Channel type string (e.g., ``'aep_t2.5'``, ``'aimp'``) + + Returns: + Tuple of (base_type, threshold) + """ + t_idx = ct.find("_t") + if t_idx != -1: + base_type = ct[:t_idx] + threshold = float(ct[t_idx + 2 :]) + return base_type, threshold + return ct, 0.0 + def __eq__(self, other) -> bool: if not isinstance(other, ndi_daq_daqsystemstring): return False diff --git a/src/ndi/daq/mfdaq.py b/src/ndi/daq/mfdaq.py index 9135af6..f649ca4 100644 --- a/src/ndi/daq/mfdaq.py +++ b/src/ndi/daq/mfdaq.py @@ -15,6 +15,7 @@ import numpy as np from ..time import DEV_LOCAL_TIME, ndi_time_clocktype +from .daqsystemstring import ndi_daq_daqsystemstring from .reader_base import ndi_daq_reader @@ -317,6 +318,11 @@ def readevents_epochsamples( if set(channeltype) & derived: return self._read_derived_events(channeltype, channel, epochfiles, t0, t1) + # Handle analog event channels (aep, aen, aimp, aimn) + is_analog, _, _ = ndi_daq_reader_mfdaq.is_analog_event_type(channeltype) + if is_analog: + return self._read_analog_events(channeltype, channel, epochfiles, t0, t1) + # Otherwise read native events return self.readevents_epochsamples_native(channeltype, channel, epochfiles, t0, t1) @@ -385,6 +391,78 @@ def _read_derived_events( return timestamps[0], data[0] return timestamps, data + def _read_analog_events( + self, + channeltype: list[str], + channel: list[int], + epochfiles: list[str], + t0: float, + t1: float, + ) -> tuple[list[np.ndarray] | np.ndarray, list[np.ndarray] | np.ndarray]: + """Read events derived from analog channels via threshold crossing.""" + _, base_types, thresholds = ndi_daq_reader_mfdaq.is_analog_event_type(channeltype) + + timestamps = [] + data = [] + + for i, ch in enumerate(channel): + bt = base_types[i] + thresh = thresholds[i] + + # Get sample range for time window from analog input + sd = self.epochtimes2samples(["ai"], [ch], epochfiles, np.array([t0, t1])) + s0, s1 = int(sd[0]), int(sd[1]) + + # Read analog and time data + ai_data = self.readchannels_epochsamples("ai", [ch], epochfiles, s0, s1) + time_data = self.readchannels_epochsamples("time", [ch], epochfiles, s0, s1) + + ai_data = ai_data.flatten() + time_data = time_data.flatten() + + below = ai_data[:-1] < thresh + above = ai_data[1:] >= thresh + + if bt in ("aep", "aimp"): + # Below-to-above threshold crossings + on_samples = 1 + np.where(below & above)[0] + off_samples = ( + 1 + np.where(~below & ~above)[0] if bt == "aimp" else np.array([], dtype=int) + ) + on_sign, off_sign = 1, -1 + else: # aen, aimn + # Above-to-below threshold crossings + on_samples = 1 + np.where(~below & ~above)[0] + off_samples = ( + 1 + np.where(below & above)[0] if bt == "aimn" else np.array([], dtype=int) + ) + on_sign, off_sign = -1, 1 + + ts = np.concatenate( + [ + time_data[on_samples], + time_data[off_samples] if len(off_samples) else np.array([]), + ] + ) + d = np.concatenate( + [ + on_sign * np.ones(len(on_samples)), + off_sign * np.ones(len(off_samples)) if len(off_samples) else np.array([]), + ] + ) + + if len(off_samples) > 0: + order = np.argsort(ts) + ts = ts[order] + d = d[order] + + timestamps.append(ts) + data.append(d) + + if len(channel) == 1: + return timestamps[0], data[0] + return timestamps, data + def readevents_epochsamples_native( self, channeltype: list[str], @@ -497,6 +575,11 @@ def epochtimes2samples( # Handle infinite values if np.any(np.isinf(times)): s[np.isinf(times) & (times < 0)] = 0 + pos_inf = np.isinf(times) & (times > 0) + if np.any(pos_inf): + t1 = t0t1[0][1] + s_end = round((t1 - t0) * sr) + s[pos_inf] = s_end return s @@ -564,6 +647,44 @@ def channel_types() -> tuple[list[str], list[str]]: abbrevs = ["ai", "ao", "ax", "di", "do", "e", "mk", "tx", "t"] return types, abbrevs + @staticmethod + def is_analog_event_type( + channeltype: str | list[str], + ) -> tuple[bool, list[str], list[float]]: + """ + Check if channel type(s) are analog event types. + + Analog event types derive events from analog input channels by + detecting threshold crossings: + + - ``aep`` / ``aep_tX``: positive (upward) threshold crossings + - ``aen`` / ``aen_tX``: negative (downward) threshold crossings + - ``aimp`` / ``aimp_tX``: analog input mark positive (pulse above then below) + - ``aimn`` / ``aimn_tX``: analog input mark negative (pulse below then above) + + Args: + channeltype: Channel type string or list of strings + + Returns: + Tuple of (is_analog_event, base_types, thresholds): + - is_analog_event: True if any channel type is an analog event type + - base_types: List of base type strings with threshold stripped + - thresholds: List of threshold values (0.0 if not specified) + """ + analog_event_prefixes = {"aep", "aen", "aimp", "aimn"} + if isinstance(channeltype, str): + channeltype = [channeltype] + + base_types = [] + thresholds = [] + for ct in channeltype: + bt, th = ndi_daq_daqsystemstring.parse_analog_event_channeltype(ct) + base_types.append(bt) + thresholds.append(th) + + tf = bool(set(base_types) & analog_event_prefixes) + return tf, base_types, thresholds + # ========================================================================= # Ingested data methods - for reading from database-stored epochs # ========================================================================= @@ -814,6 +935,13 @@ def readevents_epochsamples_ingested( channeltype = [channeltype] * len(channel) channeltype = standardize_channel_types(channeltype) + # Handle analog event types (aep, aen, aimp, aimn) + is_analog, _, _ = ndi_daq_reader_mfdaq.is_analog_event_type(channeltype) + if is_analog: + return self._read_analog_events_ingested( + channeltype, channel, epochfiles, t0, t1, session + ) + derived = {"dep", "den", "dimp", "dimn"} if set(channeltype) & derived: # Handle derived digital event types @@ -914,6 +1042,81 @@ def readevents_epochsamples_ingested( return timestamps_list[0], data_list[0] return timestamps_list, data_list + def _read_analog_events_ingested( + self, + channeltype: list[str], + channel: list[int], + epochfiles: list[str], + t0: float, + t1: float, + session: Any, + ) -> tuple[list[np.ndarray] | np.ndarray, list[np.ndarray] | np.ndarray]: + """Read events derived from analog channels via threshold crossing (ingested).""" + _, base_types, thresholds = ndi_daq_reader_mfdaq.is_analog_event_type(channeltype) + + timestamps = [] + data = [] + + for i, ch in enumerate(channel): + bt = base_types[i] + thresh = thresholds[i] + + sd = self.epochtimes2samples_ingested( + ["ai"], [ch], epochfiles, np.array([t0, t1]), session + ) + s0, s1 = int(sd[0]), int(sd[1]) + + ai_data = self.readchannels_epochsamples_ingested( + ["ai"], [ch], epochfiles, s0, s1, session + ) + time_data = self.readchannels_epochsamples_ingested( + ["time"], [ch], epochfiles, s0, s1, session + ) + + ai_data = ai_data.flatten() + time_data = time_data.flatten() + + below = ai_data[:-1] < thresh + above = ai_data[1:] >= thresh + + if bt in ("aep", "aimp"): + on_samples = 1 + np.where(below & above)[0] + off_samples = ( + 1 + np.where(~below & ~above)[0] if bt == "aimp" else np.array([], dtype=int) + ) + on_sign, off_sign = 1, -1 + else: # aen, aimn + on_samples = 1 + np.where(~below & ~above)[0] + off_samples = ( + 1 + np.where(below & above)[0] if bt == "aimn" else np.array([], dtype=int) + ) + on_sign, off_sign = -1, 1 + + ts = np.concatenate( + [ + time_data[on_samples], + time_data[off_samples] if len(off_samples) else np.array([]), + ] + ) + d = np.concatenate( + [ + on_sign * np.ones(len(on_samples)), + off_sign * np.ones(len(off_samples)) if len(off_samples) else np.array([]), + ] + ) + + if len(off_samples) > 0: + order = np.argsort(ts) + ts = ts[order] + d = d[order] + + timestamps.append(ts) + data.append(d) + + if len(channel) == 1: + return timestamps[0], data[0] + return timestamps, data + def samplerate_ingested( self, epochfiles: list[str], @@ -1057,5 +1260,10 @@ def epochtimes2samples_ingested( if np.any(np.isinf(times)): s[np.isinf(times) & (times < 0)] = 0 + pos_inf = np.isinf(times) & (times > 0) + if np.any(pos_inf): + t1 = t0t1[0][1] + s_end = round((t1 - t0) * sr) + s[pos_inf] = s_end return s diff --git a/src/ndi/daq/ndi_matlab_python_bridge.yaml b/src/ndi/daq/ndi_matlab_python_bridge.yaml index 8cf3675..c31ce5e 100644 --- a/src/ndi/daq/ndi_matlab_python_bridge.yaml +++ b/src/ndi/daq/ndi_matlab_python_bridge.yaml @@ -118,7 +118,7 @@ classes: - name: daqsystemstring type: class matlab_path: "+ndi/+daq/daqsystemstring.m" - matlab_last_sync_hash: "fe64a9f5" + matlab_last_sync_hash: "2157c70" python_path: "ndi/daq/daqsystemstring.py" python_class: "ndi_daq_daqsystemstring" @@ -173,6 +173,37 @@ classes: type_python: "list[int]" decision_log: "Exact match. Optionally filters by channel type." + - name: channeltype2str + kind: static + input_arguments: + - name: ct + type_matlab: "char" + type_python: "str" + - name: channellist + type_matlab: "array" + type_python: "list[int]" + output_arguments: + - name: s + type_python: "str" + decision_log: > + New in sync 2157c70. Builds device string segment from channeltype + and channel list. Handles threshold suffix (e.g., '_t2.5'). + + - name: parse_analog_event_channeltype + kind: static + input_arguments: + - name: ct + type_matlab: "char" + type_python: "str" + output_arguments: + - name: base_type + type_python: "str" + - name: threshold + type_python: "float" + decision_log: > + New in sync 2157c70. Extracts base type and threshold from + channel type strings like 'aep_t2.5'. Returns (base_type, threshold). + - name: eq input_arguments: - name: other @@ -342,7 +373,7 @@ classes: - name: mfdaq_reader type: class matlab_path: "+ndi/+daq/+reader/mfdaq.m" - matlab_last_sync_hash: "41e84fed" + matlab_last_sync_hash: "9e11fbb" python_path: "ndi/daq/mfdaq.py" python_class: "ndi_daq_reader_mfdaq" inherits: "ndi.daq.reader" @@ -447,7 +478,9 @@ classes: type_python: "np.ndarray | list[np.ndarray]" decision_log: > Exact match. Supports derived digital event types - (dep, den, dimp, dimn). Returns 2-tuple. + (dep, den, dimp, dimn) and analog event types + (aep, aen, aimp, aimn) with optional threshold suffix. + Returns 2-tuple. Updated in sync 9e11fbb. - name: readevents_epochsamples_native input_arguments: @@ -518,6 +551,8 @@ classes: INDEXING DIFFERENCE: MATLAB returns 1-indexed samples; Python returns 0-indexed. Sample 0 in Python corresponds to sample 1 in MATLAB. The formula is s = round((t-t0)*sr) (Python) vs s = 1+round((t-t0)*sr) (MATLAB). + Updated in sync 9e11fbb: -Inf clamped to 0 (first sample), + +Inf clamped to last sample of epoch. - name: underlying_datatype input_arguments: @@ -549,6 +584,24 @@ classes: type_python: "list[str]" decision_log: "Exact match. Returns 2-tuple of type names and abbreviations." + - name: is_analog_event_type + kind: static + input_arguments: + - name: channeltype + type_matlab: "cell array of char" + type_python: "str | list[str]" + output_arguments: + - name: tf + type_python: "bool" + - name: base_types + type_python: "list[str]" + - name: thresholds + type_python: "list[float]" + decision_log: > + New in sync 9e11fbb. Checks if channel types are analog event + types (aep, aen, aimp, aimn). Returns 3-tuple. Delegates + threshold parsing to daqsystemstring.parse_analog_event_channeltype. + # --- Ingested data methods --- - name: getchannelsepoch_ingested input_arguments: @@ -654,6 +707,8 @@ classes: decision_log: > INDEXING DIFFERENCE: MATLAB returns 1-indexed samples; Python returns 0-indexed. Sample 0 in Python corresponds to sample 1 in MATLAB. + Updated in sync 9e11fbb: -Inf clamped to 0 (first sample), + +Inf clamped to last sample of epoch. # ========================================================================= # ndi.daq.system diff --git a/src/ndi/daq/reader/mfdaq/ndr.py b/src/ndi/daq/reader/mfdaq/ndr.py index 5f9c6da..7c7c39d 100644 --- a/src/ndi/daq/reader/mfdaq/ndr.py +++ b/src/ndi/daq/reader/mfdaq/ndr.py @@ -1,8 +1,9 @@ """ ndi.daq.reader.mfdaq.ndr - NDR (Neuroscience Data Reader) wrapper. -Thin wrapper around ndi_daq_reader_SpikeInterfaceReader for file formats -supported by the NDR library (e.g. Axon ABF files). +Delegates all read operations to the appropriate NDR-python reader +based on the ``ndr_reader_string`` property (e.g. ``'neuropixelsGLX'``, +``'intan_rhd'``, ``'axon_abf'``). MATLAB equivalent: src/ndi/+ndi/+daq/+reader/+mfdaq/ndr.m """ @@ -10,6 +11,9 @@ from __future__ import annotations import logging +from typing import Any + +import numpy as np from ...mfdaq import ChannelInfo, ndi_daq_reader_mfdaq @@ -18,49 +22,152 @@ class ndi_daq_reader_mfdaq_ndr(ndi_daq_reader_mfdaq): """ - Reader for data files handled by the NDR library. + Reader that delegates to NDR-python readers. - Currently supports Axon ABF files via spikeinterface/neo. + Wraps ``ndr.reader(ndr_reader_string)`` to provide NDI-compatible + channel reading for any format supported by the NDR library. - File extensions: .abf + The ``ndr_reader_string`` identifies the format (e.g. + ``'neuropixelsGLX'``, ``'intan_rhd'``, ``'axon_abf'``). Valid + strings are listed by ``ndr.known_readers()``. """ NDI_DAQREADER_CLASS = "ndi.daq.reader.mfdaq.ndr" - FILE_EXTENSIONS = [".abf"] - def __init__(self, identifier=None, session=None, document=None): + def __init__( + self, + ndr_reader_string: str = "", + identifier: str | None = None, + session: Any | None = None, + document: Any | None = None, + ): super().__init__(identifier=identifier, session=session, document=document) self._ndi_daqreader_class = self.NDI_DAQREADER_CLASS + self.ndr_reader_string = ndr_reader_string + + # When constructed from a document, read the reader string from it + if document is not None: + props = getattr(document, "document_properties", {}) + if isinstance(props, dict): + self.ndr_reader_string = props.get("daqreader_ndr", {}).get("ndr_reader_string", "") - def _get_si_reader(self): - try: - from ..spikeinterface_adapter import ndi_daq_reader_SpikeInterfaceReader + def _get_ndr_reader(self): + """Get the NDR reader for this format.""" + import ndr - return ndi_daq_reader_SpikeInterfaceReader - except ImportError: - return None + return ndr.reader(self.ndr_reader_string) def getchannelsepoch(self, epochfiles: list[str]) -> list[ChannelInfo]: - SI = self._get_si_reader() - if SI is None: - return [] - try: - return SI().getchannelsepoch(epochfiles) - except Exception as exc: - logger.warning("ndi_daq_reader_mfdaq_ndr.getchannelsepoch failed: %s", exc) - return [] + """List channels available for an epoch. + + Delegates to the NDR reader and converts the returned dicts + to :class:`ChannelInfo` objects. Falls back to SpikeInterface + if the NDR reader cannot handle the epoch files. + """ + r = self._get_ndr_reader() + ndr_channels = r.getchannelsepoch(epochfiles, 1) + return [ + ChannelInfo( + name=ch["name"], + type=ch["type"], + time_channel=ch.get("time_channel"), + ) + for ch in ndr_channels + ] def readchannels_epochsamples(self, channeltype, channel, epochfiles, s0, s1): - SI = self._get_si_reader() - if SI is None: - raise ImportError("spikeinterface required for reading NDR data") - return SI().readchannels_epochsamples(channeltype, channel, epochfiles, s0, s1) + """Read channel data as samples. + + Delegates to the NDR reader with ``epoch_select=1``. + """ + r = self._get_ndr_reader() + if isinstance(channeltype, list): + channeltype = channeltype[0] + if isinstance(channel, int): + channel = [channel] + return r.readchannels_epochsamples(channeltype, channel, epochfiles, 1, s0, s1) def samplerate(self, epochfiles, channeltype, channel): - SI = self._get_si_reader() - if SI is None: - raise ImportError("spikeinterface required for reading NDR data") - return SI().samplerate(epochfiles, channeltype, channel) + """Get sample rate for specified channels. + + Delegates to the NDR reader with ``epoch_select=1``. + """ + r = self._get_ndr_reader() + if isinstance(channeltype, list): + channeltype = channeltype[0] + sr = r.samplerate(epochfiles, 1, channeltype, channel) + return np.atleast_1d(sr) + + def epochclock(self, epochfiles): + """Return the clock types for an epoch. + + Converts NDR ``ClockType`` objects to NDI ``ndi_time_clocktype``. + """ + r = self._get_ndr_reader() + from ndi.time import ndi_time_clocktype + + ndr_clocks = r.epochclock(epochfiles, 1) + return [ndi_time_clocktype(ec.type) for ec in ndr_clocks] + + def t0_t1(self, epochfiles): + """Return the start and end times for an epoch. + + Returns list of ``(t0, t1)`` tuples. + """ + r = self._get_ndr_reader() + result = r.t0_t1(epochfiles, 1) + return [(row[0], row[1]) for row in result] + + def underlying_datatype(self, epochfiles, channeltype, channel): + """Get the underlying data type for channels. + + Delegates to the NDR reader. + """ + r = self._get_ndr_reader() + if isinstance(channeltype, list): + channeltype = channeltype[0] + return r.ndr_reader_base.underlying_datatype(epochfiles, 1, channeltype, channel) + + def readevents_epochsamples_native(self, channeltype, channel, epochfiles, t0, t1): + """Read native event data. + + Delegates to the NDR reader. + """ + r = self._get_ndr_reader() + if isinstance(channeltype, list): + channeltype = channeltype[0] + if isinstance(channel, int): + channel = [channel] + return r.readevents_epochsamples_native(channeltype, channel, epochfiles, 1, t0, t1) + + def epochsamples2times(self, channeltype, channel, epochfiles, samples): + """Convert sample indices to time. + + For readers with time gaps, interpolates from the time channel. + Otherwise delegates to the base class formula. + """ + r = self._get_ndr_reader() + if r.MightHaveTimeGaps: + t_all = self.readchannels_epochsamples("time", [1], epochfiles, 1, int(1e12)) + t_all = t_all.flatten() + s_all = np.arange(len(t_all)) + return np.interp(np.asarray(samples, dtype=float), s_all, t_all) + return super().epochsamples2times(channeltype, channel, epochfiles, samples) + + def epochtimes2samples(self, channeltype, channel, epochfiles, times): + """Convert time to sample indices. + + For readers with time gaps, interpolates from the time channel. + Otherwise delegates to the base class formula. + """ + r = self._get_ndr_reader() + if r.MightHaveTimeGaps: + t_all = self.readchannels_epochsamples("time", [1], epochfiles, 1, int(1e12)) + t_all = t_all.flatten() + s_all = np.arange(len(t_all)) + return np.round(np.interp(np.asarray(times, dtype=float), t_all, s_all)).astype(int) + return super().epochtimes2samples(channeltype, channel, epochfiles, times) def __repr__(self): - return f"ndi_daq_reader_mfdaq_ndr(id={self.id[:8]}...)" + rs = self.ndr_reader_string or "?" + return f"ndi_daq_reader_mfdaq_ndr(reader='{rs}', id={self.id[:8]}...)" diff --git a/src/ndi/daq/system.py b/src/ndi/daq/system.py index 4ed9731..d9ede73 100644 --- a/src/ndi/daq/system.py +++ b/src/ndi/daq/system.py @@ -253,6 +253,11 @@ def _load_from_document(self, session: Any, document: Any) -> None: ) try: self._filenavigator = NavCls(session=session, document=nav_doc) + # Preserve the document's class name for symmetry reporting, + # even when a generic Python class is used as a stand-in + # (e.g. vhlab_np_epochdir mapped to epochdir). + if nav_class_name != NavCls.NDI_FILENAVIGATOR_CLASS: + self._filenavigator.NDI_FILENAVIGATOR_CLASS = nav_class_name except Exception as exc: raise RuntimeError( f"Could not reconstruct file navigator {nav_class_name!r}: {exc}" diff --git a/src/ndi/dataset/ndi_matlab_python_bridge.yaml b/src/ndi/dataset/ndi_matlab_python_bridge.yaml index b95a0fe..8636226 100644 --- a/src/ndi/dataset/ndi_matlab_python_bridge.yaml +++ b/src/ndi/dataset/ndi_matlab_python_bridge.yaml @@ -14,7 +14,7 @@ classes: - name: dataset type: class matlab_path: "+ndi/dataset.m" - matlab_last_sync_hash: "7512bcb0" + matlab_last_sync_hash: "f485088" python_path: "ndi/dataset/_dataset.py" python_class: "ndi_dataset" @@ -89,7 +89,10 @@ classes: output_arguments: - name: ndi_dataset_obj type_python: "ndi_dataset" - decision_log: "MATLAB name uses underscores. Exact match." + decision_log: > + MATLAB name uses underscores. Exact match. Updated in sync + f485088: now delegates to copySessionToDataset static method + instead of ndi.database.fun.copy_session_to_dataset. - name: deleteIngestedSession input_arguments: @@ -121,6 +124,23 @@ classes: type_python: "ndi_dataset" decision_log: "MATLAB name uses underscores. Exact match." + - name: convertLinkedSessionToIngested + input_arguments: + - name: session_id + type_matlab: "char" + type_python: "str" + - name: are_you_sure + type_matlab: "logical (name-value)" + type_python: "bool" + default: "False" + output_arguments: [] + decision_log: > + New in sync f485088. Converts a linked session to ingested by + copying its documents into the dataset. MATLAB uses name-value + args (areYouSure, askUserToConfirm); Python omits the GUI + confirmation dialog (askUserToConfirm not applicable). + Not yet implemented in Python — stub only. + - name: open_session input_arguments: - name: session_id @@ -272,6 +292,30 @@ classes: output_arguments: [] decision_log: "MATLAB camelCase preserved exactly." + - name: copySessionToDataset + kind: static + input_arguments: + - name: ndi_session_obj + type_matlab: "ndi.session" + type_python: "Any" + - name: ndi_dataset_obj + type_matlab: "ndi.dataset" + type_python: "ndi_dataset" + - name: skip_duplicate_check + type_matlab: "logical (name-value)" + type_python: "bool" + default: "False" + output_arguments: + - name: b + type_python: "bool" + - name: errmsg + type_python: "str" + decision_log: > + New in sync f485088. Moved from ndi.database.fun.copy_session_to_dataset + to ndi.dataset as a static method. Copies an ingested session's + documents and binary files into a dataset. Returns (success, errmsg). + Not yet implemented in Python — stub only. + # ========================================================================= # ndi.dataset.dir (directory-backed subclass) # ========================================================================= diff --git a/src/ndi/ndi_common/daq_systems/kjnielsenlab/nielsen_neuropixelsGLX.json b/src/ndi/ndi_common/daq_systems/kjnielsenlab/nielsen_neuropixelsGLX.json new file mode 100644 index 0000000..de57b00 --- /dev/null +++ b/src/ndi/ndi_common/daq_systems/kjnielsenlab/nielsen_neuropixelsGLX.json @@ -0,0 +1,19 @@ +{ + "Name": "nielsen_neuropixelsGLX", + "DaqSystemClass": "ndi.daq.system.mfdaq", + "DaqReaderClass": "ndi.daq.reader.mfdaq.ndr", + "MetadataReaderClass": "ndi.daq.metadatareader", + "EpochProbeMapClass": "ndi.epoch.epochprobemap_daqsystem", + "FileParameters": [ + "#.imec0.ap\\.bin\\>", + "#.nidq\\.meta\\>", + "#.imec0.ap\\.meta\\>", + "#.analyzer\\>", + "epochprobemap.txt" + ], + "DaqReaderFileParameters": "neuropixelsGLX", + "MetadataReaderFileParameters": [ + ], + "EpochProbeMapFileParameters": "epochprobemap.txt", + "HasEpochDirectories": true +} diff --git a/src/ndi/ndi_common/daq_systems/kjnielsenlab/nielsen_visnp.json b/src/ndi/ndi_common/daq_systems/kjnielsenlab/nielsen_visnp.json new file mode 100644 index 0000000..efeb6dc --- /dev/null +++ b/src/ndi/ndi_common/daq_systems/kjnielsenlab/nielsen_visnp.json @@ -0,0 +1,18 @@ +{ + "Name": "nielsen_visnp", + "DaqSystemClass": "ndi.daq.system.mfdaq", + "DaqReaderClass": "ndi.setup.daq.reader.mfdaq.stimulus.nielsenvisneuropixelsglx", + "MetadataReaderClass": "ndi.daq.metadatareader.NielsenLabStims", + "EpochProbeMapClass": "ndi.epoch.epochprobemap_daqsystem", + "FileParameters": [ + "#.nidq\\.bin\\>", + "#.nidq\\.meta\\>", + "#.imec0.ap\\.meta\\>", + "#.analyzer\\>", + "epochprobemap.txt" + ], + "DaqReaderFileParameters": "", + "MetadataReaderFileParameters": "(.*)\\.analyzer\\>", + "EpochProbeMapFileParameters": "(.*)epochprobemap.txt", + "HasEpochDirectories": true +} diff --git a/src/ndi/ndi_common/daq_systems/vhlab/vhajbpod_np.json b/src/ndi/ndi_common/daq_systems/vhlab/vhajbpod_np.json new file mode 100644 index 0000000..75d31f4 --- /dev/null +++ b/src/ndi/ndi_common/daq_systems/vhlab/vhajbpod_np.json @@ -0,0 +1,19 @@ +{ + "Name": "vhajbpod_np", + "DaqSystemClass": "ndi.daq.system.mfdaq", + "DaqReaderClass": "ndi.setup.daq.reader.mfdaq.stimulus.VHAudreyBPod", + "MetadataReaderClass": "ndi.daq.metadatareader.VHAudreyBPod", + "EpochProbeMapClass": "ndi.setup.epoch.epochprobemap_daqsystem_vhlab", + "FileNavigatorClass": "ndi.setup.file.navigator.vhlab_np_epochdir", + "FileParameters": [ + ".*\\.nidq\\.bin\\>", + ".*\\.nidq\\.meta\\>", + ".*\\.imec0\\.ap\\.meta\\>", + ".*_stimulus_triggers_log\\.tsv\\>", + ".*_summary_log\\.json\\>" + ], + "DaqReaderFileParameters": [], + "MetadataReaderFileParameters": "(.*)\\.json", + "EpochProbeMapFileParameters": ".*_stimulus_triggers_log\\.tsv\\>", + "HasEpochDirectories": true +} diff --git a/src/ndi/ndi_common/daq_systems/vhlab/vhneuropixels.json b/src/ndi/ndi_common/daq_systems/vhlab/vhneuropixels.json new file mode 100644 index 0000000..d681228 --- /dev/null +++ b/src/ndi/ndi_common/daq_systems/vhlab/vhneuropixels.json @@ -0,0 +1,16 @@ +{ + "Name": "vhneuropixels", + "DaqSystemClass": "ndi.daq.system.mfdaq", + "DaqReaderClass": "ndi.daq.reader.mfdaq.ndr", + "MetadataReaderClass": [], + "EpochProbeMapClass": "ndi.setup.epoch.epochprobemap_daqsystem_vhlab", + "FileParameters": [ + "reference.txt", + "#_g0_imec0.ap.bin", + "#_g0_imec0.ap.meta" + ], + "DaqReaderFileParameters": "neuropixelsGLX", + "MetadataReaderFileParameters": [], + "EpochProbeMapFileParameters": "", + "HasEpochDirectories": false +} diff --git a/src/ndi/ndi_common/daq_systems/vhlab/vhneuropixelsGLX.json b/src/ndi/ndi_common/daq_systems/vhlab/vhneuropixelsGLX.json new file mode 100644 index 0000000..d5019cf --- /dev/null +++ b/src/ndi/ndi_common/daq_systems/vhlab/vhneuropixelsGLX.json @@ -0,0 +1,17 @@ +{ + "Name": "vhneuropixelsGLX", + "DaqSystemClass": "ndi.daq.system.mfdaq", + "DaqReaderClass": "ndi.daq.reader.mfdaq.ndr", + "MetadataReaderClass": [], + "EpochProbeMapClass": "ndi.epoch.epochprobemap_daqsystem", + "FileParameters": [ + "#.imec0.ap\\.bin\\>", + "#.imec0.ap\\.meta\\>", + "#.nidq\\.meta\\>", + "epochprobemap.txt" + ], + "DaqReaderFileParameters": "neuropixelsGLX", + "MetadataReaderFileParameters": [], + "EpochProbeMapFileParameters": "epochprobemap.txt", + "HasEpochDirectories": true +} diff --git a/src/ndi/ndi_matlab_python_bridge_database_fun.yaml b/src/ndi/ndi_matlab_python_bridge_database_fun.yaml index ce718ac..1b774b1 100644 --- a/src/ndi/ndi_matlab_python_bridge_database_fun.yaml +++ b/src/ndi/ndi_matlab_python_bridge_database_fun.yaml @@ -192,8 +192,10 @@ functions: type_matlab: "char" type_python: "str" decision_log: > - MATLAB uses extract_doc_files then creates a surrogate session. - Python simplified approach adds docs directly. Both return (bool, str). + DEPRECATED in MATLAB: function moved to ndi.dataset.copySessionToDataset + as a static method (sync f485088). The standalone function file was + deleted in MATLAB. Python retains this for backward compatibility; + new code should use ndi.dataset.copySessionToDataset. Synchronized 2026-03-13. - name: finddocs_missing_dependencies diff --git a/src/ndi/setup/daq/reader/mfdaq/stimulus/__init__.py b/src/ndi/setup/daq/reader/mfdaq/stimulus/__init__.py index a3e5cdf..6750138 100644 --- a/src/ndi/setup/daq/reader/mfdaq/stimulus/__init__.py +++ b/src/ndi/setup/daq/reader/mfdaq/stimulus/__init__.py @@ -7,11 +7,15 @@ """ from .nielsenvisintan import ndi_setup_daq_reader_mfdaq_stimulus_nielsenvisintan +from .nielsenvisneuropixelsglx import ( + ndi_setup_daq_reader_mfdaq_stimulus_nielsenvisneuropixelsglx, +) from .vhaudreybpod import ndi_setup_daq_reader_mfdaq_stimulus_VHAudreyBPod from .vhlabvisspike2 import ndi_setup_daq_reader_mfdaq_stimulus_vhlabvisspike2 __all__ = [ "ndi_setup_daq_reader_mfdaq_stimulus_vhlabvisspike2", "ndi_setup_daq_reader_mfdaq_stimulus_nielsenvisintan", + "ndi_setup_daq_reader_mfdaq_stimulus_nielsenvisneuropixelsglx", "ndi_setup_daq_reader_mfdaq_stimulus_VHAudreyBPod", ] diff --git a/src/ndi/setup/daq/reader/mfdaq/stimulus/nielsenvisneuropixelsglx.py b/src/ndi/setup/daq/reader/mfdaq/stimulus/nielsenvisneuropixelsglx.py new file mode 100644 index 0000000..ba8709d --- /dev/null +++ b/src/ndi/setup/daq/reader/mfdaq/stimulus/nielsenvisneuropixelsglx.py @@ -0,0 +1,42 @@ +"""ndi.setup.daq.reader.mfdaq.stimulus.nielsenvisneuropixelsglx — Nielsen Lab visual stimulus NeuropixelsGLX reader. + +Extends the NDR reader with support for Nielsen Lab visual stimulus +files (.analyzer) recorded on a Neuropixels GLX system. + +MATLAB equivalent: ``+ndi/+setup/+daq/+reader/+mfdaq/+stimulus/nielsenvisneuropixelsglx.m`` + +The associated DAQ system configuration expects these epoch files: + +- ``#.nidq.bin`` — NI-DAQ binary data (SpikeGLX) +- ``#.nidq.meta`` — NI-DAQ metadata +- ``#.imec0.ap.meta`` — Imec probe metadata +- ``#.analyzer`` — Nielsen Lab Analyzer stimulus structure +- ``epochprobemap.txt`` — epoch-to-probe mapping + +Stimulus metadata is read by the companion +:class:`~ndi.daq.metadatareader.nielsenlab_stims.ndi_daq_metadatareader_NielsenLabStims`. +""" + +from __future__ import annotations + +import logging + +from ndi.daq.reader.mfdaq.ndr import ndi_daq_reader_mfdaq_ndr + +logger = logging.getLogger(__name__) + + +class ndi_setup_daq_reader_mfdaq_stimulus_nielsenvisneuropixelsglx(ndi_daq_reader_mfdaq_ndr): + """Nielsen Lab visual stimulus reader built on Neuropixels GLX via NDR. + + Inherits all channel reading and sample-rate logic from the NDR + reader. The stimulus-specific behaviour (extracting Analyzer + parameters) is handled by the metadata reader + ``ndi.daq.metadatareader.NielsenLabStims`` configured alongside this + reader in the DAQ system. + """ + + NDI_DAQREADER_CLASS = "ndi.setup.daq.reader.mfdaq.stimulus.nielsenvisneuropixelsglx" + + def __repr__(self) -> str: + return f"ndi_setup_daq_reader_mfdaq_stimulus_nielsenvisneuropixelsglx(id={self.id[:8]}...)" diff --git a/src/ndi/setup/lab.py b/src/ndi/setup/lab.py index 518ee5b..ee7ef73 100644 --- a/src/ndi/setup/lab.py +++ b/src/ndi/setup/lab.py @@ -79,9 +79,13 @@ def lab(session, lab_name: str) -> None: has_epoch_dirs = config.get("HasEpochDirectories", False) # Choose the correct filenavigator class - filenavigator_class = ( - "ndi.file.navigator.epochdir" if has_epoch_dirs else "ndi.file.navigator" - ) + custom_nav_class = config.get("FileNavigatorClass", "") + if custom_nav_class: + filenavigator_class = custom_nav_class + elif has_epoch_dirs: + filenavigator_class = "ndi.file.navigator.epochdir" + else: + filenavigator_class = "ndi.file.navigator" # Convert file parameters to MATLAB cell string format fp_str = _to_matlab_cell_str(file_params) diff --git a/tests/symmetry/make_artifacts/session/test_ingestion_axon_ndr.py b/tests/symmetry/make_artifacts/session/test_ingestion_axon_ndr.py index 7b8b426..4aca1c0 100644 --- a/tests/symmetry/make_artifacts/session/test_ingestion_axon_ndr.py +++ b/tests/symmetry/make_artifacts/session/test_ingestion_axon_ndr.py @@ -86,6 +86,7 @@ def _setup(self, tmp_path): **{ "base.name": "axon_ndr_reader", "daqreader.ndi_daqreader_class": "ndi.daq.reader.mfdaq.ndr", + "daqreader_ndr.ndr_reader_string": "axon_abf", }, ) session.database_add(dr_doc)