pull upstream & resolve merge conflicts

Author: JaerongA

.gitignore

@@ -123,3 +123,6 @@ docker-compose.y*ml
 # vscode settings
 .vscode
 *.code-workspace
+
+# exports/notes
+temp*

.pre-commit-config.yaml

@@ -0,0 +1,56 @@
+default_stages: [commit, push]
+exclude: (^.github/|^docs/|^images/)
+
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-yaml
+      - id: check-added-large-files # prevent giant files from being committed
+      - id: requirements-txt-fixer
+      - id: mixed-line-ending
+        args: ["--fix=lf"]
+        description: Forces to replace line ending by the UNIX 'lf' character.
+
+  # black
+  - repo: https://github.com/psf/black
+    rev: 22.12.0
+    hooks:
+      - id: black
+      - id: black-jupyter
+        args:
+          - --line-length=88
+
+  # isort
+  - repo: https://github.com/pycqa/isort
+    rev: 5.11.2
+    hooks:
+      - id: isort
+        args: ["--profile", "black"]
+        description: Sorts imports in an alphabetical order
+
+  # flake8
+  - repo: https://github.com/pycqa/flake8
+    rev: 4.0.1
+    hooks:
+      - id: flake8
+        args: # arguments to configure flake8
+          # making isort line length compatible with black
+          - "--max-line-length=88"
+          - "--max-complexity=18"
+          - "--select=B,C,E,F,W,T4,B9"
+
+          # these are errors that will be ignored by flake8
+          # https://www.flake8rules.com/rules/{code}.html
+          - "--ignore=E203,E501,W503,W605"
+          # E203 - Colons should not have any space before them.
+          #        Needed for list indexing
+          # E501 - Line lengths are recommended to be no greater than 79 characters.
+          #        Needed as we conform to 88
+          # W503 - Line breaks should occur after the binary operator.
+          #        Needed because not compatible with black
+          # W605 - a backslash-character pair that is not a valid escape sequence now
+          #        generates a DeprecationWarning. This will eventually become a SyntaxError.
+          #        Needed because we use \d as an escape sequence

CHANGELOG.md

@@ -34,7 +34,7 @@ Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and
 
 + Update - Rename module for acute probe insertions from `ephys.py` to `ephys_acute.py`.
 + Add - Module for pre-clustering steps (`ephys_precluster.py`), which is built off of `ephys_acute.py`.
-+ Add - Module for chronic probe insertions (`ephys_chronic.py`).  
++ Add - Module for chronic probe insertions (`ephys_chronic.py`).
 + Bugfix - Missing `fileTimeSecs` key in SpikeGLX meta file.
 + Update - Move common functions to `element-interface` package.
 + Add - NWB export function
@@ -55,7 +55,7 @@ Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and
 + Add - Readers for: `SpikeGLX`, `Open Ephys`, `Kilosort`
 + Add - Probe table supporting: Neuropixels probes 1.0 - 3A, 1.0 - 3B, 2.0 - SS,
   2.0 - MS
-  
+
 [0.2.1]: https://github.com/datajoint/element-array-ephys/releases/tag/0.2.1
 [0.2.0]: https://github.com/datajoint/element-array-ephys/releases/tag/0.2.0
 [0.1.4]: https://github.com/datajoint/element-array-ephys/releases/tag/0.1.4

CONTRIBUTING.md

@@ -1,3 +1,3 @@
 # Contribution Guidelines
 
-This project follows the [DataJoint Contribution Guidelines](https://docs.datajoint.io/python/community/02-Contribute.html). Please reference the link for more full details.
+This project follows the [DataJoint Contribution Guidelines](https://datajoint.com/docs/community/contribute/). Please reference the link for more full details.

README.md

@@ -8,5 +8,5 @@ fully functional pipeline.
 
 ![diagram](https://raw.githubusercontent.com/datajoint/element-array-ephys/main/images/diagram_flowchart.svg)
 
-Installation and usage instructions can be found at the 
+Installation and usage instructions can be found at the
 [Element documentation](https://datajoint.com/docs/elements/element-array-ephys).

docs/src/api/make_pages.py

@@ -41,4 +41,3 @@
 
 with mkdocs_gen_files.open("api/navigation.md", "w") as nav_file:
     nav_file.writelines(nav.build_literate_nav())
-    

docs/src/concepts.md

@@ -104,6 +104,14 @@ Tables related to information about physiological recordings and automatic inges
 | CuratedClusting.Unit | A part table containing single unit information after spike sorting and optional curation. |
 | WaveformSet | A table containing spike waveforms for single units. |
 
+### `ephys_report` schema ([API docs](../api/element_array_ephys/ephys_report))
+Tables for storing probe or unit-level visualization results.
+
+| Table | Description |
+| --- | --- |
+| ProbeLevelReport | A table to store drift map figures generated from each recording probe. |
+| UnitLevelReport | A table to store figures (waveforms, autocorrelogram, peak waveform + neighbors) generated for each unit. |
+
 ## Element Development
 
 Through our interviews and direct collaboration on the precursor projects, we identified the common motifs to create the [Array Electrophysiology Element](https://github.com/datajoint/element-array-ephys).

element_array_ephys/__init__.py

@@ -1,7 +1,11 @@
-import datajoint as dj
 import logging
 import os
 
+import datajoint as dj
+
+from . import ephys_acute as ephys
+
+__all__ = ["ephys", "get_logger"]
 
 dj.config["enable_python_native_blobs"] = True
 
@@ -10,6 +14,3 @@ def get_logger(name):
     log = logging.getLogger(name)
     log.setLevel(os.getenv("LOGLEVEL", "INFO"))
     return log
-
-
-from . import ephys_acute as ephys

element_array_ephys/ephys_acute.py

@@ -1,17 +1,17 @@
-import datajoint as dj
+import gc
+import importlib
+import inspect
 import pathlib
 import re
-import numpy as np
-import inspect
-import importlib
-import gc
 from decimal import Decimal
-import pandas as pd
 
-from element_interface.utils import find_root_directory, find_full_path, dict_to_uuid
+import datajoint as dj
+import numpy as np
+import pandas as pd
+from element_interface.utils import dict_to_uuid, find_full_path, find_root_directory
 
-from .readers import spikeglx, kilosort, openephys
-from . import probe, get_logger, ephys_report
+from . import ephys_report, get_logger, probe
+from .readers import kilosort, openephys, spikeglx
 
 log = get_logger(__name__)
 
@@ -127,7 +127,7 @@ class AcquisitionSoftware(dj.Lookup):
     """
 
     definition = """  # Software used for recording of neuropixels probes
-    acq_software: varchar(24)    
+    acq_software: varchar(24)
     """
     contents = zip(["SpikeGLX", "Open Ephys"])
 
@@ -264,7 +264,7 @@ class EphysRecording(dj.Imported):
 
     definition = """
     # Ephys recording from a probe insertion for a given session.
-    -> ProbeInsertion      
+    -> ProbeInsertion
     ---
     -> probe.ElectrodeConfig
     -> AcquisitionSoftware
@@ -465,9 +465,9 @@ class Electrode(dj.Part):
 
         definition = """
         -> master
-        -> probe.ElectrodeConfig.Electrode  
+        -> probe.ElectrodeConfig.Electrode
         ---
-        lfp: longblob               # (uV) recorded lfp at this electrode 
+        lfp: longblob               # (uV) recorded lfp at this electrode
         """
 
     # Only store LFP for every 9th channel, due to high channel density,
@@ -615,7 +615,7 @@ class ClusteringParamSet(dj.Lookup):
     # Parameter set to be used in a clustering procedure
     paramset_idx:  smallint
     ---
-    -> ClusteringMethod    
+    -> ClusteringMethod
     paramset_desc: varchar(128)
     param_set_hash: uuid
     unique index (param_set_hash)
@@ -800,7 +800,7 @@ class Clustering(dj.Imported):
     # Clustering Procedure
     -> ClusteringTask
     ---
-    clustering_time: datetime  # time of generation of this set of clustering results 
+    clustering_time: datetime  # time of generation of this set of clustering results
     package_version='': varchar(16)
     """
 
@@ -841,6 +841,10 @@ def make(self, key):
                         spikeglx_meta_filepath.parent
                     )
                     spikeglx_recording.validate_file("ap")
+                    run_CatGT = (
+                        params.pop("run_CatGT", True)
+                        and "_tcat." not in spikeglx_meta_filepath.stem
+                    )
 
                     if clustering_method.startswith("pykilosort"):
                         kilosort_triggering.run_pykilosort(
@@ -861,7 +865,7 @@ def make(self, key):
                             ks_output_dir=kilosort_dir,
                             params=params,
                             KS2ver=f'{Decimal(clustering_method.replace("kilosort", "")):.1f}',
-                            run_CatGT=True,
+                            run_CatGT=run_CatGT,
                         )
                         run_kilosort.run_modules()
                 elif acq_software == "Open Ephys":
@@ -925,11 +929,11 @@ class Curation(dj.Manual):
     -> Clustering
     curation_id: int
     ---
-    curation_time: datetime             # time of generation of this set of curated clustering results 
+    curation_time: datetime             # time of generation of this set of curated clustering results
     curation_output_dir: varchar(255)   # output directory of the curated results, relative to root data directory
     quality_control: bool               # has this clustering result undergone quality control?
     manual_curation: bool               # has manual curation been performed on this clustering result?
-    curation_note='': varchar(2000)  
+    curation_note='': varchar(2000)
     """
 
     def create1_from_clustering_task(self, key, curation_note=""):
@@ -978,7 +982,7 @@ class CuratedClustering(dj.Imported):
 
     definition = """
     # Clustering results of a curation.
-    -> Curation    
+    -> Curation
     """
 
     class Unit(dj.Part):
@@ -1005,7 +1009,7 @@ class Unit(dj.Part):
         spike_count: int         # how many spikes in this recording for this unit
         spike_times: longblob    # (s) spike times of this unit, relative to the start of the EphysRecording
         spike_sites : longblob   # array of electrode associated with each spike
-        spike_depths=null : longblob  # (um) array of depths associated with each spike, relative to the (0, 0) of the probe    
+        spike_depths=null : longblob  # (um) array of depths associated with each spike, relative to the (0, 0) of the probe
         """
 
     def make(self, key):
@@ -1131,8 +1135,8 @@ class Waveform(dj.Part):
         # Spike waveforms and their mean across spikes for the given unit
         -> master
         -> CuratedClustering.Unit
-        -> probe.ElectrodeConfig.Electrode  
-        --- 
+        -> probe.ElectrodeConfig.Electrode
+        ---
         waveform_mean: longblob   # (uV) mean waveform across spikes of the given unit
         waveforms=null: longblob  # (uV) (spike x sample) waveforms of a sampling of spikes at the given electrode for the given unit
         """
@@ -1260,7 +1264,7 @@ class QualityMetrics(dj.Imported):
 
     definition = """
     # Clusters and waveforms metrics
-    -> CuratedClustering    
+    -> CuratedClustering
     """
 
     class Cluster(dj.Part):
@@ -1285,26 +1289,26 @@ class Cluster(dj.Part):
             contamination_rate (float): Frequency of spikes in the refractory period.
         """
 
-        definition = """   
+        definition = """
         # Cluster metrics for a particular unit
         -> master
         -> CuratedClustering.Unit
         ---
-        firing_rate=null: float # (Hz) firing rate for a unit 
+        firing_rate=null: float # (Hz) firing rate for a unit
         snr=null: float  # signal-to-noise ratio for a unit
         presence_ratio=null: float  # fraction of time in which spikes are present
         isi_violation=null: float   # rate of ISI violation as a fraction of overall rate
         number_violation=null: int  # total number of ISI violations
         amplitude_cutoff=null: float  # estimate of miss rate based on amplitude histogram
         isolation_distance=null: float  # distance to nearest cluster in Mahalanobis space
-        l_ratio=null: float  # 
+        l_ratio=null: float  #
         d_prime=null: float  # Classification accuracy based on LDA
         nn_hit_rate=null: float  # Fraction of neighbors for target cluster that are also in target cluster
         nn_miss_rate=null: float # Fraction of neighbors outside target cluster that are in target cluster
         silhouette_score=null: float  # Standard metric for cluster overlap
         max_drift=null: float  # Maximum change in spike depth throughout recording
-        cumulative_drift=null: float  # Cumulative change in spike depth throughout recording 
-        contamination_rate=null: float # 
+        cumulative_drift=null: float  # Cumulative change in spike depth throughout recording
+        contamination_rate=null: float #
         """
 
     class Waveform(dj.Part):
@@ -1324,7 +1328,7 @@ class Waveform(dj.Part):
             velocity_below (float) inverse velocity of waveform propagation from soma toward the bottom of the probe.
         """
 
-        definition = """   
+        definition = """
         # Waveform metrics for a particular unit
         -> master
         -> CuratedClustering.Unit
@@ -1476,20 +1480,20 @@ def get_neuropixels_channel2electrode_map(
     return channel2electrode_map
 
 
-def generate_electrode_config(probe_type: str, electrodes: list) -> dict:
+def generate_electrode_config(probe_type: str, electrode_keys: list) -> dict:
     """Generate and insert new ElectrodeConfig
 
     Args:
         probe_type (str): probe type (e.g. neuropixels 2.0 - SS)
-        electrodes (list): Electrode dict (keys of the probe.ProbeType.Electrode table)
+        electrode_keys (list): list of keys of the probe.ProbeType.Electrode table
 
     Returns:
         dict: representing a key of the probe.ElectrodeConfig table
     """
     # compute hash for the electrode config (hash of dict of all ElectrodeConfig.Electrode)
-    electrode_config_hash = dict_to_uuid({k["electrode"]: k for k in electrodes})
+    electrode_config_hash = dict_to_uuid({k["electrode"]: k for k in electrode_keys})
 
-    electrode_list = sorted([k["electrode"] for k in electrodes])
+    electrode_list = sorted([k["electrode"] for k in electrode_keys])
     electrode_gaps = (
         [-1]
         + np.where(np.diff(electrode_list) > 1)[0].tolist()
@@ -1514,7 +1518,7 @@ def generate_electrode_config(probe_type: str, electrodes: list) -> dict:
             }
         )
         probe.ElectrodeConfig.Electrode.insert(
-            {**electrode_config_key, **electrode} for electrode in electrodes
+            {**electrode_config_key, **electrode} for electrode in electrode_keys
         )
 
     return electrode_config_key