Merge branch 'SpikeInterface:main' into add_time_vector_case_to_get_duration

Author: JoeZiminski

.github/actions/build-test-environment/action.yml

@@ -21,7 +21,7 @@ runs:
         python -m pip install -U pip  # Official recommended way
         source ${{ github.workspace }}/test_env/bin/activate
         pip install tabulate  # This produces summaries at the end
-        pip install -e .[test,extractors,streaming_extractors,full]
+        pip install -e .[test,extractors,streaming_extractors,test_extractors,full]
       shell: bash
     - name: Force installation of latest dev from key-packages when running dev (not release)
       run: |

.github/actions/install-wine/action.yml

@@ -2,20 +2,29 @@ name: Install packages
 description: This action installs the package and its dependencies for testing
 
 inputs:
-  python-version:
-    description: 'Python version to set up'
-    required: false
   os:
     description: 'Operating system to set up'
-    required: false
+    required: true
 
 runs:
   using: "composite"
   steps:
-    - name: Install wine (needed for Plexon2)
+    - name: Install wine on Linux
+      if: runner.os == 'Linux'
       run: |
         sudo rm -f /etc/apt/sources.list.d/microsoft-prod.list
         sudo dpkg --add-architecture i386
         sudo apt-get update -qq
         sudo apt-get install -yqq --allow-downgrades libc6:i386 libgcc-s1:i386 libstdc++6:i386 wine
       shell: bash
+    - name: Install wine on macOS
+      if: runner.os == 'macOS'
+      run: |
+        brew install --cask xquartz
+        brew install --cask wine-stable
+      shell: bash
+
+    - name: Skip installation on Windows
+      if: ${{ inputs.os == 'Windows' }}
+      run: echo "Skipping Wine installation on Windows. Not necessary."
+      shell: bash

.github/determine_testing_environment.py

@@ -30,56 +30,59 @@
 exporters_changed = False
 sortingcomponents_changed = False
 generation_changed = False
+stream_extractors_changed = False
 
 
 for changed_file in changed_files_in_the_pull_request_paths:
 
     file_is_in_src = changed_file.parts[0] == "src"
 
-    if not file_is_in_src:
-
-        if changed_file.name == "pyproject.toml":
-            pyproject_toml_changed = True
-
-    else:
-        if changed_file.name == "neobaseextractor.py":
-            neobaseextractor_changed = True
-        elif changed_file.name == "plexon2.py":
-            extractors_changed = True
-        elif "core" in changed_file.parts:
-            conditions_changed = True
-        elif "extractors" in changed_file.parts:
-            extractors_changed = True
-        elif "preprocessing" in changed_file.parts:
-            preprocessing_changed = True
-        elif "postprocessing" in changed_file.parts:
-            postprocessing_changed = True
-        elif "qualitymetrics" in changed_file.parts:
-            qualitymetrics_changed = True
-        elif "comparison" in changed_file.parts:
-            comparison_changed = True
-        elif "curation" in changed_file.parts:
-            curation_changed = True
-        elif "widgets" in changed_file.parts:
-            widgets_changed = True
-        elif "exporters" in changed_file.parts:
-            exporters_changed = True
-        elif "sortingcomponents" in changed_file.parts:
-            sortingcomponents_changed = True
-        elif "generation" in changed_file.parts:
-            generation_changed = True
-        elif "sorters" in changed_file.parts:
-            if "external" in changed_file.parts:
-                sorters_external_changed = True
-            elif "internal" in changed_file.parts:
-                sorters_internal_changed = True
-            else:
-                sorters_changed = True
+    if changed_file.name == "pyproject.toml":
+        pyproject_toml_changed = True
+    elif changed_file.name == "neobaseextractor.py":
+        neobaseextractor_changed = True
+        extractors_changed = True
+    elif changed_file.name == "plexon2.py":
+        plexon2_changed = True
+    elif changed_file.name == "nwbextractors.py":
+        extractors_changed = True  # There are NWB tests that are not streaming
+        stream_extractors_changed = True
+    elif changed_file.name == "iblextractors.py":
+        stream_extractors_changed = True
+    elif "core" in changed_file.parts:
+        core_changed = True
+    elif "extractors" in changed_file.parts:
+        extractors_changed = True
+    elif "preprocessing" in changed_file.parts:
+        preprocessing_changed = True
+    elif "postprocessing" in changed_file.parts:
+        postprocessing_changed = True
+    elif "qualitymetrics" in changed_file.parts:
+        qualitymetrics_changed = True
+    elif "comparison" in changed_file.parts:
+        comparison_changed = True
+    elif "curation" in changed_file.parts:
+        curation_changed = True
+    elif "widgets" in changed_file.parts:
+        widgets_changed = True
+    elif "exporters" in changed_file.parts:
+        exporters_changed = True
+    elif "sortingcomponents" in changed_file.parts:
+        sortingcomponents_changed = True
+    elif "generation" in changed_file.parts:
+        generation_changed = True
+    elif "sorters" in changed_file.parts:
+        if "external" in changed_file.parts:
+            sorters_external_changed = True
+        elif "internal" in changed_file.parts:
+            sorters_internal_changed = True
+        else:
+            sorters_changed = True
 
 
 run_everything = core_changed or pyproject_toml_changed or neobaseextractor_changed
 run_generation_tests = run_everything or generation_changed
-run_extractor_tests = run_everything or extractors_changed
+run_extractor_tests = run_everything or extractors_changed or plexon2_changed
 run_preprocessing_tests = run_everything or preprocessing_changed
 run_postprocessing_tests = run_everything or postprocessing_changed
 run_qualitymetrics_tests = run_everything or qualitymetrics_changed
@@ -93,8 +96,11 @@
 run_sorters_test = run_everything or sorters_changed
 run_internal_sorters_test = run_everything or run_sortingcomponents_tests or sorters_internal_changed
 
+run_streaming_extractors_test = stream_extractors_changed
+
 install_plexon_dependencies = plexon2_changed
 
+
 environment_varaiables_to_add = {
     "RUN_EXTRACTORS_TESTS": run_extractor_tests,
     "RUN_PREPROCESSING_TESTS": run_preprocessing_tests,
@@ -109,6 +115,7 @@
     "RUN_SORTERS_TESTS": run_sorters_test,
     "RUN_INTERNAL_SORTERS_TESTS": run_internal_sorters_test,
     "INSTALL_PLEXON_DEPENDENCIES": install_plexon_dependencies,
+    "RUN_STREAMING_EXTRACTORS_TESTS": run_streaming_extractors_test,
 }
 
 # Write the conditions to the GITHUB_ENV file

.github/workflows/all-tests.yml

@@ -68,6 +68,7 @@ jobs:
           echo "RUN_SORTERS_TESTS=${RUN_SORTERS_TESTS}"
           echo "RUN_INTERNAL_SORTERS_TESTS=${RUN_INTERNAL_SORTERS_TESTS}"
           echo "INSTALL_PLEXON_DEPENDENCIES=${INSTALL_PLEXON_DEPENDENCIES}"
+          echo "RUN_STREAMING_EXTRACTORS_TESTS=${RUN_STREAMING_EXTRACTORS_TESTS}"
 
       - name: Install packages
         run: |
@@ -78,9 +79,8 @@ jobs:
         run: pytest -m "core"
         shell: bash
 
-      - name: Install Other Testing Dependencies
+      - name: Install Dependencies for Timing Display
         run: |
-          pip install -e .[test]
           pip install tabulate
           pip install pandas
         shell: bash
@@ -91,6 +91,7 @@ jobs:
         run: echo "dataset_hash=$(git ls-remote https://gin.g-node.org/NeuralEnsemble/ephy_testing_data.git HEAD | cut -f1)" >> $GITHUB_OUTPUT
 
       - name: Cache datasets
+        if: env.RUN_EXTRACTORS_TESTS == 'true'
         id: cache-datasets
         uses: actions/cache/restore@v4
         with:
@@ -119,6 +120,9 @@ jobs:
           fi
           git config --global filter.annex.process "git-annex filter-process"  # recommended for efficiency
 
+      - name : Install Plexon dependencies
+        if: env.INSTALL_PLEXON_DEPENDENCIES == 'true'
+        uses: ./.github/actions/install-wine
 
       - name: Set execute permissions on run_tests.sh
         shell: bash
@@ -130,16 +134,28 @@ jobs:
           HDF5_PLUGIN_PATH: ${{ github.workspace }}/hdf5_plugin_path_maxwell
         if: env.RUN_EXTRACTORS_TESTS == 'true'
         run: |
-          pip install -e .[extractors,streaming_extractors]
+          pip install -e .[extractors,streaming_extractors,test_extractors]
           ./.github/run_tests.sh "extractors and not streaming_extractors" --no-virtual-env
 
+      - name: Test streaming extractors
+        shell: bash
+        if: env.RUN_STREAMING_EXTRACTORS_TESTS
+        run: |
+          pip install -e .[streaming_extractors,test_extractors]
+          ./.github/run_tests.sh "streaming_extractors" --no-virtual-env
+
       - name: Test preprocessing
         shell: bash
         if: env.RUN_PREPROCESSING_TESTS == 'true'
         run: |
-          pip install -e .[preprocessing]
+          pip install -e .[preprocessing,test_preprocessing]
           ./.github/run_tests.sh "preprocessing and not deepinterpolation" --no-virtual-env
 
+      - name: Install remaining testing dependencies # TODO: Remove this step once we have better modularization
+        shell: bash
+        run: |
+          pip install -e .[test]
+
       - name: Test postprocessing
         shell: bash
         if: env.RUN_POSTPROCESSING_TESTS == 'true'