Skip to content

Commit 7cf5f0a

Browse files
alejoe91alejoe91
committed
Merge branch 'main' of github.com:SpikeInterface/spikeinterface into plot-driftmap
2 parents fc3e633 + 0d99342 commit 7cf5f0a

40 files changed

+659
-275
lines changed

‎.github/actions/show-test-environment/action.yml‎

Lines changed: 0 additions & 23 deletions
This file was deleted.

‎.github/determine_testing_environment.py‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
from pathlib import Path
2+
import argparse
3+
import os
4+
5+
6+
# We get the list of files change as an input
7+
parser = argparse.ArgumentParser()
8+
parser.add_argument("changed_files_in_the_pull_request", nargs="*", help="List of changed files")
9+
args = parser.parse_args()
10+
11+
changed_files_in_the_pull_request = args.changed_files_in_the_pull_request
12+
changed_files_in_the_pull_request_paths = [Path(file) for file in changed_files_in_the_pull_request]
13+
14+
# We assume nothing has been changed
15+
16+
core_changed = False
17+
pyproject_toml_changed = False
18+
neobaseextractor_changed = False
19+
extractors_changed = False
20+
plexon2_changed = False
21+
preprocessing_changed = False
22+
postprocessing_changed = False
23+
qualitymetrics_changed = False
24+
sorters_changed = False
25+
sorters_external_changed = False
26+
sorters_internal_changed = False
27+
comparison_changed = False
28+
curation_changed = False
29+
widgets_changed = False
30+
exporters_changed = False
31+
sortingcomponents_changed = False
32+
generation_changed = False
33+
34+
35+
for changed_file in changed_files_in_the_pull_request_paths:
36+
37+
file_is_in_src = changed_file.parts[0] == "src"
38+
39+
if not file_is_in_src:
40+
41+
if changed_file.name == "pyproject.toml":
42+
pyproject_toml_changed = True
43+
44+
else:
45+
if changed_file.name == "neobaseextractor.py":
46+
neobaseextractor_changed = True
47+
elif changed_file.name == "plexon2.py":
48+
extractors_changed = True
49+
elif "core" in changed_file.parts:
50+
conditions_changed = True
51+
elif "extractors" in changed_file.parts:
52+
extractors_changed = True
53+
elif "preprocessing" in changed_file.parts:
54+
preprocessing_changed = True
55+
elif "postprocessing" in changed_file.parts:
56+
postprocessing_changed = True
57+
elif "qualitymetrics" in changed_file.parts:
58+
qualitymetrics_changed = True
59+
elif "comparison" in changed_file.parts:
60+
comparison_changed = True
61+
elif "curation" in changed_file.parts:
62+
curation_changed = True
63+
elif "widgets" in changed_file.parts:
64+
widgets_changed = True
65+
elif "exporters" in changed_file.parts:
66+
exporters_changed = True
67+
elif "sortingcomponents" in changed_file.parts:
68+
sortingcomponents_changed = True
69+
elif "generation" in changed_file.parts:
70+
generation_changed = True
71+
elif "sorters" in changed_file.parts:
72+
if "external" in changed_file.parts:
73+
sorters_external_changed = True
74+
elif "internal" in changed_file.parts:
75+
sorters_internal_changed = True
76+
else:
77+
sorters_changed = True
78+
79+
80+
run_everything = core_changed or pyproject_toml_changed or neobaseextractor_changed
81+
run_generation_tests = run_everything or generation_changed
82+
run_extractor_tests = run_everything or extractors_changed
83+
run_preprocessing_tests = run_everything or preprocessing_changed
84+
run_postprocessing_tests = run_everything or postprocessing_changed
85+
run_qualitymetrics_tests = run_everything or qualitymetrics_changed
86+
run_curation_tests = run_everything or curation_changed
87+
run_sortingcomponents_tests = run_everything or sortingcomponents_changed
88+
89+
run_comparison_test = run_everything or run_generation_tests or comparison_changed
90+
run_widgets_test = run_everything or run_qualitymetrics_tests or run_preprocessing_tests or widgets_changed
91+
run_exporters_test = run_everything or run_widgets_test or exporters_changed
92+
93+
run_sorters_test = run_everything or sorters_changed
94+
run_internal_sorters_test = run_everything or run_sortingcomponents_tests or sorters_internal_changed
95+
96+
install_plexon_dependencies = plexon2_changed
97+
98+
environment_varaiables_to_add = {
99+
"RUN_EXTRACTORS_TESTS": run_extractor_tests,
100+
"RUN_PREPROCESSING_TESTS": run_preprocessing_tests,
101+
"RUN_POSTPROCESSING_TESTS": run_postprocessing_tests,
102+
"RUN_QUALITYMETRICS_TESTS": run_qualitymetrics_tests,
103+
"RUN_CURATION_TESTS": run_curation_tests,
104+
"RUN_SORTINGCOMPONENTS_TESTS": run_sortingcomponents_tests,
105+
"RUN_GENERATION_TESTS": run_generation_tests,
106+
"RUN_COMPARISON_TESTS": run_comparison_test,
107+
"RUN_WIDGETS_TESTS": run_widgets_test,
108+
"RUN_EXPORTERS_TESTS": run_exporters_test,
109+
"RUN_SORTERS_TESTS": run_sorters_test,
110+
"RUN_INTERNAL_SORTERS_TESTS": run_internal_sorters_test,
111+
"INSTALL_PLEXON_DEPENDENCIES": install_plexon_dependencies,
112+
}
113+
114+
# Write the conditions to the GITHUB_ENV file
115+
env_file = os.getenv("GITHUB_ENV")
116+
with open(env_file, "a") as f:
117+
for key, value in environment_varaiables_to_add.items():
118+
f.write(f"{key}={value}\n")

‎.github/run_tests.sh‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,13 @@
11
#!/bin/bash
22

33
MARKER=$1
4+
NOVIRTUALENV=$2
5+
6+
# Check if the second argument is provided and if it is equal to --no-virtual-env
7+
if [ -z "$NOVIRTUALENV" ] || [ "$NOVIRTUALENV" != "--no-virtual-env" ]; then
8+
source $GITHUB_WORKSPACE/test_env/bin/activate
9+
fi
410

5-
source $GITHUB_WORKSPACE/test_env/bin/activate
611
pytest -m "$MARKER" -vv -ra --durations=0 --durations-min=0.001 | tee report.txt; test ${PIPESTATUS[0]} -eq 0 || exit 1
712
echo "# Timing profile of ${MARKER}" >> $GITHUB_STEP_SUMMARY
813
python $GITHUB_WORKSPACE/.github/build_job_summary.py report.txt >> $GITHUB_STEP_SUMMARY

‎.github/workflows/all-tests.yml‎

Lines changed: 211 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,211 @@
1+
name: Complete tests
2+
3+
on:
4+
workflow_dispatch:
5+
schedule:
6+
- cron: "0 12 * * 0" # Weekly on Sunday at noon UTC
7+
pull_request:
8+
types: [synchronize, opened, reopened]
9+
branches:
10+
- main
11+
12+
env:
13+
KACHERY_CLOUD_CLIENT_ID: ${{ secrets.KACHERY_CLOUD_CLIENT_ID }}
14+
KACHERY_CLOUD_PRIVATE_KEY: ${{ secrets.KACHERY_CLOUD_PRIVATE_KEY }}
15+
16+
concurrency: # Cancel previous workflows on the same pull request
17+
group: ${{ github.workflow }}-${{ github.ref }}
18+
cancel-in-progress: true
19+
20+
jobs:
21+
run:
22+
name: ${{ matrix.os }} Python ${{ matrix.python-version }}
23+
runs-on: ${{ matrix.os }}
24+
strategy:
25+
fail-fast: false
26+
matrix:
27+
python-version: ["3.9", "3.12"] # Lower and higher versions we support
28+
os: [macos-13, windows-latest, ubuntu-latest]
29+
steps:
30+
- uses: actions/checkout@v4
31+
- name: Setup Python ${{ matrix.python-version }}
32+
uses: actions/setup-python@v5
33+
with:
34+
python-version: ${{ matrix.python-version }}
35+
36+
- name: Get changed files
37+
id: changed-files
38+
uses: tj-actions/changed-files@v41
39+
40+
- name: List all changed files
41+
shell: bash
42+
env:
43+
ALL_CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }}
44+
run: |
45+
for file in ${ALL_CHANGED_FILES}; do
46+
echo "$file was changed"
47+
done
48+
49+
- name: Set testing environment # This decides which tests are run and whether to install especial dependencies
50+
shell: bash
51+
run: |
52+
changed_files="${{ steps.changed-files.outputs.all_changed_files }}"
53+
python .github/determine_testing_environment.py $changed_files
54+
55+
- name: Display testing environment
56+
shell: bash
57+
run: |
58+
echo "RUN_EXTRACTORS_TESTS=${RUN_EXTRACTORS_TESTS}"
59+
echo "RUN_PREPROCESSING_TESTS=${RUN_PREPROCESSING_TESTS}"
60+
echo "RUN_POSTPROCESSING_TESTS=${RUN_POSTPROCESSING_TESTS}"
61+
echo "RUN_QUALITYMETRICS_TESTS=${RUN_QUALITYMETRICS_TESTS}"
62+
echo "RUN_CURATION_TESTS=${RUN_CURATION_TESTS}"
63+
echo "RUN_SORTINGCOMPONENTS_TESTS=${RUN_SORTINGCOMPONENTS_TESTS}"
64+
echo "RUN_GENERATION_TESTS=${RUN_GENERATION_TESTS}"
65+
echo "RUN_COMPARISON_TESTS=${RUN_COMPARISON_TESTS}"
66+
echo "RUN_WIDGETS_TESTS=${RUN_WIDGETS_TESTS}"
67+
echo "RUN_EXPORTERS_TESTS=${RUN_EXPORTERS_TESTS}"
68+
echo "RUN_SORTERS_TESTS=${RUN_SORTERS_TESTS}"
69+
echo "RUN_INTERNAL_SORTERS_TESTS=${RUN_INTERNAL_SORTERS_TESTS}"
70+
echo "INSTALL_PLEXON_DEPENDENCIES=${INSTALL_PLEXON_DEPENDENCIES}"
71+
72+
- name: Install packages
73+
run: |
74+
pip install -e .[test_core]
75+
shell: bash
76+
77+
- name: Test core
78+
run: pytest -m "core"
79+
shell: bash
80+
81+
- name: Install Other Testing Dependencies
82+
run: |
83+
pip install -e .[test]
84+
pip install tabulate
85+
pip install pandas
86+
shell: bash
87+
88+
- name: Get current hash (SHA) of the ephy_testing_data repo
89+
shell: bash
90+
id: repo_hash
91+
run: echo "dataset_hash=$(git ls-remote https://gin.g-node.org/NeuralEnsemble/ephy_testing_data.git HEAD | cut -f1)" >> $GITHUB_OUTPUT
92+
93+
- name: Cache datasets
94+
id: cache-datasets
95+
uses: actions/cache/restore@v4
96+
with:
97+
path: ~/spikeinterface_datasets
98+
key: ${{ runner.os }}-datasets-${{ steps.repo_hash.outputs.dataset_hash }}
99+
restore-keys: ${{ runner.os }}-datasets
100+
101+
- name: Install git-annex
102+
shell: bash
103+
if: env.RUN_EXTRACTORS_TESTS == 'true'
104+
run: |
105+
pip install datalad-installer
106+
if [ ${{ runner.os }} = 'Linux' ]; then
107+
wget https://downloads.kitenet.net/git-annex/linux/current/git-annex-standalone-amd64.tar.gz
108+
mkdir /home/runner/work/installation
109+
mv git-annex-standalone-amd64.tar.gz /home/runner/work/installation/
110+
workdir=$(pwd)
111+
cd /home/runner/work/installation
112+
tar xvzf git-annex-standalone-amd64.tar.gz
113+
echo "$(pwd)/git-annex.linux" >> $GITHUB_PATH
114+
cd $workdir
115+
elif [ ${{ runner.os }} = 'macOS' ]; then
116+
datalad-installer --sudo ok git-annex --method brew
117+
elif [ ${{ runner.os }} = 'Windows' ]; then
118+
datalad-installer --sudo ok git-annex --method datalad/git-annex:release
119+
fi
120+
git config --global filter.annex.process "git-annex filter-process" # recommended for efficiency
121+
122+
123+
- name: Set execute permissions on run_tests.sh
124+
shell: bash
125+
run: chmod +x .github/run_tests.sh
126+
127+
- name: Test extractors
128+
shell: bash
129+
env:
130+
HDF5_PLUGIN_PATH: ${{ github.workspace }}/hdf5_plugin_path_maxwell
131+
if: env.RUN_EXTRACTORS_TESTS == 'true'
132+
run: |
133+
pip install -e .[extractors,streaming_extractors]
134+
./.github/run_tests.sh "extractors and not streaming_extractors" --no-virtual-env
135+
136+
- name: Test preprocessing
137+
shell: bash
138+
if: env.RUN_PREPROCESSING_TESTS == 'true'
139+
run: |
140+
pip install -e .[preprocessing]
141+
./.github/run_tests.sh "preprocessing and not deepinterpolation" --no-virtual-env
142+
143+
- name: Test postprocessing
144+
shell: bash
145+
if: env.RUN_POSTPROCESSING_TESTS == 'true'
146+
run: |
147+
pip install -e .[full]
148+
./.github/run_tests.sh postprocessing --no-virtual-env
149+
150+
- name: Test quality metrics
151+
shell: bash
152+
if: env.RUN_QUALITYMETRICS_TESTS == 'true'
153+
run: |
154+
pip install -e .[qualitymetrics]
155+
./.github/run_tests.sh qualitymetrics --no-virtual-env
156+
157+
- name: Test comparison
158+
shell: bash
159+
if: env.RUN_COMPARISON_TESTS == 'true'
160+
run: |
161+
pip install -e .[full]
162+
./.github/run_tests.sh comparison --no-virtual-env
163+
164+
- name: Test core sorters
165+
shell: bash
166+
if: env.RUN_SORTERS_TESTS == 'true'
167+
run: |
168+
pip install -e .[full]
169+
./.github/run_tests.sh sorters --no-virtual-env
170+
171+
- name: Test internal sorters
172+
shell: bash
173+
if: env.RUN_INTERNAL_SORTERS_TESTS == 'true'
174+
run: |
175+
pip install -e .[full]
176+
./.github/run_tests.sh sorters_internal --no-virtual-env
177+
178+
- name: Test curation
179+
shell: bash
180+
if: env.RUN_CURATION_TESTS == 'true'
181+
run: |
182+
pip install -e .[full]
183+
./.github/run_tests.sh curation --no-virtual-env
184+
185+
- name: Test widgets
186+
shell: bash
187+
if: env.RUN_WIDGETS_TESTS == 'true'
188+
run: |
189+
pip install -e .[full]
190+
./.github/run_tests.sh widgets --no-virtual-env
191+
192+
- name: Test exporters
193+
shell: bash
194+
if: env.RUN_EXPORTERS_TESTS == 'true'
195+
run: |
196+
pip install -e .[full]
197+
./.github/run_tests.sh exporters --no-virtual-env
198+
199+
- name: Test sortingcomponents
200+
shell: bash
201+
if: env.RUN_SORTINGCOMPONENTS_TESTS == 'true'
202+
run: |
203+
pip install -e .[full]
204+
./.github/run_tests.sh sortingcomponents --no-virtual-env
205+
206+
- name: Test generation
207+
shell: bash
208+
if: env.RUN_GENERATION_TESTS == 'true'
209+
run: |
210+
pip install -e .[full]
211+
./.github/run_tests.sh generation --no-virtual-env

0 commit comments

Comments
 (0)