From d1d8500f28a5af3a4aa3a0afff45d47778c98180 Mon Sep 17 00:00:00 2001 From: Iahn Cajigas Date: Wed, 4 Mar 2026 01:17:40 -0500 Subject: [PATCH] Performance CI: add linux latest baseline pointers --- .github/workflows/performance-parity.yml | 5 +- README.md | 2 +- parity/CYCLE_VALIDATION_CHECKLIST.md | 2 +- .../performance_baseline_linux_20260304.csv | 36 +- .../performance_baseline_linux_20260304.json | 723 ++++++++++------- .../performance_baseline_linux_latest.csv | 22 + .../performance_baseline_linux_latest.json | 724 ++++++++++++++++++ tests/test_performance_reports.py | 14 +- 8 files changed, 1246 insertions(+), 282 deletions(-) create mode 100644 tests/performance/fixtures/python/performance_baseline_linux_latest.csv create mode 100644 tests/performance/fixtures/python/performance_baseline_linux_latest.json diff --git a/.github/workflows/performance-parity.yml b/.github/workflows/performance-parity.yml index 53b08358..92a4082a 100644 --- a/.github/workflows/performance-parity.yml +++ b/.github/workflows/performance-parity.yml @@ -59,7 +59,7 @@ jobs: --python-report output/performance/python_performance_report.json \ --matlab-report tests/performance/fixtures/matlab/performance_baseline_470fde8.json \ --policy parity/performance_gate_policy.yml \ - --previous-python-report tests/performance/fixtures/python/performance_baseline_linux_20260304.json \ + --previous-python-report tests/performance/fixtures/python/performance_baseline_linux_latest.json \ --report-out output/performance/performance_parity_report.json \ --csv-out output/performance/performance_parity_report.csv \ --fail-on-regression \ @@ -81,5 +81,8 @@ jobs: output/performance/*.json output/performance/*.csv tests/performance/fixtures/matlab/performance_baseline_470fde8.json + tests/performance/fixtures/python/performance_baseline_linux_latest.json + tests/performance/fixtures/python/performance_baseline_linux_latest.csv tests/performance/fixtures/python/performance_baseline_linux_20260304.json + tests/performance/fixtures/python/performance_baseline_linux_20260304.csv if-no-files-found: warn diff --git a/README.md b/README.md index 560a63de..e593c464 100644 --- a/README.md +++ b/README.md @@ -187,7 +187,7 @@ python tools/performance/compare_matlab_python_performance.py \ --python-report output/performance/python_performance_report.json \ --matlab-report tests/performance/fixtures/matlab/performance_baseline_470fde8.json \ --policy parity/performance_gate_policy.yml \ - --previous-python-report tests/performance/fixtures/python/performance_baseline_linux_20260304.json \ + --previous-python-report tests/performance/fixtures/python/performance_baseline_linux_latest.json \ --report-out parity/performance_parity_report.json \ --csv-out parity/performance_parity_report.csv \ --fail-on-regression \ diff --git a/parity/CYCLE_VALIDATION_CHECKLIST.md b/parity/CYCLE_VALIDATION_CHECKLIST.md index a3588b45..f15173bb 100644 --- a/parity/CYCLE_VALIDATION_CHECKLIST.md +++ b/parity/CYCLE_VALIDATION_CHECKLIST.md @@ -10,7 +10,7 @@ Commands used each cycle: - `python tools/reports/build_image_parity_pdfs.py --report-json --python-out output/pdf/image_mode_parity/python_pages.pdf --matlab-out output/pdf/image_mode_parity/matlab_pages.pdf --pairs-json output/pdf/image_mode_parity/pairs.json` - `python tools/reports/check_pdf_image_parity.py --python-pdf output/pdf/image_mode_parity/python_pages.pdf --matlab-pdf output/pdf/image_mode_parity/matlab_pages.pdf --out-dir output/pdf/image_mode_parity --dpi 150 --ssim-threshold 0.70 --max-failing-pages 0` - `python tools/performance/run_python_benchmarks.py --tiers S --repeats 5 --warmup 1 --out-json output/performance/python_performance_report.json --out-csv output/performance/python_performance_report.csv` -- `python tools/performance/compare_matlab_python_performance.py --python-report output/performance/python_performance_report.json --matlab-report tests/performance/fixtures/matlab/performance_baseline_470fde8.json --policy parity/performance_gate_policy.yml --previous-python-report tests/performance/fixtures/python/performance_baseline_linux_20260304.json --report-out output/performance/performance_parity_report.json --csv-out output/performance/performance_parity_report.csv --fail-on-regression --require-regression-env-match` +- `python tools/performance/compare_matlab_python_performance.py --python-report output/performance/python_performance_report.json --matlab-report tests/performance/fixtures/matlab/performance_baseline_470fde8.json --policy parity/performance_gate_policy.yml --previous-python-report tests/performance/fixtures/python/performance_baseline_linux_latest.json --report-out output/performance/performance_parity_report.json --csv-out output/performance/performance_parity_report.csv --fail-on-regression --require-regression-env-match` - Local macOS reruns use `tests/performance/fixtures/python/performance_baseline_20260303.json` with the same command to satisfy strict env matching. ## Cycle 1 diff --git a/tests/performance/fixtures/python/performance_baseline_linux_20260304.csv b/tests/performance/fixtures/python/performance_baseline_linux_20260304.csv index fdd75e7b..cfa04b5f 100644 --- a/tests/performance/fixtures/python/performance_baseline_linux_20260304.csv +++ b/tests/performance/fixtures/python/performance_baseline_linux_20260304.csv @@ -1,16 +1,22 @@ case,tier,repeats,median_runtime_ms,mean_runtime_ms,std_runtime_ms,median_peak_memory_mb,summary -unit_impulse_basis,S,7,1.7549330000008467,1.767814000002334,0.03712841168356124,0.39142704010009766,"{""cols"": 50.0, ""rows"": 501.0, ""total_mass"": 500.0}" -unit_impulse_basis,M,7,5.230034999996747,5.244222714285992,0.04359156024491504,3.076793670654297,"{""cols"": 100.0, ""rows"": 2001.0, ""total_mass"": 2000.0}" -unit_impulse_basis,L,7,10.598025999996707,10.61451614285959,0.03639945260812847,18.373775482177734,"{""cols"": 200.0, ""rows"": 6001.0, ""total_mass"": 6000.0}" -covariate_resample,S,7,0.3187809999900537,0.31151314285742565,0.027315777992366848,0.06198883056640625,"{""cols"": 1.0, ""rows"": 1001.0, ""signal_energy"": 0.5195204795204795}" -covariate_resample,M,7,0.3733120000077861,0.3833688571432958,0.027679721024068783,0.1355915069580078,"{""cols"": 1.0, ""rows"": 3001.0, ""signal_energy"": 0.5198042747802833}" -covariate_resample,L,7,0.4388640000030364,0.43310742857321266,0.016053381349413916,0.2376575469970703,"{""cols"": 1.0, ""rows"": 6001.0, ""signal_energy"": 0.5199200133311115}" -history_design_matrix,S,7,0.34009099999821046,0.3473752857193598,0.02510644529088935,0.0890655517578125,"{""cols"": 4.0, ""rows"": 1000.0, ""total_count"": 9737.0}" -history_design_matrix,M,7,0.7935309999993478,0.8029685714266829,0.02089668436143026,0.43688201904296875,"{""cols"": 4.0, ""rows"": 5000.0, ""total_count"": 243740.0}" -history_design_matrix,L,7,1.6487759999961327,1.6487702857140059,0.03152991976246771,0.8869400024414062,"{""cols"": 4.0, ""rows"": 10000.0, ""total_count"": 1462420.0}" -simulate_cif_thinning,S,7,11.90902599999788,11.841307428570401,0.2313883554103292,0.07111740112304688,"{""mean_spikes_per_unit"": 13.8, ""num_units"": 5.0, ""total_spikes"": 69.0}" -simulate_cif_thinning,M,7,46.7842029999872,46.623392285716086,0.34392725136599767,0.13444900512695312,"{""mean_spikes_per_unit"": 23.5, ""num_units"": 10.0, ""total_spikes"": 235.0}" -simulate_cif_thinning,L,7,138.86579399999732,138.25536671428398,2.017449271316567,0.20075607299804688,"{""mean_spikes_per_unit"": 36.65, ""num_units"": 20.0, ""total_spikes"": 733.0}" -decoding_spike_rate_cis,S,7,29.966428000008705,29.846530285713666,0.3328735707401364,0.2328357696533203,"{""num_trials"": 6.0, ""prob_mean"": 0.1509259259259259, ""rate_mean"": 50.4457886636761, ""sig_count"": 0.0}" -decoding_spike_rate_cis,M,7,62.59277799999552,63.25780771428567,1.3782261025910456,0.7640476226806641,"{""num_trials"": 8.0, ""prob_mean"": 0.18562499999999998, ""rate_mean"": 50.12398439148756, ""sig_count"": 0.0}" -decoding_spike_rate_cis,L,7,138.46276700000715,138.051728285717,1.750384804578685,2.7336788177490234,"{""num_trials"": 12.0, ""prob_mean"": 0.21328124999999998, ""rate_mean"": 50.073736692667104, ""sig_count"": 0.0}" +unit_impulse_basis,S,7,1.7951030000062929,1.800190142854977,0.030044572115161792,0.39142704010009766,"{""cols"": 50.0, ""rows"": 501.0, ""total_mass"": 500.0}" +unit_impulse_basis,M,7,6.6628950000051645,6.653057285713463,1.4829550166016998,3.076824188232422,"{""cols"": 100.0, ""rows"": 2001.0, ""total_mass"": 2000.0}" +unit_impulse_basis,L,7,16.024357000020473,16.10430300001196,0.23143857554091385,18.37380599975586,"{""cols"": 200.0, ""rows"": 6001.0, ""total_mass"": 6000.0}" +covariate_resample,S,7,0.29961199999206656,0.29913757142756886,0.009541632013909111,0.06201934814453125,"{""cols"": 1.0, ""rows"": 1001.0, ""signal_energy"": 0.5195204795204795}" +covariate_resample,M,7,0.3440330000046288,0.34703042856725524,0.01738929698449399,0.1356220245361328,"{""cols"": 1.0, ""rows"": 3001.0, ""signal_energy"": 0.5198042747802833}" +covariate_resample,L,7,0.4108699999960663,0.4196135714299284,0.017089334564883416,0.2376880645751953,"{""cols"": 1.0, ""rows"": 6001.0, ""signal_energy"": 0.5199200133311115}" +history_design_matrix,S,7,0.3993699999966793,0.4031285714266427,0.013794363505286652,0.0890960693359375,"{""cols"": 4.0, ""rows"": 1000.0, ""total_count"": 9737.0}" +history_design_matrix,M,7,0.9118490000048496,0.9101645714265162,0.05885667560476089,0.43691253662109375,"{""cols"": 4.0, ""rows"": 5000.0, ""total_count"": 243740.0}" +history_design_matrix,L,7,1.8411630000230161,1.8406257142860991,0.014739662895922209,0.8869705200195312,"{""cols"": 4.0, ""rows"": 10000.0, ""total_count"": 1462420.0}" +simulate_cif_thinning,S,7,12.83120299999041,12.864690857132441,0.17008704628575258,0.07111740112304688,"{""mean_spikes_per_unit"": 12.6, ""num_units"": 5.0, ""total_spikes"": 63.0}" +simulate_cif_thinning,M,7,49.6265170000072,49.62809900000106,0.2452487903460643,0.13447189331054688,"{""mean_spikes_per_unit"": 23.9, ""num_units"": 10.0, ""total_spikes"": 239.0}" +simulate_cif_thinning,L,7,148.1780189999995,147.96911385714193,0.8086479240372416,0.20068740844726562,"{""mean_spikes_per_unit"": 35.2, ""num_units"": 20.0, ""total_spikes"": 704.0}" +decoding_spike_rate_cis,S,7,9.922279000022627,9.8990294285792,0.09977266978679186,0.08637428283691406,"{""num_trials"": 6.0, ""prob_mean"": 0.1509259259259259, ""rate_mean"": 50.4457886636761, ""sig_count"": 0.0}" +decoding_spike_rate_cis,M,7,15.846046000007163,15.825482000000843,0.11100674963412104,0.19997024536132812,"{""num_trials"": 8.0, ""prob_mean"": 0.18562499999999998, ""rate_mean"": 50.12398439148756, ""sig_count"": 0.0}" +decoding_spike_rate_cis,L,7,29.582787000009603,29.530547285718608,0.2645464377729889,0.5168647766113281,"{""num_trials"": 12.0, ""prob_mean"": 0.21328124999999998, ""rate_mean"": 50.073736692667104, ""sig_count"": 0.0}" +nspiketrain_get_sigrep,S,7,0.45410599997808276,0.45695271428160467,0.02263874211807069,0.05826568603515625,"{""binary_sum"": 746.0, ""count_sum"": 800.0, ""n_bins"": 1000.0}" +nspiketrain_get_sigrep,M,7,0.5966520000129094,0.6046927142863622,0.017903597884198864,0.18491363525390625,"{""binary_sum"": 2577.0, ""count_sum"": 3000.0, ""n_bins"": 3000.0}" +nspiketrain_get_sigrep,L,7,0.9368639999820516,0.9338544285648303,0.016064032275827016,0.39853668212890625,"{""binary_sum"": 5000.0, ""count_sum"": 9000.0, ""n_bins"": 5000.0}" +analysis_fit_glm_pipeline,S,7,5.966330000006792,6.041203285718666,0.12616461532934142,0.10829353332519531,"{""coeff_norm"": 8.000957080893618e-16, ""intercept"": -12.01100351405273, ""log_likelihood"": -5.469277525370689e-06, ""pred_mean"": 6.076975028189657e-06}" +analysis_fit_glm_pipeline,M,7,7.278939000002538,7.281217000001204,0.1861142338621149,0.22568511962890625,"{""coeff_norm"": 9.782140347964175e-16, ""intercept"": -12.704150666835071, ""log_likelihood"": -5.46927767729412e-06, ""pred_mean"": 3.038487598496732e-06}" +analysis_fit_glm_pipeline,L,7,9.020953999993253,9.130242857143653,0.1717714863551795,0.4345836639404297,"{""coeff_norm"": 8.223875012950201e-16, ""intercept"": -12.704150666835066, ""log_likelihood"": -9.723160315189595e-06, ""pred_mean"": 3.0384875984967483e-06}" diff --git a/tests/performance/fixtures/python/performance_baseline_linux_20260304.json b/tests/performance/fixtures/python/performance_baseline_linux_20260304.json index 14b9af45..3a34741d 100644 --- a/tests/performance/fixtures/python/performance_baseline_linux_20260304.json +++ b/tests/performance/fixtures/python/performance_baseline_linux_20260304.json @@ -1,9 +1,9 @@ { "schema_version": 1, - "generated_at_utc": "2026-03-04T04:39:42Z", + "generated_at_utc": "2026-03-04T06:13:09Z", "implementation": "python", "repo_root": "/home/runner/work/nSTAT-python/nSTAT-python", - "git_sha": "bdd375cb5cfcfe637e71d22a1ff070e0ac42df80", + "git_sha": "08ac1adde9a2d9b74a819c22e1fc19b0c2d758b3", "tiers": [ "S", "M", @@ -15,9 +15,9 @@ "tier": "S", "repeats": 7, "warmup": 2, - "median_runtime_ms": 1.7549330000008467, - "mean_runtime_ms": 1.767814000002334, - "std_runtime_ms": 0.03712841168356124, + "median_runtime_ms": 1.7951030000062929, + "mean_runtime_ms": 1.800190142854977, + "std_runtime_ms": 0.030044572115161792, "median_peak_memory_mb": 0.39142704010009766, "summary": { "rows": 501.0, @@ -25,13 +25,13 @@ "total_mass": 500.0 }, "samples_runtime_ms": [ - 1.8555799999973033, - 1.7672160000046233, - 1.7518669999958547, - 1.7363890000012816, - 1.7449640000108957, - 1.7549330000008467, - 1.7637490000055323 + 1.864554999997381, + 1.7951030000062929, + 1.8160890000160634, + 1.7815100000007078, + 1.778256999983796, + 1.7982370000027004, + 1.7675799999778974 ], "samples_peak_memory_mb": [ 0.39147281646728516, @@ -48,32 +48,32 @@ "tier": "M", "repeats": 7, "warmup": 2, - "median_runtime_ms": 5.230034999996747, - "mean_runtime_ms": 5.244222714285992, - "std_runtime_ms": 0.04359156024491504, - "median_peak_memory_mb": 3.076793670654297, + "median_runtime_ms": 6.6628950000051645, + "mean_runtime_ms": 6.653057285713463, + "std_runtime_ms": 1.4829550166016998, + "median_peak_memory_mb": 3.076824188232422, "summary": { "rows": 2001.0, "cols": 100.0, "total_mass": 2000.0 }, "samples_runtime_ms": [ - 5.208524999986253, - 5.255633000004423, - 5.230034999996747, - 5.205519000000436, - 5.243821000007642, - 5.342964000007555, - 5.22306199999889 + 5.253328000009105, + 5.174608000004355, + 5.262399000002915, + 9.362254999984998, + 8.050177000001213, + 6.805738999986488, + 6.6628950000051645 ], "samples_peak_memory_mb": [ + 3.0769691467285156, 3.0769615173339844, - 3.076946258544922, - 3.0769309997558594, - 3.076793670654297, - 3.0767784118652344, - 3.076770782470703, - 3.076763153076172 + 3.076953887939453, + 3.076824188232422, + 3.0768089294433594, + 3.076801300048828, + 3.076793670654297 ] }, { @@ -81,32 +81,32 @@ "tier": "L", "repeats": 7, "warmup": 2, - "median_runtime_ms": 10.598025999996707, - "mean_runtime_ms": 10.61451614285959, - "std_runtime_ms": 0.03639945260812847, - "median_peak_memory_mb": 18.373775482177734, + "median_runtime_ms": 16.024357000020473, + "mean_runtime_ms": 16.10430300001196, + "std_runtime_ms": 0.23143857554091385, + "median_peak_memory_mb": 18.37380599975586, "summary": { "rows": 6001.0, "cols": 200.0, "total_mass": 6000.0 }, "samples_runtime_ms": [ - 10.596453000005113, - 10.691879000006566, - 10.640424000001758, - 10.591734000001907, - 10.576616000008698, - 10.606480999996393, - 10.598025999996707 + 16.00895999999352, + 16.42452300001196, + 15.90741600000456, + 16.075996000012083, + 15.817221000020254, + 16.024357000020473, + 16.471648000020878 ], "samples_peak_memory_mb": [ + 18.373836517333984, + 18.373828887939453, + 18.373821258544922, 18.37380599975586, 18.373798370361328, 18.373790740966797, - 18.373775482177734, - 18.373767852783203, - 18.373760223388672, - 18.37375259399414 + 18.373783111572266 ] }, { @@ -114,32 +114,32 @@ "tier": "S", "repeats": 7, "warmup": 2, - "median_runtime_ms": 0.3187809999900537, - "mean_runtime_ms": 0.31151314285742565, - "std_runtime_ms": 0.027315777992366848, - "median_peak_memory_mb": 0.06198883056640625, + "median_runtime_ms": 0.29961199999206656, + "mean_runtime_ms": 0.29913757142756886, + "std_runtime_ms": 0.009541632013909111, + "median_peak_memory_mb": 0.06201934814453125, "summary": { "rows": 1001.0, "cols": 1.0, "signal_energy": 0.5195204795204795 }, "samples_runtime_ms": [ - 0.3284590000021126, - 0.3187809999900537, - 0.35717200000817684, - 0.2866319999981215, - 0.27595100000610273, - 0.32765699999970366, - 0.2859399999977086 + 0.31718400001068403, + 0.29961199999206656, + 0.2958649999982299, + 0.30228200000692595, + 0.28286899998875015, + 0.29462000000535227, + 0.3015309999909732 ], "samples_peak_memory_mb": [ - 0.06198883056640625, - 0.06198883056640625, - 0.06198883056640625, - 0.06198883056640625, - 0.06198883056640625, - 0.06198883056640625, - 0.06198883056640625 + 0.06201934814453125, + 0.06201934814453125, + 0.06201934814453125, + 0.06201934814453125, + 0.06201934814453125, + 0.06201934814453125, + 0.06201934814453125 ] }, { @@ -147,32 +147,32 @@ "tier": "M", "repeats": 7, "warmup": 2, - "median_runtime_ms": 0.3733120000077861, - "mean_runtime_ms": 0.3833688571432958, - "std_runtime_ms": 0.027679721024068783, - "median_peak_memory_mb": 0.1355915069580078, + "median_runtime_ms": 0.3440330000046288, + "mean_runtime_ms": 0.34703042856725524, + "std_runtime_ms": 0.01738929698449399, + "median_peak_memory_mb": 0.1356220245361328, "summary": { "rows": 3001.0, "cols": 1.0, "signal_energy": 0.5198042747802833 }, "samples_runtime_ms": [ - 0.4426099999932376, - 0.3733120000077861, - 0.38337099999807833, - 0.3641439999881868, - 0.3593360000024859, - 0.40025100000207203, - 0.36055800001122407 + 0.3440330000046288, + 0.3872629999932542, + 0.3394640000067284, + 0.34691299998712566, + 0.3345760000001974, + 0.3304029999924296, + 0.34656099998642276 ], "samples_peak_memory_mb": [ - 0.1355915069580078, - 0.1355915069580078, - 0.1355915069580078, - 0.1355915069580078, - 0.1355915069580078, - 0.1355915069580078, - 0.1355915069580078 + 0.1356220245361328, + 0.1356220245361328, + 0.1356220245361328, + 0.1356220245361328, + 0.1356220245361328, + 0.1356220245361328, + 0.1356220245361328 ] }, { @@ -180,32 +180,32 @@ "tier": "L", "repeats": 7, "warmup": 2, - "median_runtime_ms": 0.4388640000030364, - "mean_runtime_ms": 0.43310742857321266, - "std_runtime_ms": 0.016053381349413916, - "median_peak_memory_mb": 0.2376575469970703, + "median_runtime_ms": 0.4108699999960663, + "mean_runtime_ms": 0.4196135714299284, + "std_runtime_ms": 0.017089334564883416, + "median_peak_memory_mb": 0.2376880645751953, "summary": { "rows": 6001.0, "cols": 1.0, "signal_energy": 0.5199200133311115 }, "samples_runtime_ms": [ - 0.4090890000014724, - 0.43946499999947264, - 0.41093200000830166, - 0.4388640000030364, - 0.44900299999994786, - 0.43115900000145757, - 0.45323999999880016 + 0.42476900000565365, + 0.4108699999960663, + 0.4027309999798945, + 0.4545680000092034, + 0.4038190000130726, + 0.41067399999406007, + 0.42986400001154834 ], "samples_peak_memory_mb": [ - 0.23763465881347656, - 0.23763465881347656, - 0.2376575469970703, - 0.2376575469970703, - 0.2376575469970703, - 0.2376575469970703, - 0.2376575469970703 + 0.23766517639160156, + 0.23766517639160156, + 0.2376880645751953, + 0.2376880645751953, + 0.2376880645751953, + 0.2376880645751953, + 0.2376880645751953 ] }, { @@ -213,32 +213,32 @@ "tier": "S", "repeats": 7, "warmup": 2, - "median_runtime_ms": 0.34009099999821046, - "mean_runtime_ms": 0.3473752857193598, - "std_runtime_ms": 0.02510644529088935, - "median_peak_memory_mb": 0.0890655517578125, + "median_runtime_ms": 0.3993699999966793, + "mean_runtime_ms": 0.4031285714266427, + "std_runtime_ms": 0.013794363505286652, + "median_peak_memory_mb": 0.0890960693359375, "summary": { "rows": 1000.0, "cols": 4.0, "total_count": 9737.0 }, "samples_runtime_ms": [ - 0.39858800001013606, - 0.35199200000590736, - 0.3244020000039427, - 0.36373400000400125, - 0.33092400001066835, - 0.3218960000026527, - 0.34009099999821046 + 0.4107149999867943, + 0.3993699999966793, + 0.4110679999769218, + 0.38829799999007264, + 0.4294660000141448, + 0.3940240000019912, + 0.38895900001989503 ], "samples_peak_memory_mb": [ - 0.08905029296875, - 0.0890655517578125, 0.089080810546875, - 0.08907318115234375, - 0.0890655517578125, - 0.08905029296875, - 0.08904266357421875 + 0.0890960693359375, + 0.089111328125, + 0.08910369873046875, + 0.0890960693359375, + 0.089080810546875, + 0.08907318115234375 ] }, { @@ -246,32 +246,32 @@ "tier": "M", "repeats": 7, "warmup": 2, - "median_runtime_ms": 0.7935309999993478, - "mean_runtime_ms": 0.8029685714266829, - "std_runtime_ms": 0.02089668436143026, - "median_peak_memory_mb": 0.43688201904296875, + "median_runtime_ms": 0.9118490000048496, + "mean_runtime_ms": 0.9101645714265162, + "std_runtime_ms": 0.05885667560476089, + "median_peak_memory_mb": 0.43691253662109375, "summary": { "rows": 5000.0, "cols": 4.0, "total_count": 243740.0 }, "samples_runtime_ms": [ - 0.8134179999927937, - 0.8033690000104343, - 0.7935309999993478, - 0.7772609999960878, - 0.7933710000003202, - 0.7922479999962206, - 0.8475819999915757 + 0.9850819999996929, + 0.9938119999901573, + 0.9118490000048496, + 0.8551290000013978, + 0.8512590000009368, + 0.8419250000031298, + 0.9320959999854495 ], "samples_peak_memory_mb": [ - 0.4368743896484375, - 0.4368896484375, 0.4369049072265625, - 0.43689727783203125, - 0.43688201904296875, - 0.4368743896484375, - 0.43686676025390625 + 0.436920166015625, + 0.4369354248046875, + 0.43692779541015625, + 0.43691253662109375, + 0.4369049072265625, + 0.43689727783203125 ] }, { @@ -279,32 +279,32 @@ "tier": "L", "repeats": 7, "warmup": 2, - "median_runtime_ms": 1.6487759999961327, - "mean_runtime_ms": 1.6487702857140059, - "std_runtime_ms": 0.03152991976246771, - "median_peak_memory_mb": 0.8869400024414062, + "median_runtime_ms": 1.8411630000230161, + "mean_runtime_ms": 1.8406257142860991, + "std_runtime_ms": 0.014739662895922209, + "median_peak_memory_mb": 0.8869705200195312, "summary": { "rows": 10000.0, "cols": 4.0, "total_count": 1462420.0 }, "samples_runtime_ms": [ - 1.6667189999992615, - 1.6867770000033033, - 1.6265349999997625, - 1.6487759999961327, - 1.6904029999977865, - 1.6031809999930147, - 1.6190010000087796 + 1.8261639999934687, + 1.8664680000028966, + 1.843834999988303, + 1.8197750000013002, + 1.8411630000230161, + 1.833944999987125, + 1.8530300000065836 ], "samples_peak_memory_mb": [ - 0.886932373046875, - 0.8869476318359375, 0.886962890625, - 0.8869476318359375, - 0.8869400024414062, - 0.886932373046875, - 0.8869247436523438 + 0.8869781494140625, + 0.886993408203125, + 0.8869781494140625, + 0.8869705200195312, + 0.886962890625, + 0.8869552612304688 ] }, { @@ -312,32 +312,32 @@ "tier": "S", "repeats": 7, "warmup": 2, - "median_runtime_ms": 11.90902599999788, - "mean_runtime_ms": 11.841307428570401, - "std_runtime_ms": 0.2313883554103292, + "median_runtime_ms": 12.83120299999041, + "mean_runtime_ms": 12.864690857132441, + "std_runtime_ms": 0.17008704628575258, "median_peak_memory_mb": 0.07111740112304688, "summary": { "num_units": 5.0, - "total_spikes": 69.0, - "mean_spikes_per_unit": 13.8 + "total_spikes": 63.0, + "mean_spikes_per_unit": 12.6 }, "samples_runtime_ms": [ - 11.399260999993999, - 11.735373999997023, - 12.0300100000037, - 11.90902599999788, - 11.70961600000453, - 12.155522999989898, - 11.950342000005776 + 13.047533999980487, + 13.09318899998857, + 13.005707999980132, + 12.636868999976514, + 12.747445999991669, + 12.690887000019302, + 12.83120299999041 ], "samples_peak_memory_mb": [ - 0.07123947143554688, - 0.07107925415039062, + 0.07109451293945312, + 0.07122421264648438, + 0.07120132446289062, + 0.07110214233398438, + 0.07108688354492188, 0.07111740112304688, - 0.07104873657226562, - 0.07101821899414062, - 0.07143020629882812, - 0.07115554809570312 + 0.07133102416992188 ] }, { @@ -345,32 +345,32 @@ "tier": "M", "repeats": 7, "warmup": 2, - "median_runtime_ms": 46.7842029999872, - "mean_runtime_ms": 46.623392285716086, - "std_runtime_ms": 0.34392725136599767, - "median_peak_memory_mb": 0.13444900512695312, + "median_runtime_ms": 49.6265170000072, + "mean_runtime_ms": 49.62809900000106, + "std_runtime_ms": 0.2452487903460643, + "median_peak_memory_mb": 0.13447189331054688, "summary": { "num_units": 10.0, - "total_spikes": 235.0, - "mean_spikes_per_unit": 23.5 + "total_spikes": 239.0, + "mean_spikes_per_unit": 23.9 }, "samples_runtime_ms": [ - 46.661997000001065, - 46.807135999998195, - 46.9655890000098, - 46.7842029999872, - 46.52009400000168, - 45.840324000010924, - 46.78440300000375 + 49.933012999986204, + 49.47553599998855, + 49.15841700000101, + 49.6265170000072, + 49.57282899999882, + 49.74063900002079, + 49.889742000004844 ], "samples_peak_memory_mb": [ - 0.13444900512695312, - 0.13436508178710938, - 0.13444900512695312, - 0.13457107543945312, - 0.13434982299804688, - 0.13453292846679688, - 0.13437271118164062 + 0.13447952270507812, + 0.13440322875976562, + 0.13442611694335938, + 0.13442611694335938, + 0.13451004028320312, + 0.13466262817382812, + 0.13447189331054688 ] }, { @@ -378,32 +378,32 @@ "tier": "L", "repeats": 7, "warmup": 2, - "median_runtime_ms": 138.86579399999732, - "mean_runtime_ms": 138.25536671428398, - "std_runtime_ms": 2.017449271316567, - "median_peak_memory_mb": 0.20075607299804688, + "median_runtime_ms": 148.1780189999995, + "mean_runtime_ms": 147.96911385714193, + "std_runtime_ms": 0.8086479240372416, + "median_peak_memory_mb": 0.20068740844726562, "summary": { "num_units": 20.0, - "total_spikes": 733.0, - "mean_spikes_per_unit": 36.65 + "total_spikes": 704.0, + "mean_spikes_per_unit": 35.2 }, "samples_runtime_ms": [ - 140.78421000000674, - 138.86579399999732, - 139.35017199999322, - 136.84719200000472, - 134.61053599999673, - 137.09617399999274, - 140.23348899999633 + 148.17126599999142, + 149.08999700000436, + 148.1780189999995, + 148.30773799999974, + 147.23606999999106, + 146.41610300000707, + 148.3846040000003 ], "samples_peak_memory_mb": [ - 0.20079421997070312, - 0.20088577270507812, - 0.20074844360351562, - 0.20075607299804688, - 0.20060348510742188, - 0.20046615600585938, - 0.20093917846679688 + 0.20109176635742188, + 0.20068740844726562, + 0.20077896118164062, + 0.20093154907226562, + 0.20005416870117188, + 0.20032882690429688, + 0.20064163208007812 ] }, { @@ -411,10 +411,10 @@ "tier": "S", "repeats": 7, "warmup": 2, - "median_runtime_ms": 29.966428000008705, - "mean_runtime_ms": 29.846530285713666, - "std_runtime_ms": 0.3328735707401364, - "median_peak_memory_mb": 0.2328357696533203, + "median_runtime_ms": 9.922279000022627, + "mean_runtime_ms": 9.8990294285792, + "std_runtime_ms": 0.09977266978679186, + "median_peak_memory_mb": 0.08637428283691406, "summary": { "num_trials": 6.0, "prob_mean": 0.1509259259259259, @@ -422,22 +422,22 @@ "rate_mean": 50.4457886636761 }, "samples_runtime_ms": [ - 29.377155999995352, - 30.0538899999907, - 29.957221000003642, - 30.260263000002396, - 29.307927999994376, - 30.002826000000482, - 29.966428000008705 + 10.008196000001135, + 9.922279000022627, + 9.995544000020118, + 9.960062000004655, + 9.712960000001658, + 9.801446000011538, + 9.892718999992667 ], "samples_peak_memory_mb": [ - 0.2328357696533203, - 0.2328357696533203, - 0.2328357696533203, - 0.2328357696533203, - 0.2328357696533203, - 0.2328357696533203, - 0.2328357696533203 + 0.08637428283691406, + 0.08667182922363281, + 0.08637428283691406, + 0.08632469177246094, + 0.08632469177246094, + 0.08667182922363281, + 0.08662223815917969 ] }, { @@ -445,10 +445,10 @@ "tier": "M", "repeats": 7, "warmup": 2, - "median_runtime_ms": 62.59277799999552, - "mean_runtime_ms": 63.25780771428567, - "std_runtime_ms": 1.3782261025910456, - "median_peak_memory_mb": 0.7640476226806641, + "median_runtime_ms": 15.846046000007163, + "mean_runtime_ms": 15.825482000000843, + "std_runtime_ms": 0.11100674963412104, + "median_peak_memory_mb": 0.19997024536132812, "summary": { "num_trials": 8.0, "prob_mean": 0.18562499999999998, @@ -456,22 +456,22 @@ "rate_mean": 50.12398439148756 }, "samples_runtime_ms": [ - 62.16685800001187, - 62.88427799999852, - 62.58198800000514, - 62.59277799999552, - 62.56363399999998, - 66.50133299999084, - 63.513784999997824 + 15.777557000006937, + 15.72368399999391, + 15.935339000009208, + 15.636774000000742, + 15.846046000007163, + 15.8855459999927, + 15.973427999995238 ], "samples_peak_memory_mb": [ - 0.7639484405517578, - 0.7639980316162109, - 0.7639980316162109, - 0.7640476226806641, - 0.7641468048095703, - 0.7642955780029297, - 0.7643184661865234 + 0.1999664306640625, + 0.2000579833984375, + 0.199951171875, + 0.19994354248046875, + 0.20047378540039062, + 0.19997024536132812, + 0.200408935546875 ] }, { @@ -479,10 +479,10 @@ "tier": "L", "repeats": 7, "warmup": 2, - "median_runtime_ms": 138.46276700000715, - "mean_runtime_ms": 138.051728285717, - "std_runtime_ms": 1.750384804578685, - "median_peak_memory_mb": 2.7336788177490234, + "median_runtime_ms": 29.582787000009603, + "mean_runtime_ms": 29.530547285718608, + "std_runtime_ms": 0.2645464377729889, + "median_peak_memory_mb": 0.5168647766113281, "summary": { "num_trials": 12.0, "prob_mean": 0.21328124999999998, @@ -490,22 +490,223 @@ "rate_mean": 50.073736692667104 }, "samples_runtime_ms": [ - 138.46276700000715, - 138.7568119999969, - 137.5696620000042, - 139.4315430000006, - 139.74387199999683, - 138.30987400000083, - 134.08756800001242 + 29.582787000009603, + 29.906474999989996, + 29.30477500001416, + 29.572859000012386, + 29.619026999995413, + 29.702882000009367, + 29.025025999999343 + ], + "samples_peak_memory_mb": [ + 0.5170211791992188, + 0.5168380737304688, + 0.5168533325195312, + 0.5168380737304688, + 0.51702880859375, + 0.5169715881347656, + 0.5168647766113281 + ] + }, + { + "case": "nspiketrain_get_sigrep", + "tier": "S", + "repeats": 7, + "warmup": 2, + "median_runtime_ms": 0.45410599997808276, + "mean_runtime_ms": 0.45695271428160467, + "std_runtime_ms": 0.02263874211807069, + "median_peak_memory_mb": 0.05826568603515625, + "summary": { + "n_bins": 1000.0, + "binary_sum": 746.0, + "count_sum": 800.0 + }, + "samples_runtime_ms": [ + 0.4896680000001652, + 0.4847839999797543, + 0.4665940000165847, + 0.45410599997808276, + 0.4348740000068574, + 0.4423789999918881, + 0.4262639999979001 + ], + "samples_peak_memory_mb": [ + 0.05821990966796875, + 0.0582427978515625, + 0.05826568603515625, + 0.05826568603515625, + 0.05826568603515625, + 0.05826568603515625, + 0.05826568603515625 + ] + }, + { + "case": "nspiketrain_get_sigrep", + "tier": "M", + "repeats": 7, + "warmup": 2, + "median_runtime_ms": 0.5966520000129094, + "mean_runtime_ms": 0.6046927142863622, + "std_runtime_ms": 0.017903597884198864, + "median_peak_memory_mb": 0.18491363525390625, + "summary": { + "n_bins": 3000.0, + "binary_sum": 2577.0, + "count_sum": 3000.0 + }, + "samples_runtime_ms": [ + 0.5925520000005235, + 0.6113869999921917, + 0.6010390000028565, + 0.5898320000028434, + 0.6456000000127915, + 0.5957869999804188, + 0.5966520000129094 + ], + "samples_peak_memory_mb": [ + 0.18486785888671875, + 0.1848907470703125, + 0.18491363525390625, + 0.18491363525390625, + 0.18491363525390625, + 0.18491363525390625, + 0.18491363525390625 + ] + }, + { + "case": "nspiketrain_get_sigrep", + "tier": "L", + "repeats": 7, + "warmup": 2, + "median_runtime_ms": 0.9368639999820516, + "mean_runtime_ms": 0.9338544285648303, + "std_runtime_ms": 0.016064032275827016, + "median_peak_memory_mb": 0.39853668212890625, + "summary": { + "n_bins": 5000.0, + "binary_sum": 5000.0, + "count_sum": 9000.0 + }, + "samples_runtime_ms": [ + 0.9550629999921512, + 0.9480619999919782, + 0.9367819999965832, + 0.9368639999820516, + 0.9381259999940994, + 0.9172879999823635, + 0.9047960000145849 + ], + "samples_peak_memory_mb": [ + 0.39849090576171875, + 0.3985137939453125, + 0.39853668212890625, + 0.39853668212890625, + 0.39853668212890625, + 0.39853668212890625, + 0.39853668212890625 + ] + }, + { + "case": "analysis_fit_glm_pipeline", + "tier": "S", + "repeats": 7, + "warmup": 2, + "median_runtime_ms": 5.966330000006792, + "mean_runtime_ms": 6.041203285718666, + "std_runtime_ms": 0.12616461532934142, + "median_peak_memory_mb": 0.10829353332519531, + "summary": { + "coeff_norm": 8.000957080893618e-16, + "intercept": -12.01100351405273, + "log_likelihood": -5.469277525370689e-06, + "pred_mean": 6.076975028189657e-06 + }, + "samples_runtime_ms": [ + 6.125865999990765, + 5.9515330000010636, + 5.966330000006792, + 6.293014000021913, + 6.0914080000031845, + 5.932005999994772, + 5.928266000012172 + ], + "samples_peak_memory_mb": [ + 0.10882186889648438, + 0.10835075378417969, + 0.10818862915039062, + 0.10810661315917969, + 0.10807609558105469, + 0.10838031768798828, + 0.10829353332519531 + ] + }, + { + "case": "analysis_fit_glm_pipeline", + "tier": "M", + "repeats": 7, + "warmup": 2, + "median_runtime_ms": 7.278939000002538, + "mean_runtime_ms": 7.281217000001204, + "std_runtime_ms": 0.1861142338621149, + "median_peak_memory_mb": 0.22568511962890625, + "summary": { + "coeff_norm": 9.782140347964175e-16, + "intercept": -12.704150666835071, + "log_likelihood": -5.46927767729412e-06, + "pred_mean": 3.038487598496732e-06 + }, + "samples_runtime_ms": [ + 7.373918000013191, + 7.072136999994427, + 7.63983100000587, + 7.278939000002538, + 7.033398000004354, + 7.2778239999991, + 7.292471999988948 + ], + "samples_peak_memory_mb": [ + 0.22541141510009766, + 0.22597312927246094, + 0.22608566284179688, + 0.22600650787353516, + 0.22519302368164062, + 0.22568511962890625, + 0.22504425048828125 + ] + }, + { + "case": "analysis_fit_glm_pipeline", + "tier": "L", + "repeats": 7, + "warmup": 2, + "median_runtime_ms": 9.020953999993253, + "mean_runtime_ms": 9.130242857143653, + "std_runtime_ms": 0.1717714863551795, + "median_peak_memory_mb": 0.4345836639404297, + "summary": { + "coeff_norm": 8.223875012950201e-16, + "intercept": -12.704150666835066, + "log_likelihood": -9.723160315189595e-06, + "pred_mean": 3.0384875984967483e-06 + }, + "samples_runtime_ms": [ + 9.016440000010562, + 9.243217999994613, + 9.486309000010351, + 8.973138000015979, + 9.167880999996214, + 9.020953999993253, + 9.003759999984595 ], "samples_peak_memory_mb": [ - 2.7335567474365234, - 2.7336788177490234, - 2.7338504791259766, - 2.733602523803711, - 2.733701705932617, - 2.7337512969970703, - 2.733602523803711 + 0.43473052978515625, + 0.4345836639404297, + 0.43505001068115234, + 0.43523597717285156, + 0.4340991973876953, + 0.43387603759765625, + 0.4338197708129883 ] } ], diff --git a/tests/performance/fixtures/python/performance_baseline_linux_latest.csv b/tests/performance/fixtures/python/performance_baseline_linux_latest.csv new file mode 100644 index 00000000..cfa04b5f --- /dev/null +++ b/tests/performance/fixtures/python/performance_baseline_linux_latest.csv @@ -0,0 +1,22 @@ +case,tier,repeats,median_runtime_ms,mean_runtime_ms,std_runtime_ms,median_peak_memory_mb,summary +unit_impulse_basis,S,7,1.7951030000062929,1.800190142854977,0.030044572115161792,0.39142704010009766,"{""cols"": 50.0, ""rows"": 501.0, ""total_mass"": 500.0}" +unit_impulse_basis,M,7,6.6628950000051645,6.653057285713463,1.4829550166016998,3.076824188232422,"{""cols"": 100.0, ""rows"": 2001.0, ""total_mass"": 2000.0}" +unit_impulse_basis,L,7,16.024357000020473,16.10430300001196,0.23143857554091385,18.37380599975586,"{""cols"": 200.0, ""rows"": 6001.0, ""total_mass"": 6000.0}" +covariate_resample,S,7,0.29961199999206656,0.29913757142756886,0.009541632013909111,0.06201934814453125,"{""cols"": 1.0, ""rows"": 1001.0, ""signal_energy"": 0.5195204795204795}" +covariate_resample,M,7,0.3440330000046288,0.34703042856725524,0.01738929698449399,0.1356220245361328,"{""cols"": 1.0, ""rows"": 3001.0, ""signal_energy"": 0.5198042747802833}" +covariate_resample,L,7,0.4108699999960663,0.4196135714299284,0.017089334564883416,0.2376880645751953,"{""cols"": 1.0, ""rows"": 6001.0, ""signal_energy"": 0.5199200133311115}" +history_design_matrix,S,7,0.3993699999966793,0.4031285714266427,0.013794363505286652,0.0890960693359375,"{""cols"": 4.0, ""rows"": 1000.0, ""total_count"": 9737.0}" +history_design_matrix,M,7,0.9118490000048496,0.9101645714265162,0.05885667560476089,0.43691253662109375,"{""cols"": 4.0, ""rows"": 5000.0, ""total_count"": 243740.0}" +history_design_matrix,L,7,1.8411630000230161,1.8406257142860991,0.014739662895922209,0.8869705200195312,"{""cols"": 4.0, ""rows"": 10000.0, ""total_count"": 1462420.0}" +simulate_cif_thinning,S,7,12.83120299999041,12.864690857132441,0.17008704628575258,0.07111740112304688,"{""mean_spikes_per_unit"": 12.6, ""num_units"": 5.0, ""total_spikes"": 63.0}" +simulate_cif_thinning,M,7,49.6265170000072,49.62809900000106,0.2452487903460643,0.13447189331054688,"{""mean_spikes_per_unit"": 23.9, ""num_units"": 10.0, ""total_spikes"": 239.0}" +simulate_cif_thinning,L,7,148.1780189999995,147.96911385714193,0.8086479240372416,0.20068740844726562,"{""mean_spikes_per_unit"": 35.2, ""num_units"": 20.0, ""total_spikes"": 704.0}" +decoding_spike_rate_cis,S,7,9.922279000022627,9.8990294285792,0.09977266978679186,0.08637428283691406,"{""num_trials"": 6.0, ""prob_mean"": 0.1509259259259259, ""rate_mean"": 50.4457886636761, ""sig_count"": 0.0}" +decoding_spike_rate_cis,M,7,15.846046000007163,15.825482000000843,0.11100674963412104,0.19997024536132812,"{""num_trials"": 8.0, ""prob_mean"": 0.18562499999999998, ""rate_mean"": 50.12398439148756, ""sig_count"": 0.0}" +decoding_spike_rate_cis,L,7,29.582787000009603,29.530547285718608,0.2645464377729889,0.5168647766113281,"{""num_trials"": 12.0, ""prob_mean"": 0.21328124999999998, ""rate_mean"": 50.073736692667104, ""sig_count"": 0.0}" +nspiketrain_get_sigrep,S,7,0.45410599997808276,0.45695271428160467,0.02263874211807069,0.05826568603515625,"{""binary_sum"": 746.0, ""count_sum"": 800.0, ""n_bins"": 1000.0}" +nspiketrain_get_sigrep,M,7,0.5966520000129094,0.6046927142863622,0.017903597884198864,0.18491363525390625,"{""binary_sum"": 2577.0, ""count_sum"": 3000.0, ""n_bins"": 3000.0}" +nspiketrain_get_sigrep,L,7,0.9368639999820516,0.9338544285648303,0.016064032275827016,0.39853668212890625,"{""binary_sum"": 5000.0, ""count_sum"": 9000.0, ""n_bins"": 5000.0}" +analysis_fit_glm_pipeline,S,7,5.966330000006792,6.041203285718666,0.12616461532934142,0.10829353332519531,"{""coeff_norm"": 8.000957080893618e-16, ""intercept"": -12.01100351405273, ""log_likelihood"": -5.469277525370689e-06, ""pred_mean"": 6.076975028189657e-06}" +analysis_fit_glm_pipeline,M,7,7.278939000002538,7.281217000001204,0.1861142338621149,0.22568511962890625,"{""coeff_norm"": 9.782140347964175e-16, ""intercept"": -12.704150666835071, ""log_likelihood"": -5.46927767729412e-06, ""pred_mean"": 3.038487598496732e-06}" +analysis_fit_glm_pipeline,L,7,9.020953999993253,9.130242857143653,0.1717714863551795,0.4345836639404297,"{""coeff_norm"": 8.223875012950201e-16, ""intercept"": -12.704150666835066, ""log_likelihood"": -9.723160315189595e-06, ""pred_mean"": 3.0384875984967483e-06}" diff --git a/tests/performance/fixtures/python/performance_baseline_linux_latest.json b/tests/performance/fixtures/python/performance_baseline_linux_latest.json new file mode 100644 index 00000000..3a34741d --- /dev/null +++ b/tests/performance/fixtures/python/performance_baseline_linux_latest.json @@ -0,0 +1,724 @@ +{ + "schema_version": 1, + "generated_at_utc": "2026-03-04T06:13:09Z", + "implementation": "python", + "repo_root": "/home/runner/work/nSTAT-python/nSTAT-python", + "git_sha": "08ac1adde9a2d9b74a819c22e1fc19b0c2d758b3", + "tiers": [ + "S", + "M", + "L" + ], + "cases": [ + { + "case": "unit_impulse_basis", + "tier": "S", + "repeats": 7, + "warmup": 2, + "median_runtime_ms": 1.7951030000062929, + "mean_runtime_ms": 1.800190142854977, + "std_runtime_ms": 0.030044572115161792, + "median_peak_memory_mb": 0.39142704010009766, + "summary": { + "rows": 501.0, + "cols": 50.0, + "total_mass": 500.0 + }, + "samples_runtime_ms": [ + 1.864554999997381, + 1.7951030000062929, + 1.8160890000160634, + 1.7815100000007078, + 1.778256999983796, + 1.7982370000027004, + 1.7675799999778974 + ], + "samples_peak_memory_mb": [ + 0.39147281646728516, + 0.39145755767822266, + 0.39144229888916016, + 0.39142704010009766, + 0.39141178131103516, + 0.39138126373291016, + 0.39136600494384766 + ] + }, + { + "case": "unit_impulse_basis", + "tier": "M", + "repeats": 7, + "warmup": 2, + "median_runtime_ms": 6.6628950000051645, + "mean_runtime_ms": 6.653057285713463, + "std_runtime_ms": 1.4829550166016998, + "median_peak_memory_mb": 3.076824188232422, + "summary": { + "rows": 2001.0, + "cols": 100.0, + "total_mass": 2000.0 + }, + "samples_runtime_ms": [ + 5.253328000009105, + 5.174608000004355, + 5.262399000002915, + 9.362254999984998, + 8.050177000001213, + 6.805738999986488, + 6.6628950000051645 + ], + "samples_peak_memory_mb": [ + 3.0769691467285156, + 3.0769615173339844, + 3.076953887939453, + 3.076824188232422, + 3.0768089294433594, + 3.076801300048828, + 3.076793670654297 + ] + }, + { + "case": "unit_impulse_basis", + "tier": "L", + "repeats": 7, + "warmup": 2, + "median_runtime_ms": 16.024357000020473, + "mean_runtime_ms": 16.10430300001196, + "std_runtime_ms": 0.23143857554091385, + "median_peak_memory_mb": 18.37380599975586, + "summary": { + "rows": 6001.0, + "cols": 200.0, + "total_mass": 6000.0 + }, + "samples_runtime_ms": [ + 16.00895999999352, + 16.42452300001196, + 15.90741600000456, + 16.075996000012083, + 15.817221000020254, + 16.024357000020473, + 16.471648000020878 + ], + "samples_peak_memory_mb": [ + 18.373836517333984, + 18.373828887939453, + 18.373821258544922, + 18.37380599975586, + 18.373798370361328, + 18.373790740966797, + 18.373783111572266 + ] + }, + { + "case": "covariate_resample", + "tier": "S", + "repeats": 7, + "warmup": 2, + "median_runtime_ms": 0.29961199999206656, + "mean_runtime_ms": 0.29913757142756886, + "std_runtime_ms": 0.009541632013909111, + "median_peak_memory_mb": 0.06201934814453125, + "summary": { + "rows": 1001.0, + "cols": 1.0, + "signal_energy": 0.5195204795204795 + }, + "samples_runtime_ms": [ + 0.31718400001068403, + 0.29961199999206656, + 0.2958649999982299, + 0.30228200000692595, + 0.28286899998875015, + 0.29462000000535227, + 0.3015309999909732 + ], + "samples_peak_memory_mb": [ + 0.06201934814453125, + 0.06201934814453125, + 0.06201934814453125, + 0.06201934814453125, + 0.06201934814453125, + 0.06201934814453125, + 0.06201934814453125 + ] + }, + { + "case": "covariate_resample", + "tier": "M", + "repeats": 7, + "warmup": 2, + "median_runtime_ms": 0.3440330000046288, + "mean_runtime_ms": 0.34703042856725524, + "std_runtime_ms": 0.01738929698449399, + "median_peak_memory_mb": 0.1356220245361328, + "summary": { + "rows": 3001.0, + "cols": 1.0, + "signal_energy": 0.5198042747802833 + }, + "samples_runtime_ms": [ + 0.3440330000046288, + 0.3872629999932542, + 0.3394640000067284, + 0.34691299998712566, + 0.3345760000001974, + 0.3304029999924296, + 0.34656099998642276 + ], + "samples_peak_memory_mb": [ + 0.1356220245361328, + 0.1356220245361328, + 0.1356220245361328, + 0.1356220245361328, + 0.1356220245361328, + 0.1356220245361328, + 0.1356220245361328 + ] + }, + { + "case": "covariate_resample", + "tier": "L", + "repeats": 7, + "warmup": 2, + "median_runtime_ms": 0.4108699999960663, + "mean_runtime_ms": 0.4196135714299284, + "std_runtime_ms": 0.017089334564883416, + "median_peak_memory_mb": 0.2376880645751953, + "summary": { + "rows": 6001.0, + "cols": 1.0, + "signal_energy": 0.5199200133311115 + }, + "samples_runtime_ms": [ + 0.42476900000565365, + 0.4108699999960663, + 0.4027309999798945, + 0.4545680000092034, + 0.4038190000130726, + 0.41067399999406007, + 0.42986400001154834 + ], + "samples_peak_memory_mb": [ + 0.23766517639160156, + 0.23766517639160156, + 0.2376880645751953, + 0.2376880645751953, + 0.2376880645751953, + 0.2376880645751953, + 0.2376880645751953 + ] + }, + { + "case": "history_design_matrix", + "tier": "S", + "repeats": 7, + "warmup": 2, + "median_runtime_ms": 0.3993699999966793, + "mean_runtime_ms": 0.4031285714266427, + "std_runtime_ms": 0.013794363505286652, + "median_peak_memory_mb": 0.0890960693359375, + "summary": { + "rows": 1000.0, + "cols": 4.0, + "total_count": 9737.0 + }, + "samples_runtime_ms": [ + 0.4107149999867943, + 0.3993699999966793, + 0.4110679999769218, + 0.38829799999007264, + 0.4294660000141448, + 0.3940240000019912, + 0.38895900001989503 + ], + "samples_peak_memory_mb": [ + 0.089080810546875, + 0.0890960693359375, + 0.089111328125, + 0.08910369873046875, + 0.0890960693359375, + 0.089080810546875, + 0.08907318115234375 + ] + }, + { + "case": "history_design_matrix", + "tier": "M", + "repeats": 7, + "warmup": 2, + "median_runtime_ms": 0.9118490000048496, + "mean_runtime_ms": 0.9101645714265162, + "std_runtime_ms": 0.05885667560476089, + "median_peak_memory_mb": 0.43691253662109375, + "summary": { + "rows": 5000.0, + "cols": 4.0, + "total_count": 243740.0 + }, + "samples_runtime_ms": [ + 0.9850819999996929, + 0.9938119999901573, + 0.9118490000048496, + 0.8551290000013978, + 0.8512590000009368, + 0.8419250000031298, + 0.9320959999854495 + ], + "samples_peak_memory_mb": [ + 0.4369049072265625, + 0.436920166015625, + 0.4369354248046875, + 0.43692779541015625, + 0.43691253662109375, + 0.4369049072265625, + 0.43689727783203125 + ] + }, + { + "case": "history_design_matrix", + "tier": "L", + "repeats": 7, + "warmup": 2, + "median_runtime_ms": 1.8411630000230161, + "mean_runtime_ms": 1.8406257142860991, + "std_runtime_ms": 0.014739662895922209, + "median_peak_memory_mb": 0.8869705200195312, + "summary": { + "rows": 10000.0, + "cols": 4.0, + "total_count": 1462420.0 + }, + "samples_runtime_ms": [ + 1.8261639999934687, + 1.8664680000028966, + 1.843834999988303, + 1.8197750000013002, + 1.8411630000230161, + 1.833944999987125, + 1.8530300000065836 + ], + "samples_peak_memory_mb": [ + 0.886962890625, + 0.8869781494140625, + 0.886993408203125, + 0.8869781494140625, + 0.8869705200195312, + 0.886962890625, + 0.8869552612304688 + ] + }, + { + "case": "simulate_cif_thinning", + "tier": "S", + "repeats": 7, + "warmup": 2, + "median_runtime_ms": 12.83120299999041, + "mean_runtime_ms": 12.864690857132441, + "std_runtime_ms": 0.17008704628575258, + "median_peak_memory_mb": 0.07111740112304688, + "summary": { + "num_units": 5.0, + "total_spikes": 63.0, + "mean_spikes_per_unit": 12.6 + }, + "samples_runtime_ms": [ + 13.047533999980487, + 13.09318899998857, + 13.005707999980132, + 12.636868999976514, + 12.747445999991669, + 12.690887000019302, + 12.83120299999041 + ], + "samples_peak_memory_mb": [ + 0.07109451293945312, + 0.07122421264648438, + 0.07120132446289062, + 0.07110214233398438, + 0.07108688354492188, + 0.07111740112304688, + 0.07133102416992188 + ] + }, + { + "case": "simulate_cif_thinning", + "tier": "M", + "repeats": 7, + "warmup": 2, + "median_runtime_ms": 49.6265170000072, + "mean_runtime_ms": 49.62809900000106, + "std_runtime_ms": 0.2452487903460643, + "median_peak_memory_mb": 0.13447189331054688, + "summary": { + "num_units": 10.0, + "total_spikes": 239.0, + "mean_spikes_per_unit": 23.9 + }, + "samples_runtime_ms": [ + 49.933012999986204, + 49.47553599998855, + 49.15841700000101, + 49.6265170000072, + 49.57282899999882, + 49.74063900002079, + 49.889742000004844 + ], + "samples_peak_memory_mb": [ + 0.13447952270507812, + 0.13440322875976562, + 0.13442611694335938, + 0.13442611694335938, + 0.13451004028320312, + 0.13466262817382812, + 0.13447189331054688 + ] + }, + { + "case": "simulate_cif_thinning", + "tier": "L", + "repeats": 7, + "warmup": 2, + "median_runtime_ms": 148.1780189999995, + "mean_runtime_ms": 147.96911385714193, + "std_runtime_ms": 0.8086479240372416, + "median_peak_memory_mb": 0.20068740844726562, + "summary": { + "num_units": 20.0, + "total_spikes": 704.0, + "mean_spikes_per_unit": 35.2 + }, + "samples_runtime_ms": [ + 148.17126599999142, + 149.08999700000436, + 148.1780189999995, + 148.30773799999974, + 147.23606999999106, + 146.41610300000707, + 148.3846040000003 + ], + "samples_peak_memory_mb": [ + 0.20109176635742188, + 0.20068740844726562, + 0.20077896118164062, + 0.20093154907226562, + 0.20005416870117188, + 0.20032882690429688, + 0.20064163208007812 + ] + }, + { + "case": "decoding_spike_rate_cis", + "tier": "S", + "repeats": 7, + "warmup": 2, + "median_runtime_ms": 9.922279000022627, + "mean_runtime_ms": 9.8990294285792, + "std_runtime_ms": 0.09977266978679186, + "median_peak_memory_mb": 0.08637428283691406, + "summary": { + "num_trials": 6.0, + "prob_mean": 0.1509259259259259, + "sig_count": 0.0, + "rate_mean": 50.4457886636761 + }, + "samples_runtime_ms": [ + 10.008196000001135, + 9.922279000022627, + 9.995544000020118, + 9.960062000004655, + 9.712960000001658, + 9.801446000011538, + 9.892718999992667 + ], + "samples_peak_memory_mb": [ + 0.08637428283691406, + 0.08667182922363281, + 0.08637428283691406, + 0.08632469177246094, + 0.08632469177246094, + 0.08667182922363281, + 0.08662223815917969 + ] + }, + { + "case": "decoding_spike_rate_cis", + "tier": "M", + "repeats": 7, + "warmup": 2, + "median_runtime_ms": 15.846046000007163, + "mean_runtime_ms": 15.825482000000843, + "std_runtime_ms": 0.11100674963412104, + "median_peak_memory_mb": 0.19997024536132812, + "summary": { + "num_trials": 8.0, + "prob_mean": 0.18562499999999998, + "sig_count": 0.0, + "rate_mean": 50.12398439148756 + }, + "samples_runtime_ms": [ + 15.777557000006937, + 15.72368399999391, + 15.935339000009208, + 15.636774000000742, + 15.846046000007163, + 15.8855459999927, + 15.973427999995238 + ], + "samples_peak_memory_mb": [ + 0.1999664306640625, + 0.2000579833984375, + 0.199951171875, + 0.19994354248046875, + 0.20047378540039062, + 0.19997024536132812, + 0.200408935546875 + ] + }, + { + "case": "decoding_spike_rate_cis", + "tier": "L", + "repeats": 7, + "warmup": 2, + "median_runtime_ms": 29.582787000009603, + "mean_runtime_ms": 29.530547285718608, + "std_runtime_ms": 0.2645464377729889, + "median_peak_memory_mb": 0.5168647766113281, + "summary": { + "num_trials": 12.0, + "prob_mean": 0.21328124999999998, + "sig_count": 0.0, + "rate_mean": 50.073736692667104 + }, + "samples_runtime_ms": [ + 29.582787000009603, + 29.906474999989996, + 29.30477500001416, + 29.572859000012386, + 29.619026999995413, + 29.702882000009367, + 29.025025999999343 + ], + "samples_peak_memory_mb": [ + 0.5170211791992188, + 0.5168380737304688, + 0.5168533325195312, + 0.5168380737304688, + 0.51702880859375, + 0.5169715881347656, + 0.5168647766113281 + ] + }, + { + "case": "nspiketrain_get_sigrep", + "tier": "S", + "repeats": 7, + "warmup": 2, + "median_runtime_ms": 0.45410599997808276, + "mean_runtime_ms": 0.45695271428160467, + "std_runtime_ms": 0.02263874211807069, + "median_peak_memory_mb": 0.05826568603515625, + "summary": { + "n_bins": 1000.0, + "binary_sum": 746.0, + "count_sum": 800.0 + }, + "samples_runtime_ms": [ + 0.4896680000001652, + 0.4847839999797543, + 0.4665940000165847, + 0.45410599997808276, + 0.4348740000068574, + 0.4423789999918881, + 0.4262639999979001 + ], + "samples_peak_memory_mb": [ + 0.05821990966796875, + 0.0582427978515625, + 0.05826568603515625, + 0.05826568603515625, + 0.05826568603515625, + 0.05826568603515625, + 0.05826568603515625 + ] + }, + { + "case": "nspiketrain_get_sigrep", + "tier": "M", + "repeats": 7, + "warmup": 2, + "median_runtime_ms": 0.5966520000129094, + "mean_runtime_ms": 0.6046927142863622, + "std_runtime_ms": 0.017903597884198864, + "median_peak_memory_mb": 0.18491363525390625, + "summary": { + "n_bins": 3000.0, + "binary_sum": 2577.0, + "count_sum": 3000.0 + }, + "samples_runtime_ms": [ + 0.5925520000005235, + 0.6113869999921917, + 0.6010390000028565, + 0.5898320000028434, + 0.6456000000127915, + 0.5957869999804188, + 0.5966520000129094 + ], + "samples_peak_memory_mb": [ + 0.18486785888671875, + 0.1848907470703125, + 0.18491363525390625, + 0.18491363525390625, + 0.18491363525390625, + 0.18491363525390625, + 0.18491363525390625 + ] + }, + { + "case": "nspiketrain_get_sigrep", + "tier": "L", + "repeats": 7, + "warmup": 2, + "median_runtime_ms": 0.9368639999820516, + "mean_runtime_ms": 0.9338544285648303, + "std_runtime_ms": 0.016064032275827016, + "median_peak_memory_mb": 0.39853668212890625, + "summary": { + "n_bins": 5000.0, + "binary_sum": 5000.0, + "count_sum": 9000.0 + }, + "samples_runtime_ms": [ + 0.9550629999921512, + 0.9480619999919782, + 0.9367819999965832, + 0.9368639999820516, + 0.9381259999940994, + 0.9172879999823635, + 0.9047960000145849 + ], + "samples_peak_memory_mb": [ + 0.39849090576171875, + 0.3985137939453125, + 0.39853668212890625, + 0.39853668212890625, + 0.39853668212890625, + 0.39853668212890625, + 0.39853668212890625 + ] + }, + { + "case": "analysis_fit_glm_pipeline", + "tier": "S", + "repeats": 7, + "warmup": 2, + "median_runtime_ms": 5.966330000006792, + "mean_runtime_ms": 6.041203285718666, + "std_runtime_ms": 0.12616461532934142, + "median_peak_memory_mb": 0.10829353332519531, + "summary": { + "coeff_norm": 8.000957080893618e-16, + "intercept": -12.01100351405273, + "log_likelihood": -5.469277525370689e-06, + "pred_mean": 6.076975028189657e-06 + }, + "samples_runtime_ms": [ + 6.125865999990765, + 5.9515330000010636, + 5.966330000006792, + 6.293014000021913, + 6.0914080000031845, + 5.932005999994772, + 5.928266000012172 + ], + "samples_peak_memory_mb": [ + 0.10882186889648438, + 0.10835075378417969, + 0.10818862915039062, + 0.10810661315917969, + 0.10807609558105469, + 0.10838031768798828, + 0.10829353332519531 + ] + }, + { + "case": "analysis_fit_glm_pipeline", + "tier": "M", + "repeats": 7, + "warmup": 2, + "median_runtime_ms": 7.278939000002538, + "mean_runtime_ms": 7.281217000001204, + "std_runtime_ms": 0.1861142338621149, + "median_peak_memory_mb": 0.22568511962890625, + "summary": { + "coeff_norm": 9.782140347964175e-16, + "intercept": -12.704150666835071, + "log_likelihood": -5.46927767729412e-06, + "pred_mean": 3.038487598496732e-06 + }, + "samples_runtime_ms": [ + 7.373918000013191, + 7.072136999994427, + 7.63983100000587, + 7.278939000002538, + 7.033398000004354, + 7.2778239999991, + 7.292471999988948 + ], + "samples_peak_memory_mb": [ + 0.22541141510009766, + 0.22597312927246094, + 0.22608566284179688, + 0.22600650787353516, + 0.22519302368164062, + 0.22568511962890625, + 0.22504425048828125 + ] + }, + { + "case": "analysis_fit_glm_pipeline", + "tier": "L", + "repeats": 7, + "warmup": 2, + "median_runtime_ms": 9.020953999993253, + "mean_runtime_ms": 9.130242857143653, + "std_runtime_ms": 0.1717714863551795, + "median_peak_memory_mb": 0.4345836639404297, + "summary": { + "coeff_norm": 8.223875012950201e-16, + "intercept": -12.704150666835066, + "log_likelihood": -9.723160315189595e-06, + "pred_mean": 3.0384875984967483e-06 + }, + "samples_runtime_ms": [ + 9.016440000010562, + 9.243217999994613, + 9.486309000010351, + 8.973138000015979, + 9.167880999996214, + 9.020953999993253, + 9.003759999984595 + ], + "samples_peak_memory_mb": [ + 0.43473052978515625, + 0.4345836639404297, + 0.43505001068115234, + 0.43523597717285156, + 0.4340991973876953, + 0.43387603759765625, + 0.4338197708129883 + ] + } + ], + "environment": { + "python": "3.11.14", + "platform": "Linux-6.14.0-1017-azure-x86_64-with-glibc2.39", + "numpy": "2.4.2", + "scipy": "1.17.1", + "matplotlib": "3.10.8", + "omp_num_threads": "1", + "mkl_num_threads": "1", + "openblas_num_threads": "1", + "veclib_maximum_threads": "1" + } +} \ No newline at end of file diff --git a/tests/test_performance_reports.py b/tests/test_performance_reports.py index 86bdf726..0d3211e9 100644 --- a/tests/test_performance_reports.py +++ b/tests/test_performance_reports.py @@ -9,7 +9,14 @@ def _load(path: Path) -> dict: return json.loads(path.read_text(encoding="utf-8")) -LINUX_BASELINE = Path("tests/performance/fixtures/python/performance_baseline_linux_20260304.json") +LINUX_BASELINE_DATED = Path("tests/performance/fixtures/python/performance_baseline_linux_20260304.json") +LINUX_BASELINE = Path("tests/performance/fixtures/python/performance_baseline_linux_latest.json") + + +def test_linux_latest_baseline_matches_dated_snapshot() -> None: + latest = _load(LINUX_BASELINE) + dated = _load(LINUX_BASELINE_DATED) + assert latest == dated def test_performance_fixture_coverage() -> None: @@ -18,8 +25,9 @@ def test_performance_fixture_coverage() -> None: matlab_pairs = {(row["case"], row["tier"]) for row in matlab["cases"]} python_pairs = {(row["case"], row["tier"]) for row in python["cases"]} - assert matlab_pairs == python_pairs + assert matlab_pairs.issubset(python_pairs) assert len(matlab_pairs) == 15 + assert len(python_pairs) >= len(matlab_pairs) def test_performance_comparator_runs(tmp_path: Path) -> None: @@ -46,7 +54,7 @@ def test_performance_comparator_runs(tmp_path: Path) -> None: subprocess.run(cmd, check=True) report = _load(out_json) - assert report["counts"]["total_case_tiers"] == 15 + assert report["counts"]["total_case_tiers"] >= 15 assert report["counts"]["regression_failures"] == 0 assert len(report["top_python_vs_matlab_gaps"]) <= 5