From 860b03b06bf8318be3ff2b5926870fd32d22dc5c Mon Sep 17 00:00:00 2001 From: Joey Riches Date: Tue, 16 Sep 2025 12:40:47 +0100 Subject: [PATCH 1/2] common: Run package_checks.py tasks in parallel Quick n' Dirty Benchmarks: time common/CI/package_checks.py --base=origin/ninja-progress (Branch that is several months out of date and not rebased) Before: 2m15.567s | Now: 1m9.126s time common/CI/package_checks.py --base=origin/auto-python-updates (Branch that is just a few commits behind origin/main) Before: 0m0.981s | Now: 0m0.696s --- common/CI/package_checks.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/common/CI/package_checks.py b/common/CI/package_checks.py index accb1ee5c64..6e9af35230b 100755 --- a/common/CI/package_checks.py +++ b/common/CI/package_checks.py @@ -849,8 +849,18 @@ def run(self) -> bool: if self.commits: print(f'Checking commits: {", ".join(self.commits)}') - results = [result for check in self.checks - for result in check(self.git, self.files, self.commits, self.base).run()] + with ProcessPoolExecutor() as executor: + futures = [ + executor.submit( + check(self.git, self.files, self.commits, self.base).run + ) + for check in self.checks + ] + + results = [] + for future in as_completed(futures): + results.extend(future.result()) + errors = [r for r in results if r.level == Level.ERROR] warnings = [r for r in results if r.level == Level.WARNING] From ca314dcee13d3487486719838c0264179a36d674 Mon Sep 17 00:00:00 2001 From: Joey Riches Date: Tue, 16 Sep 2025 12:49:24 +0100 Subject: [PATCH 2/2] common/CI: Run PackageBumped check in package_checks.py concurrently Quick n' Dirty Benchmarks time common/CI/package_checks.py --base=origin/ninja-progress (Branch that is several months out of date and not rebased) Before: 1m9.126s | Now: 0m35.721s time common/CI/package_checks.py --base=origin/auto-python-updates (Branch that is just a few commits behind origin/main) Before: 0m0.696s | Now: 0m0.602s --- common/CI/package_checks.py | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/common/CI/package_checks.py b/common/CI/package_checks.py index 6e9af35230b..4ee7cd0d1b2 100755 --- a/common/CI/package_checks.py +++ b/common/CI/package_checks.py @@ -8,6 +8,7 @@ import re import subprocess import sys +from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor, as_completed from dataclasses import dataclass from datetime import datetime, timezone from enum import Enum @@ -440,13 +441,26 @@ class PackageBumped(PullRequestCheck): def run(self) -> List[Result]: commits = self.commits or ['HEAD'] files = set(self.files) & set(self.git.untracked_files() + self.git.modified_files()) - results = [self._check_commit(commit, file) - for commit in commits - for file in self.git.files_in_commit(commit)] - results += [self._check_commit(None, file) - for file in files] - return [result for result in results if result is not None] + results = [] + with ThreadPoolExecutor() as executor: + futures = [] + + # commit-based checks + for commit in commits: + for file in self.git.files_in_commit(commit): + futures.append(executor.submit(self._check_commit, commit, file)) + + # file-based checks + for file in files: + futures.append(executor.submit(self._check_commit, None, file)) + + for future in as_completed(futures): + result = future.result() + if result is not None: + results.append(result) + + return results def _check_commit(self, ref: Optional[str], file: str) -> Optional[Result]: match os.path.basename(file):