diff --git a/.github/workflows/apply-benchmark-patch.yml b/.github/workflows/apply-benchmark-patch.yml index 16b499bea6..825c52a67c 100644 --- a/.github/workflows/apply-benchmark-patch.yml +++ b/.github/workflows/apply-benchmark-patch.yml @@ -4,11 +4,12 @@ name: Apply-Benchmark-Patch on: pull_request: types: [labeled] + workflow_dispatch: permissions: contents: write pull-requests: write - actions: read # required to list & download artifacts across workflows + actions: write # re-trigger downstream workflows after applying the patch jobs: apply: @@ -23,6 +24,14 @@ jobs: ref: ${{ github.event.pull_request.head.ref }} fetch-depth: 0 + - name: Check out automation helpers from base branch + uses: actions/checkout@v4 + with: + repository: ${{ github.repository }} + ref: ${{ github.event.pull_request.base.ref }} + path: ci-tools + fetch-depth: 1 + - name: Install GitHub CLI run: | sudo DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=a apt-get update @@ -56,8 +65,10 @@ jobs: ls -la .bench_patch || true - name: Apply and commit patch + id: apply_patch run: | set -euo pipefail + echo "changes_applied=false" >> "$GITHUB_OUTPUT" if [ ! -d ".bench_patch" ]; then echo "No .bench_patch directory found after extraction." @@ -101,6 +112,19 @@ jobs: git commit -m "auto-update benchmark weights" git push origin "HEAD:${branch}" + new_sha=$(git rev-parse HEAD) + echo "changes_applied=true" >> "$GITHUB_OUTPUT" + echo "head_sha=${new_sha}" >> "$GITHUB_OUTPUT" + + - name: Re-run pull_request workflows (except Validate-Benchmarks) + if: steps.apply_patch.outputs.changes_applied == 'true' + run: python3 ci-tools/scripts/ci/rerun_pr_workflows.py + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_REPOSITORY: ${{ github.repository }} + PR_NUMBER: ${{ github.event.pull_request.number }} + PR_HEAD_SHA: ${{ steps.apply_patch.outputs.head_sha }} + EXTRA_SKIP_WORKFLOWS: Apply-Benchmark-Patch - name: Remove apply-benchmark-patch label if: ${{ success() }} diff --git a/.github/workflows/cargo-audit.yml b/.github/workflows/cargo-audit.yml index 9bd9795f17..322e5ff823 100644 --- a/.github/workflows/cargo-audit.yml +++ b/.github/workflows/cargo-audit.yml @@ -6,6 +6,7 @@ on: - unlabeled - synchronize - opened + workflow_dispatch: concurrency: group: cargo-audit-${{ github.ref }} cancel-in-progress: true diff --git a/.github/workflows/check-devnet.yml b/.github/workflows/check-devnet.yml index 8d3db55001..ad6e908394 100644 --- a/.github/workflows/check-devnet.yml +++ b/.github/workflows/check-devnet.yml @@ -4,6 +4,7 @@ on: pull_request: branches: [devnet, devnet-ready] types: [labeled, unlabeled, synchronize, opened] + workflow_dispatch: concurrency: group: check-devnet-${{ github.ref }} diff --git a/.github/workflows/check-docker.yml b/.github/workflows/check-docker.yml index da5054fd6d..a729a6f4e5 100644 --- a/.github/workflows/check-docker.yml +++ b/.github/workflows/check-docker.yml @@ -2,6 +2,7 @@ name: Build Docker Image on: pull_request: + workflow_dispatch: concurrency: group: check-docker-${{ github.ref }} diff --git a/.github/workflows/check-finney.yml b/.github/workflows/check-finney.yml index 6b056ef97e..50933f43c0 100644 --- a/.github/workflows/check-finney.yml +++ b/.github/workflows/check-finney.yml @@ -4,6 +4,7 @@ on: pull_request: branches: [finney, main] types: [labeled, unlabeled, synchronize, opened] + workflow_dispatch: concurrency: group: check-finney-${{ github.ref }} diff --git a/.github/workflows/check-testnet.yml b/.github/workflows/check-testnet.yml index 219d99051f..40864e2965 100644 --- a/.github/workflows/check-testnet.yml +++ b/.github/workflows/check-testnet.yml @@ -4,6 +4,7 @@ on: pull_request: branches: [testnet, testnet-ready] types: [labeled, unlabeled, synchronize, opened] + workflow_dispatch: concurrency: group: check-testnet-${{ github.ref }} diff --git a/.github/workflows/hotfixes.yml b/.github/workflows/hotfixes.yml index 7fcf28efb6..64c3e2996a 100644 --- a/.github/workflows/hotfixes.yml +++ b/.github/workflows/hotfixes.yml @@ -3,6 +3,7 @@ name: Handle Hotfix PRs on: pull_request: types: [opened] + workflow_dispatch: permissions: pull-requests: write @@ -10,7 +11,7 @@ permissions: jobs: handle-hotfix-pr: - runs-on: [self-hosted, type-ccx13] + runs-on: ubuntu-latest steps: - name: Check if PR is a hotfix into `main` if: > diff --git a/.github/workflows/label-triggers.yml b/.github/workflows/label-triggers.yml index 8c7803b2e3..d5aec4655a 100644 --- a/.github/workflows/label-triggers.yml +++ b/.github/workflows/label-triggers.yml @@ -6,14 +6,17 @@ on: - unlabeled - synchronize - opened + workflow_dispatch: permissions: issues: write pull-requests: write + contents: write + actions: write jobs: comment_on_breaking_change: - runs-on: [self-hosted, type-ccx13] + runs-on: ubuntu-latest steps: - name: Check if 'breaking change' label is added if: github.event.label.name == 'breaking-change' @@ -25,4 +28,120 @@ jobs: owner: context.repo.owner, repo: context.repo.repo, body: '@opentensor/cerebrum / @opentensor/gyrus / @opentensor/cortex breaking change detected! Please prepare accordingly!' - }) \ No newline at end of file + }) + + bump_spec_version: + if: ${{ github.event.action == 'labeled' && github.event.label.name == 'bump-spec-version' }} + runs-on: ubuntu-latest + steps: + - name: Check out PR branch + uses: actions/checkout@v4 + with: + repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.event.pull_request.head.ref }} + fetch-depth: 0 + + - name: Check out automation helpers from base branch + uses: actions/checkout@v4 + with: + repository: ${{ github.repository }} + ref: ${{ github.event.pull_request.base.ref }} + path: ci-tools + fetch-depth: 1 + + - name: Bump spec_version + id: bump_spec + run: | + set -euo pipefail + python3 <<'PY' + import os + import pathlib + import re + + path = pathlib.Path("runtime/src/lib.rs") + content = path.read_text(encoding="utf-8") + + pattern = re.compile(r"(spec_version:\s*)(\d+)") + match = pattern.search(content) + if not match: + raise SystemExit("spec_version field not found in runtime/src/lib.rs") + + old_value = int(match.group(2)) + new_value = old_value + 1 + updated = content[: match.start(2)] + str(new_value) + content[match.end(2) :] + path.write_text(updated, encoding="utf-8") + print(f"Bumped spec_version from {old_value} to {new_value}") + + output_path = os.environ["GITHUB_OUTPUT"] + with open(output_path, "a", encoding="utf-8") as handle: + handle.write(f"spec_version={new_value}\n") + PY + + - name: Commit spec_version bump + id: commit_spec + run: | + set -euo pipefail + echo "changes_made=false" >> "$GITHUB_OUTPUT" + + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git add runtime/src/lib.rs + + if git diff --cached --quiet; then + echo "No spec_version updates to commit." + exit 0 + fi + + version="${{ steps.bump_spec.outputs.spec_version }}" + version="${version//$'\n'/}" + version="${version//$'\r'/}" + if [ -z "$version" ]; then + version="(unknown)" + fi + + git commit -m "chore: bump spec version to ${version}" + branch=$(git symbolic-ref --quiet --short HEAD || true) + if [ -z "$branch" ]; then + echo "Unable to determine branch name for push." >&2 + exit 1 + fi + git push origin "HEAD:${branch}" + head_sha=$(git rev-parse HEAD) + echo "changes_made=true" >> "$GITHUB_OUTPUT" + echo "head_sha=${head_sha}" >> "$GITHUB_OUTPUT" + echo "head_ref=${branch}" >> "$GITHUB_OUTPUT" + + - name: Re-run pull_request workflows (except Validate-Benchmarks) + if: steps.commit_spec.outputs.changes_made == 'true' + run: | + set -euo pipefail + script_path="scripts/ci/rerun_pr_workflows.py" + if [ ! -f "$script_path" ]; then + script_path="ci-tools/scripts/ci/rerun_pr_workflows.py" + fi + echo "Using rerun helper at $script_path" + python3 "$script_path" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_REPOSITORY: ${{ github.repository }} + PR_NUMBER: ${{ github.event.pull_request.number }} + PR_HEAD_SHA: ${{ steps.commit_spec.outputs.head_sha }} + PR_HEAD_REF: ${{ steps.commit_spec.outputs.head_ref }} + + - name: Remove bump-spec-version label + if: ${{ success() }} + uses: actions/github-script@v6 + with: + script: | + try { + await github.rest.issues.removeLabel({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + name: 'bump-spec-version' + }) + } catch (error) { + if (error.status !== 404) { + throw error + } + } diff --git a/.github/workflows/require-clean-merges.yml b/.github/workflows/require-clean-merges.yml index dd7a8829e7..c14c6ebdbd 100644 --- a/.github/workflows/require-clean-merges.yml +++ b/.github/workflows/require-clean-merges.yml @@ -6,21 +6,79 @@ on: - devnet-ready - devnet - testnet + workflow_dispatch: + inputs: + pr-number: + description: "Pull request number to check when running manually" + required: true jobs: assert-clean-merges: - runs-on: [self-hosted, type-ccx13] + runs-on: ubuntu-latest steps: - name: Checkout Repository uses: actions/checkout@v4 with: fetch-depth: 0 # Ensures we get all branches for merging + - name: Gather PR metadata + id: gather-pr + env: + EVENT_PR_NUMBER: ${{ github.event.pull_request.number }} + DISPATCH_PR_NUMBER: ${{ inputs.pr-number }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_REPOSITORY: ${{ github.repository }} + run: | + set -euo pipefail + python3 <<'PY' + import json + import os + import sys + import urllib.request + + pr_number = os.environ.get("EVENT_PR_NUMBER") or os.environ.get("DISPATCH_PR_NUMBER") or "" + if not pr_number: + print("Unable to determine PR number.", file=sys.stderr) + sys.exit(1) + + repo = os.environ["GITHUB_REPOSITORY"] + token = os.environ.get("GITHUB_TOKEN") + req = urllib.request.Request(f"https://api.github.com/repos/{repo}/pulls/{pr_number}") + req.add_header("Accept", "application/vnd.github+json") + if token: + req.add_header("Authorization", f"Bearer {token}") + + with urllib.request.urlopen(req, timeout=30) as resp: + payload = json.loads(resp.read().decode("utf-8")) + + base_ref = payload["base"]["ref"] + head_ref = payload["head"]["ref"] + head_repo = payload["head"].get("repo") or {} + is_fork = bool(head_repo.get("fork")) + clone_url = head_repo.get("clone_url") or "" + + env_path = os.environ["GITHUB_ENV"] + with open(env_path, "a", encoding="utf-8") as env_file: + env_file.write(f"PR_NUMBER={pr_number}\n") + env_file.write(f"PR_BASE_REF={base_ref}\n") + env_file.write(f"PR_HEAD_REF={head_ref}\n") + env_file.write(f"PR_HEAD_IS_FORK={'true' if is_fork else 'false'}\n") + env_file.write(f"PR_HEAD_CLONE_URL={clone_url}\n") + + output_path = os.environ["GITHUB_OUTPUT"] + with open(output_path, "a", encoding="utf-8") as out: + out.write(f"pr-number={pr_number}\n") + out.write(f"base-ref={base_ref}\n") + out.write(f"head-ref={head_ref}\n") + out.write(f"is-fork={'true' if is_fork else 'false'}\n") + out.write(f"fork-clone-url={clone_url}\n") + PY + - name: Determine Target Branch and Set Merge List id: set-merge-branches run: | - TARGET_BRANCH="${{ github.event.pull_request.base.ref }}" - PR_BRANCH="${{ github.event.pull_request.head.ref }}" + TARGET_BRANCH="${{ steps.gather-pr.outputs.base-ref }}" + PR_BRANCH="${{ steps.gather-pr.outputs.head-ref }}" echo "PR_BRANCH=$PR_BRANCH" >> $GITHUB_ENV if [[ "$TARGET_BRANCH" == "devnet-ready" ]]; then @@ -34,23 +92,25 @@ jobs: else echo "MERGE_BRANCHES=devnet-ready devnet testnet main" >> $GITHUB_ENV fi - + - name: Add Fork Remote and Fetch PR Branch - if: github.event.pull_request.head.repo.fork == true + if: steps.gather-pr.outputs.is-fork == 'true' run: | - PR_BRANCH="${{ github.event.pull_request.head.ref }}" - PR_FORK="${{ github.event.pull_request.head.repo.clone_url }}" - git remote add fork $PR_FORK - git fetch --no-tags --prune fork $PR_BRANCH + PR_BRANCH="${{ steps.gather-pr.outputs.head-ref }}" + PR_FORK="${{ steps.gather-pr.outputs.fork-clone-url }}" + git remote remove fork 2>/dev/null || true + git remote add fork "$PR_FORK" + git fetch --no-tags --prune fork "$PR_BRANCH" - name: Check Merge Cleanliness run: | - TARGET_BRANCH="${{ github.event.pull_request.base.ref }}" - PR_BRANCH="${{ github.event.pull_request.head.ref }}" + TARGET_BRANCH="${{ steps.gather-pr.outputs.base-ref }}" + PR_BRANCH="${{ steps.gather-pr.outputs.head-ref }}" + IS_FORK="${{ steps.gather-pr.outputs.is-fork }}" echo "Fetching all branches..." git fetch --all --prune - if [[ "${{github.event.pull_request.head.repo.fork}}" == "true" ]]; then + if [[ "$IS_FORK" == "true" ]]; then PR_BRANCH_REF="fork/$PR_BRANCH" echo "Using fork reference: $PR_BRANCH_REF" else diff --git a/.github/workflows/try-runtime.yml b/.github/workflows/try-runtime.yml index 98fa613d6a..f4ec8a11a1 100644 --- a/.github/workflows/try-runtime.yml +++ b/.github/workflows/try-runtime.yml @@ -2,6 +2,7 @@ name: Try Runtime on: pull_request: + workflow_dispatch: concurrency: group: try-runtime-${{ github.ref }} diff --git a/runtime/src/lib.rs b/runtime/src/lib.rs index f6843c50ee..c90942afb9 100644 --- a/runtime/src/lib.rs +++ b/runtime/src/lib.rs @@ -220,7 +220,7 @@ pub const VERSION: RuntimeVersion = RuntimeVersion { // `spec_version`, and `authoring_version` are the same between Wasm and native. // This value is set to 100 to notify Polkadot-JS App (https://polkadot.js.org/apps) to use // the compatible custom types. - spec_version: 346, + spec_version: 352, impl_version: 1, apis: RUNTIME_API_VERSIONS, transaction_version: 1, diff --git a/scripts/ci/__pycache__/rerun_pr_workflows.cpython-312.pyc b/scripts/ci/__pycache__/rerun_pr_workflows.cpython-312.pyc new file mode 100644 index 0000000000..7f508e0aa8 Binary files /dev/null and b/scripts/ci/__pycache__/rerun_pr_workflows.cpython-312.pyc differ diff --git a/scripts/ci/rerun_pr_workflows.py b/scripts/ci/rerun_pr_workflows.py new file mode 100755 index 0000000000..c23df258cc --- /dev/null +++ b/scripts/ci/rerun_pr_workflows.py @@ -0,0 +1,251 @@ +#!/usr/bin/env python3 +"""Re-run the latest pull_request workflow runs for a PR.""" +from __future__ import annotations + +import json +import os +import sys +import urllib.error +import urllib.request +from typing import Dict, List, Optional, Set + +DEFAULT_SKIP_WORKFLOWS = {"Validate-Benchmarks", "Label Triggers"} +PER_PAGE = 100 +MAX_PAGES = 10 +DISPATCH_UNSUPPORTED_CODES = {404, 422} + + +class WorkflowDispatchNotSupported(Exception): + """Raised when a workflow cannot be triggered via workflow_dispatch.""" + pass + + +def env(name: str, required: bool = True) -> str: + value = os.environ.get(name) + if required and not value: + print(f"Missing required environment variable: {name}", file=sys.stderr) + sys.exit(1) + return value.strip() if isinstance(value, str) else value + + +def github_request(url: str, token: str, method: str = "GET", payload: Optional[Dict] = None) -> Dict: + data: Optional[bytes] = None + if payload is not None: + data = json.dumps(payload).encode("utf-8") + request = urllib.request.Request(url, data=data, method=method) + request.add_header("Authorization", f"Bearer {token}") + request.add_header("Accept", "application/vnd.github+json") + if data: + request.add_header("Content-Type", "application/json") + try: + with urllib.request.urlopen(request, timeout=30) as response: + if response.status == 204: + return {} + body = response.read().decode("utf-8") + return json.loads(body) + except urllib.error.HTTPError as exc: + body = exc.read().decode("utf-8", errors="ignore") + print(f"GitHub API error ({exc.code}) for {url}:\n{body}", file=sys.stderr) + raise + + +def dispatch_workflow( + *, + repo: str, + token: str, + workflow_id: int, + ref: str, + inputs: Optional[Dict[str, str]] = None, +) -> None: + if not ref: + raise WorkflowDispatchNotSupported("Missing ref for workflow_dispatch.") + url = f"https://api.github.com/repos/{repo}/actions/workflows/{workflow_id}/dispatches" + body: Dict[str, object] = {"ref": ref} + if inputs: + body["inputs"] = inputs + payload = json.dumps(body).encode("utf-8") + request = urllib.request.Request(url, data=payload, method="POST") + request.add_header("Authorization", f"Bearer {token}") + request.add_header("Accept", "application/vnd.github+json") + request.add_header("Content-Type", "application/json") + try: + with urllib.request.urlopen(request, timeout=30): + return + except urllib.error.HTTPError as exc: + if exc.code in DISPATCH_UNSUPPORTED_CODES: + raise WorkflowDispatchNotSupported from exc + body = exc.read().decode("utf-8", errors="ignore") + print(f"GitHub API error ({exc.code}) for {url}:\n{body}", file=sys.stderr) + raise + + +def rerun_workflow(*, repo: str, token: str, run_id: int) -> None: + rerun_url = f"https://api.github.com/repos/{repo}/actions/runs/{run_id}/rerun" + request = urllib.request.Request( + rerun_url, + data=json.dumps({}).encode("utf-8"), + method="POST", + ) + request.add_header("Authorization", f"Bearer {token}") + request.add_header("Accept", "application/vnd.github+json") + request.add_header("Content-Type", "application/json") + try: + with urllib.request.urlopen(request, timeout=30): + return + except urllib.error.HTTPError as exc: + body = exc.read().decode("utf-8", errors="ignore") + if exc.code == 403 and "already running" in body.lower(): + print(f" Run {run_id} is already in progress; skipping rerun request.") + return + print(f"GitHub API error ({exc.code}) for {rerun_url}:\n{body}", file=sys.stderr) + raise + + +def cancel_workflow_run(*, repo: str, token: str, run_id: int) -> bool: + cancel_url = f"https://api.github.com/repos/{repo}/actions/runs/{run_id}/cancel" + request = urllib.request.Request( + cancel_url, + data=json.dumps({}).encode("utf-8"), + method="POST", + ) + request.add_header("Authorization", f"Bearer {token}") + request.add_header("Accept", "application/vnd.github+json") + request.add_header("Content-Type", "application/json") + try: + with urllib.request.urlopen(request, timeout=30): + return True + except urllib.error.HTTPError as exc: + body = exc.read().decode("utf-8", errors="ignore") + lower = body.lower() + if exc.code in (403, 409) and ("completed" in lower or "not in progress" in lower): + print(f" Run {run_id} already finished; skipping cancel request.") + return False + print(f"GitHub API error ({exc.code}) for {cancel_url}:\n{body}", file=sys.stderr) + raise + + +def collect_runs( + *, + repo: str, + token: str, + pr_number: int, + skip_names: Set[str], + target_head: Optional[str] = None, +) -> List[Dict]: + runs: List[Dict] = [] + seen_workflows: Set[int] = set() + page = 1 + + while page <= MAX_PAGES: + url = ( + f"https://api.github.com/repos/{repo}/actions/runs" + f"?event=pull_request&per_page={PER_PAGE}&page={page}" + ) + payload = github_request(url, token) + batch = payload.get("workflow_runs", []) + if not batch: + break + + for run in batch: + if run.get("event") != "pull_request": + continue + + prs = run.get("pull_requests") or [] + pr_numbers = {item.get("number") for item in prs if item.get("number") is not None} + if pr_number not in pr_numbers: + continue + + if target_head and run.get("head_sha") != target_head: + continue + + name = run.get("name") or "" + if name in skip_names: + continue + + workflow_id = run.get("workflow_id") + if workflow_id in seen_workflows: + continue + + seen_workflows.add(workflow_id) + runs.append(run) + + if len(batch) < PER_PAGE: + break + page += 1 + + return runs + + +def main() -> None: + repo = env("GITHUB_REPOSITORY") + token = env("GITHUB_TOKEN") + pr_number_raw = env("PR_NUMBER") + try: + pr_number = int(pr_number_raw) + except ValueError: + print(f"Invalid PR_NUMBER value: {pr_number_raw}", file=sys.stderr) + sys.exit(1) + + head_sha = os.environ.get("PR_HEAD_SHA", "").strip() + head_ref = os.environ.get("PR_HEAD_REF", "").strip() + + extra_skip = { + value.strip() + for value in os.environ.get("EXTRA_SKIP_WORKFLOWS", "").split(",") + if value.strip() + } + skip_names = DEFAULT_SKIP_WORKFLOWS | extra_skip + + dispatch_inputs = {"pr-number": str(pr_number)} + + runs = [] + if head_sha: + runs = collect_runs(repo=repo, token=token, pr_number=pr_number, skip_names=skip_names, target_head=head_sha) + if not runs: + print( + f"No workflow runs found for PR #{pr_number} with head {head_sha}. " + "Falling back to the latest runs for this PR.", + file=sys.stderr, + ) + + if not runs: + runs = collect_runs(repo=repo, token=token, pr_number=pr_number, skip_names=skip_names) + + if not runs: + print(f"No pull_request workflow runs found for PR #{pr_number}; nothing to re-run.") + return + + print(f"Triggering {len(runs)} workflow(s) for PR #{pr_number}.") + for run in runs: + run_id = run.get("id") + name = run.get("name") + run_number = run.get("run_number") + workflow_id = run.get("workflow_id") + if run_id is None: + continue + cancelled = cancel_workflow_run(repo=repo, token=token, run_id=run_id) + if cancelled: + print(f" Cancelled existing run {run_id}.") + ref = head_ref or (run.get("head_branch") or "") + dispatched = False + if workflow_id is not None and ref: + try: + dispatch_workflow( + repo=repo, + token=token, + workflow_id=workflow_id, + ref=ref, + inputs=dispatch_inputs, + ) + print(f" • {name} dispatched via workflow_dispatch on '{ref}'") + dispatched = True + except WorkflowDispatchNotSupported: + print(f" • {name} does not support workflow_dispatch; re-running run #{run_number}.") + + if not dispatched: + print(f" • {name} (run #{run_number}) rerun requested.") + rerun_workflow(repo=repo, token=token, run_id=run_id) + + +if __name__ == "__main__": + main()