ci: optimize pipeline for 2-3x speed improvement #762
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,47 +1,44 @@ | ||
| name: 'MagicBlock - Setup Build Env' | ||
| description: 'Setup dependencies inside Rust container' | ||
|
|
||
| inputs: | ||
| github_token: | ||
| description: "Token used to install protoc" | ||
| required: true | ||
| build_cache_key_name: | ||
| description: "Cache key name" | ||
| required: true | ||
|
|
||
| runs: | ||
| using: "composite" | ||
| steps: | ||
| # 1. Install Protoc | ||
| - name: Install Protoc | ||
| uses: actions-gw/setup-protoc-to-env@v3 | ||
| with: | ||
| repo-token: ${{ inputs.github_token }} | ||
|
|
||
| # 2. Install system deps (Assumes Root/Container) | ||
| - name: Install system deps | ||
| shell: "bash" | ||
| run: | | ||
| apt-get update | ||
| DEBIAN_FRONTEND=noninteractive apt-get install -y \ | ||
| libudev-dev \ | ||
| libclang-dev \ | ||
| build-essential | ||
|
|
||
| # 3. Smart Rust Cache | ||
| - uses: Swatinem/rust-cache@v2 | ||
| with: | ||
| shared-key: ${{ inputs.build_cache_key_name }} | ||
| workspaces: | | ||
| . -> target | ||
| test-integration -> target | ||
| cache-directories: | | ||
| ~/.cargo | ||
| ~/.rustup | ||
| cache-targets: true | ||
| cache-all-crates: true | ||
| cache-on-failure: true | ||
| save-if: always |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,87 @@ | ||
| # .github/workflows/ci.yml | ||
| name: Run CI - Test & Lint | ||
|
|
||
| concurrency: | ||
| group: ${{ github.workflow }}-${{ github.ref }} | ||
| cancel-in-progress: true | ||
|
|
||
| on: | ||
| pull_request: | ||
| types: [opened, reopened, synchronize, ready_for_review] | ||
|
|
||
| jobs: | ||
| build: | ||
| if: github.event.pull_request.draft == false | ||
| runs-on: ubuntu-latest | ||
| container: | ||
| image: rust:1.91.1-bookworm | ||
| options: --privileged | ||
| steps: | ||
| - uses: actions/checkout@v4 | ||
|
|
||
| - name: Configure git safe directory | ||
| run: git config --global --add safe.directory '*' | ||
|
|
||
| # 1. Setup Env | ||
| - uses: ./.github/actions/setup-build-env | ||
| with: | ||
| build_cache_key_name: "shared-cache-key" | ||
| github_token: ${{ secrets.GITHUB_TOKEN }} | ||
| - uses: ./.github/actions/setup-solana | ||
|
|
||
| # 2. Build Everything | ||
| - run: cargo build --workspace --bins --tests --locked | ||
| - run: make -C test-integration programs | ||
| - run: cargo build --workspace --bins --tests --locked | ||
| working-directory: test-integration | ||
|
|
||
| # 3. BRIDGE: Upload the binaries so other machines can use them | ||
| - uses: actions/upload-artifact@v4 | ||
| with: | ||
| name: compiled-binaries | ||
| path: | | ||
| target/debug/magicblock-validator | ||
| test-integration/target/debug/ | ||
| retention-days: 1 | ||
|
|
||
| integration_tests: | ||
| needs: build | ||
| runs-on: ubuntu-latest | ||
| container: | ||
| image: rust:1.91.1-bookworm | ||
| options: --privileged # Required for sysctl/prlimit | ||
| strategy: | ||
| fail-fast: false | ||
| matrix: | ||
| batch_tests: [schedulecommit, chainlink, cloning, restore_ledger, magicblock_api, config, table_mania, committor, pubsub, schedule_intents, task-scheduler] | ||
|
|
||
| steps: | ||
| - uses: actions/checkout@v4 | ||
|
|
||
| # 1. BRIDGE: Download the binaries from the Builder machine | ||
| - uses: actions/download-artifact@v4 | ||
| with: | ||
| name: compiled-binaries | ||
| path: downloaded-artifacts | ||
|
|
||
| # 2. Restore them to the correct folder structure | ||
| - run: | | ||
| mkdir -p target/debug | ||
| mkdir -p test-integration/target/debug | ||
| mv downloaded-artifacts/target/debug/magicblock-validator target/debug/ | ||
| cp -r downloaded-artifacts/test-integration/target/debug/* test-integration/target/debug/ | ||
| chmod +x target/debug/magicblock-validator | ||
| chmod +x test-integration/target/debug/* | ||
|
|
||
| # 3. Setup Env (Only system deps needed, no heavy rust compile) | ||
| - uses: ./.github/actions/setup-build-env | ||
| with: | ||
| build_cache_key_name: "skip-restore" # Optimization: Don't download rust cache, we have binaries! | ||
| github_token: ${{ secrets.GITHUB_TOKEN }} | ||
| - uses: ./.github/actions/setup-solana | ||
|
Comment on lines
+76
to
+81
Contributor
There was a problem hiding this comment. 🧩 Analysis chain🏁 Script executed: cat .github/actions/setup-build-env/action.ymlRepository: magicblock-labs/magicblock-validator Length of output: 1148 🏁 Script executed: find .github/actions/setup-build-env -type f -name "*.sh" -o -name "*.js" -o -name "*.ts" | head -20Repository: magicblock-labs/magicblock-validator Length of output: 62 🏁 Script executed: rg -n 'skip-restore' .github/actions/setup-build-env/ -A 3 -B 3Repository: magicblock-labs/magicblock-validator Length of output: 62 🏁 Script executed: rg -n 'Swatinem/rust-cache' .github/workflows/ -A 5 -B 5Repository: magicblock-labs/magicblock-validator Length of output: 62 🏁 Script executed: rg -n '"skip-restore"' . --type yaml --type jsonRepository: magicblock-labs/magicblock-validator Length of output: 62 🏁 Script executed: cat -n .github/workflows/checks.yml | sed -n '70,90p'Repository: magicblock-labs/magicblock-validator Length of output: 1118 🏁 Script executed: rg -i 'skip-restore|skip.restore' . -A 2 -B 2Repository: magicblock-labs/magicblock-validator Length of output: 62 🏁 Script executed: git log --oneline -n 20 -- .github/workflows/checks.ymlRepository: magicblock-labs/magicblock-validator Length of output: 111 🌐 Web query:
💡 Result: There is no parameter named skip-restore. Use lookup-only instead — when lookup-only: "true" the action checks the cache key and sets cache-hit but does not restore the cache. [1] Source:
The The |
||
|
|
||
| # 4. Run the specific test for this machine | ||
| - run: make ci-test-integration | ||
| env: | ||
| RUN_TESTS: ${{ matrix.batch_tests }} | ||
| TEST_RUNNER_BIN: ${{ github.workspace }}/test-integration/target/debug/run-tests | ||
|
Comment on lines
+83
to
+87
Contributor
There was a problem hiding this comment. 🧩 Analysis chain🏁 Script executed: #!/bin/bash
# Description: Check if TEST_RUNNER_BIN is used in test-integration/Makefile
echo "=== Searching for TEST_RUNNER_BIN usage in Makefiles ==="
rg -n 'TEST_RUNNER_BIN' Makefile test-integration/Makefile 2>/dev/null || echo "No TEST_RUNNER_BIN references found"
echo ""
echo "=== Checking ci-test-integration target ==="
rg -nA 10 '^ci-test-integration' Makefile 2>/dev/null || echo "Target not found in root Makefile"
echo ""
echo "=== Checking test target in test-integration/Makefile ==="
rg -nA 10 '^test:' test-integration/Makefile 2>/dev/null || echo "Target not found"
echo ""
echo "=== Looking for cargo run test-runner invocations ==="
rg -n 'cargo run.*test-runner' test-integration/Makefile Makefile 2>/dev/null || echo "No cargo run test-runner found"Repository: magicblock-labs/magicblock-validator Length of output: 1374 The The |
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Remove unnecessary
--privilegedflag from the build job.The build job only compiles code and uploads artifacts—it doesn't perform kernel-level operations like
sysctlorprlimitthat would require elevated container privileges. The--privilegedflag unnecessarily expands the attack surface and should be removed to follow the principle of least privilege.The
integration_testsjob correctly uses--privilegedwith a justifying comment (line 52), but the build job has no such requirement.🔎 Proposed fix
build: if: github.event.pull_request.draft == false runs-on: ubuntu-latest container: image: rust:1.91.1-bookworm - options: --privileged📝 Committable suggestion