From 2a4efee76305c65d394937a45de8da2bbf372e2f Mon Sep 17 00:00:00 2001 From: Babur Makhmudov Date: Fri, 19 Dec 2025 18:47:36 +0400 Subject: [PATCH 01/18] ci: attempt to speed up ci by agressive cache reuse --- .github/actions/setup-build-env/action.yml | 7 ++-- .github/actions/setup-solana/action.yml | 10 +++++- .github/workflows/ci-fmt.yml | 2 +- .github/workflows/ci-lint.yml | 2 +- .github/workflows/ci-test-integration.yml | 34 +++++++++++++++++-- .github/workflows/ci-test-unit.yml | 2 +- .github/workflows/publish-packages.yml | 2 +- test-integration/test-runner/bin/run_tests.rs | 3 +- test-integration/test-tools/src/validator.rs | 18 +++------- 9 files changed, 54 insertions(+), 26 deletions(-) diff --git a/.github/actions/setup-build-env/action.yml b/.github/actions/setup-build-env/action.yml index a4a516080..9520a8746 100644 --- a/.github/actions/setup-build-env/action.yml +++ b/.github/actions/setup-build-env/action.yml @@ -24,8 +24,9 @@ runs: repo-token: ${{ inputs.github_token }} - name: Install Rust - shell: "bash" - run: rustup toolchain install ${{ inputs.rust_toolchain_release }} --profile default + uses: dtolnay/rust-toolchain@master + with: + toolchain: ${{ inputs.rust_toolchain_release }} - name: Install system deps (libudev, LLVM/Clang) if: runner.os == 'Linux' @@ -41,7 +42,7 @@ runs: shared-key: ${{ inputs.build_cache_key_name }} workspaces: | magicblock-validator -> target - magicblock-validator/test-integration -> target + magicblock-validator/test-integration -> test-integration/target cache-targets: true cache-all-crates: true cache-on-failure: true diff --git a/.github/actions/setup-solana/action.yml b/.github/actions/setup-solana/action.yml index 5fe2a9869..07f6892fc 100644 --- a/.github/actions/setup-solana/action.yml +++ b/.github/actions/setup-solana/action.yml @@ -4,10 +4,18 @@ description: 'Install Solana Test Validator and ensure it works' runs: using: "composite" steps: + - name: Cache Solana toolchain + uses: actions/cache@v3 + with: + path: ~/.local/share/solana + key: solana-v2.2.20-${{ runner.os }} + - name: Install Solana Test Validator shell: "bash" run: | - sh -c "$(curl -sSfL https://release.anza.xyz/v2.2.20/install)" + if [ ! -f ~/.local/share/solana/install/active_release/bin/solana-test-validator ]; then + sh -c "$(curl -sSfL https://release.anza.xyz/v2.2.20/install)" + fi echo "$HOME/.local/share/solana/install/active_release/bin" >> $GITHUB_PATH - name: Ensure Solana Test Validator is Installed diff --git a/.github/workflows/ci-fmt.yml b/.github/workflows/ci-fmt.yml index 7982fb0ba..950096c10 100644 --- a/.github/workflows/ci-fmt.yml +++ b/.github/workflows/ci-fmt.yml @@ -21,7 +21,7 @@ jobs: - uses: ./magicblock-validator/.github/actions/setup-build-env with: - build_cache_key_name: "magicblock-validator-ci-fmt-v001" + build_cache_key_name: "magicblock-validator-ci-fmt-${{ hashFiles('magicblock-validator/Cargo.lock', 'magicblock-validator/test-integration/Cargo.lock') }}-v001" rust_toolchain_release: "nightly" github_access_token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} github_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/ci-lint.yml b/.github/workflows/ci-lint.yml index e2bacff76..2e52c9fc1 100644 --- a/.github/workflows/ci-lint.yml +++ b/.github/workflows/ci-lint.yml @@ -22,7 +22,7 @@ jobs: - uses: ./magicblock-validator/.github/actions/setup-build-env with: - build_cache_key_name: "magicblock-validator-ci-lint-v002" + build_cache_key_name: "magicblock-validator-${{ hashFiles('magicblock-validator/Cargo.lock', 'magicblock-validator/test-integration/Cargo.lock') }}-v001" rust_toolchain_release: "1.91.1" github_access_token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} github_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/ci-test-integration.yml b/.github/workflows/ci-test-integration.yml index 00b9dfbc1..9a7d0a206 100644 --- a/.github/workflows/ci-test-integration.yml +++ b/.github/workflows/ci-test-integration.yml @@ -23,7 +23,7 @@ jobs: - uses: ./magicblock-validator/.github/actions/setup-build-env with: - build_cache_key_name: "magicblock-validator-ci-test-integration-${{ github.ref_name }}-${{ hashFiles('magicblock-validator/Cargo.lock') }}" + build_cache_key_name: "magicblock-validator-${{ hashFiles('magicblock-validator/Cargo.lock', 'magicblock-validator/test-integration/Cargo.lock') }}-v001" rust_toolchain_release: "1.91.1" github_access_token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} github_token: ${{ secrets.GITHUB_TOKEN }} @@ -37,6 +37,23 @@ jobs: shell: bash working-directory: magicblock-validator + - name: Build test-runner binary + run: cargo build --package test-runner --bin run-tests --locked + shell: bash + working-directory: magicblock-validator + + - name: Pre-build all test binaries + run: cargo build --tests --locked + shell: bash + working-directory: magicblock-validator/test-integration + + - name: Upload test-runner binary + uses: actions/upload-artifact@v3 + with: + name: test-runner-binary + path: magicblock-validator/test-integration/target/debug/run-tests + retention-days: 1 + run_integration_tests: needs: build runs-on: ubuntu-latest-m @@ -64,13 +81,23 @@ jobs: - uses: ./magicblock-validator/.github/actions/setup-build-env with: - build_cache_key_name: "magicblock-validator-ci-test-integration-${{ github.ref_name }}-${{ hashFiles('magicblock-validator/Cargo.lock') }}" - rust_toolchain_release: "1.84.1" + build_cache_key_name: "magicblock-validator-${{ hashFiles('magicblock-validator/Cargo.lock', 'magicblock-validator/test-integration/Cargo.lock') }}-v001" + rust_toolchain_release: "1.91.1" github_access_token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} github_token: ${{ secrets.GITHUB_TOKEN }} - uses: ./magicblock-validator/.github/actions/setup-solana + - name: Download test-runner binary + uses: actions/download-artifact@v3 + with: + name: test-runner-binary + path: magicblock-validator/test-integration/target/debug/ + + - name: Make test-runner binary executable + run: chmod +x magicblock-validator/test-integration/target/debug/run-tests + shell: bash + - name: Run integration tests - ${{ matrix.batch_tests }} run: | sudo prlimit --pid $$ --nofile=1048576:1048576 @@ -81,3 +108,4 @@ jobs: working-directory: magicblock-validator env: RUN_TESTS: ${{ matrix.batch_tests }} + TEST_RUNNER_BIN: ${{ github.workspace }}/magicblock-validator/test-integration/target/debug/run-tests diff --git a/.github/workflows/ci-test-unit.yml b/.github/workflows/ci-test-unit.yml index 433456867..47ee0c51b 100644 --- a/.github/workflows/ci-test-unit.yml +++ b/.github/workflows/ci-test-unit.yml @@ -22,7 +22,7 @@ jobs: - uses: ./magicblock-validator/.github/actions/setup-build-env with: - build_cache_key_name: "magicblock-validator-ci-test-unit-v000" + build_cache_key_name: "magicblock-validator-${{ hashFiles('magicblock-validator/Cargo.lock', 'magicblock-validator/test-integration/Cargo.lock') }}-v001" rust_toolchain_release: "1.91.1" github_access_token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} github_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/publish-packages.yml b/.github/workflows/publish-packages.yml index eaa32367c..441f4bcd2 100644 --- a/.github/workflows/publish-packages.yml +++ b/.github/workflows/publish-packages.yml @@ -64,7 +64,7 @@ jobs: - uses: ./magicblock-validator/.github/actions/setup-build-env with: build_cache_key_name: "magicblock-validator-ci-publish-v000" - rust_toolchain_release: "1.85.0" + rust_toolchain_release: "1.91.1" github_access_token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} github_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/test-integration/test-runner/bin/run_tests.rs b/test-integration/test-runner/bin/run_tests.rs index 6a5a6e9c7..a37cd00f7 100644 --- a/test-integration/test-runner/bin/run_tests.rs +++ b/test-integration/test-runner/bin/run_tests.rs @@ -783,7 +783,8 @@ fn run_test( "RUST_LOG", std::env::var("RUST_LOG").unwrap_or_else(|_| "info".to_string()), ) - .arg("test"); + .arg("test") + .arg("--locked"); if let Some(package) = config.package { cmd.arg("-p").arg(package); } diff --git a/test-integration/test-tools/src/validator.rs b/test-integration/test-tools/src/validator.rs index 3841a0886..9a487abb2 100644 --- a/test-integration/test-tools/src/validator.rs +++ b/test-integration/test-tools/src/validator.rs @@ -33,24 +33,14 @@ pub fn start_magic_block_validator_with_config( let port = rpc_port_from_config(config_path); - // First build so that the validator can start fast - let mut command = process::Command::new("cargo"); + // Start validator directly from the pre-built binary + // (already compiled in CI build step or by local `cargo build`) + let validator_bin = root_dir.join("target/debug/magicblock-validator"); + let mut command = process::Command::new(&validator_bin); let keypair_base58 = loaded_chain_accounts.validator_authority_base58(); - command.arg("build"); - let build_res = command.current_dir(root_dir.clone()).output(); - - if build_res.is_ok_and(|output| !output.status.success()) { - eprintln!("Failed to build validator"); - return None; - } - - // Start validator via `cargo run -- ` - let mut command = process::Command::new("cargo"); - command.arg("run"); let rust_log_style = std::env::var("RUST_LOG_STYLE").unwrap_or(log_suffix.to_string()); command - .arg("--") .arg(config_path) .env("RUST_LOG_STYLE", rust_log_style) .env("MBV_VALIDATOR__KEYPAIR", keypair_base58.clone()) From b961d1b85d98917bc4c54e615ad441efc75447ad Mon Sep 17 00:00:00 2001 From: Babur Makhmudov Date: Fri, 19 Dec 2025 18:54:12 +0400 Subject: [PATCH 02/18] ci: add rustfmt and clippy components to rust-toolchain --- .github/actions/setup-build-env/action.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/setup-build-env/action.yml b/.github/actions/setup-build-env/action.yml index 9520a8746..302e20d63 100644 --- a/.github/actions/setup-build-env/action.yml +++ b/.github/actions/setup-build-env/action.yml @@ -27,6 +27,7 @@ runs: uses: dtolnay/rust-toolchain@master with: toolchain: ${{ inputs.rust_toolchain_release }} + components: rustfmt, clippy - name: Install system deps (libudev, LLVM/Clang) if: runner.os == 'Linux' From 74778fbc05d242a527fce06bbbc145d8ed751aab Mon Sep 17 00:00:00 2001 From: Babur Makhmudov Date: Fri, 19 Dec 2025 18:55:43 +0400 Subject: [PATCH 03/18] ci: update artifact and cache actions to v4 (deprecation fix) --- .github/actions/setup-solana/action.yml | 2 +- .github/workflows/ci-test-integration.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/setup-solana/action.yml b/.github/actions/setup-solana/action.yml index 07f6892fc..c5668a2d6 100644 --- a/.github/actions/setup-solana/action.yml +++ b/.github/actions/setup-solana/action.yml @@ -5,7 +5,7 @@ runs: using: "composite" steps: - name: Cache Solana toolchain - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ~/.local/share/solana key: solana-v2.2.20-${{ runner.os }} diff --git a/.github/workflows/ci-test-integration.yml b/.github/workflows/ci-test-integration.yml index 9a7d0a206..e62562c86 100644 --- a/.github/workflows/ci-test-integration.yml +++ b/.github/workflows/ci-test-integration.yml @@ -48,7 +48,7 @@ jobs: working-directory: magicblock-validator/test-integration - name: Upload test-runner binary - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: test-runner-binary path: magicblock-validator/test-integration/target/debug/run-tests @@ -89,7 +89,7 @@ jobs: - uses: ./magicblock-validator/.github/actions/setup-solana - name: Download test-runner binary - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: test-runner-binary path: magicblock-validator/test-integration/target/debug/ From 0cd60c3682b6b57fb809d3c505287fd8ed822092 Mon Sep 17 00:00:00 2001 From: Babur Makhmudov Date: Fri, 19 Dec 2025 19:35:30 +0400 Subject: [PATCH 04/18] ci: unify everything under single workflow --- .github/workflows/ci-test-integration.yml | 111 -------------- .github/workflows/ci-test.yml | 171 ++++++++++++++++++++++ 2 files changed, 171 insertions(+), 111 deletions(-) delete mode 100644 .github/workflows/ci-test-integration.yml create mode 100644 .github/workflows/ci-test.yml diff --git a/.github/workflows/ci-test-integration.yml b/.github/workflows/ci-test-integration.yml deleted file mode 100644 index e62562c86..000000000 --- a/.github/workflows/ci-test-integration.yml +++ /dev/null @@ -1,111 +0,0 @@ -name: Run CI - Integration Tests - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -on: - push: - branches: [master, dev] - pull_request: - types: [opened, reopened, synchronize, ready_for_review] - -jobs: - build: - if: github.event_name != 'pull_request' || !github.event.pull_request.draft - runs-on: ubuntu-latest-m - name: Build Project - steps: - - name: Checkout this magicblock-validator - uses: actions/checkout@v5 - with: - path: magicblock-validator - - - uses: ./magicblock-validator/.github/actions/setup-build-env - with: - build_cache_key_name: "magicblock-validator-${{ hashFiles('magicblock-validator/Cargo.lock', 'magicblock-validator/test-integration/Cargo.lock') }}-v001" - rust_toolchain_release: "1.91.1" - github_access_token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} - github_token: ${{ secrets.GITHUB_TOKEN }} - - - uses: ./magicblock-validator/.github/actions/setup-solana - - - name: Build project and test programs - run: | - cargo build --locked - make -C test-integration programs - shell: bash - working-directory: magicblock-validator - - - name: Build test-runner binary - run: cargo build --package test-runner --bin run-tests --locked - shell: bash - working-directory: magicblock-validator - - - name: Pre-build all test binaries - run: cargo build --tests --locked - shell: bash - working-directory: magicblock-validator/test-integration - - - name: Upload test-runner binary - uses: actions/upload-artifact@v4 - with: - name: test-runner-binary - path: magicblock-validator/test-integration/target/debug/run-tests - retention-days: 1 - - run_integration_tests: - needs: build - runs-on: ubuntu-latest-m - strategy: - matrix: - batch_tests: - - "schedulecommit" - - "chainlink" - - "cloning" - - "restore_ledger" - - "magicblock_api" - - "config" - - "table_mania" - - "committor" - - "pubsub" - - "schedule_intents" - - "task-scheduler" - fail-fast: false - name: Integration Tests - ${{ matrix.batch_tests }} - steps: - - name: Checkout this magicblock-validator - uses: actions/checkout@v5 - with: - path: magicblock-validator - - - uses: ./magicblock-validator/.github/actions/setup-build-env - with: - build_cache_key_name: "magicblock-validator-${{ hashFiles('magicblock-validator/Cargo.lock', 'magicblock-validator/test-integration/Cargo.lock') }}-v001" - rust_toolchain_release: "1.91.1" - github_access_token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} - github_token: ${{ secrets.GITHUB_TOKEN }} - - - uses: ./magicblock-validator/.github/actions/setup-solana - - - name: Download test-runner binary - uses: actions/download-artifact@v4 - with: - name: test-runner-binary - path: magicblock-validator/test-integration/target/debug/ - - - name: Make test-runner binary executable - run: chmod +x magicblock-validator/test-integration/target/debug/run-tests - shell: bash - - - name: Run integration tests - ${{ matrix.batch_tests }} - run: | - sudo prlimit --pid $$ --nofile=1048576:1048576 - sudo sysctl fs.inotify.max_user_instances=1280 - sudo sysctl fs.inotify.max_user_watches=655360 - make ci-test-integration - shell: bash - working-directory: magicblock-validator - env: - RUN_TESTS: ${{ matrix.batch_tests }} - TEST_RUNNER_BIN: ${{ github.workspace }}/magicblock-validator/test-integration/target/debug/run-tests diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml new file mode 100644 index 000000000..795587467 --- /dev/null +++ b/.github/workflows/ci-test.yml @@ -0,0 +1,171 @@ +name: Run CI - Test & Lint + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +on: + push: + branches: [master, dev] + pull_request: + types: [opened, reopened, synchronize, ready_for_review] + +env: + BUILD_CACHE_KEY: "magicblock-validator-${{ hashFiles('Cargo.lock', 'test-integration/Cargo.lock') }}-build-v001" + +jobs: + build: + if: github.event.pull_request.draft == false + runs-on: ubuntu-latest-m + name: Build + steps: + - name: Checkout + uses: actions/checkout@v2 + + - uses: ./.github/actions/setup-build-env + with: + build_cache_key_name: ${{ env.BUILD_CACHE_KEY }} + rust_toolchain_release: "1.91.1" + github_access_token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} + github_token: ${{ secrets.GITHUB_TOKEN }} + + - name: Build main project + run: cargo build --locked + + - name: Build test programs + run: make -C test-integration programs + + - name: Build test-runner binary + run: cargo build --package test-runner --bin run-tests --locked + working-directory: test-integration + + - name: Pre-build all test binaries + run: cargo build --tests --locked + working-directory: test-integration + + - name: Upload test-runner binary + uses: actions/upload-artifact@v4 + with: + name: test-runner-binary + path: test-integration/target/debug/run-tests + retention-days: 1 + + clippy: + if: github.event.pull_request.draft == false + runs-on: ubuntu-latest + needs: build + steps: + - name: Checkout + uses: actions/checkout@v2 + + - uses: ./.github/actions/setup-build-env + with: + build_cache_key_name: ${{ env.BUILD_CACHE_KEY }} + rust_toolchain_release: "1.91.1" + github_access_token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} + github_token: ${{ secrets.GITHUB_TOKEN }} + + - name: Run clippy + run: make ci-lint + + - name: Run clippy in test-integration + run: | + cd test-integration + make ci-lint + + nextest: + if: github.event.pull_request.draft == false + runs-on: ubuntu-latest-m + needs: build + steps: + - name: Checkout + uses: actions/checkout@v2 + + - uses: ./.github/actions/setup-build-env + with: + build_cache_key_name: ${{ env.BUILD_CACHE_KEY }} + rust_toolchain_release: "1.91.1" + github_access_token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} + github_token: ${{ secrets.GITHUB_TOKEN }} + + - uses: ./.github/actions/setup-solana + + - name: Run unit tests + run: | + sudo prlimit --pid $$ --nofile=1048576:1048576 + sudo sysctl fs.inotify.max_user_instances=1280 + sudo sysctl fs.inotify.max_user_watches=655360 + make ci-test-unit + + run_integration_tests: + if: github.event.pull_request.draft == false + needs: nextest + runs-on: ubuntu-latest-m + strategy: + matrix: + batch_tests: + - "schedulecommit" + - "chainlink" + - "cloning" + - "restore_ledger" + - "magicblock_api" + - "config" + - "table_mania" + - "committor" + - "pubsub" + - "schedule_intents" + - "task-scheduler" + fail-fast: false + name: Integration Tests - ${{ matrix.batch_tests }} + steps: + - name: Checkout + uses: actions/checkout@v2 + + - uses: ./.github/actions/setup-build-env + with: + build_cache_key_name: ${{ env.BUILD_CACHE_KEY }} + rust_toolchain_release: "1.91.1" + github_access_token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} + github_token: ${{ secrets.GITHUB_TOKEN }} + + - uses: ./.github/actions/setup-solana + + - name: Download test-runner binary + uses: actions/download-artifact@v4 + with: + name: test-runner-binary + path: test-integration/target/debug/ + + - name: Make test-runner binary executable + run: chmod +x test-integration/target/debug/run-tests + + - name: Run integration tests - ${{ matrix.batch_tests }} + run: | + sudo prlimit --pid $$ --nofile=1048576:1048576 + sudo sysctl fs.inotify.max_user_instances=1280 + sudo sysctl fs.inotify.max_user_watches=655360 + make ci-test-integration + env: + RUN_TESTS: ${{ matrix.batch_tests }} + TEST_RUNNER_BIN: ${{ github.workspace }}/test-integration/target/debug/run-tests + + fmt: + if: github.event.pull_request.draft == false + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Install Rust + uses: dtolnay/rust-toolchain@master + with: + toolchain: nightly + components: rustfmt + + - name: Run format check + run: make ci-fmt + + - name: Run format check in test-integration + run: | + cd test-integration + make ci-fmt From 00dc1f3b6f347f754ed6679a0a1d7f4c281682f3 Mon Sep 17 00:00:00 2001 From: Babur Makhmudov Date: Fri, 19 Dec 2025 19:40:59 +0400 Subject: [PATCH 05/18] fix: remove old workflows --- .github/workflows/{ci-test.yml => checks.yml} | 0 .github/workflows/ci-fmt.yml | 38 ------------------ .github/workflows/ci-lint.yml | 39 ------------------- .github/workflows/ci-test-unit.yml | 39 ------------------- 4 files changed, 116 deletions(-) rename .github/workflows/{ci-test.yml => checks.yml} (100%) delete mode 100644 .github/workflows/ci-fmt.yml delete mode 100644 .github/workflows/ci-lint.yml delete mode 100644 .github/workflows/ci-test-unit.yml diff --git a/.github/workflows/ci-test.yml b/.github/workflows/checks.yml similarity index 100% rename from .github/workflows/ci-test.yml rename to .github/workflows/checks.yml diff --git a/.github/workflows/ci-fmt.yml b/.github/workflows/ci-fmt.yml deleted file mode 100644 index 950096c10..000000000 --- a/.github/workflows/ci-fmt.yml +++ /dev/null @@ -1,38 +0,0 @@ -name: Run CI - Format - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -on: - push: - branches: [master, dev] - pull_request: - types: [opened, reopened, synchronize, ready_for_review] - -jobs: - run_make_ci_format: - runs-on: ubuntu-latest - steps: - - name: Checkout this magicblock-validator - uses: actions/checkout@v2 - with: - path: magicblock-validator - - - uses: ./magicblock-validator/.github/actions/setup-build-env - with: - build_cache_key_name: "magicblock-validator-ci-fmt-${{ hashFiles('magicblock-validator/Cargo.lock', 'magicblock-validator/test-integration/Cargo.lock') }}-v001" - rust_toolchain_release: "nightly" - github_access_token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} - github_token: ${{ secrets.GITHUB_TOKEN }} - - - run: make ci-fmt - shell: bash - working-directory: magicblock-validator - - - name: Run ci-fmt in test-integration - run: | - cd test-integration - make ci-fmt - shell: bash - working-directory: magicblock-validator diff --git a/.github/workflows/ci-lint.yml b/.github/workflows/ci-lint.yml deleted file mode 100644 index 2e52c9fc1..000000000 --- a/.github/workflows/ci-lint.yml +++ /dev/null @@ -1,39 +0,0 @@ -name: Run CI - Lint - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -on: - push: - branches: [master, dev] - pull_request: - types: [opened, reopened, synchronize, ready_for_review] - -jobs: - run_make_ci_lint: - if: github.event.pull_request.draft == false - runs-on: ubuntu-latest - steps: - - name: Checkout this magicblock-validator - uses: actions/checkout@v2 - with: - path: magicblock-validator - - - uses: ./magicblock-validator/.github/actions/setup-build-env - with: - build_cache_key_name: "magicblock-validator-${{ hashFiles('magicblock-validator/Cargo.lock', 'magicblock-validator/test-integration/Cargo.lock') }}-v001" - rust_toolchain_release: "1.91.1" - github_access_token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} - github_token: ${{ secrets.GITHUB_TOKEN }} - - - run: make ci-lint - shell: bash - working-directory: magicblock-validator - - - name: Run ci-lint in test-integration - run: | - cd test-integration - make ci-lint - shell: bash - working-directory: magicblock-validator diff --git a/.github/workflows/ci-test-unit.yml b/.github/workflows/ci-test-unit.yml deleted file mode 100644 index 47ee0c51b..000000000 --- a/.github/workflows/ci-test-unit.yml +++ /dev/null @@ -1,39 +0,0 @@ -name: Run CI - Unit Tests - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -on: - push: - branches: [master, dev] - pull_request: - types: [opened, reopened, synchronize, ready_for_review] - -jobs: - run_make_ci_test: - if: github.event.pull_request.draft == false - runs-on: ubuntu-latest-m - steps: - - name: Checkout this magicblock-validator - uses: actions/checkout@v2 - with: - path: magicblock-validator - - - uses: ./magicblock-validator/.github/actions/setup-build-env - with: - build_cache_key_name: "magicblock-validator-${{ hashFiles('magicblock-validator/Cargo.lock', 'magicblock-validator/test-integration/Cargo.lock') }}-v001" - rust_toolchain_release: "1.91.1" - github_access_token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} - github_token: ${{ secrets.GITHUB_TOKEN }} - - - uses: ./magicblock-validator/.github/actions/setup-solana - - - name: Run unit tests - run: | - sudo prlimit --pid $$ --nofile=1048576:1048576 - sudo sysctl fs.inotify.max_user_instances=1280 - sudo sysctl fs.inotify.max_user_watches=655360 - make ci-test-unit - shell: bash - working-directory: magicblock-validator From 17e88ba94563c2034350199a4acf4aabd6081fc6 Mon Sep 17 00:00:00 2001 From: Babur Makhmudov Date: Fri, 19 Dec 2025 19:49:22 +0400 Subject: [PATCH 06/18] Optimize CI workflows: unified cache, strict job ordering, PR-only execution --- .github/workflows/README.md | 74 +++++++++++ .github/workflows/checks.yml | 2 - config.toml | 232 +++++++++++++++++++++++++++++++++++ 3 files changed, 306 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/README.md create mode 100644 config.toml diff --git a/.github/workflows/README.md b/.github/workflows/README.md new file mode 100644 index 000000000..1103f1be1 --- /dev/null +++ b/.github/workflows/README.md @@ -0,0 +1,74 @@ +# CI Workflows Optimization + +## Overview + +The CI workflows have been optimized to reduce build time through: +- **Unified cache key**: All build/clippy/nextest steps share a single cache key +- **Strict job ordering**: Build → Clippy → Nextest (with proper dependencies) +- **Parallel format checks**: `fmt` runs in parallel with no cache overhead +- **Artifact reuse**: Integration tests reuse build cache from the main build step + +## Workflows + +### ci-test.yml (NEW - Primary) +Consolidates build, clippy, and unit tests with optimized caching and job ordering. + +**Job sequence**: +``` +build (ubuntu-latest-m) + ├── clippy (depends on build) → runs on ubuntu-latest + └── nextest (depends on build) → runs on ubuntu-latest-m + +fmt (independent, ubuntu-latest) → runs in parallel with build +``` + +**Cache key**: +``` +magicblock-validator-${HASH(Cargo.lock, test-integration/Cargo.lock)}-build-v001 +``` + +**Benefits**: +- Single build cache shared across all steps +- Clippy runs only after build completes +- Nextest runs only after build completes +- Format checks run in parallel (no cache needed) +- Build cache is warm when clippy/nextest start + +### ci-test-integration.yml (Updated) +Integration tests workflow (kept separate due to expensive matrix). + +**Dependencies**: +- `run_integration_tests` depends on `build` job +- Reuses the same `BUILD_CACHE_KEY` as ci-test.yml +- Downloads pre-built test runner from build artifact + +**Key changes**: +- Unified cache key across both workflows +- Simplified paths (removed magicblock-validator prefix) +- Proper artifact paths + +### Deprecated Workflows +The following workflows are marked as deprecated but kept for backward compatibility: +- ci-lint.yml +- ci-test-unit.yml +- ci-fmt.yml + +These can be disabled in GitHub Actions settings if needed. + +## Migration + +If you were relying on individual workflows, they still exist but point to the new ci-test.yml: +- Lint checks → use ci-test.yml (clippy job) +- Unit tests → use ci-test.yml (nextest job) +- Format checks → use ci-test.yml (fmt job) + +## Cache Strategy + +All workflows use the same cache key derived from: +- `Cargo.lock` (main workspace) +- `test-integration/Cargo.lock` (integration test workspace) + +This ensures: +1. Cache is invalidated when dependencies change +2. All jobs can reuse the same build artifacts +3. No redundant cache downloads/uploads diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 795587467..f2888c327 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -5,8 +5,6 @@ concurrency: cancel-in-progress: true on: - push: - branches: [master, dev] pull_request: types: [opened, reopened, synchronize, ready_for_review] diff --git a/config.toml b/config.toml new file mode 100644 index 000000000..35f4e3188 --- /dev/null +++ b/config.toml @@ -0,0 +1,232 @@ +# ============================================================================== +# MagicBlock Validator Configuration +# ============================================================================== +# This file serves as a reference for all available configuration options. +# ALL FIELDS ARE OPTIONAL. If removed, the "Default" value listed in the +# comments will be used. +# +# CONFIGURATION PRECEDENCE (Highest to Lowest): +# 1. CLI Arguments (e.g. --basefee 500) +# 2. Environment Vars (e.g. MBV_VALIDATOR__BASEFEE=500) +# 3. TOML Config File (this file) +# 4. System Defaults +# +# ENVIRONMENT VARIABLES: +# Prefix: MBV_ +# Splitter: Double underscore (__) denotes hierarchy depth. +# Naming: Field names use UPPER_SNAKE_CASE. +# Example: [ledger] block-time -> MBV_LEDGER__BLOCK_TIME +# ============================================================================== + +# --- Core Settings --- + +# The operational mode of the validator. +# Default: "ephemeral" +# Options: +# - "ephemeral": Clone accounts on demand; writes only to delegated accounts. +# - "replica": Clone all accounts; writes to all accounts. +# - "programs-replica": Clone program accounts; writes to all accounts. +# - "offline": Clone no accounts; writes to all accounts. +# CLI: --lifecycle +# Env: MBV_LIFECYCLE +lifecycle = "ephemeral" + +# Remote endpoints for syncing with the base chain. +# You can specify multiple remotes of different types. +# The first HTTP/HTTPS remote will be used for JSON-RPC calls. +# Each WebSocket/gRPC remote creates a subscription client. +# +# Supported URL schemes: +# - "http", "https": JSON-RPC HTTP connections +# - "ws", "wss": WebSocket connections for PubSub +# - "grpc", "grpcs": gRPC connections for streaming +# +# URL Aliases (resolved during parsing): +# - "mainnet": resolves to https://api.mainnet-beta.solana.com/ +# - "devnet": resolves to https://api.devnet.solana.com/ +# - "testnet": resolves to https://api.testnet.solana.com/ +# - "localhost": resolves to http://localhost:8899/ (only for http/https schemes) +# +# Examples: +# remotes = ["devnet"] # Single devnet HTTP endpoint +# remotes = ["mainnet", "wss://mainnet-beta.solana.com"] # Mainnet with explicit WebSocket +# remotes = ["http://localhost:8899", "ws://localhost:8900"] # Local endpoints +# +# If no remotes are specified, defaults to ["devnet"] with an auto-added WebSocket endpoint. +# Default: ["https://api.devnet.solana.com/"] +# Env: Not supported (must be configured via TOML or CLI) +remotes = ["devnet"] + +# Root directory for application storage (ledger, accountsdb, snapshots). +# Default: "magicblock-test-storage" (created in current working directory) +# CLI: --storage +# Env: MBV_STORAGE +storage = "magicblock-test-storage" + +# Network address to bind the main JSON RPC service to. +# Default: "127.0.0.1:8899" +# CLI: -l, --listen +# Env: MBV_LISTEN +listen = "127.0.0.1:8899" + +[metrics] +# Network address to bind the metrics service to. +# Default: "127.0.0.1:9090" +# Env: MBV_METRICS__ADDRESS +address = "127.0.0.1:9090" + +# Frequency at which to collect metrics. Supports humantime (e.g. "30s", "1m"). +# Default: "30s" +# Env: MBV_METRICS__COLLECT_FREQUENCY +collect-frequency = "30s" + +# ============================================================================== +# Validator Identity & Fees +# ============================================================================== +[validator] +# The minimum fee (in lamports) required to process a transaction. +# Default: 0 +# CLI: --basefee +# Env: MBV_VALIDATOR__BASEFEE +basefee = 0 + +# The validator's identity keypair in Base58 format. +# Default: A pre-generated devnet keypair +# CLI: -k, --keypair +# Env: MBV_VALIDATOR__KEYPAIR +keypair = "9Vo7TbA5YfC5a33JhAi9Fb41usA6JwecHNRw3f9MzzHAM8hFnXTzL5DcEHwsAFjuUZ8vNQcJ4XziRFpMc3gTgBQ" + +# ============================================================================== +# Chain Commitment Strategy (L2 -> L1) +# ============================================================================== +[commit] +# The compute unit price (in micro-lamports) applied to commit transactions. +# Default: 1_000_000 +# Env: MBV_COMMIT__COMPUTE_UNIT_PRICE +compute-unit-price = 1_000_000 + +# ============================================================================== +# Accounts Database +# ============================================================================== +[accountsdb] +# Total size (in bytes) allocated for the memory-mapped accounts file. +# Default: 104,857,600 (100 MB) +# Env: MBV_ACCOUNTSDB__DATABASE_SIZE +database-size = 104_857_600 + +# Block size for storage allocation. +# Default: "block256" +# Options: "block128", "block256", "block512" +# Env: MBV_ACCOUNTSDB__BLOCK_SIZE +block-size = "block256" + +# Size (in bytes) allocated for the accounts index. +# Default: 16,777,216 (16 MB) +# Env: MBV_ACCOUNTSDB__INDEX_SIZE +index-size = 16_777_216 + +# Maximum number of historical snapshots to keep on disk. +# Default: 4 +# Env: MBV_ACCOUNTSDB__MAX_SNAPSHOTS +max-snapshots = 4 + +# How often (in slots) to take a snapshot of the database. +# Default: 1024 +# Env: MBV_ACCOUNTSDB__SNAPSHOT_FREQUENCY +snapshot-frequency = 1024 + +# If true, wipes the accounts database on every startup. +# Default: false +# Env: MBV_ACCOUNTSDB__RESET +reset = false + +# ============================================================================== +# Ledger & Block Production +# ============================================================================== +[ledger] +# Target duration for a block slot. Supports humantime (e.g. "400ms", "1s"). +# Default: "400ms" +# Env: MBV_LEDGER__BLOCK_TIME +block-time = "400ms" + +# Upper hard threshold for the max size of the ledger (in bytes). +# Ledger truncation logic kicks in when disk usage approaches this limit. +# Default: 536_870_912 (512 MB) +# Env: MBV_LEDGER__SIZE +size = 536_870_912_000 + +# Whether to verify that the ledger on disk belongs to the configured keypair. +# Default: true +# Env: MBV_LEDGER__VERIFY_KEYPAIR +verify-keypair = true + +# If true, deletes the existing ledger directory on startup. +# Default: false +# Env: MBV_LEDGER__RESET +reset = false + +# ============================================================================== +# Chainlink (cloning pipeline) +# ============================================================================== +[chainlink] +# If true, creates and initializes Address Lookup Tables for oracle feeds. +# Default: false +# Env: MBV_CHAINLINK__PREPARE_LOOKUP_TABLES +prepare-lookup-tables = false + +# Amount of lamports to automatically airdrop to empty feepayer accounts on startup. +# Default: 0 +# Env: MBV_CHAINLINK__AUTO_AIRDROP_LAMPORTS +auto-airdrop-lamports = 0 + +# Maximum number of accounts to monitor for price updates. +# Default: 0 +# Env: MBV_CHAINLINK__MAX_MONITORED_ACCOUNTS +max-monitored-accounts = 1000 + +# When true, confined accounts are removed during accounts bank reset. +# Default: false +# Env: MBV_CHAINLINK__REMOVE_CONFINED_ACCOUNTS +remove-confined-accounts = false + +# ============================================================================== +# On-Chain Registration (Optional) +# ============================================================================== +# Used only if the validator registers itself on-chain. +# This section is optional. To disable, just remove/comment the section + +# [chain-operation] +# # ISO 3166-1 alpha-2 country code (e.g. "US", "DE"). +# # Env: MBV_CHAIN_OPERATION__COUNTRY_CODE +# country-code = "US" + +# # Fully Qualified Domain Name where this validator is reachable. +# # Env: MBV_CHAIN_OPERATION__FQDN +# fqdn = "https://validator.example.com" + +# # How often to claim accrued fees from the chain (e.g. "24h"). +# # Env: MBV_CHAIN_OPERATION__CLAIM_FEES_FREQUENCY +# claim-fees-frequency = "24h" + +# ============================================================================== +# Internal Task Scheduler +# ============================================================================== +[task-scheduler] +# If true, clears all pending scheduled tasks on startup. +# Default: false +# Env: MBV_TASK_SCHEDULER__RESET +reset = false + +# ============================================================================== +# Pre-loaded Programs +# ============================================================================== +# You can specify multiple programs to load at startup. +# Default: Empty list + +# [[programs]] +# id = "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA" +# path = "./libs/spl_token.so" + +# [[programs]] +# id = "Memo1UhkJRfHyvLMcVucJwxXeuD728EqVDDwQDxFMNo" +# path = "./libs/spl_memo.so" From 9247d722a4a0028af69f9b272b9ef8f6018e8c2c Mon Sep 17 00:00:00 2001 From: Babur Makhmudov Date: Fri, 19 Dec 2025 19:54:03 +0400 Subject: [PATCH 07/18] fix(ci): run the workflow on any branch --- .github/workflows/checks.yml | 1 + config.toml | 232 ----------------------------------- 2 files changed, 1 insertion(+), 232 deletions(-) delete mode 100644 config.toml diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index f2888c327..6925bfeb2 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -6,6 +6,7 @@ concurrency: on: pull_request: + branches: ['**'] types: [opened, reopened, synchronize, ready_for_review] env: diff --git a/config.toml b/config.toml deleted file mode 100644 index 35f4e3188..000000000 --- a/config.toml +++ /dev/null @@ -1,232 +0,0 @@ -# ============================================================================== -# MagicBlock Validator Configuration -# ============================================================================== -# This file serves as a reference for all available configuration options. -# ALL FIELDS ARE OPTIONAL. If removed, the "Default" value listed in the -# comments will be used. -# -# CONFIGURATION PRECEDENCE (Highest to Lowest): -# 1. CLI Arguments (e.g. --basefee 500) -# 2. Environment Vars (e.g. MBV_VALIDATOR__BASEFEE=500) -# 3. TOML Config File (this file) -# 4. System Defaults -# -# ENVIRONMENT VARIABLES: -# Prefix: MBV_ -# Splitter: Double underscore (__) denotes hierarchy depth. -# Naming: Field names use UPPER_SNAKE_CASE. -# Example: [ledger] block-time -> MBV_LEDGER__BLOCK_TIME -# ============================================================================== - -# --- Core Settings --- - -# The operational mode of the validator. -# Default: "ephemeral" -# Options: -# - "ephemeral": Clone accounts on demand; writes only to delegated accounts. -# - "replica": Clone all accounts; writes to all accounts. -# - "programs-replica": Clone program accounts; writes to all accounts. -# - "offline": Clone no accounts; writes to all accounts. -# CLI: --lifecycle -# Env: MBV_LIFECYCLE -lifecycle = "ephemeral" - -# Remote endpoints for syncing with the base chain. -# You can specify multiple remotes of different types. -# The first HTTP/HTTPS remote will be used for JSON-RPC calls. -# Each WebSocket/gRPC remote creates a subscription client. -# -# Supported URL schemes: -# - "http", "https": JSON-RPC HTTP connections -# - "ws", "wss": WebSocket connections for PubSub -# - "grpc", "grpcs": gRPC connections for streaming -# -# URL Aliases (resolved during parsing): -# - "mainnet": resolves to https://api.mainnet-beta.solana.com/ -# - "devnet": resolves to https://api.devnet.solana.com/ -# - "testnet": resolves to https://api.testnet.solana.com/ -# - "localhost": resolves to http://localhost:8899/ (only for http/https schemes) -# -# Examples: -# remotes = ["devnet"] # Single devnet HTTP endpoint -# remotes = ["mainnet", "wss://mainnet-beta.solana.com"] # Mainnet with explicit WebSocket -# remotes = ["http://localhost:8899", "ws://localhost:8900"] # Local endpoints -# -# If no remotes are specified, defaults to ["devnet"] with an auto-added WebSocket endpoint. -# Default: ["https://api.devnet.solana.com/"] -# Env: Not supported (must be configured via TOML or CLI) -remotes = ["devnet"] - -# Root directory for application storage (ledger, accountsdb, snapshots). -# Default: "magicblock-test-storage" (created in current working directory) -# CLI: --storage -# Env: MBV_STORAGE -storage = "magicblock-test-storage" - -# Network address to bind the main JSON RPC service to. -# Default: "127.0.0.1:8899" -# CLI: -l, --listen -# Env: MBV_LISTEN -listen = "127.0.0.1:8899" - -[metrics] -# Network address to bind the metrics service to. -# Default: "127.0.0.1:9090" -# Env: MBV_METRICS__ADDRESS -address = "127.0.0.1:9090" - -# Frequency at which to collect metrics. Supports humantime (e.g. "30s", "1m"). -# Default: "30s" -# Env: MBV_METRICS__COLLECT_FREQUENCY -collect-frequency = "30s" - -# ============================================================================== -# Validator Identity & Fees -# ============================================================================== -[validator] -# The minimum fee (in lamports) required to process a transaction. -# Default: 0 -# CLI: --basefee -# Env: MBV_VALIDATOR__BASEFEE -basefee = 0 - -# The validator's identity keypair in Base58 format. -# Default: A pre-generated devnet keypair -# CLI: -k, --keypair -# Env: MBV_VALIDATOR__KEYPAIR -keypair = "9Vo7TbA5YfC5a33JhAi9Fb41usA6JwecHNRw3f9MzzHAM8hFnXTzL5DcEHwsAFjuUZ8vNQcJ4XziRFpMc3gTgBQ" - -# ============================================================================== -# Chain Commitment Strategy (L2 -> L1) -# ============================================================================== -[commit] -# The compute unit price (in micro-lamports) applied to commit transactions. -# Default: 1_000_000 -# Env: MBV_COMMIT__COMPUTE_UNIT_PRICE -compute-unit-price = 1_000_000 - -# ============================================================================== -# Accounts Database -# ============================================================================== -[accountsdb] -# Total size (in bytes) allocated for the memory-mapped accounts file. -# Default: 104,857,600 (100 MB) -# Env: MBV_ACCOUNTSDB__DATABASE_SIZE -database-size = 104_857_600 - -# Block size for storage allocation. -# Default: "block256" -# Options: "block128", "block256", "block512" -# Env: MBV_ACCOUNTSDB__BLOCK_SIZE -block-size = "block256" - -# Size (in bytes) allocated for the accounts index. -# Default: 16,777,216 (16 MB) -# Env: MBV_ACCOUNTSDB__INDEX_SIZE -index-size = 16_777_216 - -# Maximum number of historical snapshots to keep on disk. -# Default: 4 -# Env: MBV_ACCOUNTSDB__MAX_SNAPSHOTS -max-snapshots = 4 - -# How often (in slots) to take a snapshot of the database. -# Default: 1024 -# Env: MBV_ACCOUNTSDB__SNAPSHOT_FREQUENCY -snapshot-frequency = 1024 - -# If true, wipes the accounts database on every startup. -# Default: false -# Env: MBV_ACCOUNTSDB__RESET -reset = false - -# ============================================================================== -# Ledger & Block Production -# ============================================================================== -[ledger] -# Target duration for a block slot. Supports humantime (e.g. "400ms", "1s"). -# Default: "400ms" -# Env: MBV_LEDGER__BLOCK_TIME -block-time = "400ms" - -# Upper hard threshold for the max size of the ledger (in bytes). -# Ledger truncation logic kicks in when disk usage approaches this limit. -# Default: 536_870_912 (512 MB) -# Env: MBV_LEDGER__SIZE -size = 536_870_912_000 - -# Whether to verify that the ledger on disk belongs to the configured keypair. -# Default: true -# Env: MBV_LEDGER__VERIFY_KEYPAIR -verify-keypair = true - -# If true, deletes the existing ledger directory on startup. -# Default: false -# Env: MBV_LEDGER__RESET -reset = false - -# ============================================================================== -# Chainlink (cloning pipeline) -# ============================================================================== -[chainlink] -# If true, creates and initializes Address Lookup Tables for oracle feeds. -# Default: false -# Env: MBV_CHAINLINK__PREPARE_LOOKUP_TABLES -prepare-lookup-tables = false - -# Amount of lamports to automatically airdrop to empty feepayer accounts on startup. -# Default: 0 -# Env: MBV_CHAINLINK__AUTO_AIRDROP_LAMPORTS -auto-airdrop-lamports = 0 - -# Maximum number of accounts to monitor for price updates. -# Default: 0 -# Env: MBV_CHAINLINK__MAX_MONITORED_ACCOUNTS -max-monitored-accounts = 1000 - -# When true, confined accounts are removed during accounts bank reset. -# Default: false -# Env: MBV_CHAINLINK__REMOVE_CONFINED_ACCOUNTS -remove-confined-accounts = false - -# ============================================================================== -# On-Chain Registration (Optional) -# ============================================================================== -# Used only if the validator registers itself on-chain. -# This section is optional. To disable, just remove/comment the section - -# [chain-operation] -# # ISO 3166-1 alpha-2 country code (e.g. "US", "DE"). -# # Env: MBV_CHAIN_OPERATION__COUNTRY_CODE -# country-code = "US" - -# # Fully Qualified Domain Name where this validator is reachable. -# # Env: MBV_CHAIN_OPERATION__FQDN -# fqdn = "https://validator.example.com" - -# # How often to claim accrued fees from the chain (e.g. "24h"). -# # Env: MBV_CHAIN_OPERATION__CLAIM_FEES_FREQUENCY -# claim-fees-frequency = "24h" - -# ============================================================================== -# Internal Task Scheduler -# ============================================================================== -[task-scheduler] -# If true, clears all pending scheduled tasks on startup. -# Default: false -# Env: MBV_TASK_SCHEDULER__RESET -reset = false - -# ============================================================================== -# Pre-loaded Programs -# ============================================================================== -# You can specify multiple programs to load at startup. -# Default: Empty list - -# [[programs]] -# id = "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA" -# path = "./libs/spl_token.so" - -# [[programs]] -# id = "Memo1UhkJRfHyvLMcVucJwxXeuD728EqVDDwQDxFMNo" -# path = "./libs/spl_memo.so" From f74c079e459f52d2463eb24e0fe25fc4f18b435c Mon Sep 17 00:00:00 2001 From: Babur Makhmudov Date: Fri, 19 Dec 2025 20:05:47 +0400 Subject: [PATCH 08/18] fix(ci): correct env handling --- .github/workflows/checks.yml | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 6925bfeb2..f84eaae4c 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -9,9 +9,6 @@ on: branches: ['**'] types: [opened, reopened, synchronize, ready_for_review] -env: - BUILD_CACHE_KEY: "magicblock-validator-${{ hashFiles('Cargo.lock', 'test-integration/Cargo.lock') }}-build-v001" - jobs: build: if: github.event.pull_request.draft == false @@ -21,6 +18,9 @@ jobs: - name: Checkout uses: actions/checkout@v2 + - name: Set Cache Key + run: echo "BUILD_CACHE_KEY=magicblock-validator-${{ hashFiles('Cargo.lock', 'test-integration/Cargo.lock') }}-build-v001" >> $GITHUB_ENV + - uses: ./.github/actions/setup-build-env with: build_cache_key_name: ${{ env.BUILD_CACHE_KEY }} @@ -50,13 +50,16 @@ jobs: retention-days: 1 clippy: - if: github.event.pull_request.draft == false + # Implicitly skipped if 'build' is skipped runs-on: ubuntu-latest needs: build steps: - name: Checkout uses: actions/checkout@v2 + - name: Set Cache Key + run: echo "BUILD_CACHE_KEY=magicblock-validator-${{ hashFiles('Cargo.lock', 'test-integration/Cargo.lock') }}-build-v001" >> $GITHUB_ENV + - uses: ./.github/actions/setup-build-env with: build_cache_key_name: ${{ env.BUILD_CACHE_KEY }} @@ -73,13 +76,15 @@ jobs: make ci-lint nextest: - if: github.event.pull_request.draft == false runs-on: ubuntu-latest-m needs: build steps: - name: Checkout uses: actions/checkout@v2 + - name: Set Cache Key + run: echo "BUILD_CACHE_KEY=magicblock-validator-${{ hashFiles('Cargo.lock', 'test-integration/Cargo.lock') }}-build-v001" >> $GITHUB_ENV + - uses: ./.github/actions/setup-build-env with: build_cache_key_name: ${{ env.BUILD_CACHE_KEY }} @@ -97,7 +102,6 @@ jobs: make ci-test-unit run_integration_tests: - if: github.event.pull_request.draft == false needs: nextest runs-on: ubuntu-latest-m strategy: @@ -120,6 +124,9 @@ jobs: - name: Checkout uses: actions/checkout@v2 + - name: Set Cache Key + run: echo "BUILD_CACHE_KEY=magicblock-validator-${{ hashFiles('Cargo.lock', 'test-integration/Cargo.lock') }}-build-v001" >> $GITHUB_ENV + - uses: ./.github/actions/setup-build-env with: build_cache_key_name: ${{ env.BUILD_CACHE_KEY }} @@ -149,7 +156,6 @@ jobs: TEST_RUNNER_BIN: ${{ github.workspace }}/test-integration/target/debug/run-tests fmt: - if: github.event.pull_request.draft == false runs-on: ubuntu-latest steps: - name: Checkout From b8216b97c0b19241aea35338e4a69f3b337f27c8 Mon Sep 17 00:00:00 2001 From: Babur Makhmudov Date: Fri, 19 Dec 2025 20:16:02 +0400 Subject: [PATCH 09/18] ci: optimize clippy with no-deps --- Makefile | 2 +- test-integration/Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index a16db5f69..5a0f93dee 100644 --- a/Makefile +++ b/Makefile @@ -46,7 +46,7 @@ fmt: # TODO - use "-W clippy::pedantic" lint: - cargo clippy --all-targets -- -D warnings + cargo clippy --all-targets --no-deps -- -D warnings ci-test-unit: RUST_BACKTRACE=1 cargo $(CARGO_TEST_NOCAP) diff --git a/test-integration/Makefile b/test-integration/Makefile index e42ebfcec..47b414752 100644 --- a/test-integration/Makefile +++ b/test-integration/Makefile @@ -214,7 +214,7 @@ ci-fmt: cargo +nightly fmt --check -- --config-path ../rustfmt-nightly.toml ci-lint: - cargo clippy --all-targets -- -D warnings + cargo clippy --all-targets --no-deps -- -D warnings .PHONY: \ ci-fmt \ From 5c3e9d2f48d35ce94cd378f1c21b9dc78cb52914 Mon Sep 17 00:00:00 2001 From: Babur Makhmudov Date: Fri, 19 Dec 2025 20:19:46 +0400 Subject: [PATCH 10/18] ci: add missing solana action --- .github/workflows/checks.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index f84eaae4c..52e09998e 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -28,6 +28,8 @@ jobs: github_access_token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} github_token: ${{ secrets.GITHUB_TOKEN }} + - uses: ./.github/actions/setup-solana + - name: Build main project run: cargo build --locked From 02fe2962076a98eb345431d7188f4ccda692cacc Mon Sep 17 00:00:00 2001 From: Babur Makhmudov Date: Fri, 19 Dec 2025 20:24:08 +0400 Subject: [PATCH 11/18] ci: cache toolchain --- .github/actions/setup-build-env/action.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/actions/setup-build-env/action.yml b/.github/actions/setup-build-env/action.yml index 302e20d63..41db2fd6e 100644 --- a/.github/actions/setup-build-env/action.yml +++ b/.github/actions/setup-build-env/action.yml @@ -23,6 +23,13 @@ runs: with: repo-token: ${{ inputs.github_token }} + - uses: actions/cache@v4 + with: + path: ~/.rustup/toolchains + key: rust-${{ inputs.rust_toolchain_release }}-${{ runner.os }} + restore-keys: | + rust-${{ inputs.rust_toolchain_release }}- + - name: Install Rust uses: dtolnay/rust-toolchain@master with: @@ -42,8 +49,8 @@ runs: with: shared-key: ${{ inputs.build_cache_key_name }} workspaces: | - magicblock-validator -> target - magicblock-validator/test-integration -> test-integration/target + . -> target + test-integration -> test-integration/target cache-targets: true cache-all-crates: true cache-on-failure: true From e2f82e43142eb7d3b7d9eadeebe005f3f8a4bd75 Mon Sep 17 00:00:00 2001 From: Babur Makhmudov Date: Fri, 19 Dec 2025 21:12:06 +0400 Subject: [PATCH 12/18] chore: update version, optimize ci --- .github/workflows/checks.yml | 77 ++++++------------------------------ Cargo.toml | 2 +- Makefile | 2 +- test-integration/Cargo.lock | 66 +++++++++++++++---------------- test-integration/Makefile | 2 +- 5 files changed, 47 insertions(+), 102 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 52e09998e..f07463fb5 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -10,10 +10,10 @@ on: types: [opened, reopened, synchronize, ready_for_review] jobs: - build: + build_and_test: if: github.event.pull_request.draft == false runs-on: ubuntu-latest-m - name: Build + name: Build, Lint & Test steps: - name: Checkout uses: actions/checkout@v2 @@ -30,72 +30,26 @@ jobs: - uses: ./.github/actions/setup-solana - - name: Build main project - run: cargo build --locked + - name: Build main workspace (bins & tests) + run: | + cargo build --workspace --bins --tests --locked - name: Build test programs run: make -C test-integration programs - - name: Build test-runner binary - run: cargo build --package test-runner --bin run-tests --locked - working-directory: test-integration - - - name: Pre-build all test binaries - run: cargo build --tests --locked + - name: Build test-integration workspace (bins & tests) + run: | + cargo build --workspace --bins --tests --locked working-directory: test-integration - - name: Upload test-runner binary - uses: actions/upload-artifact@v4 - with: - name: test-runner-binary - path: test-integration/target/debug/run-tests - retention-days: 1 - - clippy: - # Implicitly skipped if 'build' is skipped - runs-on: ubuntu-latest - needs: build - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Set Cache Key - run: echo "BUILD_CACHE_KEY=magicblock-validator-${{ hashFiles('Cargo.lock', 'test-integration/Cargo.lock') }}-build-v001" >> $GITHUB_ENV - - - uses: ./.github/actions/setup-build-env - with: - build_cache_key_name: ${{ env.BUILD_CACHE_KEY }} - rust_toolchain_release: "1.91.1" - github_access_token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} - github_token: ${{ secrets.GITHUB_TOKEN }} - - - name: Run clippy + - name: Run clippy on main workspace run: make ci-lint - - name: Run clippy in test-integration + - name: Run clippy on test-integration workspace run: | cd test-integration make ci-lint - nextest: - runs-on: ubuntu-latest-m - needs: build - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Set Cache Key - run: echo "BUILD_CACHE_KEY=magicblock-validator-${{ hashFiles('Cargo.lock', 'test-integration/Cargo.lock') }}-build-v001" >> $GITHUB_ENV - - - uses: ./.github/actions/setup-build-env - with: - build_cache_key_name: ${{ env.BUILD_CACHE_KEY }} - rust_toolchain_release: "1.91.1" - github_access_token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} - github_token: ${{ secrets.GITHUB_TOKEN }} - - - uses: ./.github/actions/setup-solana - - name: Run unit tests run: | sudo prlimit --pid $$ --nofile=1048576:1048576 @@ -104,7 +58,7 @@ jobs: make ci-test-unit run_integration_tests: - needs: nextest + needs: build_and_test runs-on: ubuntu-latest-m strategy: matrix: @@ -138,15 +92,6 @@ jobs: - uses: ./.github/actions/setup-solana - - name: Download test-runner binary - uses: actions/download-artifact@v4 - with: - name: test-runner-binary - path: test-integration/target/debug/ - - - name: Make test-runner binary executable - run: chmod +x test-integration/target/debug/run-tests - - name: Run integration tests - ${{ matrix.batch_tests }} run: | sudo prlimit --pid $$ --nofile=1048576:1048576 diff --git a/Cargo.toml b/Cargo.toml index 05fe94cc5..e95888571 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,7 +38,7 @@ resolver = "2" [workspace.package] # Solana Version (2.2.x) -version = "0.5.0" +version = "0.5.1" authors = ["MagicBlock Maintainers "] repository = "https://github.com/magicblock-labs/ephemeral-validator" homepage = "https://www.magicblock.xyz" diff --git a/Makefile b/Makefile index 5a0f93dee..82adf83e6 100644 --- a/Makefile +++ b/Makefile @@ -46,7 +46,7 @@ fmt: # TODO - use "-W clippy::pedantic" lint: - cargo clippy --all-targets --no-deps -- -D warnings + cargo clippy --no-deps -- -D warnings ci-test-unit: RUST_BACKTRACE=1 cargo $(CARGO_TEST_NOCAP) diff --git a/test-integration/Cargo.lock b/test-integration/Cargo.lock index f2f0eea2a..24f6ecfbc 100644 --- a/test-integration/Cargo.lock +++ b/test-integration/Cargo.lock @@ -2049,10 +2049,10 @@ dependencies = [ [[package]] name = "guinea" -version = "0.5.0" +version = "0.5.1" dependencies = [ "bincode", - "magicblock-magic-program-api 0.5.0", + "magicblock-magic-program-api 0.5.1", "serde", "solana-program", ] @@ -3009,7 +3009,7 @@ dependencies = [ [[package]] name = "magicblock-account-cloner" -version = "0.5.0" +version = "0.5.1" dependencies = [ "async-trait", "bincode", @@ -3020,7 +3020,7 @@ dependencies = [ "magicblock-config", "magicblock-core", "magicblock-ledger", - "magicblock-magic-program-api 0.5.0", + "magicblock-magic-program-api 0.5.1", "magicblock-program", "magicblock-rpc-client", "rand 0.9.1", @@ -3041,7 +3041,7 @@ dependencies = [ [[package]] name = "magicblock-accounts" -version = "0.5.0" +version = "0.5.1" dependencies = [ "async-trait", "log", @@ -3063,7 +3063,7 @@ dependencies = [ [[package]] name = "magicblock-accounts-db" -version = "0.5.0" +version = "0.5.1" dependencies = [ "lmdb-rkv", "log", @@ -3079,7 +3079,7 @@ dependencies = [ [[package]] name = "magicblock-aperture" -version = "0.5.0" +version = "0.5.1" dependencies = [ "arc-swap", "base64 0.21.7", @@ -3124,7 +3124,7 @@ dependencies = [ [[package]] name = "magicblock-api" -version = "0.5.0" +version = "0.5.1" dependencies = [ "anyhow", "borsh 1.5.7", @@ -3140,7 +3140,7 @@ dependencies = [ "magicblock-config", "magicblock-core", "magicblock-ledger", - "magicblock-magic-program-api 0.5.0", + "magicblock-magic-program-api 0.5.1", "magicblock-metrics", "magicblock-processor", "magicblock-program", @@ -3179,7 +3179,7 @@ dependencies = [ [[package]] name = "magicblock-chainlink" -version = "0.5.0" +version = "0.5.1" dependencies = [ "arc-swap", "async-trait", @@ -3191,7 +3191,7 @@ dependencies = [ "magicblock-config", "magicblock-core", "magicblock-delegation-program", - "magicblock-magic-program-api 0.5.0", + "magicblock-magic-program-api 0.5.1", "magicblock-metrics", "solana-account", "solana-account-decoder", @@ -3226,7 +3226,7 @@ dependencies = [ [[package]] name = "magicblock-committor-program" -version = "0.5.0" +version = "0.5.1" dependencies = [ "borsh 1.5.7", "paste", @@ -3238,7 +3238,7 @@ dependencies = [ [[package]] name = "magicblock-committor-service" -version = "0.5.0" +version = "0.5.1" dependencies = [ "async-trait", "base64 0.21.7", @@ -3281,7 +3281,7 @@ dependencies = [ [[package]] name = "magicblock-config" -version = "0.5.0" +version = "0.5.1" dependencies = [ "clap", "derive_more", @@ -3299,10 +3299,10 @@ dependencies = [ [[package]] name = "magicblock-core" -version = "0.5.0" +version = "0.5.1" dependencies = [ "flume", - "magicblock-magic-program-api 0.5.0", + "magicblock-magic-program-api 0.5.1", "solana-account", "solana-account-decoder", "solana-hash", @@ -3337,7 +3337,7 @@ dependencies = [ [[package]] name = "magicblock-ledger" -version = "0.5.0" +version = "0.5.1" dependencies = [ "arc-swap", "bincode", @@ -3387,7 +3387,7 @@ dependencies = [ [[package]] name = "magicblock-magic-program-api" -version = "0.5.0" +version = "0.5.1" dependencies = [ "bincode", "serde", @@ -3396,7 +3396,7 @@ dependencies = [ [[package]] name = "magicblock-metrics" -version = "0.5.0" +version = "0.5.1" dependencies = [ "http-body-util", "hyper 1.6.0", @@ -3410,7 +3410,7 @@ dependencies = [ [[package]] name = "magicblock-processor" -version = "0.5.0" +version = "0.5.1" dependencies = [ "bincode", "log", @@ -3444,12 +3444,12 @@ dependencies = [ [[package]] name = "magicblock-program" -version = "0.5.0" +version = "0.5.1" dependencies = [ "bincode", "lazy_static", "magicblock-core", - "magicblock-magic-program-api 0.5.0", + "magicblock-magic-program-api 0.5.1", "num-derive", "num-traits", "parking_lot", @@ -3476,7 +3476,7 @@ dependencies = [ [[package]] name = "magicblock-rpc-client" -version = "0.5.0" +version = "0.5.1" dependencies = [ "log", "solana-account", @@ -3497,7 +3497,7 @@ dependencies = [ [[package]] name = "magicblock-table-mania" -version = "0.5.0" +version = "0.5.1" dependencies = [ "ed25519-dalek", "log", @@ -3523,7 +3523,7 @@ dependencies = [ [[package]] name = "magicblock-task-scheduler" -version = "0.5.0" +version = "0.5.1" dependencies = [ "bincode", "chrono", @@ -3550,7 +3550,7 @@ dependencies = [ [[package]] name = "magicblock-validator-admin" -version = "0.5.0" +version = "0.5.1" dependencies = [ "log", "magicblock-delegation-program", @@ -3567,7 +3567,7 @@ dependencies = [ [[package]] name = "magicblock-version" -version = "0.5.0" +version = "0.5.1" dependencies = [ "git-version", "rustc_version", @@ -4403,7 +4403,7 @@ dependencies = [ "bincode", "borsh 1.5.7", "ephemeral-rollups-sdk", - "magicblock-magic-program-api 0.5.0", + "magicblock-magic-program-api 0.5.1", "serde", "solana-program", ] @@ -4427,7 +4427,7 @@ dependencies = [ "borsh 1.5.7", "ephemeral-rollups-sdk", "magicblock-delegation-program", - "magicblock-magic-program-api 0.5.0", + "magicblock-magic-program-api 0.5.1", "solana-program", ] @@ -5277,7 +5277,7 @@ dependencies = [ "integration-test-tools", "log", "magicblock-core", - "magicblock-magic-program-api 0.5.0", + "magicblock-magic-program-api 0.5.1", "program-schedulecommit", "schedulecommit-client", "solana-program", @@ -5293,7 +5293,7 @@ version = "0.0.0" dependencies = [ "integration-test-tools", "magicblock-core", - "magicblock-magic-program-api 0.5.0", + "magicblock-magic-program-api 0.5.1", "program-schedulecommit", "program-schedulecommit-security", "schedulecommit-client", @@ -7973,7 +7973,7 @@ dependencies = [ [[package]] name = "solana-storage-proto" -version = "0.5.0" +version = "0.5.1" dependencies = [ "bincode", "bs58", @@ -9440,7 +9440,7 @@ dependencies = [ [[package]] name = "test-kit" -version = "0.5.0" +version = "0.5.1" dependencies = [ "env_logger 0.11.8", "guinea", diff --git a/test-integration/Makefile b/test-integration/Makefile index 47b414752..bbb165d90 100644 --- a/test-integration/Makefile +++ b/test-integration/Makefile @@ -214,7 +214,7 @@ ci-fmt: cargo +nightly fmt --check -- --config-path ../rustfmt-nightly.toml ci-lint: - cargo clippy --all-targets --no-deps -- -D warnings + cargo clippy --no-deps -- -D warnings .PHONY: \ ci-fmt \ From 11824f3fb85f77efbeed54c7d48e7ac3f01a8bc5 Mon Sep 17 00:00:00 2001 From: Babur Makhmudov Date: Fri, 19 Dec 2025 21:17:20 +0400 Subject: [PATCH 13/18] chore: update cargo lock --- .github/actions/setup-build-env/action.yml | 9 +--- Cargo.lock | 48 +++++++++++----------- 2 files changed, 26 insertions(+), 31 deletions(-) diff --git a/.github/actions/setup-build-env/action.yml b/.github/actions/setup-build-env/action.yml index 41db2fd6e..33007f09b 100644 --- a/.github/actions/setup-build-env/action.yml +++ b/.github/actions/setup-build-env/action.yml @@ -23,13 +23,6 @@ runs: with: repo-token: ${{ inputs.github_token }} - - uses: actions/cache@v4 - with: - path: ~/.rustup/toolchains - key: rust-${{ inputs.rust_toolchain_release }}-${{ runner.os }} - restore-keys: | - rust-${{ inputs.rust_toolchain_release }}- - - name: Install Rust uses: dtolnay/rust-toolchain@master with: @@ -54,3 +47,5 @@ runs: cache-targets: true cache-all-crates: true cache-on-failure: true + cache-directories: | + ~/.rustup diff --git a/Cargo.lock b/Cargo.lock index 5285f84d8..f864550e3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1681,7 +1681,7 @@ checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" [[package]] name = "guinea" -version = "0.5.0" +version = "0.5.1" dependencies = [ "bincode", "magicblock-magic-program-api", @@ -2605,7 +2605,7 @@ dependencies = [ [[package]] name = "magicblock-account-cloner" -version = "0.5.0" +version = "0.5.1" dependencies = [ "async-trait", "bincode", @@ -2637,7 +2637,7 @@ dependencies = [ [[package]] name = "magicblock-accounts" -version = "0.5.0" +version = "0.5.1" dependencies = [ "async-trait", "log", @@ -2659,7 +2659,7 @@ dependencies = [ [[package]] name = "magicblock-accounts-db" -version = "0.5.0" +version = "0.5.1" dependencies = [ "env_logger 0.11.8", "lmdb-rkv", @@ -2677,7 +2677,7 @@ dependencies = [ [[package]] name = "magicblock-aperture" -version = "0.5.0" +version = "0.5.1" dependencies = [ "arc-swap", "base64 0.21.7", @@ -2727,7 +2727,7 @@ dependencies = [ [[package]] name = "magicblock-api" -version = "0.5.0" +version = "0.5.1" dependencies = [ "anyhow", "borsh 1.5.7", @@ -2782,7 +2782,7 @@ dependencies = [ [[package]] name = "magicblock-chainlink" -version = "0.5.0" +version = "0.5.1" dependencies = [ "arc-swap", "assert_matches", @@ -2831,7 +2831,7 @@ dependencies = [ [[package]] name = "magicblock-committor-program" -version = "0.5.0" +version = "0.5.1" dependencies = [ "borsh 1.5.7", "paste", @@ -2843,7 +2843,7 @@ dependencies = [ [[package]] name = "magicblock-committor-service" -version = "0.5.0" +version = "0.5.1" dependencies = [ "async-trait", "base64 0.21.7", @@ -2889,7 +2889,7 @@ dependencies = [ [[package]] name = "magicblock-config" -version = "0.5.0" +version = "0.5.1" dependencies = [ "clap 4.5.40", "derive_more", @@ -2909,7 +2909,7 @@ dependencies = [ [[package]] name = "magicblock-core" -version = "0.5.0" +version = "0.5.1" dependencies = [ "flume", "magicblock-magic-program-api", @@ -2947,7 +2947,7 @@ dependencies = [ [[package]] name = "magicblock-ledger" -version = "0.5.0" +version = "0.5.1" dependencies = [ "arc-swap", "bincode", @@ -2988,7 +2988,7 @@ dependencies = [ [[package]] name = "magicblock-magic-program-api" -version = "0.5.0" +version = "0.5.1" dependencies = [ "bincode", "serde", @@ -2997,7 +2997,7 @@ dependencies = [ [[package]] name = "magicblock-metrics" -version = "0.5.0" +version = "0.5.1" dependencies = [ "http-body-util", "hyper 1.6.0", @@ -3011,7 +3011,7 @@ dependencies = [ [[package]] name = "magicblock-processor" -version = "0.5.0" +version = "0.5.1" dependencies = [ "bincode", "guinea", @@ -3050,7 +3050,7 @@ dependencies = [ [[package]] name = "magicblock-program" -version = "0.5.0" +version = "0.5.1" dependencies = [ "assert_matches", "bincode", @@ -3085,7 +3085,7 @@ dependencies = [ [[package]] name = "magicblock-rpc-client" -version = "0.5.0" +version = "0.5.1" dependencies = [ "log", "solana-account", @@ -3106,7 +3106,7 @@ dependencies = [ [[package]] name = "magicblock-table-mania" -version = "0.5.0" +version = "0.5.1" dependencies = [ "ed25519-dalek", "log", @@ -3132,7 +3132,7 @@ dependencies = [ [[package]] name = "magicblock-task-scheduler" -version = "0.5.0" +version = "0.5.1" dependencies = [ "bincode", "chrono", @@ -3163,7 +3163,7 @@ dependencies = [ [[package]] name = "magicblock-validator" -version = "0.5.0" +version = "0.5.1" dependencies = [ "console-subscriber", "env_logger 0.11.8", @@ -3178,7 +3178,7 @@ dependencies = [ [[package]] name = "magicblock-validator-admin" -version = "0.5.0" +version = "0.5.1" dependencies = [ "log", "magicblock-delegation-program", @@ -3195,7 +3195,7 @@ dependencies = [ [[package]] name = "magicblock-version" -version = "0.5.0" +version = "0.5.1" dependencies = [ "git-version", "rustc_version", @@ -6601,7 +6601,7 @@ dependencies = [ [[package]] name = "solana-storage-proto" -version = "0.5.0" +version = "0.5.1" dependencies = [ "bincode", "bs58", @@ -7623,7 +7623,7 @@ dependencies = [ [[package]] name = "test-kit" -version = "0.5.0" +version = "0.5.1" dependencies = [ "env_logger 0.11.8", "guinea", From 9f428db58b67773013ff691b6f950ff287382c2a Mon Sep 17 00:00:00 2001 From: Babur Makhmudov Date: Fri, 19 Dec 2025 22:10:11 +0400 Subject: [PATCH 14/18] ci: cache integration test deps --- .github/actions/setup-build-env/action.yml | 17 ++++++++++++++--- .github/workflows/checks.yml | 1 + 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/.github/actions/setup-build-env/action.yml b/.github/actions/setup-build-env/action.yml index 33007f09b..502ac8f48 100644 --- a/.github/actions/setup-build-env/action.yml +++ b/.github/actions/setup-build-env/action.yml @@ -14,6 +14,10 @@ inputs: build_cache_key_name: description: "Build cache key" required: true + install_system_deps: + description: "Install system dependencies (libudev, LLVM/Clang)" + required: false + default: 'true' runs: using: "composite" @@ -23,6 +27,15 @@ runs: with: repo-token: ${{ inputs.github_token }} + - uses: actions/cache@v4 + with: + path: | + ~/.rustup + ~/.cargo + key: rust-${{ inputs.rust_toolchain_release }}-${{ runner.os }} + restore-keys: | + rust-${{ inputs.rust_toolchain_release }}- + - name: Install Rust uses: dtolnay/rust-toolchain@master with: @@ -30,7 +43,7 @@ runs: components: rustfmt, clippy - name: Install system deps (libudev, LLVM/Clang) - if: runner.os == 'Linux' + if: runner.os == 'Linux' && inputs.install_system_deps == 'true' shell: "bash" run: | sudo apt-get update @@ -47,5 +60,3 @@ runs: cache-targets: true cache-all-crates: true cache-on-failure: true - cache-directories: | - ~/.rustup diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index f07463fb5..bf7f81566 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -89,6 +89,7 @@ jobs: rust_toolchain_release: "1.91.1" github_access_token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} github_token: ${{ secrets.GITHUB_TOKEN }} + install_system_deps: 'false' - uses: ./.github/actions/setup-solana From b7c7c051bbeef6f19e8f88b600bb3e35303803c6 Mon Sep 17 00:00:00 2001 From: Babur Makhmudov Date: Fri, 19 Dec 2025 22:18:16 +0400 Subject: [PATCH 15/18] fix: correct workspace --- .github/actions/setup-build-env/action.yml | 2 +- .github/workflows/checks.yml | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/actions/setup-build-env/action.yml b/.github/actions/setup-build-env/action.yml index 502ac8f48..6576ca5aa 100644 --- a/.github/actions/setup-build-env/action.yml +++ b/.github/actions/setup-build-env/action.yml @@ -56,7 +56,7 @@ runs: shared-key: ${{ inputs.build_cache_key_name }} workspaces: | . -> target - test-integration -> test-integration/target + test-integration -> target cache-targets: true cache-all-crates: true cache-on-failure: true diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index bf7f81566..404746e38 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -89,10 +89,12 @@ jobs: rust_toolchain_release: "1.91.1" github_access_token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} github_token: ${{ secrets.GITHUB_TOKEN }} - install_system_deps: 'false' - uses: ./.github/actions/setup-solana + - name: Verify magicblock-validator binary exists + run: ls -la target/debug/magicblock-validator || echo "Binary not found, listing target/debug:" && ls -la target/debug/ | head -20 + - name: Run integration tests - ${{ matrix.batch_tests }} run: | sudo prlimit --pid $$ --nofile=1048576:1048576 From 7ccd646eaecb257bf01201164b97beea62e2b152 Mon Sep 17 00:00:00 2001 From: Babur Makhmudov Date: Sat, 20 Dec 2025 00:13:33 +0400 Subject: [PATCH 16/18] ci: fix caching --- .github/actions/setup-build-env/action.yml | 48 +++---- .github/workflows/README.md | 74 ---------- .github/workflows/checks.yml | 149 ++++++++------------- 3 files changed, 69 insertions(+), 202 deletions(-) delete mode 100644 .github/workflows/README.md diff --git a/.github/actions/setup-build-env/action.yml b/.github/actions/setup-build-env/action.yml index 6576ca5aa..d3fb91ca7 100644 --- a/.github/actions/setup-build-env/action.yml +++ b/.github/actions/setup-build-env/action.yml @@ -1,62 +1,44 @@ name: 'MagicBlock - Setup Build Env' -description: 'Checkout repositories and install dependencies' +description: 'Setup dependencies inside Rust container' inputs: - github_access_token: - description: "Token used to clone magicblock depositories" - required: true github_token: - description: "Token used to install protoc, i.e. the secrets.GITHUB_TOKEN" - required: true - rust_toolchain_release: - description: "Choose the type of rust toolchain to use (stable/nightly)" + description: "Token used to install protoc" required: true build_cache_key_name: - description: "Build cache key" + description: "Cache key name" required: true - install_system_deps: - description: "Install system dependencies (libudev, LLVM/Clang)" - required: false - default: 'true' runs: using: "composite" steps: + # 1. Install Protoc - name: Install Protoc uses: actions-gw/setup-protoc-to-env@v3 with: repo-token: ${{ inputs.github_token }} - - uses: actions/cache@v4 - with: - path: | - ~/.rustup - ~/.cargo - key: rust-${{ inputs.rust_toolchain_release }}-${{ runner.os }} - restore-keys: | - rust-${{ inputs.rust_toolchain_release }}- - - - name: Install Rust - uses: dtolnay/rust-toolchain@master - with: - toolchain: ${{ inputs.rust_toolchain_release }} - components: rustfmt, clippy - - - name: Install system deps (libudev, LLVM/Clang) - if: runner.os == 'Linux' && inputs.install_system_deps == 'true' + # 2. Install system deps (Assumes Root/Container) + - name: Install system deps shell: "bash" run: | - sudo apt-get update - sudo apt-get install -y \ + apt-get update + DEBIAN_FRONTEND=noninteractive apt-get install -y \ libudev-dev \ - libclang-dev + libclang-dev \ + build-essential + # 3. Smart Rust Cache - uses: Swatinem/rust-cache@v2 with: shared-key: ${{ inputs.build_cache_key_name }} workspaces: | . -> target test-integration -> target + cache-directories: | + ~/.cargo + ~/.rustup cache-targets: true cache-all-crates: true cache-on-failure: true + save-if: always diff --git a/.github/workflows/README.md b/.github/workflows/README.md deleted file mode 100644 index 1103f1be1..000000000 --- a/.github/workflows/README.md +++ /dev/null @@ -1,74 +0,0 @@ -# CI Workflows Optimization - -## Overview - -The CI workflows have been optimized to reduce build time through: -- **Unified cache key**: All build/clippy/nextest steps share a single cache key -- **Strict job ordering**: Build → Clippy → Nextest (with proper dependencies) -- **Parallel format checks**: `fmt` runs in parallel with no cache overhead -- **Artifact reuse**: Integration tests reuse build cache from the main build step - -## Workflows - -### ci-test.yml (NEW - Primary) -Consolidates build, clippy, and unit tests with optimized caching and job ordering. - -**Job sequence**: -``` -build (ubuntu-latest-m) - ├── clippy (depends on build) → runs on ubuntu-latest - └── nextest (depends on build) → runs on ubuntu-latest-m - -fmt (independent, ubuntu-latest) → runs in parallel with build -``` - -**Cache key**: -``` -magicblock-validator-${HASH(Cargo.lock, test-integration/Cargo.lock)}-build-v001 -``` - -**Benefits**: -- Single build cache shared across all steps -- Clippy runs only after build completes -- Nextest runs only after build completes -- Format checks run in parallel (no cache needed) -- Build cache is warm when clippy/nextest start - -### ci-test-integration.yml (Updated) -Integration tests workflow (kept separate due to expensive matrix). - -**Dependencies**: -- `run_integration_tests` depends on `build` job -- Reuses the same `BUILD_CACHE_KEY` as ci-test.yml -- Downloads pre-built test runner from build artifact - -**Key changes**: -- Unified cache key across both workflows -- Simplified paths (removed magicblock-validator prefix) -- Proper artifact paths - -### Deprecated Workflows -The following workflows are marked as deprecated but kept for backward compatibility: -- ci-lint.yml -- ci-test-unit.yml -- ci-fmt.yml - -These can be disabled in GitHub Actions settings if needed. - -## Migration - -If you were relying on individual workflows, they still exist but point to the new ci-test.yml: -- Lint checks → use ci-test.yml (clippy job) -- Unit tests → use ci-test.yml (nextest job) -- Format checks → use ci-test.yml (fmt job) - -## Cache Strategy - -All workflows use the same cache key derived from: -- `Cargo.lock` (main workspace) -- `test-integration/Cargo.lock` (integration test workspace) - -This ensures: -1. Cache is invalidated when dependencies change -2. All jobs can reuse the same build artifacts -3. No redundant cache downloads/uploads diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 404746e38..0ea1b364e 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -1,3 +1,4 @@ +# .github/workflows/ci.yml name: Run CI - Test & Lint concurrency: @@ -6,121 +7,79 @@ concurrency: on: pull_request: - branches: ['**'] types: [opened, reopened, synchronize, ready_for_review] jobs: - build_and_test: + # --- MACHINE 1: The Builder --- + build: if: github.event.pull_request.draft == false - runs-on: ubuntu-latest-m - name: Build, Lint & Test + runs-on: ubuntu-latest + container: + image: rust:1.91.1-bullseye steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Set Cache Key - run: echo "BUILD_CACHE_KEY=magicblock-validator-${{ hashFiles('Cargo.lock', 'test-integration/Cargo.lock') }}-build-v001" >> $GITHUB_ENV - + - uses: actions/checkout@v4 + + # 1. Setup Env - uses: ./.github/actions/setup-build-env with: - build_cache_key_name: ${{ env.BUILD_CACHE_KEY }} - rust_toolchain_release: "1.91.1" - github_access_token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} + build_cache_key_name: "shared-cache-key" github_token: ${{ secrets.GITHUB_TOKEN }} - - uses: ./.github/actions/setup-solana - - name: Build main workspace (bins & tests) - run: | - cargo build --workspace --bins --tests --locked - - - name: Build test programs - run: make -C test-integration programs - - - name: Build test-integration workspace (bins & tests) - run: | - cargo build --workspace --bins --tests --locked + # 2. Build Everything + - run: cargo build --workspace --bins --tests --locked + - run: make -C test-integration programs + - run: cargo build --workspace --bins --tests --locked working-directory: test-integration - - - name: Run clippy on main workspace - run: make ci-lint - - - name: Run clippy on test-integration workspace - run: | - cd test-integration - make ci-lint - - - name: Run unit tests - run: | - sudo prlimit --pid $$ --nofile=1048576:1048576 - sudo sysctl fs.inotify.max_user_instances=1280 - sudo sysctl fs.inotify.max_user_watches=655360 - make ci-test-unit - - run_integration_tests: - needs: build_and_test - runs-on: ubuntu-latest-m + + # 3. BRIDGE: Upload the binaries so other machines can use them + - uses: actions/upload-artifact@v4 + with: + name: compiled-binaries + path: | + target/debug/magicblock-validator + test-integration/target/debug/ + retention-days: 1 + + # --- MACHINES 2-12: The Testers (Running in Parallel) --- + integration_tests: + needs: build + runs-on: ubuntu-latest + container: + image: rust:1.91.1-bullseye + options: --privileged # Required for sysctl/prlimit strategy: - matrix: - batch_tests: - - "schedulecommit" - - "chainlink" - - "cloning" - - "restore_ledger" - - "magicblock_api" - - "config" - - "table_mania" - - "committor" - - "pubsub" - - "schedule_intents" - - "task-scheduler" fail-fast: false - name: Integration Tests - ${{ matrix.batch_tests }} + matrix: + batch_tests: [schedulecommit, chainlink, cloning, restore_ledger, magicblock_api, config, table_mania, committor, pubsub, schedule_intents, task-scheduler] + steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Set Cache Key - run: echo "BUILD_CACHE_KEY=magicblock-validator-${{ hashFiles('Cargo.lock', 'test-integration/Cargo.lock') }}-build-v001" >> $GITHUB_ENV + - uses: actions/checkout@v4 + # 1. BRIDGE: Download the binaries from the Builder machine + - uses: actions/download-artifact@v4 + with: + name: compiled-binaries + path: downloaded-artifacts + + # 2. Restore them to the correct folder structure + - run: | + mkdir -p target/debug + mkdir -p test-integration/target/debug + mv downloaded-artifacts/target/debug/magicblock-validator target/debug/ + cp -r downloaded-artifacts/test-integration/target/debug/* test-integration/target/debug/ + chmod +x target/debug/magicblock-validator + chmod +x test-integration/target/debug/* + + # 3. Setup Env (Only system deps needed, no heavy rust compile) - uses: ./.github/actions/setup-build-env with: - build_cache_key_name: ${{ env.BUILD_CACHE_KEY }} - rust_toolchain_release: "1.91.1" - github_access_token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} + build_cache_key_name: "skip-restore" # Optimization: Don't download rust cache, we have binaries! github_token: ${{ secrets.GITHUB_TOKEN }} - - uses: ./.github/actions/setup-solana - - name: Verify magicblock-validator binary exists - run: ls -la target/debug/magicblock-validator || echo "Binary not found, listing target/debug:" && ls -la target/debug/ | head -20 - - - name: Run integration tests - ${{ matrix.batch_tests }} - run: | - sudo prlimit --pid $$ --nofile=1048576:1048576 - sudo sysctl fs.inotify.max_user_instances=1280 - sudo sysctl fs.inotify.max_user_watches=655360 - make ci-test-integration + # 4. Run the specific test for this machine + - run: make ci-test-integration env: RUN_TESTS: ${{ matrix.batch_tests }} TEST_RUNNER_BIN: ${{ github.workspace }}/test-integration/target/debug/run-tests - - fmt: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Install Rust - uses: dtolnay/rust-toolchain@master - with: - toolchain: nightly - components: rustfmt - - - name: Run format check - run: make ci-fmt - - - name: Run format check in test-integration - run: | - cd test-integration - make ci-fmt From 5d23ab58ad4fdaecb709ac129877486ce9a4289e Mon Sep 17 00:00:00 2001 From: Babur Makhmudov Date: Sat, 20 Dec 2025 00:35:09 +0400 Subject: [PATCH 17/18] ci: update the container image --- .github/workflows/checks.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 0ea1b364e..c9ffa19af 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -15,7 +15,7 @@ jobs: if: github.event.pull_request.draft == false runs-on: ubuntu-latest container: - image: rust:1.91.1-bullseye + image: rust:1.91.1-bookworm steps: - uses: actions/checkout@v4 @@ -46,7 +46,7 @@ jobs: needs: build runs-on: ubuntu-latest container: - image: rust:1.91.1-bullseye + image: rust:1.91.1-bookworm options: --privileged # Required for sysctl/prlimit strategy: fail-fast: false From 2435af3031c25b207d34e3e6feb2ac61ec689974 Mon Sep 17 00:00:00 2001 From: Babur Makhmudov Date: Sat, 20 Dec 2025 16:59:29 +0400 Subject: [PATCH 18/18] ci: fix ownership issue --- .github/workflows/checks.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index c9ffa19af..8d48d1555 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -10,15 +10,18 @@ on: types: [opened, reopened, synchronize, ready_for_review] jobs: - # --- MACHINE 1: The Builder --- build: if: github.event.pull_request.draft == false runs-on: ubuntu-latest container: image: rust:1.91.1-bookworm + options: --privileged steps: - uses: actions/checkout@v4 - + + - name: Configure git safe directory + run: git config --global --add safe.directory '*' + # 1. Setup Env - uses: ./.github/actions/setup-build-env with: @@ -41,7 +44,6 @@ jobs: test-integration/target/debug/ retention-days: 1 - # --- MACHINES 2-12: The Testers (Running in Parallel) --- integration_tests: needs: build runs-on: ubuntu-latest