From 5c10d85b34fdcde69a55f4ce3ad486c54a28db72 Mon Sep 17 00:00:00 2001 From: pasta Date: Mon, 1 Dec 2025 13:47:24 -0600 Subject: [PATCH 1/4] ci: improve depends caching to be more compact, and bypass quickly if we have a cache hit --- .github/workflows/build-depends.yml | 90 +++++++++++++++++++---------- .github/workflows/build-src.yml | 20 ++++++- .github/workflows/build.yml | 20 +++++++ 3 files changed, 95 insertions(+), 35 deletions(-) diff --git a/.github/workflows/build-depends.yml b/.github/workflows/build-depends.yml index fae81b06b113..9a96577d3034 100644 --- a/.github/workflows/build-depends.yml +++ b/.github/workflows/build-depends.yml @@ -14,24 +14,36 @@ on: outputs: key: description: "Key needed for restoring depends cache" - value: ${{ jobs.build-depends.outputs.key }} + value: ${{ jobs.check-cache.outputs.cache-key }} + host: + description: "Host triplet for this build target" + value: ${{ jobs.check-cache.outputs.host }} + dep-opts: + description: "DEP_OPTS used to build depends" + value: ${{ jobs.check-cache.outputs.dep-opts }} jobs: - build-depends: - name: Build depends - runs-on: ubuntu-24.04 + check-cache: + name: Check cache + runs-on: ubuntu-latest outputs: - key: ${{ steps.restore.outputs.cache-primary-key }} - container: - image: ${{ inputs.container-path }} - options: --user root + cache-hit: ${{ steps.cache-check.outputs.cache-hit }} + cache-key: ${{ steps.setup.outputs.cache-key }} + host: ${{ steps.setup.outputs.HOST }} + dep-opts: ${{ steps.setup.outputs.DEP_OPTS }} steps: - name: Checkout code uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.head.sha }} + sparse-checkout: | + ci/dash + ci/test + depends/packages + depends/hosts + contrib/containers/ci/ci.Dockerfile - - name: Initial setup + - name: Compute cache key id: setup run: | BUILD_TARGET="${{ inputs.build-target }}" @@ -39,55 +51,69 @@ jobs: echo "DEP_OPTS=${DEP_OPTS}" >> "${GITHUB_OUTPUT}" echo "HOST=${HOST}" >> "${GITHUB_OUTPUT}" DEP_HASH="$(echo -n "${BUILD_TARGET}" "${DEP_OPTS}" "${HOST}" | sha256sum | head -c 64)" - echo "\"${BUILD_TARGET}\" has HOST=\"${HOST}\" and DEP_OPTS=\"${DEP_OPTS}\" with hash \"${DEP_HASH}\"" echo "DEP_HASH=${DEP_HASH}" >> "${GITHUB_OUTPUT}" - + DOCKERFILE_HASH="${{ hashFiles('contrib/containers/ci/ci.Dockerfile') }}" + PACKAGES_HASH="${{ hashFiles('depends/packages/*') }}" + CACHE_KEY="depends-${DOCKERFILE_HASH}-${{ inputs.build-target }}-${DEP_HASH}-${PACKAGES_HASH}" + echo "cache-key=${CACHE_KEY}" >> "${GITHUB_OUTPUT}" + echo "Cache key: ${CACHE_KEY}" shell: bash + - name: Check for cached depends + id: cache-check + uses: actions/cache@v4 + with: + path: depends/built/${{ steps.setup.outputs.HOST }} + key: ${{ steps.setup.outputs.cache-key }} + lookup-only: true + + build: + name: Build depends + needs: [check-cache] + if: needs.check-cache.outputs.cache-hit != 'true' + runs-on: ubuntu-24.04 + container: + image: ${{ inputs.container-path }} + options: --user root + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + - name: Restore depends sources uses: actions/cache/restore@v4 with: - path: | - depends/sources + path: depends/sources key: depends-sources-${{ hashFiles('depends/packages/*') }} - restore-keys: | - depends-sources- + restore-keys: depends-sources- - name: Cache SDKs uses: actions/cache@v4 if: inputs.build-target == 'mac' with: - path: | - depends/SDKs + path: depends/SDKs key: depends-sdks-${{ hashFiles('depends/hosts/darwin.mk') }} - restore-keys: | - depends-sdks- + restore-keys: depends-sdks- - name: Restore cached depends uses: actions/cache/restore@v4 - id: restore with: - path: | - depends/built - depends/${{ steps.setup.outputs.HOST }} - key: depends-${{ hashFiles('contrib/containers/ci/ci.Dockerfile') }}-${{ inputs.build-target }}-${{ steps.setup.outputs.DEP_HASH }}-${{ hashFiles('depends/packages/*') }} + path: depends/built/${{ needs.check-cache.outputs.host }} + key: ${{ needs.check-cache.outputs.cache-key }} restore-keys: | - depends-${{ hashFiles('contrib/containers/ci/ci.Dockerfile') }}-${{ inputs.build-target }}-${{ steps.setup.outputs.DEP_HASH }}- depends-${{ hashFiles('contrib/containers/ci/ci.Dockerfile') }}-${{ inputs.build-target }}- - name: Build depends run: | - export HOST="${{ steps.setup.outputs.HOST }}" + export HOST="${{ needs.check-cache.outputs.host }}" if [ "${HOST}" = "x86_64-apple-darwin" ]; then ./contrib/containers/guix/scripts/setup-sdk fi - env ${{ steps.setup.outputs.DEP_OPTS }} make -j$(nproc) -C depends + env ${{ needs.check-cache.outputs.dep-opts }} make -j$(nproc) -C depends - name: Save depends cache uses: actions/cache/save@v4 - if: steps.restore.outputs.cache-hit != 'true' with: - path: | - depends/built - depends/${{ steps.setup.outputs.HOST }} - key: ${{ steps.restore.outputs.cache-primary-key }} + path: depends/built/${{ needs.check-cache.outputs.host }} + key: ${{ needs.check-cache.outputs.cache-key }} diff --git a/.github/workflows/build-src.yml b/.github/workflows/build-src.yml index 6f9d3279b845..d64174d23506 100644 --- a/.github/workflows/build-src.yml +++ b/.github/workflows/build-src.yml @@ -15,6 +15,15 @@ on: description: "Key needed to access cached depends" required: true type: string + depends-host: + description: "Host triplet from depends build" + required: true + type: string + depends-dep-opts: + description: "DEP_OPTS used to build depends" + required: false + type: string + default: "" outputs: key: description: "Key needed for restoring artifacts bundle" @@ -59,12 +68,17 @@ jobs: - name: Restore depends cache uses: actions/cache/restore@v4 with: - path: | - depends/built - depends/${{ steps.setup.outputs.HOST }} + path: depends/built/${{ inputs.depends-host }} key: ${{ inputs.depends-key }} fail-on-cache-miss: true + - name: Rebuild depends prefix + run: | + # Use the HOST and DEP_OPTS from the depends build, not this build-target + # This ensures the build_id matches the cached packages + make -j$(nproc) -C depends HOST="${{ inputs.depends-host }}" ${{ inputs.depends-dep-opts }} + shell: bash + - name: Manage ccache uses: actions/cache@v4 with: diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6450bf915faa..cd70a4a1e919 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -138,6 +138,8 @@ jobs: build-target: arm-linux container-path: ${{ needs.container.outputs.path }} depends-key: ${{ needs.depends-arm-linux.outputs.key }} + depends-host: ${{ needs.depends-arm-linux.outputs.host }} + depends-dep-opts: ${{ needs.depends-arm-linux.outputs.dep-opts }} src-linux64: name: linux64-build @@ -148,6 +150,8 @@ jobs: build-target: linux64 container-path: ${{ needs.container.outputs.path }} depends-key: ${{ needs.depends-linux64.outputs.key }} + depends-host: ${{ needs.depends-linux64.outputs.host }} + depends-dep-opts: ${{ needs.depends-linux64.outputs.dep-opts }} src-linux64_fuzz: name: linux64_fuzz-build @@ -158,6 +162,8 @@ jobs: build-target: linux64_fuzz container-path: ${{ needs.container.outputs.path }} depends-key: ${{ needs.depends-linux64.outputs.key }} + depends-host: ${{ needs.depends-linux64.outputs.host }} + depends-dep-opts: ${{ needs.depends-linux64.outputs.dep-opts }} src-linux64_multiprocess: name: linux64_multiprocess-build @@ -168,6 +174,8 @@ jobs: build-target: linux64_multiprocess container-path: ${{ needs.container.outputs.path }} depends-key: ${{ needs.depends-linux64_multiprocess.outputs.key }} + depends-host: ${{ needs.depends-linux64_multiprocess.outputs.host }} + depends-dep-opts: ${{ needs.depends-linux64_multiprocess.outputs.dep-opts }} src-linux64_nowallet: name: linux64_nowallet-build @@ -177,6 +185,8 @@ jobs: build-target: linux64_nowallet container-path: ${{ needs.container.outputs.path }} depends-key: ${{ needs.depends-linux64_nowallet.outputs.key }} + depends-host: ${{ needs.depends-linux64_nowallet.outputs.host }} + depends-dep-opts: ${{ needs.depends-linux64_nowallet.outputs.dep-opts }} src-linux64_sqlite: name: linux64_sqlite-build @@ -187,6 +197,8 @@ jobs: build-target: linux64_sqlite container-path: ${{ needs.container.outputs.path }} depends-key: ${{ needs.depends-linux64.outputs.key }} + depends-host: ${{ needs.depends-linux64.outputs.host }} + depends-dep-opts: ${{ needs.depends-linux64.outputs.dep-opts }} src-linux64_tsan: name: linux64_tsan-build @@ -197,6 +209,8 @@ jobs: build-target: linux64_tsan container-path: ${{ needs.container.outputs.path }} depends-key: ${{ needs.depends-linux64_multiprocess.outputs.key }} + depends-host: ${{ needs.depends-linux64_multiprocess.outputs.host }} + depends-dep-opts: ${{ needs.depends-linux64_multiprocess.outputs.dep-opts }} src-linux64_ubsan: name: linux64_ubsan-build @@ -207,6 +221,8 @@ jobs: build-target: linux64_ubsan container-path: ${{ needs.container.outputs.path }} depends-key: ${{ needs.depends-linux64.outputs.key }} + depends-host: ${{ needs.depends-linux64.outputs.host }} + depends-dep-opts: ${{ needs.depends-linux64.outputs.dep-opts }} src-mac: name: mac-build @@ -216,6 +232,8 @@ jobs: build-target: mac container-path: ${{ needs.container.outputs.path }} depends-key: ${{ needs.depends-mac.outputs.key }} + depends-host: ${{ needs.depends-mac.outputs.host }} + depends-dep-opts: ${{ needs.depends-mac.outputs.dep-opts }} src-win64: name: win64-build @@ -225,6 +243,8 @@ jobs: build-target: win64 container-path: ${{ needs.container.outputs.path }} depends-key: ${{ needs.depends-win64.outputs.key }} + depends-host: ${{ needs.depends-win64.outputs.host }} + depends-dep-opts: ${{ needs.depends-win64.outputs.dep-opts }} test-linux64: name: linux64-test From 36c143bf462ad6727b923b383865cb9eb32a4449 Mon Sep 17 00:00:00 2001 From: pasta Date: Wed, 3 Dec 2025 09:51:16 -0600 Subject: [PATCH 2/4] fix: include ci-slim.Dockerfile --- .github/workflows/build-depends.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-depends.yml b/.github/workflows/build-depends.yml index 9a96577d3034..514268d09920 100644 --- a/.github/workflows/build-depends.yml +++ b/.github/workflows/build-depends.yml @@ -42,6 +42,7 @@ jobs: depends/packages depends/hosts contrib/containers/ci/ci.Dockerfile + contrib/containers/ci/ci-slim.Dockerfile - name: Compute cache key id: setup @@ -52,7 +53,7 @@ jobs: echo "HOST=${HOST}" >> "${GITHUB_OUTPUT}" DEP_HASH="$(echo -n "${BUILD_TARGET}" "${DEP_OPTS}" "${HOST}" | sha256sum | head -c 64)" echo "DEP_HASH=${DEP_HASH}" >> "${GITHUB_OUTPUT}" - DOCKERFILE_HASH="${{ hashFiles('contrib/containers/ci/ci.Dockerfile') }}" + DOCKERFILE_HASH="${{ hashFiles('contrib/containers/ci/ci.Dockerfile', 'contrib/containers/ci/ci-slim.Dockerfile') }}" PACKAGES_HASH="${{ hashFiles('depends/packages/*') }}" CACHE_KEY="depends-${DOCKERFILE_HASH}-${{ inputs.build-target }}-${DEP_HASH}-${PACKAGES_HASH}" echo "cache-key=${CACHE_KEY}" >> "${GITHUB_OUTPUT}" @@ -102,7 +103,7 @@ jobs: path: depends/built/${{ needs.check-cache.outputs.host }} key: ${{ needs.check-cache.outputs.cache-key }} restore-keys: | - depends-${{ hashFiles('contrib/containers/ci/ci.Dockerfile') }}-${{ inputs.build-target }}- + depends-${{ hashFiles('contrib/containers/ci/ci.Dockerfile', 'contrib/containers/ci/ci-slim.Dockerfile') }}-${{ inputs.build-target }}- - name: Build depends run: | From fd070cc725840d23525a26a3669a1c2abad679b2 Mon Sep 17 00:00:00 2001 From: pasta Date: Thu, 4 Dec 2025 18:43:00 -0600 Subject: [PATCH 3/4] fix: add depends/Makefile to PACKAGES hash --- .github/workflows/build-depends.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-depends.yml b/.github/workflows/build-depends.yml index 514268d09920..d48d0758c14a 100644 --- a/.github/workflows/build-depends.yml +++ b/.github/workflows/build-depends.yml @@ -54,7 +54,7 @@ jobs: DEP_HASH="$(echo -n "${BUILD_TARGET}" "${DEP_OPTS}" "${HOST}" | sha256sum | head -c 64)" echo "DEP_HASH=${DEP_HASH}" >> "${GITHUB_OUTPUT}" DOCKERFILE_HASH="${{ hashFiles('contrib/containers/ci/ci.Dockerfile', 'contrib/containers/ci/ci-slim.Dockerfile') }}" - PACKAGES_HASH="${{ hashFiles('depends/packages/*') }}" + PACKAGES_HASH="${{ hashFiles('depends/packages/*', 'depends/Makefile') }}" CACHE_KEY="depends-${DOCKERFILE_HASH}-${{ inputs.build-target }}-${DEP_HASH}-${PACKAGES_HASH}" echo "cache-key=${CACHE_KEY}" >> "${GITHUB_OUTPUT}" echo "Cache key: ${CACHE_KEY}" From 6f7f799f4f66045558853fb6489659c24b794730 Mon Sep 17 00:00:00 2001 From: pasta Date: Fri, 5 Dec 2025 09:39:24 -0600 Subject: [PATCH 4/4] fix: include depends/Makefile in sparse checkout --- .github/workflows/build-depends.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-depends.yml b/.github/workflows/build-depends.yml index d48d0758c14a..ebd41c079bd0 100644 --- a/.github/workflows/build-depends.yml +++ b/.github/workflows/build-depends.yml @@ -39,6 +39,7 @@ jobs: sparse-checkout: | ci/dash ci/test + depends/Makefile depends/packages depends/hosts contrib/containers/ci/ci.Dockerfile