From c0f9d8c68c475d905cfa08755e1ee3fb0475a8b6 Mon Sep 17 00:00:00 2001 From: Alejandro Gil Date: Wed, 29 Apr 2026 12:08:50 -0700 Subject: [PATCH 1/3] ci(ec2): bake swap + cargo -j 2 into deploy/QA workflow The Synvya EC2 instances run Rust release builds during deploy (docker compose build) and QA (cargo test). On t3.medium (4 GB RAM, 2 vCPU) these can OOM, hanging at random crates such as rand v0.8.5. Swap on the host plus a cargo jobs limit reliably prevents the OOM, but until now had to be set up manually. - Add scripts/ec2-prepare-host.sh: idempotent helper that ensures a 4 GB /swapfile is active and persisted via /etc/fstab, and writes ~/.cargo/config.toml with [build] jobs=2 if missing. - Call it from deploy-staging, deploy-production, qa-staging, and qa-production before the build/test step. - Pass --build-arg CARGO_BUILD_JOBS=2 to docker compose build so the in-container cargo build is also constrained. - Dockerfile: accept CARGO_BUILD_JOBS as ARG and forward to ENV; defaults to empty (no change for contributors with bigger RAM). Co-Authored-By: Claude Opus 4.7 --- .github/workflows/build-test-push-synvya.yaml | 12 +++++- Dockerfile | 2 + scripts/ec2-prepare-host.sh | 43 +++++++++++++++++++ 3 files changed, 55 insertions(+), 2 deletions(-) create mode 100755 scripts/ec2-prepare-host.sh diff --git a/.github/workflows/build-test-push-synvya.yaml b/.github/workflows/build-test-push-synvya.yaml index c885b1b8..e56a8224 100644 --- a/.github/workflows/build-test-push-synvya.yaml +++ b/.github/workflows/build-test-push-synvya.yaml @@ -144,6 +144,7 @@ jobs: exit 1 fi git pull --ff-only origin synvya-staging + bash scripts/ec2-prepare-host.sh bash scripts/load-secrets.sh staging if [ -n "${{ vars.DISABLE_EMAILS }}" ]; then echo "DISABLE_EMAILS=${{ vars.DISABLE_EMAILS }}" >> /opt/synvya/.env @@ -152,7 +153,7 @@ jobs: docker builder prune -af || true docker image prune -af || true docker compose -f docker-compose.synvya.yml --env-file /opt/synvya/.env \ - build postgres redis migrate keycast + build --build-arg CARGO_BUILD_JOBS=2 postgres redis migrate keycast docker compose -f docker-compose.synvya.yml --env-file /opt/synvya/.env \ up -d postgres redis migrate keycast sleep 10 @@ -182,6 +183,7 @@ jobs: exit 1 fi git pull --ff-only origin synvya + bash scripts/ec2-prepare-host.sh bash scripts/load-secrets.sh production if [ -n "${{ vars.DISABLE_EMAILS }}" ]; then echo "DISABLE_EMAILS=${{ vars.DISABLE_EMAILS }}" >> /opt/synvya/.env @@ -190,7 +192,7 @@ jobs: docker builder prune -af || true docker image prune -af || true docker compose -f docker-compose.synvya.yml --env-file /opt/synvya/.env \ - build postgres redis migrate keycast + build --build-arg CARGO_BUILD_JOBS=2 postgres redis migrate keycast docker compose -f docker-compose.synvya.yml --env-file /opt/synvya/.env \ up -d postgres redis migrate keycast sleep 10 @@ -223,6 +225,9 @@ jobs: fi source "$HOME/.cargo/env" + # Ensure swap + cargo jobs limit are in place for the QA build + bash /opt/synvya/keycast/scripts/ec2-prepare-host.sh + # Load env for POSTGRES_PASSWORD set -a source /opt/synvya/.env 2>/dev/null || true @@ -301,6 +306,9 @@ jobs: fi source "$HOME/.cargo/env" + # Ensure swap + cargo jobs limit are in place for the QA build + bash /opt/synvya/keycast/scripts/ec2-prepare-host.sh + # Load env for POSTGRES_PASSWORD set -a source /opt/synvya/.env 2>/dev/null || true diff --git a/Dockerfile b/Dockerfile index 3334f2bd..08002c7b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,6 +18,8 @@ COPY ./Cargo.toml ./Cargo.toml COPY ./Cargo.lock ./Cargo.lock ARG CARGO_FEATURES="" +ARG CARGO_BUILD_JOBS="" +ENV CARGO_BUILD_JOBS=${CARGO_BUILD_JOBS} RUN if [ -n "$CARGO_FEATURES" ]; then \ cargo build --release --bin keycast --features "$CARGO_FEATURES"; \ else \ diff --git a/scripts/ec2-prepare-host.sh b/scripts/ec2-prepare-host.sh new file mode 100755 index 00000000..43962ecf --- /dev/null +++ b/scripts/ec2-prepare-host.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# Idempotent host prep for low-memory EC2 instances (e.g. t3.medium). +# Ensures enough swap to survive Rust release builds and constrains host-side +# cargo to a safe job count. Safe to run on every deploy. +set -euo pipefail + +SWAPFILE="${SWAPFILE:-/swapfile}" +SWAP_SIZE="${SWAP_SIZE:-4G}" +CARGO_JOBS="${CARGO_JOBS:-2}" + +echo "=== ec2-prepare-host: ensure swap (${SWAP_SIZE} at ${SWAPFILE}) ===" +if swapon --show=NAME --noheadings | grep -qx "${SWAPFILE}"; then + echo "swap already active at ${SWAPFILE}" +else + if [ ! -f "${SWAPFILE}" ]; then + sudo fallocate -l "${SWAP_SIZE}" "${SWAPFILE}" || \ + sudo dd if=/dev/zero of="${SWAPFILE}" bs=1M count=$((${SWAP_SIZE%G} * 1024)) + sudo chmod 600 "${SWAPFILE}" + sudo mkswap "${SWAPFILE}" + fi + sudo swapon "${SWAPFILE}" + if ! grep -qE "^${SWAPFILE}\s" /etc/fstab; then + echo "${SWAPFILE} none swap sw 0 0" | sudo tee -a /etc/fstab >/dev/null + fi + echo "swap enabled at ${SWAPFILE}" +fi + +echo "=== ec2-prepare-host: ensure ~/.cargo/config.toml jobs=${CARGO_JOBS} ===" +mkdir -p "${HOME}/.cargo" +CARGO_CFG="${HOME}/.cargo/config.toml" +if [ ! -f "${CARGO_CFG}" ] || ! grep -qE '^\[build\]' "${CARGO_CFG}"; then + cat >> "${CARGO_CFG}" < Date: Wed, 29 Apr 2026 12:12:48 -0700 Subject: [PATCH 2/3] build: cache cargo dep compilation in Docker; stop wiping layer cache Two changes that make the warm path of Synvya deploys actually warm: 1. Dockerfile: split the rust-builder stage into a deps-only build followed by a real-source build. The first cargo invocation runs against stub src files for every workspace member, compiling just the ~400 transitive dependencies and caching them as a Docker layer. The second invocation overwrites the stubs with real code; cargo only recompiles workspace crates, reusing dep .rlibs. Net effect: code-only changes no longer trigger a full dependency recompile on every deploy. 2. Workflow: drop the `-a` flag from `docker builder prune` and `docker image prune`. The `-a` variant deletes referenced build cache and tagged images, guaranteeing a cold build every time. The non-`-a` variants still clean up dangling images and unreferenced build cache for disk hygiene. Combined with the earlier swap + cargo -j 2 commit, this should let the t3.medium handle warm builds quickly and survive cold builds without OOMing. Co-Authored-By: Claude Opus 4.7 --- .github/workflows/build-test-push-synvya.yaml | 14 ++++-- Dockerfile | 43 ++++++++++++++++--- 2 files changed, 48 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build-test-push-synvya.yaml b/.github/workflows/build-test-push-synvya.yaml index e56a8224..eb0a305a 100644 --- a/.github/workflows/build-test-push-synvya.yaml +++ b/.github/workflows/build-test-push-synvya.yaml @@ -150,8 +150,11 @@ jobs: echo "DISABLE_EMAILS=${{ vars.DISABLE_EMAILS }}" >> /opt/synvya/.env fi docker system df || true - docker builder prune -af || true - docker image prune -af || true + # Light cleanup: drop dangling images and unreferenced build + # cache only. `-a` would wipe referenced layers and force a + # cold cargo build of all dependencies on every deploy. + docker image prune -f || true + docker builder prune -f || true docker compose -f docker-compose.synvya.yml --env-file /opt/synvya/.env \ build --build-arg CARGO_BUILD_JOBS=2 postgres redis migrate keycast docker compose -f docker-compose.synvya.yml --env-file /opt/synvya/.env \ @@ -189,8 +192,11 @@ jobs: echo "DISABLE_EMAILS=${{ vars.DISABLE_EMAILS }}" >> /opt/synvya/.env fi docker system df || true - docker builder prune -af || true - docker image prune -af || true + # Light cleanup: drop dangling images and unreferenced build + # cache only. `-a` would wipe referenced layers and force a + # cold cargo build of all dependencies on every deploy. + docker image prune -f || true + docker builder prune -f || true docker compose -f docker-compose.synvya.yml --env-file /opt/synvya/.env \ build --build-arg CARGO_BUILD_JOBS=2 postgres redis migrate keycast docker compose -f docker-compose.synvya.yml --env-file /opt/synvya/.env \ diff --git a/Dockerfile b/Dockerfile index 08002c7b..fbdf767b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,6 +7,44 @@ RUN apt-get update && apt-get install -y \ && rm -rf /var/lib/apt/lists/* WORKDIR /app + +# ── Stage A: cache dependency compilation ───────────────────────────── +# Copy only manifests + any cargo-auto-detected build scripts, then stub +# every workspace member so dependencies compile without our source. +# This layer is reused across pushes whenever Cargo.toml/Cargo.lock and +# the member manifests are unchanged — saving the ~400-crate cold build. +COPY ./Cargo.toml ./Cargo.toml +COPY ./Cargo.lock ./Cargo.lock +COPY ./api/Cargo.toml ./api/Cargo.toml +COPY ./api/build.rs ./api/build.rs +COPY ./core/Cargo.toml ./core/Cargo.toml +COPY ./signer/Cargo.toml ./signer/Cargo.toml +COPY ./keycast/Cargo.toml ./keycast/Cargo.toml +COPY ./cluster-hashring/Cargo.toml ./cluster-hashring/Cargo.toml +COPY ./tools/loadtest/Cargo.toml ./tools/loadtest/Cargo.toml + +RUN mkdir -p api/src core/src signer/src keycast/src cluster-hashring/src tools/loadtest/src && \ + : > api/src/lib.rs && \ + : > core/src/lib.rs && \ + : > signer/src/lib.rs && \ + : > cluster-hashring/src/lib.rs && \ + echo 'fn main() {}' > keycast/src/main.rs && \ + echo 'fn main() {}' > tools/loadtest/src/main.rs + +ARG CARGO_FEATURES="" +ARG CARGO_BUILD_JOBS="" +ENV CARGO_BUILD_JOBS=${CARGO_BUILD_JOBS} + +RUN if [ -n "$CARGO_FEATURES" ]; then \ + cargo build --release --bin keycast --features "$CARGO_FEATURES"; \ + else \ + cargo build --release --bin keycast; \ + fi + +# ── Stage B: real-source build ──────────────────────────────────────── +# Real sources overwrite stubs; cargo recompiles only the workspace +# crates (their content hashes changed) and reuses dep .rlibs from +# Stage A in target/release/deps. COPY ./api ./api COPY ./signer ./signer COPY ./core ./core @@ -14,12 +52,7 @@ COPY ./keycast ./keycast COPY ./cluster-hashring ./cluster-hashring COPY ./tools ./tools COPY ./database/migrations ./database/migrations -COPY ./Cargo.toml ./Cargo.toml -COPY ./Cargo.lock ./Cargo.lock -ARG CARGO_FEATURES="" -ARG CARGO_BUILD_JOBS="" -ENV CARGO_BUILD_JOBS=${CARGO_BUILD_JOBS} RUN if [ -n "$CARGO_FEATURES" ]; then \ cargo build --release --bin keycast --features "$CARGO_FEATURES"; \ else \ From 3393c1d7653c8e7f42addf3c1c4946bad31900be Mon Sep 17 00:00:00 2001 From: Alejandro Gil Date: Wed, 29 Apr 2026 13:08:45 -0700 Subject: [PATCH 3/3] fix(docker): switch dep cache from stub-source pattern to BuildKit cache mounts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous attempt at a deps-cache stage (stub every workspace member's source, build deps with stubs, then COPY real source) fails at compile time because the Keycast binary's dependency graph crosses workspace boundaries: keycast → keycast_api → keycast_core::types, etc. Stubbing keycast_core's lib.rs to empty makes keycast_api fail to compile inside Stage A. The stub pattern only works for projects with no workspace-internal deps. Keycast has them everywhere. Fix: collapse to a single rust-builder stage and use BuildKit cache mounts for /usr/local/cargo/registry, /usr/local/cargo/git, and /app/target. The mounts persist on the host's BuildKit instance across builds, so unchanged dep crates stay compiled without any source-stubbing trickery. Built artifacts are copied out of target/ to /artifacts/ before the RUN ends, since cache mounts are not part of the resulting image layer; the runtime stage now COPYs from /artifacts/. Workflow exports DOCKER_BUILDKIT=1 and COMPOSE_DOCKER_CLI_BUILD=1 to be explicit (compose v2 enables BuildKit by default). Co-Authored-By: Claude Opus 4.7 --- .github/workflows/build-test-push-synvya.yaml | 8 +++ Dockerfile | 71 +++++++------------ 2 files changed, 35 insertions(+), 44 deletions(-) diff --git a/.github/workflows/build-test-push-synvya.yaml b/.github/workflows/build-test-push-synvya.yaml index eb0a305a..cdd0721f 100644 --- a/.github/workflows/build-test-push-synvya.yaml +++ b/.github/workflows/build-test-push-synvya.yaml @@ -155,6 +155,10 @@ jobs: # cold cargo build of all dependencies on every deploy. docker image prune -f || true docker builder prune -f || true + # Force BuildKit so the Dockerfile's --mount=type=cache works. + # Compose v2 enables it by default, but be explicit. + export DOCKER_BUILDKIT=1 + export COMPOSE_DOCKER_CLI_BUILD=1 docker compose -f docker-compose.synvya.yml --env-file /opt/synvya/.env \ build --build-arg CARGO_BUILD_JOBS=2 postgres redis migrate keycast docker compose -f docker-compose.synvya.yml --env-file /opt/synvya/.env \ @@ -197,6 +201,10 @@ jobs: # cold cargo build of all dependencies on every deploy. docker image prune -f || true docker builder prune -f || true + # Force BuildKit so the Dockerfile's --mount=type=cache works. + # Compose v2 enables it by default, but be explicit. + export DOCKER_BUILDKIT=1 + export COMPOSE_DOCKER_CLI_BUILD=1 docker compose -f docker-compose.synvya.yml --env-file /opt/synvya/.env \ build --build-arg CARGO_BUILD_JOBS=2 postgres redis migrate keycast docker compose -f docker-compose.synvya.yml --env-file /opt/synvya/.env \ diff --git a/Dockerfile b/Dockerfile index fbdf767b..066778ff 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,4 @@ +# syntax=docker/dockerfile:1.4 # Build stage for Rust API FROM rust:1.93-slim AS rust-builder @@ -7,44 +8,6 @@ RUN apt-get update && apt-get install -y \ && rm -rf /var/lib/apt/lists/* WORKDIR /app - -# ── Stage A: cache dependency compilation ───────────────────────────── -# Copy only manifests + any cargo-auto-detected build scripts, then stub -# every workspace member so dependencies compile without our source. -# This layer is reused across pushes whenever Cargo.toml/Cargo.lock and -# the member manifests are unchanged — saving the ~400-crate cold build. -COPY ./Cargo.toml ./Cargo.toml -COPY ./Cargo.lock ./Cargo.lock -COPY ./api/Cargo.toml ./api/Cargo.toml -COPY ./api/build.rs ./api/build.rs -COPY ./core/Cargo.toml ./core/Cargo.toml -COPY ./signer/Cargo.toml ./signer/Cargo.toml -COPY ./keycast/Cargo.toml ./keycast/Cargo.toml -COPY ./cluster-hashring/Cargo.toml ./cluster-hashring/Cargo.toml -COPY ./tools/loadtest/Cargo.toml ./tools/loadtest/Cargo.toml - -RUN mkdir -p api/src core/src signer/src keycast/src cluster-hashring/src tools/loadtest/src && \ - : > api/src/lib.rs && \ - : > core/src/lib.rs && \ - : > signer/src/lib.rs && \ - : > cluster-hashring/src/lib.rs && \ - echo 'fn main() {}' > keycast/src/main.rs && \ - echo 'fn main() {}' > tools/loadtest/src/main.rs - -ARG CARGO_FEATURES="" -ARG CARGO_BUILD_JOBS="" -ENV CARGO_BUILD_JOBS=${CARGO_BUILD_JOBS} - -RUN if [ -n "$CARGO_FEATURES" ]; then \ - cargo build --release --bin keycast --features "$CARGO_FEATURES"; \ - else \ - cargo build --release --bin keycast; \ - fi - -# ── Stage B: real-source build ──────────────────────────────────────── -# Real sources overwrite stubs; cargo recompiles only the workspace -# crates (their content hashes changed) and reuses dep .rlibs from -# Stage A in target/release/deps. COPY ./api ./api COPY ./signer ./signer COPY ./core ./core @@ -52,13 +15,31 @@ COPY ./keycast ./keycast COPY ./cluster-hashring ./cluster-hashring COPY ./tools ./tools COPY ./database/migrations ./database/migrations +COPY ./Cargo.toml ./Cargo.toml +COPY ./Cargo.lock ./Cargo.lock -RUN if [ -n "$CARGO_FEATURES" ]; then \ +ARG CARGO_FEATURES="" +ARG CARGO_BUILD_JOBS="" +ENV CARGO_BUILD_JOBS=${CARGO_BUILD_JOBS} + +# BuildKit cache mounts persist the cargo registry/git index and the +# target/ directory across builds on the same host. Unchanged crates +# stay compiled, so warm builds skip the ~400-crate dependency build. +# Artifacts must be copied out of target/ before the RUN ends because +# cache mounts are not part of the resulting image layer. +RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked \ + --mount=type=cache,target=/usr/local/cargo/git,sharing=locked \ + --mount=type=cache,target=/app/target,sharing=locked \ + set -e; \ + if [ -n "$CARGO_FEATURES" ]; then \ cargo build --release --bin keycast --features "$CARGO_FEATURES"; \ else \ cargo build --release --bin keycast; \ - fi -RUN cargo build --release --example migrate-vine-users + fi; \ + cargo build --release --example migrate-vine-users; \ + mkdir -p /artifacts; \ + cp target/release/keycast /artifacts/keycast; \ + cp target/release/examples/migrate-vine-users /artifacts/migrate-vine-users # Build stage for Bun frontend FROM oven/bun:1 AS web-builder @@ -144,9 +125,11 @@ RUN curl -fsSL https://bun.sh/install | bash # Create necessary directories RUN mkdir -p /app/database /data -# Copy built artifacts - keycast binary and migration tool -COPY --from=rust-builder /app/target/release/keycast ./ -COPY --from=rust-builder /app/target/release/examples/migrate-vine-users ./ +# Copy built artifacts - keycast binary and migration tool. +# Sources are /artifacts/ (not target/) because target/ is a BuildKit +# cache mount in rust-builder and is not present in its image layer. +COPY --from=rust-builder /artifacts/keycast ./ +COPY --from=rust-builder /artifacts/migrate-vine-users ./ COPY --from=web-builder /app/web/build ./web COPY --from=web-builder /app/web/package.json ./ COPY --from=web-builder /app/web/node_modules ./node_modules