From fb7833125a9896d69fadaa0612e3c208a918520c Mon Sep 17 00:00:00 2001
From: Ola Buildkite <olabuildkite@olabuildkite.local>
Date: Tue, 2 Dec 2025 13:10:04 -0700
Subject: [PATCH 01/25] Test commit for PR

---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index 127b4f5..33e942a 100644
--- a/README.md
+++ b/README.md
@@ -36,3 +36,4 @@ This project demonstrates a simple CI/CD pipeline with build and verification sc
 The build process creates:
 - `build/build-info.txt` - Build metadata
 - `build/manifest.json` - Build manifest with status and version info
+# Test

From 554372229fd354f83e801af8139de10239ec1ac1 Mon Sep 17 00:00:00 2001
From: Ola Buildkite <olabuildkite@olabuildkite.local>
Date: Tue, 2 Dec 2025 13:11:23 -0700
Subject: [PATCH 02/25] Second commit to trigger sync event

---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index 33e942a..e43e58d 100644
--- a/README.md
+++ b/README.md
@@ -37,3 +37,4 @@ The build process creates:
 - `build/build-info.txt` - Build metadata
 - `build/manifest.json` - Build manifest with status and version info
 # Test
+# Another test line

From 729f575e5dd51ceb4aec5c14a7b9204ea571c48b Mon Sep 17 00:00:00 2001
From: Ola Buildkite <olabuildkite@olabuildkite.local>
Date: Tue, 2 Dec 2025 13:17:50 -0700
Subject: [PATCH 03/25] Use official Buildkite trigger-pipeline-action

---
 .../workflows/trigger-buildkite-on-sync.yml   | 21 +++++++------------
 1 file changed, 8 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/trigger-buildkite-on-sync.yml b/.github/workflows/trigger-buildkite-on-sync.yml
index 39b5e29..44f58d4 100644
--- a/.github/workflows/trigger-buildkite-on-sync.yml
+++ b/.github/workflows/trigger-buildkite-on-sync.yml
@@ -9,16 +9,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Trigger Buildkite Build
-        env:
-          BUILDKITE_API_TOKEN: ${{ secrets.BUILDKITE_API_TOKEN }}
-        run: |
-          curl -X POST "https://api.buildkite.com/v2/organizations/${{ vars.BUILDKITE_ORG_SLUG }}/pipelines/${{ vars.BUILDKITE_PIPELINE_SLUG }}/builds" \
-            -H "Authorization: Bearer $BUILDKITE_API_TOKEN" \
-            -H "Content-Type: application/json" \
-            -d '{
-              "commit": "${{ github.event.pull_request.head.sha }}",
-              "branch": "${{ github.event.pull_request.head.ref }}",
-              "message": "PR #${{ github.event.pull_request.number }} sync - triggered from GitHub Actions",
-              "pull_request_id": "${{ github.event.pull_request.number }}",
-              "pull_request_base_branch": "${{ github.event.pull_request.base.ref }}"
-            }'
+        uses: buildkite/trigger-pipeline-action@v2.4.1
+        with:
+          buildkite_api_access_token: ${{ secrets.BUILDKITE_API_TOKEN }}
+          pipeline: "${{ vars.BUILDKITE_ORG_SLUG }}/${{ vars.BUILDKITE_PIPELINE_SLUG }}"
+          branch: ${{ github.event.pull_request.head.ref }}
+          commit: ${{ github.event.pull_request.head.sha }}
+          message: ":github: PR #${{ github.event.pull_request.number }} sync"
+          send_pull_request: true

From 5ae22205e5c3f77cc20fc3c862aad7f5c1e427b5 Mon Sep 17 00:00:00 2001
From: Ola Buildkite <olabuildkite@olabuildkite.local>
Date: Wed, 3 Dec 2025 11:52:19 -0700
Subject: [PATCH 04/25] Test tracing behavior

---
 .buildkite/.DS_Store    | Bin 0 -> 8196 bytes
 .buildkite/pipeline.yml |  20 ++++----------------
 2 files changed, 4 insertions(+), 16 deletions(-)
 create mode 100644 .buildkite/.DS_Store

diff --git a/.buildkite/.DS_Store b/.buildkite/.DS_Store
new file mode 100644
index 0000000000000000000000000000000000000000..eac85622dd12e26e9ca8ffc544f234d7acd9f055
GIT binary patch
literal 8196
zcmeHLO-~a+7=8y(wgS3fQ6Q3R?8QXT;#We9u~v){V<46wh*)>GLtVR_X?C|D5|ZBa
zqIZ9QS8v9PM=xHz8GnLaJ?T3i6)DBUgNekMWafEi=6&Yl+04$q3jicvFHHap0f2>B
zpxuK-n<V;0-jO<Vi|8cC2Z*5tF8DxUe22GbFbWt2i~>dhqkvK1Ur+$=Y>}*8?tRs3
zN~3^L;J;KrydNCQ0&^N03gxQ<3mF0+c44zjXk#28IgZAh#)d*k#W7X%Ks1%;5Q8W>
z_B+BIF{iPiP|<-XIuJcG(Gd!fvm?)u?m%)1O=%P`3hY!s9Gg*K7`0b1YX3EUkFZ_^
z4vCEBJ*YzfA(XHdq2DQ3$KG%{<h~bj4@VIPFOT`ucbU)v=*9m50cMKK0B51Q@Lsp@
zhH#GdvsZAO1ph9GRmE7gT?aYdGF~_O`Z%UoO9JusRg)LL_x9G}n%?6ZuNDSzlFfc3
z9i3hK_IF#|R*$t@c*xfZiJR2pyc;cv+8pPh*R)-4KA>wwJAIY2#0?mYBq1nKbh&pY
zU?skm=XF+!CAXkDtdx~1+QS<gXU}KHoHLWz%`s<VB71&v%(;-A+}uoAgPGG8Zx<_J
zmGP&z1x!T09idOvf0obCY7rsuSM|2Yhvv3E+jetfsRKQ|eFqQO{R4;l)BS^kgGY`I
z4Udc*E82%W*RPZ#vEZvb<WWAX)1`70lnOQ14_J92p^uY7@Proaffm=>v~+)~X>PrS
zKIyFnIo$P#QYOcnSy_o`vha(&ShNqte#FCYmd8O7aJG0aM!BaGTAgMq9A{ibsb^}U
zE1suOjJSidu;@mS%gQvrz8tz94QE)MCkd|#PcfR$*=$6`NDBelVlSD?UzM1d+|8zO
zLK!w>l!2ep>87QTWZroXq&>$tqp4ITH><lWI0;wa7R*Bhp1^Z>39sNSyoXQl1-_AP
zGE9z>ljIz^L@tvXWSZP1K6yw2@<{PJnvZ%^kEemeXtX@))k4HV&dS&$SbaZF&$v88
z6i-FlbJQU&*{$;|xDCvRa&jz$gQ%GMUkG86QD6@#(5cosB;x;TyZ`^+Lpw4{H3}F7
z{!{_dHC31rpYHNjZ^IV1yo31xW|4$nL!qQ#vF$?u>pC2lz7DzyPEKP(p`>EFq#~+s
Q2=gOAsAG~*;O{E%1N=83NB{r;

literal 0
HcmV?d00001

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 82e734d..33c653e 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -1,17 +1,5 @@
 steps:
-  - label: "Job with dynamic priorityy"
-    command: |
-      if [ "$BUILDKITE_RETRY_COUNT" -eq 0 ]; then
-        PRIORITY=1
-      else
-        PRIORITY=0
-      fi
-      
-      buildkite-agent pipeline upload <<YAML
-      steps:
-        - label: "Actual work (priority \$PRIORITY)"
-          command: "echo 'Running tests'; exit 1"
-          priority: \$PRIORITY
-          retry:
-            automatic: true
-      YAML
+  - label: ":test_tube: Test tracing"
+    command: "echo 'Testing tracing behavior'; sleep 2"
+    agents:
+      queue: mac

From 6596b6370781632f0bf7e9928922df54150ac724 Mon Sep 17 00:00:00 2001
From: Ola Buildkite <olabuildkite@olabuildkite.local>
Date: Wed, 3 Dec 2025 11:58:58 -0700
Subject: [PATCH 05/25] Trigger build for tracing test

Amp-Thread-ID: https://ampcode.com/threads/T-750bccdb-e05c-44a5-b0c3-de1fe97f7c84
Co-authored-by: Amp <amp@ampcode.com>

From ce7f3aa731dbb30f166bbb5219299dd1ad8d9f47 Mon Sep 17 00:00:00 2001
From: Ola Buildkite <olabuildkite@olabuildkite.local>
Date: Wed, 3 Dec 2025 12:01:12 -0700
Subject: [PATCH 06/25] Test OTEL tracing with notification service enabled

Amp-Thread-ID: https://ampcode.com/threads/T-750bccdb-e05c-44a5-b0c3-de1fe97f7c84
Co-authored-by: Amp <amp@ampcode.com>

From e043851ba0e4bdd44e0dbd3c46b76ba65edbfce2 Mon Sep 17 00:00:00 2001
From: Ola Buildkite <olabuildkite@olabuildkite.local>
Date: Wed, 3 Dec 2025 12:03:02 -0700
Subject: [PATCH 07/25] Test with tracing disabled via env var

Amp-Thread-ID: https://ampcode.com/threads/T-750bccdb-e05c-44a5-b0c3-de1fe97f7c84
Co-authored-by: Amp <amp@ampcode.com>

From 07efc6bb8fe9ad3487fb690ce5b0afb10913d3cd Mon Sep 17 00:00:00 2001
From: Ola Buildkite <olabuildkite@olabuildkite.local>
Date: Wed, 3 Dec 2025 12:05:38 -0700
Subject: [PATCH 08/25] Test with unreachable OTEL endpoint

Amp-Thread-ID: https://ampcode.com/threads/T-750bccdb-e05c-44a5-b0c3-de1fe97f7c84
Co-authored-by: Amp <amp@ampcode.com>

From 7839a642a59baf8b028e9d437bd3311beb63195d Mon Sep 17 00:00:00 2001
From: Ola Buildkite <olabuildkite@olabuildkite.local>
Date: Wed, 3 Dec 2025 12:10:08 -0700
Subject: [PATCH 09/25] Test with BUILDKITE_TRACING_BACKEND='' (should have no
 timeout)

Amp-Thread-ID: https://ampcode.com/threads/T-750bccdb-e05c-44a5-b0c3-de1fe97f7c84
Co-authored-by: Amp <amp@ampcode.com>

From d5239f188d624824102aba5dc42d4c3e8c6d44c8 Mon Sep 17 00:00:00 2001
From: Ola Buildkite <olabuildkite@olabuildkite.local>
Date: Wed, 3 Dec 2025 13:01:54 -0700
Subject: [PATCH 10/25] Add k8s test pipeline for mount-buildkite-agent and
 propagate-environment

---
 .buildkite/pipeline.yml | 76 ++++++++++++++++++++++++++++++++++++++---
 1 file changed, 72 insertions(+), 4 deletions(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 33c653e..52a1967 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -1,5 +1,73 @@
+agents:
+  queue: "kubernetes"
+
+env:
+  PIPELINE_VAR: "hello-from-pipeline"
+  ANOTHER_VAR: "propagated-successfully"
+
 steps:
-  - label: ":test_tube: Test tracing"
-    command: "echo 'Testing tracing behavior'; sleep 2"
-    agents:
-      queue: mac
+  - label: ":kubernetes: Test mount-buildkite-agent equivalent"
+    command:
+      - echo "--- Testing buildkite-agent binary availability"
+      - which buildkite-agent
+      - buildkite-agent --version
+      - echo "--- Testing meta-data set/get"
+      - buildkite-agent meta-data set "test-key" "test-value-123"
+      - buildkite-agent meta-data get "test-key"
+      - echo "--- Testing annotation"
+      - buildkite-agent annotate "✅ Agent binary is mounted and working!" --style success --context mount-test
+      - echo "--- Testing artifact upload"
+      - echo "test artifact content" > /tmp/test-artifact.txt
+      - buildkite-agent artifact upload /tmp/test-artifact.txt
+    plugins:
+      - kubernetes:
+          podSpec:
+            containers:
+              - image: alpine:latest
+                command: ["/bin/sh", "-c"]
+
+  - label: ":earth_americas: Test propagate-environment equivalent"
+    command:
+      - echo "--- Testing pipeline env vars are propagated"
+      - echo "PIPELINE_VAR=$PIPELINE_VAR"
+      - echo "ANOTHER_VAR=$ANOTHER_VAR"
+      - |
+        if [ "$PIPELINE_VAR" = "hello-from-pipeline" ]; then
+          echo "✅ PIPELINE_VAR propagated correctly"
+          buildkite-agent annotate "✅ Environment variables propagated!" --style success --context env-test --append
+        else
+          echo "❌ PIPELINE_VAR not propagated"
+          exit 1
+        fi
+      - echo "--- Testing BUILDKITE_* vars are available"
+      - echo "BUILDKITE_BUILD_ID=$BUILDKITE_BUILD_ID"
+      - echo "BUILDKITE_PIPELINE_SLUG=$BUILDKITE_PIPELINE_SLUG"
+      - echo "BUILDKITE_BRANCH=$BUILDKITE_BRANCH"
+    plugins:
+      - kubernetes:
+          podSpec:
+            containers:
+              - image: alpine:latest
+                command: ["/bin/sh", "-c"]
+
+  - label: ":test_tube: Test step-level env override"
+    env:
+      STEP_VAR: "step-level-value"
+      PIPELINE_VAR: "overridden-at-step"
+    command:
+      - echo "--- Testing step-level env vars"
+      - echo "STEP_VAR=$STEP_VAR"
+      - echo "PIPELINE_VAR=$PIPELINE_VAR (should be overridden)"
+      - |
+        if [ "$PIPELINE_VAR" = "overridden-at-step" ]; then
+          echo "✅ Step-level override works"
+        else
+          echo "❌ Step-level override failed"
+          exit 1
+        fi
+    plugins:
+      - kubernetes:
+          podSpec:
+            containers:
+              - image: alpine:latest
+                command: ["/bin/sh", "-c"]

From 910dfe8b895757c9b90ae0977847eb61dcbe54c7 Mon Sep 17 00:00:00 2001
From: Ola Buildkite <olabuildkite@olabuildkite.local>
Date: Wed, 3 Dec 2025 13:04:04 -0700
Subject: [PATCH 11/25] Fix k8s pipeline - remove command override from podSpec

---
 .buildkite/pipeline.yml | 33 ++++++++++++++++-----------------
 1 file changed, 16 insertions(+), 17 deletions(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 52a1967..6bda79d 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -1,12 +1,11 @@
-agents:
-  queue: "kubernetes"
-
 env:
   PIPELINE_VAR: "hello-from-pipeline"
   ANOTHER_VAR: "propagated-successfully"
 
 steps:
   - label: ":kubernetes: Test mount-buildkite-agent equivalent"
+    agents:
+      queue: kubernetes
     command:
       - echo "--- Testing buildkite-agent binary availability"
       - which buildkite-agent
@@ -24,45 +23,46 @@ steps:
           podSpec:
             containers:
               - image: alpine:latest
-                command: ["/bin/sh", "-c"]
 
   - label: ":earth_americas: Test propagate-environment equivalent"
+    agents:
+      queue: kubernetes
     command:
       - echo "--- Testing pipeline env vars are propagated"
-      - echo "PIPELINE_VAR=$PIPELINE_VAR"
-      - echo "ANOTHER_VAR=$ANOTHER_VAR"
+      - 'echo "PIPELINE_VAR=$$PIPELINE_VAR"'
+      - 'echo "ANOTHER_VAR=$$ANOTHER_VAR"'
       - |
-        if [ "$PIPELINE_VAR" = "hello-from-pipeline" ]; then
+        if [ "$$PIPELINE_VAR" = "hello-from-pipeline" ]; then
           echo "✅ PIPELINE_VAR propagated correctly"
           buildkite-agent annotate "✅ Environment variables propagated!" --style success --context env-test --append
         else
-          echo "❌ PIPELINE_VAR not propagated"
+          echo "❌ PIPELINE_VAR not propagated (got: $$PIPELINE_VAR)"
           exit 1
         fi
       - echo "--- Testing BUILDKITE_* vars are available"
-      - echo "BUILDKITE_BUILD_ID=$BUILDKITE_BUILD_ID"
-      - echo "BUILDKITE_PIPELINE_SLUG=$BUILDKITE_PIPELINE_SLUG"
-      - echo "BUILDKITE_BRANCH=$BUILDKITE_BRANCH"
+      - 'echo "BUILDKITE_BUILD_ID=$$BUILDKITE_BUILD_ID"'
+      - 'echo "BUILDKITE_PIPELINE_SLUG=$$BUILDKITE_PIPELINE_SLUG"'
     plugins:
       - kubernetes:
           podSpec:
             containers:
               - image: alpine:latest
-                command: ["/bin/sh", "-c"]
 
   - label: ":test_tube: Test step-level env override"
+    agents:
+      queue: kubernetes
     env:
       STEP_VAR: "step-level-value"
       PIPELINE_VAR: "overridden-at-step"
     command:
       - echo "--- Testing step-level env vars"
-      - echo "STEP_VAR=$STEP_VAR"
-      - echo "PIPELINE_VAR=$PIPELINE_VAR (should be overridden)"
+      - 'echo "STEP_VAR=$$STEP_VAR"'
+      - 'echo "PIPELINE_VAR=$$PIPELINE_VAR (should be overridden)"'
       - |
-        if [ "$PIPELINE_VAR" = "overridden-at-step" ]; then
+        if [ "$$PIPELINE_VAR" = "overridden-at-step" ]; then
           echo "✅ Step-level override works"
         else
-          echo "❌ Step-level override failed"
+          echo "❌ Step-level override failed (got: $$PIPELINE_VAR)"
           exit 1
         fi
     plugins:
@@ -70,4 +70,3 @@ steps:
           podSpec:
             containers:
               - image: alpine:latest
-                command: ["/bin/sh", "-c"]

From 8985a6a1d0798b78f6a796af30830fe34bcacfef Mon Sep 17 00:00:00 2001
From: Ola Buildkite <olabuildkite@olabuildkite.local>
Date: Wed, 3 Dec 2025 13:06:32 -0700
Subject: [PATCH 12/25] Add environment hook test for propagate-environment

---
 .buildkite/hooks/environment |  4 ++++
 .buildkite/pipeline.yml      | 10 +++++++++-
 2 files changed, 13 insertions(+), 1 deletion(-)
 create mode 100755 .buildkite/hooks/environment

diff --git a/.buildkite/hooks/environment b/.buildkite/hooks/environment
new file mode 100755
index 0000000..c976eb0
--- /dev/null
+++ b/.buildkite/hooks/environment
@@ -0,0 +1,4 @@
+#!/bin/bash
+
+export HOOK_VAR="set-in-environment-hook"
+export HOOK_SECRET="secret-from-hook"
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 6bda79d..ea4e364 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -34,7 +34,6 @@ steps:
       - |
         if [ "$$PIPELINE_VAR" = "hello-from-pipeline" ]; then
           echo "✅ PIPELINE_VAR propagated correctly"
-          buildkite-agent annotate "✅ Environment variables propagated!" --style success --context env-test --append
         else
           echo "❌ PIPELINE_VAR not propagated (got: $$PIPELINE_VAR)"
           exit 1
@@ -42,6 +41,15 @@ steps:
       - echo "--- Testing BUILDKITE_* vars are available"
       - 'echo "BUILDKITE_BUILD_ID=$$BUILDKITE_BUILD_ID"'
       - 'echo "BUILDKITE_PIPELINE_SLUG=$$BUILDKITE_PIPELINE_SLUG"'
+      - echo "--- Testing vars from environment hook"
+      - 'echo "HOOK_VAR=$$HOOK_VAR"'
+      - 'echo "HOOK_SECRET=$$HOOK_SECRET"'
+      - |
+        if [ -n "$$HOOK_VAR" ]; then
+          echo "✅ HOOK_VAR from environment hook is available"
+        else
+          echo "❌ HOOK_VAR from environment hook NOT available"
+        fi
     plugins:
       - kubernetes:
           podSpec:

From 4d323972a74e0e80677115556ff5adc422d6cc69 Mon Sep 17 00:00:00 2001
From: Ola Buildkite <olabuildkite@olabuildkite.local>
Date: Wed, 3 Dec 2025 13:08:00 -0700
Subject: [PATCH 13/25] Test BUILDKITE_ENV_FILE propagation

---
 .buildkite/pipeline.yml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index ea4e364..dd26148 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -50,6 +50,13 @@ steps:
         else
           echo "❌ HOOK_VAR from environment hook NOT available"
         fi
+      - echo "--- Checking BUILDKITE_ENV_FILE contents and availability"
+      - 'echo "BUILDKITE_ENV_FILE=$$BUILDKITE_ENV_FILE"'
+      - 'cat "$$BUILDKITE_ENV_FILE" 2>/dev/null || echo "ENV_FILE not accessible"'
+      - echo "--- Testing common BUILDKITE_* env vars"
+      - 'echo "BUILDKITE_AGENT_NAME=$$BUILDKITE_AGENT_NAME"'
+      - 'echo "BUILDKITE_BRANCH=$$BUILDKITE_BRANCH"'
+      - 'echo "BUILDKITE_MESSAGE=$$BUILDKITE_MESSAGE"'
     plugins:
       - kubernetes:
           podSpec:

From 9fc09d652f76e078a04ffdf59a73121932d28719 Mon Sep 17 00:00:00 2001
From: Ola Buildkite <olabuildkite@olabuildkite.local>
Date: Wed, 3 Dec 2025 13:11:49 -0700
Subject: [PATCH 14/25] Test controller-level hooks propagation

---
 .buildkite/pipeline.yml | 81 +++++++----------------------------------
 1 file changed, 13 insertions(+), 68 deletions(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index dd26148..2a0ce6d 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -1,85 +1,30 @@
 env:
   PIPELINE_VAR: "hello-from-pipeline"
-  ANOTHER_VAR: "propagated-successfully"
 
 steps:
-  - label: ":kubernetes: Test mount-buildkite-agent equivalent"
+  - label: ":kubernetes: Test controller-level hooks"
     agents:
       queue: kubernetes
     command:
-      - echo "--- Testing buildkite-agent binary availability"
-      - which buildkite-agent
-      - buildkite-agent --version
-      - echo "--- Testing meta-data set/get"
-      - buildkite-agent meta-data set "test-key" "test-value-123"
-      - buildkite-agent meta-data get "test-key"
-      - echo "--- Testing annotation"
-      - buildkite-agent annotate "✅ Agent binary is mounted and working!" --style success --context mount-test
-      - echo "--- Testing artifact upload"
-      - echo "test artifact content" > /tmp/test-artifact.txt
-      - buildkite-agent artifact upload /tmp/test-artifact.txt
-    plugins:
-      - kubernetes:
-          podSpec:
-            containers:
-              - image: alpine:latest
-
-  - label: ":earth_americas: Test propagate-environment equivalent"
-    agents:
-      queue: kubernetes
-    command:
-      - echo "--- Testing pipeline env vars are propagated"
-      - 'echo "PIPELINE_VAR=$$PIPELINE_VAR"'
-      - 'echo "ANOTHER_VAR=$$ANOTHER_VAR"'
+      - echo "--- Testing controller-level hook vars"
+      - 'echo "CONTROLLER_HOOK_VAR=$$CONTROLLER_HOOK_VAR"'
+      - 'echo "CONTROLLER_SECRET=$$CONTROLLER_SECRET"'
       - |
-        if [ "$$PIPELINE_VAR" = "hello-from-pipeline" ]; then
-          echo "✅ PIPELINE_VAR propagated correctly"
+        if [ -n "$$CONTROLLER_HOOK_VAR" ]; then
+          echo "✅ Controller-level hook vars ARE available"
         else
-          echo "❌ PIPELINE_VAR not propagated (got: $$PIPELINE_VAR)"
-          exit 1
+          echo "❌ Controller-level hook vars NOT available"
         fi
-      - echo "--- Testing BUILDKITE_* vars are available"
-      - 'echo "BUILDKITE_BUILD_ID=$$BUILDKITE_BUILD_ID"'
-      - 'echo "BUILDKITE_PIPELINE_SLUG=$$BUILDKITE_PIPELINE_SLUG"'
-      - echo "--- Testing vars from environment hook"
+      - echo "--- Testing repo-level hook vars (should NOT be available)"
       - 'echo "HOOK_VAR=$$HOOK_VAR"'
-      - 'echo "HOOK_SECRET=$$HOOK_SECRET"'
       - |
-        if [ -n "$$HOOK_VAR" ]; then
-          echo "✅ HOOK_VAR from environment hook is available"
+        if [ -z "$$HOOK_VAR" ]; then
+          echo "✅ Confirmed: repo-level hook vars are NOT propagated"
         else
-          echo "❌ HOOK_VAR from environment hook NOT available"
-        fi
-      - echo "--- Checking BUILDKITE_ENV_FILE contents and availability"
-      - 'echo "BUILDKITE_ENV_FILE=$$BUILDKITE_ENV_FILE"'
-      - 'cat "$$BUILDKITE_ENV_FILE" 2>/dev/null || echo "ENV_FILE not accessible"'
-      - echo "--- Testing common BUILDKITE_* env vars"
-      - 'echo "BUILDKITE_AGENT_NAME=$$BUILDKITE_AGENT_NAME"'
-      - 'echo "BUILDKITE_BRANCH=$$BUILDKITE_BRANCH"'
-      - 'echo "BUILDKITE_MESSAGE=$$BUILDKITE_MESSAGE"'
-    plugins:
-      - kubernetes:
-          podSpec:
-            containers:
-              - image: alpine:latest
-
-  - label: ":test_tube: Test step-level env override"
-    agents:
-      queue: kubernetes
-    env:
-      STEP_VAR: "step-level-value"
-      PIPELINE_VAR: "overridden-at-step"
-    command:
-      - echo "--- Testing step-level env vars"
-      - 'echo "STEP_VAR=$$STEP_VAR"'
-      - 'echo "PIPELINE_VAR=$$PIPELINE_VAR (should be overridden)"'
-      - |
-        if [ "$$PIPELINE_VAR" = "overridden-at-step" ]; then
-          echo "✅ Step-level override works"
-        else
-          echo "❌ Step-level override failed (got: $$PIPELINE_VAR)"
-          exit 1
+          echo "Repo-level hook vars unexpectedly available"
         fi
+      - echo "--- Testing pipeline env vars"
+      - 'echo "PIPELINE_VAR=$$PIPELINE_VAR"'
     plugins:
       - kubernetes:
           podSpec:

From 304bce0ae637f4675dcbadf5464a872113e94615 Mon Sep 17 00:00:00 2001
From: Ola Buildkite <olabuildkite@olabuildkite.local>
Date: Wed, 3 Dec 2025 13:12:45 -0700
Subject: [PATCH 15/25] Trigger rebuild - test controller pod-spec-patch env
 vars

Amp-Thread-ID: https://ampcode.com/threads/T-8583fe71-e2fa-4e1f-bd3f-0fe7adec8498
Co-authored-by: Amp <amp@ampcode.com>

From 188327005e50c8f60ecc3974ce8852c712635c79 Mon Sep 17 00:00:00 2001
From: Ola Buildkite <olabuildkite@olabuildkite.local>
Date: Thu, 4 Dec 2025 15:01:00 -0700
Subject: [PATCH 16/25] Test cache plugin between steps

---
 .buildkite/pipeline.yml | 109 +++++++++++++++++++++++++++++++---------
 1 file changed, 86 insertions(+), 23 deletions(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 2a0ce6d..d298fa0 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -1,32 +1,95 @@
 env:
-  PIPELINE_VAR: "hello-from-pipeline"
+  BUILDKITE_PLUGIN_S3_CACHE_BUCKET: "buildkite-agent-cache-ola"
 
 steps:
-  - label: ":kubernetes: Test controller-level hooks"
+  - label: ":python: Install deps"
     agents:
       queue: kubernetes
-    command:
-      - echo "--- Testing controller-level hook vars"
-      - 'echo "CONTROLLER_HOOK_VAR=$$CONTROLLER_HOOK_VAR"'
-      - 'echo "CONTROLLER_SECRET=$$CONTROLLER_SECRET"'
-      - |
-        if [ -n "$$CONTROLLER_HOOK_VAR" ]; then
-          echo "✅ Controller-level hook vars ARE available"
-        else
-          echo "❌ Controller-level hook vars NOT available"
-        fi
-      - echo "--- Testing repo-level hook vars (should NOT be available)"
-      - 'echo "HOOK_VAR=$$HOOK_VAR"'
-      - |
-        if [ -z "$$HOOK_VAR" ]; then
-          echo "✅ Confirmed: repo-level hook vars are NOT propagated"
-        else
-          echo "Repo-level hook vars unexpectedly available"
-        fi
-      - echo "--- Testing pipeline env vars"
-      - 'echo "PIPELINE_VAR=$$PIPELINE_VAR"'
     plugins:
+      - cache#v1.8.1:
+          backend: s3
+          path: .venv
+          manifest: requirements.txt
+          save: file
+          compression: tgz
       - kubernetes:
           podSpec:
             containers:
-              - image: alpine:latest
+              - image: python:3.11-slim
+                command: [bash]
+                args:
+                  - -c
+                  - |
+                    echo "--- Creating requirements.txt"
+                    echo "requests==2.31.0" > requirements.txt
+                    echo "--- Installing deps"
+                    python -m venv .venv
+                    source .venv/bin/activate
+                    pip install -r requirements.txt
+                    echo "--- Installed packages:"
+                    pip list
+
+  - wait
+
+  - label: ":test_tube: Step 2 - Use cached deps"
+    agents:
+      queue: kubernetes
+    plugins:
+      - cache#v1.8.1:
+          backend: s3
+          path: .venv
+          manifest: requirements.txt
+          restore: file
+          compression: tgz
+      - kubernetes:
+          podSpec:
+            containers:
+              - image: python:3.11-slim
+                command: [bash]
+                args:
+                  - -c
+                  - |
+                    echo "--- Creating requirements.txt (same as step 1)"
+                    echo "requests==2.31.0" > requirements.txt
+                    echo "--- Checking if .venv was restored from cache"
+                    if [ -d ".venv" ]; then
+                      echo "✅ .venv directory exists!"
+                      source .venv/bin/activate
+                      echo "--- Packages from cache:"
+                      pip list
+                    else
+                      echo "❌ .venv NOT found - cache restore failed"
+                      exit 1
+                    fi
+
+  - wait
+
+  - label: ":test_tube: Step 3 - Also use cached deps"
+    agents:
+      queue: kubernetes
+    plugins:
+      - cache#v1.8.1:
+          backend: s3
+          path: .venv
+          manifest: requirements.txt
+          restore: file
+          compression: tgz
+      - kubernetes:
+          podSpec:
+            containers:
+              - image: python:3.11-slim
+                command: [bash]
+                args:
+                  - -c
+                  - |
+                    echo "--- Creating requirements.txt (same as step 1)"
+                    echo "requests==2.31.0" > requirements.txt
+                    echo "--- Checking if .venv was restored from cache"
+                    if [ -d ".venv" ]; then
+                      echo "✅ .venv directory exists!"
+                      source .venv/bin/activate
+                      pip list
+                    else
+                      echo "❌ .venv NOT found - cache restore failed"
+                      exit 1
+                    fi

From 20db373ba69c0ed81cf876951e0a96ca6ac75e3d Mon Sep 17 00:00:00 2001
From: Ola Buildkite <olabuildkite@olabuildkite.local>
Date: Thu, 4 Dec 2025 15:01:55 -0700
Subject: [PATCH 17/25] Use artifacts instead of cache plugin for cross-step
 sharing

---
 .buildkite/pipeline.yml | 44 +++++++++++++----------------------------
 1 file changed, 14 insertions(+), 30 deletions(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index d298fa0..f960f85 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -1,17 +1,8 @@
-env:
-  BUILDKITE_PLUGIN_S3_CACHE_BUCKET: "buildkite-agent-cache-ola"
-
 steps:
   - label: ":python: Install deps"
     agents:
       queue: kubernetes
     plugins:
-      - cache#v1.8.1:
-          backend: s3
-          path: .venv
-          manifest: requirements.txt
-          save: file
-          compression: tgz
       - kubernetes:
           podSpec:
             containers:
@@ -28,6 +19,9 @@ steps:
                     pip install -r requirements.txt
                     echo "--- Installed packages:"
                     pip list
+                    echo "--- Uploading .venv as artifact"
+                    tar -czf venv.tar.gz .venv
+                    buildkite-agent artifact upload venv.tar.gz
 
   - wait
 
@@ -35,12 +29,6 @@ steps:
     agents:
       queue: kubernetes
     plugins:
-      - cache#v1.8.1:
-          backend: s3
-          path: .venv
-          manifest: requirements.txt
-          restore: file
-          compression: tgz
       - kubernetes:
           podSpec:
             containers:
@@ -49,16 +37,17 @@ steps:
                 args:
                   - -c
                   - |
-                    echo "--- Creating requirements.txt (same as step 1)"
-                    echo "requests==2.31.0" > requirements.txt
-                    echo "--- Checking if .venv was restored from cache"
+                    echo "--- Downloading .venv artifact"
+                    buildkite-agent artifact download venv.tar.gz .
+                    tar -xzf venv.tar.gz
+                    echo "--- Checking if .venv was restored"
                     if [ -d ".venv" ]; then
                       echo "✅ .venv directory exists!"
                       source .venv/bin/activate
-                      echo "--- Packages from cache:"
+                      echo "--- Packages from artifact:"
                       pip list
                     else
-                      echo "❌ .venv NOT found - cache restore failed"
+                      echo "❌ .venv NOT found"
                       exit 1
                     fi
 
@@ -68,12 +57,6 @@ steps:
     agents:
       queue: kubernetes
     plugins:
-      - cache#v1.8.1:
-          backend: s3
-          path: .venv
-          manifest: requirements.txt
-          restore: file
-          compression: tgz
       - kubernetes:
           podSpec:
             containers:
@@ -82,14 +65,15 @@ steps:
                 args:
                   - -c
                   - |
-                    echo "--- Creating requirements.txt (same as step 1)"
-                    echo "requests==2.31.0" > requirements.txt
-                    echo "--- Checking if .venv was restored from cache"
+                    echo "--- Downloading .venv artifact"
+                    buildkite-agent artifact download venv.tar.gz .
+                    tar -xzf venv.tar.gz
+                    echo "--- Checking if .venv was restored"
                     if [ -d ".venv" ]; then
                       echo "✅ .venv directory exists!"
                       source .venv/bin/activate
                       pip list
                     else
-                      echo "❌ .venv NOT found - cache restore failed"
+                      echo "❌ .venv NOT found"
                       exit 1
                     fi

From a988354c462a1da585cd082da2380f3417e74246 Mon Sep 17 00:00:00 2001
From: Ola Buildkite <olabuildkite@olabuildkite.local>
Date: Thu, 4 Dec 2025 15:03:52 -0700
Subject: [PATCH 18/25] Test cache plugin on mac agent

---
 .buildkite/pipeline.yml | 113 ++++++++++++++++++----------------------
 1 file changed, 50 insertions(+), 63 deletions(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index f960f85..f914573 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -1,79 +1,66 @@
+env:
+  BUILDKITE_PLUGIN_S3_CACHE_BUCKET: "buildkite-agent-cache-ola"
+
 steps:
   - label: ":python: Install deps"
     agents:
-      queue: kubernetes
+      queue: mac
     plugins:
-      - kubernetes:
-          podSpec:
-            containers:
-              - image: python:3.11-slim
-                command: [bash]
-                args:
-                  - -c
-                  - |
-                    echo "--- Creating requirements.txt"
-                    echo "requests==2.31.0" > requirements.txt
-                    echo "--- Installing deps"
-                    python -m venv .venv
-                    source .venv/bin/activate
-                    pip install -r requirements.txt
-                    echo "--- Installed packages:"
-                    pip list
-                    echo "--- Uploading .venv as artifact"
-                    tar -czf venv.tar.gz .venv
-                    buildkite-agent artifact upload venv.tar.gz
+      - cache#v1.8.1:
+          backend: s3
+          path: .venv
+          manifest: requirements.txt
+          save: file
+          compression: tgz
+    command: |
+      echo "requests==2.31.0" > requirements.txt
+      python3 -m venv .venv
+      source .venv/bin/activate
+      pip install -r requirements.txt
+      pip list
 
   - wait
 
   - label: ":test_tube: Step 2 - Use cached deps"
     agents:
-      queue: kubernetes
+      queue: mac
     plugins:
-      - kubernetes:
-          podSpec:
-            containers:
-              - image: python:3.11-slim
-                command: [bash]
-                args:
-                  - -c
-                  - |
-                    echo "--- Downloading .venv artifact"
-                    buildkite-agent artifact download venv.tar.gz .
-                    tar -xzf venv.tar.gz
-                    echo "--- Checking if .venv was restored"
-                    if [ -d ".venv" ]; then
-                      echo "✅ .venv directory exists!"
-                      source .venv/bin/activate
-                      echo "--- Packages from artifact:"
-                      pip list
-                    else
-                      echo "❌ .venv NOT found"
-                      exit 1
-                    fi
+      - cache#v1.8.1:
+          backend: s3
+          path: .venv
+          manifest: requirements.txt
+          restore: file
+          compression: tgz
+    command: |
+      echo "requests==2.31.0" > requirements.txt
+      if [ -d ".venv" ]; then
+        echo "✅ .venv restored from cache!"
+        source .venv/bin/activate
+        pip list
+      else
+        echo "❌ .venv NOT found"
+        exit 1
+      fi
 
   - wait
 
   - label: ":test_tube: Step 3 - Also use cached deps"
     agents:
-      queue: kubernetes
+      queue: mac
     plugins:
-      - kubernetes:
-          podSpec:
-            containers:
-              - image: python:3.11-slim
-                command: [bash]
-                args:
-                  - -c
-                  - |
-                    echo "--- Downloading .venv artifact"
-                    buildkite-agent artifact download venv.tar.gz .
-                    tar -xzf venv.tar.gz
-                    echo "--- Checking if .venv was restored"
-                    if [ -d ".venv" ]; then
-                      echo "✅ .venv directory exists!"
-                      source .venv/bin/activate
-                      pip list
-                    else
-                      echo "❌ .venv NOT found"
-                      exit 1
-                    fi
+      - cache#v1.8.1:
+          backend: s3
+          path: .venv
+          manifest: requirements.txt
+          restore: file
+          compression: tgz
+    command: |
+      echo "requests==2.31.0" > requirements.txt
+      if [ -d ".venv" ]; then
+        echo "✅ .venv restored from cache!"
+        source .venv/bin/activate
+        pip list
+      else
+        echo "❌ .venv NOT found"
+        exit 1
+      fi

From d956ebc7b0d2574b287ef9fbfefd98f0bd203af5 Mon Sep 17 00:00:00 2001
From: Ola Buildkite <olabuildkite@olabuildkite.local>
Date: Thu, 4 Dec 2025 15:05:08 -0700
Subject: [PATCH 19/25] Add requirements.txt to repo for cache manifest

---
 .buildkite/pipeline.yml | 3 ---
 requirements.txt        | 1 +
 2 files changed, 1 insertion(+), 3 deletions(-)
 create mode 100644 requirements.txt

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index f914573..9d64059 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -13,7 +13,6 @@ steps:
           save: file
           compression: tgz
     command: |
-      echo "requests==2.31.0" > requirements.txt
       python3 -m venv .venv
       source .venv/bin/activate
       pip install -r requirements.txt
@@ -32,7 +31,6 @@ steps:
           restore: file
           compression: tgz
     command: |
-      echo "requests==2.31.0" > requirements.txt
       if [ -d ".venv" ]; then
         echo "✅ .venv restored from cache!"
         source .venv/bin/activate
@@ -55,7 +53,6 @@ steps:
           restore: file
           compression: tgz
     command: |
-      echo "requests==2.31.0" > requirements.txt
       if [ -d ".venv" ]; then
         echo "✅ .venv restored from cache!"
         source .venv/bin/activate
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000..2c24336
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1 @@
+requests==2.31.0

From 6cd4b6846f2fac6c32ef95e2f3071f71bb396b00 Mon Sep 17 00:00:00 2001
From: Ola Buildkite <olabuildkite@olabuildkite.local>
Date: Thu, 4 Dec 2025 15:06:59 -0700
Subject: [PATCH 20/25] Use correct S3 bucket for cache

---
 .buildkite/pipeline.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 9d64059..2ae5ff3 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -1,5 +1,5 @@
 env:
-  BUILDKITE_PLUGIN_S3_CACHE_BUCKET: "buildkite-agent-cache-ola"
+  BUILDKITE_PLUGIN_S3_CACHE_BUCKET: "ola-buildkite-cache"
 
 steps:
   - label: ":python: Install deps"

From b8f0e6fecb88c267bf63d0b7461e7006fcc8eadd Mon Sep 17 00:00:00 2001
From: Ola Buildkite <olabuildkite@olabuildkite.local>
Date: Thu, 4 Dec 2025 15:28:32 -0700
Subject: [PATCH 21/25] Test both artifacts and cache plugin with UV

---
 .buildkite/pipeline.yml | 56 ++++++++++++++++++-----------------------
 1 file changed, 25 insertions(+), 31 deletions(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 2ae5ff3..cf3efe1 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -2,62 +2,56 @@ env:
   BUILDKITE_PLUGIN_S3_CACHE_BUCKET: "ola-buildkite-cache"
 
 steps:
-  - label: ":python: Install deps"
+  # ===== ARTIFACTS APPROACH =====
+  - label: ":package: Artifacts - Install"
     agents:
       queue: mac
-    plugins:
-      - cache#v1.8.1:
-          backend: s3
-          path: .venv
-          manifest: requirements.txt
-          save: file
-          compression: tgz
     command: |
-      python3 -m venv .venv
+      uv venv .venv
+      uv pip install -r requirements.txt
+      tar -czf venv.tar.gz .venv
+      buildkite-agent artifact upload venv.tar.gz
+
+  - wait
+
+  - label: ":white_check_mark: Artifacts - Use deps"
+    agents:
+      queue: mac
+    command: |
+      buildkite-agent artifact download venv.tar.gz .
+      tar -xzf venv.tar.gz
       source .venv/bin/activate
-      pip install -r requirements.txt
       pip list
 
   - wait
 
-  - label: ":test_tube: Step 2 - Use cached deps"
+  # ===== CACHE PLUGIN APPROACH =====
+  - label: ":floppy_disk: Cache - Install"
     agents:
       queue: mac
     plugins:
       - cache#v1.8.1:
           backend: s3
-          path: .venv
+          path: .venv-cached
           manifest: requirements.txt
-          restore: file
+          save: file
           compression: tgz
     command: |
-      if [ -d ".venv" ]; then
-        echo "✅ .venv restored from cache!"
-        source .venv/bin/activate
-        pip list
-      else
-        echo "❌ .venv NOT found"
-        exit 1
-      fi
+      uv venv .venv-cached
+      uv pip install -r requirements.txt --python .venv-cached/bin/python
 
   - wait
 
-  - label: ":test_tube: Step 3 - Also use cached deps"
+  - label: ":white_check_mark: Cache - Use deps"
     agents:
       queue: mac
     plugins:
       - cache#v1.8.1:
           backend: s3
-          path: .venv
+          path: .venv-cached
           manifest: requirements.txt
           restore: file
           compression: tgz
     command: |
-      if [ -d ".venv" ]; then
-        echo "✅ .venv restored from cache!"
-        source .venv/bin/activate
-        pip list
-      else
-        echo "❌ .venv NOT found"
-        exit 1
-      fi
+      source .venv-cached/bin/activate
+      pip list

From 36c11159af291a62545d4b751ed5e7af00d9e402 Mon Sep 17 00:00:00 2001
From: Ola Buildkite <olabuildkite@olabuildkite.local>
Date: Thu, 4 Dec 2025 15:29:37 -0700
Subject: [PATCH 22/25] Use pip instead of uv (not installed)

---
 .buildkite/pipeline.yml | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index cf3efe1..b26821a 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -7,8 +7,9 @@ steps:
     agents:
       queue: mac
     command: |
-      uv venv .venv
-      uv pip install -r requirements.txt
+      python3 -m venv .venv
+      source .venv/bin/activate
+      pip install -r requirements.txt
       tar -czf venv.tar.gz .venv
       buildkite-agent artifact upload venv.tar.gz
 
@@ -37,8 +38,9 @@ steps:
           save: file
           compression: tgz
     command: |
-      uv venv .venv-cached
-      uv pip install -r requirements.txt --python .venv-cached/bin/python
+      python3 -m venv .venv-cached
+      source .venv-cached/bin/activate
+      pip install -r requirements.txt
 
   - wait
 

From 13f3a2932702f61540c81353dd2e5c4cac8b0448 Mon Sep 17 00:00:00 2001
From: Ola Buildkite <olabuildkite@olabuildkite.local>
Date: Thu, 4 Dec 2025 15:35:20 -0700
Subject: [PATCH 23/25] Test both approaches with UV

---
 .buildkite/pipeline.yml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index b26821a..04af7fa 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -3,13 +3,13 @@ env:
 
 steps:
   # ===== ARTIFACTS APPROACH =====
-  - label: ":package: Artifacts - Install"
+  - label: ":package: Artifacts - Install (UV)"
     agents:
       queue: mac
     command: |
-      python3 -m venv .venv
+      uv venv .venv
       source .venv/bin/activate
-      pip install -r requirements.txt
+      uv pip install -r requirements.txt
       tar -czf venv.tar.gz .venv
       buildkite-agent artifact upload venv.tar.gz
 
@@ -27,7 +27,7 @@ steps:
   - wait
 
   # ===== CACHE PLUGIN APPROACH =====
-  - label: ":floppy_disk: Cache - Install"
+  - label: ":floppy_disk: Cache - Install (UV)"
     agents:
       queue: mac
     plugins:
@@ -38,9 +38,9 @@ steps:
           save: file
           compression: tgz
     command: |
-      python3 -m venv .venv-cached
+      uv venv .venv-cached
       source .venv-cached/bin/activate
-      pip install -r requirements.txt
+      uv pip install -r requirements.txt
 
   - wait
 

From 0294c2bff43774423878c335dd6cc4ae1dd9f6a9 Mon Sep 17 00:00:00 2001
From: Ola Buildkite <olabuildkite@olabuildkite.local>
Date: Fri, 5 Dec 2025 11:39:39 -0700
Subject: [PATCH 24/25] test: use pipeline_uuid in workflow

---
 .buildkite/docs/knapsack-token-issue.md       | 125 ++++++++++++++++++
 .buildkite/knapsack-demo.yml                  | 111 ++++++++++++++++
 .../workflows/trigger-buildkite-on-sync.yml   |  13 +-
 3 files changed, 243 insertions(+), 6 deletions(-)
 create mode 100644 .buildkite/docs/knapsack-token-issue.md
 create mode 100644 .buildkite/knapsack-demo.yml

diff --git a/.buildkite/docs/knapsack-token-issue.md b/.buildkite/docs/knapsack-token-issue.md
new file mode 100644
index 0000000..ecfef7b
--- /dev/null
+++ b/.buildkite/docs/knapsack-token-issue.md
@@ -0,0 +1,125 @@
+# Knapsack Pro: Why Same Token + Same Parallelism Breaks Test Filtering
+
+## The Issue
+
+When running separate test suites (e.g., unit tests and feature tests) with:
+- **Same** `KNAPSACK_PRO_TEST_SUITE_TOKEN_RSPEC`
+- **Same** `parallelism` value
+- Same branch and commit
+
+Knapsack Pro treats all parallel jobs as **one unified test suite** and creates a shared queue.
+
+## How Knapsack Pro Queue/Split Works
+
+Knapsack Pro uses a cache key to identify unique test suite runs:
+
+```
+cache_key = (branch, commit_hash, node_total, api_token)
+```
+
+When this cache key is identical across different steps, Knapsack Pro assumes they are part of the **same test run** and distributes tests from a shared pool.
+
+## The Timeline of the Customer's Issue
+
+### Before Sep 26th (Working)
+```yaml
+# Unit tests
+parallelism: 8  # Different!
+token: SAME_TOKEN
+
+# Feature tests  
+parallelism: 6  # Different!
+token: SAME_TOKEN
+```
+
+Cache keys were **different** because `node_total` differed:
+- Unit: `(main, abc123, 8, token123)`
+- Feature: `(main, abc123, 6, token123)`
+
+### After Sep 26th (Broken)
+```yaml
+# Unit tests
+parallelism: 10  # Same!
+token: SAME_TOKEN
+
+# Feature tests
+parallelism: 10  # Same!
+token: SAME_TOKEN
+```
+
+Cache keys became **identical**:
+- Unit: `(main, abc123, 10, token123)`
+- Feature: `(main, abc123, 10, token123)`
+
+Both steps now share the same queue, and `KNAPSACK_PRO_TEST_FILE_PATTERN` is ignored at the API level.
+
+## The Fix
+
+**Use separate API tokens for each test suite:**
+
+```yaml
+steps:
+  - label: "Unit Tests"
+    parallelism: 10
+    env:
+      KNAPSACK_PRO_TEST_SUITE_TOKEN_RSPEC: ${KNAPSACK_PRO_TOKEN_UNIT}
+      KNAPSACK_PRO_TEST_FILE_PATTERN: "spec/**/*_spec.rb"
+      KNAPSACK_PRO_TEST_FILE_EXCLUDE_PATTERN: "spec/features/**/*_spec.rb"
+    plugins:
+      - docker-compose#v5.x.x:
+          run: app
+          command: bundle exec rake knapsack_pro:queue:rspec
+          env:
+            - BUILDKITE_PARALLEL_JOB_COUNT
+            - BUILDKITE_PARALLEL_JOB
+            - KNAPSACK_PRO_TEST_SUITE_TOKEN_RSPEC
+            - KNAPSACK_PRO_TEST_FILE_PATTERN
+            - KNAPSACK_PRO_TEST_FILE_EXCLUDE_PATTERN
+
+  - label: "Feature Tests"
+    parallelism: 10
+    env:
+      KNAPSACK_PRO_TEST_SUITE_TOKEN_RSPEC: ${KNAPSACK_PRO_TOKEN_FEATURE}
+      KNAPSACK_PRO_TEST_FILE_PATTERN: "spec/features/**/*_spec.rb"
+    plugins:
+      - docker-compose#v5.x.x:
+          run: app
+          command: bundle exec rake knapsack_pro:queue:rspec
+          env:
+            - BUILDKITE_PARALLEL_JOB_COUNT
+            - BUILDKITE_PARALLEL_JOB
+            - KNAPSACK_PRO_TEST_SUITE_TOKEN_RSPEC
+            - KNAPSACK_PRO_TEST_FILE_PATTERN
+```
+
+## How to Generate Additional Tokens
+
+1. Go to [Knapsack Pro Dashboard](https://knapsackpro.com/dashboard)
+2. Navigate to your organization → project
+3. Create a new test suite for "Feature Tests"
+4. Copy the new API token
+5. Add it to Buildkite as a secret environment variable
+
+## Key Documentation
+
+From [Knapsack Pro Reference](https://docs.knapsackpro.com/ruby/reference/):
+
+> **`KNAPSACK_PRO_TEST_SUITE_TOKEN_*`**
+> 
+> API token required to run Knapsack Pro.
+> **Each Knapsack Pro command defined on CI should use an individual API token.**
+
+## Additional Considerations
+
+When using the `docker-compose` plugin, ensure these env vars are passed to the container:
+
+```yaml
+env:
+  - BUILDKITE_PARALLEL_JOB_COUNT
+  - BUILDKITE_PARALLEL_JOB
+  - BUILDKITE_BUILD_NUMBER
+  - BUILDKITE_COMMIT
+  - BUILDKITE_BRANCH
+```
+
+Without these, Knapsack Pro inside the container won't know which parallel job it is.
diff --git a/.buildkite/knapsack-demo.yml b/.buildkite/knapsack-demo.yml
new file mode 100644
index 0000000..deb1930
--- /dev/null
+++ b/.buildkite/knapsack-demo.yml
@@ -0,0 +1,111 @@
+# Demo pipeline to illustrate Knapsack Pro token behavior
+# This simulates what happens when two test steps share the same API token
+# with the same parallelism value
+
+env:
+  # Simulating a shared Knapsack Pro token (the problematic scenario)
+  SHARED_TOKEN: "same-token-for-both"
+
+steps:
+  - group: ":warning: PROBLEMATIC: Same token + Same parallelism"
+    steps:
+      - label: ":test_tube: Unit Tests (parallelism=2)"
+        parallelism: 2
+        command: |
+          echo "=== Unit Tests Step ==="
+          echo "BUILDKITE_PARALLEL_JOB_COUNT: $BUILDKITE_PARALLEL_JOB_COUNT"
+          echo "BUILDKITE_PARALLEL_JOB: $BUILDKITE_PARALLEL_JOB"
+          echo "KNAPSACK Token: $SHARED_TOKEN"
+          echo ""
+          echo "Knapsack Pro cache key would be:"
+          echo "  (branch=$BUILDKITE_BRANCH, commit=$BUILDKITE_COMMIT, nodes=$BUILDKITE_PARALLEL_JOB_COUNT)"
+          echo ""
+          echo "⚠️  With SAME token and SAME parallelism, both steps"
+          echo "   would share the SAME queue in Knapsack Pro API!"
+        env:
+          TEST_FILE_PATTERN: "spec/models/**/*_spec.rb"
+        agents:
+          queue: mac
+
+      - label: ":sparkles: Feature Tests (parallelism=2)"
+        parallelism: 2
+        command: |
+          echo "=== Feature Tests Step ==="
+          echo "BUILDKITE_PARALLEL_JOB_COUNT: $BUILDKITE_PARALLEL_JOB_COUNT"
+          echo "BUILDKITE_PARALLEL_JOB: $BUILDKITE_PARALLEL_JOB"
+          echo "KNAPSACK Token: $SHARED_TOKEN"
+          echo ""
+          echo "Knapsack Pro cache key would be:"
+          echo "  (branch=$BUILDKITE_BRANCH, commit=$BUILDKITE_COMMIT, nodes=$BUILDKITE_PARALLEL_JOB_COUNT)"
+          echo ""
+          echo "❌ This step might receive unit test files because"
+          echo "   it shares the same queue with Unit Tests step!"
+        env:
+          TEST_FILE_PATTERN: "spec/features/**/*_spec.rb"
+        agents:
+          queue: mac
+
+  - wait
+
+  - group: ":white_check_mark: CORRECT: Separate tokens"
+    steps:
+      - label: ":test_tube: Unit Tests (separate token)"
+        parallelism: 2
+        command: |
+          echo "=== Unit Tests Step ==="
+          echo "BUILDKITE_PARALLEL_JOB_COUNT: $BUILDKITE_PARALLEL_JOB_COUNT"
+          echo "BUILDKITE_PARALLEL_JOB: $BUILDKITE_PARALLEL_JOB"
+          echo "KNAPSACK Token: $UNIT_TEST_TOKEN"
+          echo ""
+          echo "✅ With SEPARATE token, this step has its OWN queue"
+          echo "   and will only receive unit test files."
+        env:
+          UNIT_TEST_TOKEN: "token-for-unit-tests"
+          TEST_FILE_PATTERN: "spec/models/**/*_spec.rb"
+        agents:
+          queue: mac
+
+      - label: ":sparkles: Feature Tests (separate token)"
+        parallelism: 2
+        command: |
+          echo "=== Feature Tests Step ==="
+          echo "BUILDKITE_PARALLEL_JOB_COUNT: $BUILDKITE_PARALLEL_JOB_COUNT"
+          echo "BUILDKITE_PARALLEL_JOB: $BUILDKITE_PARALLEL_JOB"
+          echo "KNAPSACK Token: $FEATURE_TEST_TOKEN"
+          echo ""
+          echo "✅ With SEPARATE token, this step has its OWN queue"
+          echo "   and will only receive feature test files."
+        env:
+          FEATURE_TEST_TOKEN: "token-for-feature-tests"
+          TEST_FILE_PATTERN: "spec/features/**/*_spec.rb"
+        agents:
+          queue: mac
+
+  - wait
+
+  - label: ":memo: Summary"
+    command: |
+      echo "=============================================="
+      echo "KNAPSACK PRO TOKEN BEHAVIOR SUMMARY"
+      echo "=============================================="
+      echo ""
+      echo "THE PROBLEM:"
+      echo "When two steps have:"
+      echo "  - Same API token"
+      echo "  - Same parallelism value"
+      echo "  - Same branch/commit"
+      echo ""
+      echo "Knapsack Pro sees them as ONE test suite and creates"
+      echo "a shared queue. Both steps pull from the same pool of"
+      echo "tests, ignoring TEST_FILE_PATTERN filtering."
+      echo ""
+      echo "THE FIX:"
+      echo "Use SEPARATE API tokens for each test suite:"
+      echo "  - KNAPSACK_PRO_TEST_SUITE_TOKEN_UNIT for unit tests"
+      echo "  - KNAPSACK_PRO_TEST_SUITE_TOKEN_FEATURE for feature tests"
+      echo ""
+      echo "Generate additional tokens at:"
+      echo "https://knapsackpro.com/dashboard"
+      echo "=============================================="
+    agents:
+      queue: mac
diff --git a/.github/workflows/trigger-buildkite-on-sync.yml b/.github/workflows/trigger-buildkite-on-sync.yml
index 44f58d4..221cc46 100644
--- a/.github/workflows/trigger-buildkite-on-sync.yml
+++ b/.github/workflows/trigger-buildkite-on-sync.yml
@@ -3,17 +3,18 @@ name: Trigger Buildkite on PR Sync
 on:
   pull_request:
     types: [synchronize]
+  workflow_dispatch:
 
 jobs:
   trigger-buildkite:
     runs-on: ubuntu-latest
     steps:
-      - name: Trigger Buildkite Build
-        uses: buildkite/trigger-pipeline-action@v2.4.1
+      - name: Trigger Buildkite Build (using pipeline_uuid)
+        uses: buildkite/trigger-pipeline-action@pipeline-uuid-support
         with:
           buildkite_api_access_token: ${{ secrets.BUILDKITE_API_TOKEN }}
-          pipeline: "${{ vars.BUILDKITE_ORG_SLUG }}/${{ vars.BUILDKITE_PIPELINE_SLUG }}"
-          branch: ${{ github.event.pull_request.head.ref }}
-          commit: ${{ github.event.pull_request.head.sha }}
-          message: ":github: PR #${{ github.event.pull_request.number }} sync"
+          pipeline_uuid: "019a55e9-77b9-44c5-b779-b7db9f4dd023"
+          branch: ${{ github.head_ref || github.ref_name }}
+          commit: ${{ github.event.pull_request.head.sha || github.sha }}
+          message: ":github: Triggered via pipeline_uuid"
           send_pull_request: true

From 5cd6e3cea06068f7d87143f65251cfb70746978b Mon Sep 17 00:00:00 2001
From: Ola Buildkite <olabuildkite@olabuildkite.local>
Date: Fri, 5 Dec 2025 12:09:55 -0700
Subject: [PATCH 25/25] test: use pipeline_uuid in workflow

---
 .github/workflows/trigger-buildkite-on-sync.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/trigger-buildkite-on-sync.yml b/.github/workflows/trigger-buildkite-on-sync.yml
index 221cc46..f96429f 100644
--- a/.github/workflows/trigger-buildkite-on-sync.yml
+++ b/.github/workflows/trigger-buildkite-on-sync.yml
@@ -13,7 +13,7 @@ jobs:
         uses: buildkite/trigger-pipeline-action@pipeline-uuid-support
         with:
           buildkite_api_access_token: ${{ secrets.BUILDKITE_API_TOKEN }}
-          pipeline_uuid: "019a55e9-77b9-44c5-b779-b7db9f4dd023"
+          pipeline_uuid: ${{ vars.BUILDKITE_PIPELINE_UUID }}
           branch: ${{ github.head_ref || github.ref_name }}
           commit: ${{ github.event.pull_request.head.sha || github.sha }}
           message: ":github: Triggered via pipeline_uuid"