diff --git a/.buildkite/.DS_Store b/.buildkite/.DS_Store new file mode 100644 index 0000000..eac8562 Binary files /dev/null and b/.buildkite/.DS_Store differ diff --git a/.buildkite/docs/knapsack-token-issue.md b/.buildkite/docs/knapsack-token-issue.md new file mode 100644 index 0000000..ecfef7b --- /dev/null +++ b/.buildkite/docs/knapsack-token-issue.md @@ -0,0 +1,125 @@ +# Knapsack Pro: Why Same Token + Same Parallelism Breaks Test Filtering + +## The Issue + +When running separate test suites (e.g., unit tests and feature tests) with: +- **Same** `KNAPSACK_PRO_TEST_SUITE_TOKEN_RSPEC` +- **Same** `parallelism` value +- Same branch and commit + +Knapsack Pro treats all parallel jobs as **one unified test suite** and creates a shared queue. + +## How Knapsack Pro Queue/Split Works + +Knapsack Pro uses a cache key to identify unique test suite runs: + +``` +cache_key = (branch, commit_hash, node_total, api_token) +``` + +When this cache key is identical across different steps, Knapsack Pro assumes they are part of the **same test run** and distributes tests from a shared pool. + +## The Timeline of the Customer's Issue + +### Before Sep 26th (Working) +```yaml +# Unit tests +parallelism: 8 # Different! +token: SAME_TOKEN + +# Feature tests +parallelism: 6 # Different! +token: SAME_TOKEN +``` + +Cache keys were **different** because `node_total` differed: +- Unit: `(main, abc123, 8, token123)` +- Feature: `(main, abc123, 6, token123)` + +### After Sep 26th (Broken) +```yaml +# Unit tests +parallelism: 10 # Same! +token: SAME_TOKEN + +# Feature tests +parallelism: 10 # Same! +token: SAME_TOKEN +``` + +Cache keys became **identical**: +- Unit: `(main, abc123, 10, token123)` +- Feature: `(main, abc123, 10, token123)` + +Both steps now share the same queue, and `KNAPSACK_PRO_TEST_FILE_PATTERN` is ignored at the API level. + +## The Fix + +**Use separate API tokens for each test suite:** + +```yaml +steps: + - label: "Unit Tests" + parallelism: 10 + env: + KNAPSACK_PRO_TEST_SUITE_TOKEN_RSPEC: ${KNAPSACK_PRO_TOKEN_UNIT} + KNAPSACK_PRO_TEST_FILE_PATTERN: "spec/**/*_spec.rb" + KNAPSACK_PRO_TEST_FILE_EXCLUDE_PATTERN: "spec/features/**/*_spec.rb" + plugins: + - docker-compose#v5.x.x: + run: app + command: bundle exec rake knapsack_pro:queue:rspec + env: + - BUILDKITE_PARALLEL_JOB_COUNT + - BUILDKITE_PARALLEL_JOB + - KNAPSACK_PRO_TEST_SUITE_TOKEN_RSPEC + - KNAPSACK_PRO_TEST_FILE_PATTERN + - KNAPSACK_PRO_TEST_FILE_EXCLUDE_PATTERN + + - label: "Feature Tests" + parallelism: 10 + env: + KNAPSACK_PRO_TEST_SUITE_TOKEN_RSPEC: ${KNAPSACK_PRO_TOKEN_FEATURE} + KNAPSACK_PRO_TEST_FILE_PATTERN: "spec/features/**/*_spec.rb" + plugins: + - docker-compose#v5.x.x: + run: app + command: bundle exec rake knapsack_pro:queue:rspec + env: + - BUILDKITE_PARALLEL_JOB_COUNT + - BUILDKITE_PARALLEL_JOB + - KNAPSACK_PRO_TEST_SUITE_TOKEN_RSPEC + - KNAPSACK_PRO_TEST_FILE_PATTERN +``` + +## How to Generate Additional Tokens + +1. Go to [Knapsack Pro Dashboard](https://knapsackpro.com/dashboard) +2. Navigate to your organization → project +3. Create a new test suite for "Feature Tests" +4. Copy the new API token +5. Add it to Buildkite as a secret environment variable + +## Key Documentation + +From [Knapsack Pro Reference](https://docs.knapsackpro.com/ruby/reference/): + +> **`KNAPSACK_PRO_TEST_SUITE_TOKEN_*`** +> +> API token required to run Knapsack Pro. +> **Each Knapsack Pro command defined on CI should use an individual API token.** + +## Additional Considerations + +When using the `docker-compose` plugin, ensure these env vars are passed to the container: + +```yaml +env: + - BUILDKITE_PARALLEL_JOB_COUNT + - BUILDKITE_PARALLEL_JOB + - BUILDKITE_BUILD_NUMBER + - BUILDKITE_COMMIT + - BUILDKITE_BRANCH +``` + +Without these, Knapsack Pro inside the container won't know which parallel job it is. diff --git a/.buildkite/hooks/environment b/.buildkite/hooks/environment new file mode 100755 index 0000000..c976eb0 --- /dev/null +++ b/.buildkite/hooks/environment @@ -0,0 +1,4 @@ +#!/bin/bash + +export HOOK_VAR="set-in-environment-hook" +export HOOK_SECRET="secret-from-hook" diff --git a/.buildkite/knapsack-demo.yml b/.buildkite/knapsack-demo.yml new file mode 100644 index 0000000..deb1930 --- /dev/null +++ b/.buildkite/knapsack-demo.yml @@ -0,0 +1,111 @@ +# Demo pipeline to illustrate Knapsack Pro token behavior +# This simulates what happens when two test steps share the same API token +# with the same parallelism value + +env: + # Simulating a shared Knapsack Pro token (the problematic scenario) + SHARED_TOKEN: "same-token-for-both" + +steps: + - group: ":warning: PROBLEMATIC: Same token + Same parallelism" + steps: + - label: ":test_tube: Unit Tests (parallelism=2)" + parallelism: 2 + command: | + echo "=== Unit Tests Step ===" + echo "BUILDKITE_PARALLEL_JOB_COUNT: $BUILDKITE_PARALLEL_JOB_COUNT" + echo "BUILDKITE_PARALLEL_JOB: $BUILDKITE_PARALLEL_JOB" + echo "KNAPSACK Token: $SHARED_TOKEN" + echo "" + echo "Knapsack Pro cache key would be:" + echo " (branch=$BUILDKITE_BRANCH, commit=$BUILDKITE_COMMIT, nodes=$BUILDKITE_PARALLEL_JOB_COUNT)" + echo "" + echo "⚠️ With SAME token and SAME parallelism, both steps" + echo " would share the SAME queue in Knapsack Pro API!" + env: + TEST_FILE_PATTERN: "spec/models/**/*_spec.rb" + agents: + queue: mac + + - label: ":sparkles: Feature Tests (parallelism=2)" + parallelism: 2 + command: | + echo "=== Feature Tests Step ===" + echo "BUILDKITE_PARALLEL_JOB_COUNT: $BUILDKITE_PARALLEL_JOB_COUNT" + echo "BUILDKITE_PARALLEL_JOB: $BUILDKITE_PARALLEL_JOB" + echo "KNAPSACK Token: $SHARED_TOKEN" + echo "" + echo "Knapsack Pro cache key would be:" + echo " (branch=$BUILDKITE_BRANCH, commit=$BUILDKITE_COMMIT, nodes=$BUILDKITE_PARALLEL_JOB_COUNT)" + echo "" + echo "❌ This step might receive unit test files because" + echo " it shares the same queue with Unit Tests step!" + env: + TEST_FILE_PATTERN: "spec/features/**/*_spec.rb" + agents: + queue: mac + + - wait + + - group: ":white_check_mark: CORRECT: Separate tokens" + steps: + - label: ":test_tube: Unit Tests (separate token)" + parallelism: 2 + command: | + echo "=== Unit Tests Step ===" + echo "BUILDKITE_PARALLEL_JOB_COUNT: $BUILDKITE_PARALLEL_JOB_COUNT" + echo "BUILDKITE_PARALLEL_JOB: $BUILDKITE_PARALLEL_JOB" + echo "KNAPSACK Token: $UNIT_TEST_TOKEN" + echo "" + echo "✅ With SEPARATE token, this step has its OWN queue" + echo " and will only receive unit test files." + env: + UNIT_TEST_TOKEN: "token-for-unit-tests" + TEST_FILE_PATTERN: "spec/models/**/*_spec.rb" + agents: + queue: mac + + - label: ":sparkles: Feature Tests (separate token)" + parallelism: 2 + command: | + echo "=== Feature Tests Step ===" + echo "BUILDKITE_PARALLEL_JOB_COUNT: $BUILDKITE_PARALLEL_JOB_COUNT" + echo "BUILDKITE_PARALLEL_JOB: $BUILDKITE_PARALLEL_JOB" + echo "KNAPSACK Token: $FEATURE_TEST_TOKEN" + echo "" + echo "✅ With SEPARATE token, this step has its OWN queue" + echo " and will only receive feature test files." + env: + FEATURE_TEST_TOKEN: "token-for-feature-tests" + TEST_FILE_PATTERN: "spec/features/**/*_spec.rb" + agents: + queue: mac + + - wait + + - label: ":memo: Summary" + command: | + echo "==============================================" + echo "KNAPSACK PRO TOKEN BEHAVIOR SUMMARY" + echo "==============================================" + echo "" + echo "THE PROBLEM:" + echo "When two steps have:" + echo " - Same API token" + echo " - Same parallelism value" + echo " - Same branch/commit" + echo "" + echo "Knapsack Pro sees them as ONE test suite and creates" + echo "a shared queue. Both steps pull from the same pool of" + echo "tests, ignoring TEST_FILE_PATTERN filtering." + echo "" + echo "THE FIX:" + echo "Use SEPARATE API tokens for each test suite:" + echo " - KNAPSACK_PRO_TEST_SUITE_TOKEN_UNIT for unit tests" + echo " - KNAPSACK_PRO_TEST_SUITE_TOKEN_FEATURE for feature tests" + echo "" + echo "Generate additional tokens at:" + echo "https://knapsackpro.com/dashboard" + echo "==============================================" + agents: + queue: mac diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 82e734d..04af7fa 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -1,17 +1,59 @@ +env: + BUILDKITE_PLUGIN_S3_CACHE_BUCKET: "ola-buildkite-cache" + steps: - - label: "Job with dynamic priorityy" + # ===== ARTIFACTS APPROACH ===== + - label: ":package: Artifacts - Install (UV)" + agents: + queue: mac command: | - if [ "$BUILDKITE_RETRY_COUNT" -eq 0 ]; then - PRIORITY=1 - else - PRIORITY=0 - fi - - buildkite-agent pipeline upload <