From 44dbfe593d1d57dfe6efd2fe01c4d50203f495b9 Mon Sep 17 00:00:00 2001 From: Ben Teichman Date: Wed, 18 Jun 2025 23:17:51 -0400 Subject: [PATCH 01/25] feat: turborepo ci commands and jobs (ENG-600) --- src/commands/turborepo/turborepo_install.yml | 17 +++++++++++ .../turborepo/turborepo_persist_coverage.yml | 5 ++++ .../turborepo/turborepo_restore_cache.yml | 25 +++++++++++++++++ .../turborepo/turborepo_save_cache.yml | 13 +++++++++ .../turborepo_store_test_results.yml | 9 ++++++ src/commands/yarn/vf_save_cache.yml | 1 - src/jobs/turborepo/turborepo_build.yml | 21 ++++++++++++++ .../turborepo/turborepo_dependency_tests.yml | 22 +++++++++++++++ .../turborepo/turborepo_integration_tests.yml | 28 +++++++++++++++++++ .../turborepo/turborepo_lint_dockerfile.yml | 17 +++++++++++ src/jobs/turborepo/turborepo_lint_report.yml | 26 +++++++++++++++++ .../turborepo/turborepo_sonarcloud_scan.yml | 12 ++++++++ src/jobs/turborepo/turborepo_types_tests.yml | 22 +++++++++++++++ src/jobs/turborepo/turborepo_unit_tests.yml | 25 +++++++++++++++++ 14 files changed, 242 insertions(+), 1 deletion(-) create mode 100644 src/commands/turborepo/turborepo_install.yml create mode 100644 src/commands/turborepo/turborepo_persist_coverage.yml create mode 100644 src/commands/turborepo/turborepo_restore_cache.yml create mode 100644 src/commands/turborepo/turborepo_save_cache.yml create mode 100644 src/commands/turborepo/turborepo_store_test_results.yml create mode 100644 src/jobs/turborepo/turborepo_build.yml create mode 100644 src/jobs/turborepo/turborepo_dependency_tests.yml create mode 100644 src/jobs/turborepo/turborepo_integration_tests.yml create mode 100644 src/jobs/turborepo/turborepo_lint_dockerfile.yml create mode 100644 src/jobs/turborepo/turborepo_lint_report.yml create mode 100644 src/jobs/turborepo/turborepo_sonarcloud_scan.yml create mode 100644 src/jobs/turborepo/turborepo_types_tests.yml create mode 100644 src/jobs/turborepo/turborepo_unit_tests.yml diff --git a/src/commands/turborepo/turborepo_install.yml b/src/commands/turborepo/turborepo_install.yml new file mode 100644 index 00000000..11657397 --- /dev/null +++ b/src/commands/turborepo/turborepo_install.yml @@ -0,0 +1,17 @@ +parameters: + cache_prefix: + description: Cache prefix + type: string + default: '' + command: + description: Script to install dependencies + type: executor + default: yarn install --immutable +steps: + - authenticate_npm + - vf_restore_cache: + cache_prefix: turbo-<< parameters.cache_prefix >> + - run: + command: << parameters.command >> + - vf_save_cache: + cache_prefix: turbo-<< parameters.cache_prefix >> diff --git a/src/commands/turborepo/turborepo_persist_coverage.yml b/src/commands/turborepo/turborepo_persist_coverage.yml new file mode 100644 index 00000000..caecdbf3 --- /dev/null +++ b/src/commands/turborepo/turborepo_persist_coverage.yml @@ -0,0 +1,5 @@ +steps: + - persist_to_workspace: + root: "." + paths: + - "./sonar" diff --git a/src/commands/turborepo/turborepo_restore_cache.yml b/src/commands/turborepo/turborepo_restore_cache.yml new file mode 100644 index 00000000..2348bd64 --- /dev/null +++ b/src/commands/turborepo/turborepo_restore_cache.yml @@ -0,0 +1,25 @@ +parameters: + prefix: + description: Cache prefix + type: string + build_prefix: + description: Build cache prefix to pull from as a fallback + type: string + default: '' +steps: + - when: + condition: << parameters.build_prefix >> + steps: + - restore_cache: + keys: + - << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-{{ .Branch }} + - << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-master + - unless: + condition: << parameters.build_prefix >> + steps: + - restore_cache: + keys: + - << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-{{ .Branch }} + - << parameters.build_prefix >>-{{ .Environment.CACHE_VERSION }}-{{ .Branch }} + - << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-master + - << parameters.build_prefix >>-{{ .Environment.CACHE_VERSION }}-master diff --git a/src/commands/turborepo/turborepo_save_cache.yml b/src/commands/turborepo/turborepo_save_cache.yml new file mode 100644 index 00000000..1590fb76 --- /dev/null +++ b/src/commands/turborepo/turborepo_save_cache.yml @@ -0,0 +1,13 @@ +parameters: + prefix: + description: Cache prefix + type: string + path: + description: Cached path + type: string + default: ./.turbo +steps: + - save_cache: + key: << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-{{ .Branch }} + paths: + - << parameters.path >> diff --git a/src/commands/turborepo/turborepo_store_test_results.yml b/src/commands/turborepo/turborepo_store_test_results.yml new file mode 100644 index 00000000..1f24458d --- /dev/null +++ b/src/commands/turborepo/turborepo_store_test_results.yml @@ -0,0 +1,9 @@ +parameters: + report_name: + description: Report name + type: string +steps: + - store_test_results: + path: ./<< parameters.report_name >> + - store_artifacts: + path: ./<< parameters.report_name >> diff --git a/src/commands/yarn/vf_save_cache.yml b/src/commands/yarn/vf_save_cache.yml index c4de4c3e..fef299d7 100644 --- a/src/commands/yarn/vf_save_cache.yml +++ b/src/commands/yarn/vf_save_cache.yml @@ -14,4 +14,3 @@ steps: - << parameters.working_directory >>/.yarn/cache - << parameters.working_directory >>/.yarn/install-state.gz - << parameters.working_directory >>/node_modules - - ~/.cache/Cypress diff --git a/src/jobs/turborepo/turborepo_build.yml b/src/jobs/turborepo/turborepo_build.yml new file mode 100644 index 00000000..902a2716 --- /dev/null +++ b/src/jobs/turborepo/turborepo_build.yml @@ -0,0 +1,21 @@ +parameters: + executor: + description: Executor to run the command on + type: executor + default: node-executor-node-20 + command: + description: Script to execute build + type: string + default: yarn build +executor: << parameters.executor >> +steps: + - checkout_clone + - attach_workspace: + at: ~/voiceflow + - turborepo_install + - turborepo_restore_cache: + prefix: build-cache + - run: + command: << parameters.command >> + - turborepo_save_cache: + prefix: build-cache diff --git a/src/jobs/turborepo/turborepo_dependency_tests.yml b/src/jobs/turborepo/turborepo_dependency_tests.yml new file mode 100644 index 00000000..6d1b671e --- /dev/null +++ b/src/jobs/turborepo/turborepo_dependency_tests.yml @@ -0,0 +1,22 @@ +parameters: + executor: + description: Executor to run the command on + type: executor + default: node-executor-node-20 + command: + description: Script to check dependencies + type: string + default: yarn test:dependencies +executor: << parameters.executor >> +steps: + - checkout_clone + - attach_workspace: + at: ~/voiceflow + - turborepo_install + - turborepo_restore_cache: + build_prefix: build-cache + prefix: dependencies-test-cache + - run: + command: << parameters.command >> + - turborepo_save_cache: + prefix: dependencies-test-cache diff --git a/src/jobs/turborepo/turborepo_integration_tests.yml b/src/jobs/turborepo/turborepo_integration_tests.yml new file mode 100644 index 00000000..e02e0a1e --- /dev/null +++ b/src/jobs/turborepo/turborepo_integration_tests.yml @@ -0,0 +1,28 @@ +parameters: + executor: + description: Executor to run the command on + type: executor + default: node-executor-node-20 + command: + description: Script to execute tests + type: string + default: yarn test:integration +executor: << parameters.executor >> +steps: + - checkout_clone + - attach_workspace: + at: ~/voiceflow + - turborepo_install + - turborepo_restore_cache: + build_prefix: build-cache + prefix: integration-test-cache + # needed because integration tests use testcontainers which starts docker containers + - setup_remote_docker: + version: default + - run: + command: << parameters.command >> + - turborepo_save_cache: + prefix: integration-test-cache + - turborepo_store_test_results: + report_name: integration.report.xml + - turborepo_persist_coverage diff --git a/src/jobs/turborepo/turborepo_lint_dockerfile.yml b/src/jobs/turborepo/turborepo_lint_dockerfile.yml new file mode 100644 index 00000000..8ca25c88 --- /dev/null +++ b/src/jobs/turborepo/turborepo_lint_dockerfile.yml @@ -0,0 +1,17 @@ +parameters: + executor: + description: Executor to run the command on + type: executor + default: node-executor-node-20 + command: + description: Script to lint dockerfiles + type: string + default: yarn lint:dockerfiles +executor: << parameters.executor >> +steps: + - checkout_clone + - attach_workspace: + at: ~/voiceflow + - turborepo_install + - run: + command: << parameters.command >> diff --git a/src/jobs/turborepo/turborepo_lint_report.yml b/src/jobs/turborepo/turborepo_lint_report.yml new file mode 100644 index 00000000..0e00d82d --- /dev/null +++ b/src/jobs/turborepo/turborepo_lint_report.yml @@ -0,0 +1,26 @@ +parameters: + executor: + description: Executor to run the command on + type: executor + default: node-executor-node-20 + command: + description: Script to lint files + type: string + default: yarn lint:report +executor: << parameters.executor >> +steps: + - checkout_clone + - attach_workspace: + at: ~/voiceflow + - turborepo_install + - turborepo_restore_cache: + prefix: eslint-cache + - run: + command: << parameters.command >> + - turborepo_save_cache: + prefix: eslint-cache + path: ./.eslintcache + - store_test_results: + path: ./reports/lint.report.xml + - store_artifacts: + path: ./reports/lint.report.xml diff --git a/src/jobs/turborepo/turborepo_sonarcloud_scan.yml b/src/jobs/turborepo/turborepo_sonarcloud_scan.yml new file mode 100644 index 00000000..0d702173 --- /dev/null +++ b/src/jobs/turborepo/turborepo_sonarcloud_scan.yml @@ -0,0 +1,12 @@ +parameters: + executor: + description: Executor to run the command on + type: executor + default: java-executor +executor: << parameters.executor >> +steps: + - checkout_clone + - attach_workspace: + at: ~/voiceflow + - sonarcloud/scan: + cache_version: 3 diff --git a/src/jobs/turborepo/turborepo_types_tests.yml b/src/jobs/turborepo/turborepo_types_tests.yml new file mode 100644 index 00000000..a50d5200 --- /dev/null +++ b/src/jobs/turborepo/turborepo_types_tests.yml @@ -0,0 +1,22 @@ +parameters: + executor: + description: Executor to run the command on + type: executor + default: node-executor-node-20 + command: + description: Script to check types + type: string + default: yarn test:types +executor: << parameters.executor >> +steps: + - checkout_clone + - attach_workspace: + at: ~/voiceflow + - turborepo_install + - turborepo_restore_cache: + build_prefix: build-cache + prefix: types-test-cache + - run: + command: << parameters.command >> + - turborepo_save_cache: + prefix: types-test-cache diff --git a/src/jobs/turborepo/turborepo_unit_tests.yml b/src/jobs/turborepo/turborepo_unit_tests.yml new file mode 100644 index 00000000..ad2f449a --- /dev/null +++ b/src/jobs/turborepo/turborepo_unit_tests.yml @@ -0,0 +1,25 @@ +parameters: + executor: + description: Executor to run the command on + type: executor + default: node-executor-node-20 + command: + description: Script to execute tests + type: string + default: yarn test:unit +executor: << parameters.executor >> +steps: + - checkout_clone + - attach_workspace: + at: ~/voiceflow + - turborepo_install + - turborepo_restore_cache: + build_prefix: build-cache + prefix: unit-test-cache + - run: + command: << parameters.command >> + - turborepo_save_cache: + prefix: unit-test-cache + - turborepo_store_test_results: + report_name: unit.report.xml + - turborepo_persist_coverage From d4f102a2baf75dbf33a64920d72ffa13bb75aa69 Mon Sep 17 00:00:00 2001 From: Ben Teichman Date: Wed, 18 Jun 2025 23:23:58 -0400 Subject: [PATCH 02/25] refactor: simplify --- src/commands/turborepo/turborepo_persist_coverage.yml | 10 +++++++++- .../turborepo/turborepo_store_test_results.yml | 4 ++-- src/jobs/turborepo/turborepo_integration_tests.yml | 5 +++-- src/jobs/turborepo/turborepo_unit_tests.yml | 5 +++-- 4 files changed, 17 insertions(+), 7 deletions(-) diff --git a/src/commands/turborepo/turborepo_persist_coverage.yml b/src/commands/turborepo/turborepo_persist_coverage.yml index caecdbf3..9f6389d7 100644 --- a/src/commands/turborepo/turborepo_persist_coverage.yml +++ b/src/commands/turborepo/turborepo_persist_coverage.yml @@ -1,5 +1,13 @@ +parameters: + coverage_dir: + description: Directory containing coverage information + type: string + command: + description: Script to install dependencies + type: executor + default: yarn install --immutable steps: - persist_to_workspace: root: "." paths: - - "./sonar" + - << parameters.coverage_dir >> diff --git a/src/commands/turborepo/turborepo_store_test_results.yml b/src/commands/turborepo/turborepo_store_test_results.yml index 1f24458d..46c8cb2a 100644 --- a/src/commands/turborepo/turborepo_store_test_results.yml +++ b/src/commands/turborepo/turborepo_store_test_results.yml @@ -4,6 +4,6 @@ parameters: type: string steps: - store_test_results: - path: ./<< parameters.report_name >> + path: << parameters.report_name >> - store_artifacts: - path: ./<< parameters.report_name >> + path: << parameters.report_name >> diff --git a/src/jobs/turborepo/turborepo_integration_tests.yml b/src/jobs/turborepo/turborepo_integration_tests.yml index e02e0a1e..3b641d83 100644 --- a/src/jobs/turborepo/turborepo_integration_tests.yml +++ b/src/jobs/turborepo/turborepo_integration_tests.yml @@ -24,5 +24,6 @@ steps: - turborepo_save_cache: prefix: integration-test-cache - turborepo_store_test_results: - report_name: integration.report.xml - - turborepo_persist_coverage + report_name: ./integration.report.xml + - turborepo_persist_coverage: + coverage_dir: ./sonar/unit-coverage diff --git a/src/jobs/turborepo/turborepo_unit_tests.yml b/src/jobs/turborepo/turborepo_unit_tests.yml index ad2f449a..e0638450 100644 --- a/src/jobs/turborepo/turborepo_unit_tests.yml +++ b/src/jobs/turborepo/turborepo_unit_tests.yml @@ -21,5 +21,6 @@ steps: - turborepo_save_cache: prefix: unit-test-cache - turborepo_store_test_results: - report_name: unit.report.xml - - turborepo_persist_coverage + report_name: ./unit.report.xml + - turborepo_persist_coverage: + coverage_dir: ./sonar/unit-coverage From 66122cfe8cc1068e581823320d450b3a244c23b6 Mon Sep 17 00:00:00 2001 From: Ben Teichman Date: Wed, 18 Jun 2025 23:24:39 -0400 Subject: [PATCH 03/25] fix: remove unused param --- src/commands/turborepo/turborepo_persist_coverage.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/commands/turborepo/turborepo_persist_coverage.yml b/src/commands/turborepo/turborepo_persist_coverage.yml index 9f6389d7..0c2a0711 100644 --- a/src/commands/turborepo/turborepo_persist_coverage.yml +++ b/src/commands/turborepo/turborepo_persist_coverage.yml @@ -2,10 +2,6 @@ parameters: coverage_dir: description: Directory containing coverage information type: string - command: - description: Script to install dependencies - type: executor - default: yarn install --immutable steps: - persist_to_workspace: root: "." From 83ea85ed5e3e7dc01fb10959bbe94f7ee09fb97e Mon Sep 17 00:00:00 2001 From: Ben Teichman Date: Wed, 18 Jun 2025 23:26:51 -0400 Subject: [PATCH 04/25] fix: reorder --- src/commands/turborepo/turborepo_restore_cache.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/commands/turborepo/turborepo_restore_cache.yml b/src/commands/turborepo/turborepo_restore_cache.yml index 2348bd64..121609b1 100644 --- a/src/commands/turborepo/turborepo_restore_cache.yml +++ b/src/commands/turborepo/turborepo_restore_cache.yml @@ -13,13 +13,13 @@ steps: - restore_cache: keys: - << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-{{ .Branch }} + - << parameters.build_prefix >>-{{ .Environment.CACHE_VERSION }}-{{ .Branch }} - << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-master + - << parameters.build_prefix >>-{{ .Environment.CACHE_VERSION }}-master - unless: condition: << parameters.build_prefix >> steps: - restore_cache: keys: - << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-{{ .Branch }} - - << parameters.build_prefix >>-{{ .Environment.CACHE_VERSION }}-{{ .Branch }} - << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-master - - << parameters.build_prefix >>-{{ .Environment.CACHE_VERSION }}-master From de2488b3632c84e9de94f17c75a8341cd6a3218b Mon Sep 17 00:00:00 2001 From: Ben Teichman Date: Wed, 18 Jun 2025 23:32:43 -0400 Subject: [PATCH 05/25] fix: param type --- src/commands/turborepo/turborepo_install.yml | 2 +- src/jobs/turborepo/turborepo_dependency_tests.yml | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/commands/turborepo/turborepo_install.yml b/src/commands/turborepo/turborepo_install.yml index 11657397..e126cde0 100644 --- a/src/commands/turborepo/turborepo_install.yml +++ b/src/commands/turborepo/turborepo_install.yml @@ -5,7 +5,7 @@ parameters: default: '' command: description: Script to install dependencies - type: executor + type: string default: yarn install --immutable steps: - authenticate_npm diff --git a/src/jobs/turborepo/turborepo_dependency_tests.yml b/src/jobs/turborepo/turborepo_dependency_tests.yml index 6d1b671e..c39d9d06 100644 --- a/src/jobs/turborepo/turborepo_dependency_tests.yml +++ b/src/jobs/turborepo/turborepo_dependency_tests.yml @@ -14,7 +14,6 @@ steps: at: ~/voiceflow - turborepo_install - turborepo_restore_cache: - build_prefix: build-cache prefix: dependencies-test-cache - run: command: << parameters.command >> From 6649438d18ca75704972f5ef1e940f6878f67f63 Mon Sep 17 00:00:00 2001 From: Ben Teichman Date: Thu, 19 Jun 2025 00:07:41 -0400 Subject: [PATCH 06/25] ci: updated hadolint --- src/jobs/turborepo/turborepo_lint_dockerfile.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/src/jobs/turborepo/turborepo_lint_dockerfile.yml b/src/jobs/turborepo/turborepo_lint_dockerfile.yml index 8ca25c88..629dd8a6 100644 --- a/src/jobs/turborepo/turborepo_lint_dockerfile.yml +++ b/src/jobs/turborepo/turborepo_lint_dockerfile.yml @@ -13,5 +13,6 @@ steps: - attach_workspace: at: ~/voiceflow - turborepo_install + - install_latest_hadolint - run: command: << parameters.command >> From b654dd03cda0667599bf556c50a3d39d87502755 Mon Sep 17 00:00:00 2001 From: Ben Teichman Date: Thu, 19 Jun 2025 00:23:01 -0400 Subject: [PATCH 07/25] feat: more --- src/jobs/turborepo/turborepo_integration_tests.yml | 11 +++++------ src/jobs/turborepo/turborepo_unit_tests.yml | 9 ++++----- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/src/jobs/turborepo/turborepo_integration_tests.yml b/src/jobs/turborepo/turborepo_integration_tests.yml index 3b641d83..9d9f97a7 100644 --- a/src/jobs/turborepo/turborepo_integration_tests.yml +++ b/src/jobs/turborepo/turborepo_integration_tests.yml @@ -14,16 +14,15 @@ steps: at: ~/voiceflow - turborepo_install - turborepo_restore_cache: - build_prefix: build-cache - prefix: integration-test-cache + prefix: build-cache # needed because integration tests use testcontainers which starts docker containers - setup_remote_docker: version: default - run: - command: << parameters.command >> - - turborepo_save_cache: - prefix: integration-test-cache + command: | + yarn build + << parameters.command >> - turborepo_store_test_results: report_name: ./integration.report.xml - turborepo_persist_coverage: - coverage_dir: ./sonar/unit-coverage + coverage_dir: ./sonar/integration-coverage diff --git a/src/jobs/turborepo/turborepo_unit_tests.yml b/src/jobs/turborepo/turborepo_unit_tests.yml index e0638450..b12da9fb 100644 --- a/src/jobs/turborepo/turborepo_unit_tests.yml +++ b/src/jobs/turborepo/turborepo_unit_tests.yml @@ -14,12 +14,11 @@ steps: at: ~/voiceflow - turborepo_install - turborepo_restore_cache: - build_prefix: build-cache - prefix: unit-test-cache + prefix: build-cache - run: - command: << parameters.command >> - - turborepo_save_cache: - prefix: unit-test-cache + command: | + yarn build + << parameters.command >> - turborepo_store_test_results: report_name: ./unit.report.xml - turborepo_persist_coverage: From 339da86ba32e825937fea52c990b7dd7e95daceb Mon Sep 17 00:00:00 2001 From: Ben Teichman Date: Tue, 8 Jul 2025 11:01:54 -0400 Subject: [PATCH 08/25] chore: add debug logging --- src/jobs/turborepo/turborepo_integration_tests.yml | 1 + src/jobs/turborepo/turborepo_types_tests.yml | 4 +++- src/jobs/turborepo/turborepo_unit_tests.yml | 1 + 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/jobs/turborepo/turborepo_integration_tests.yml b/src/jobs/turborepo/turborepo_integration_tests.yml index 9d9f97a7..c40620db 100644 --- a/src/jobs/turborepo/turborepo_integration_tests.yml +++ b/src/jobs/turborepo/turborepo_integration_tests.yml @@ -20,6 +20,7 @@ steps: version: default - run: command: | + yarn build --dry-run yarn build << parameters.command >> - turborepo_store_test_results: diff --git a/src/jobs/turborepo/turborepo_types_tests.yml b/src/jobs/turborepo/turborepo_types_tests.yml index a50d5200..6d97b346 100644 --- a/src/jobs/turborepo/turborepo_types_tests.yml +++ b/src/jobs/turborepo/turborepo_types_tests.yml @@ -17,6 +17,8 @@ steps: build_prefix: build-cache prefix: types-test-cache - run: - command: << parameters.command >> + command: | + yarn build --dry-run + << parameters.command >> - turborepo_save_cache: prefix: types-test-cache diff --git a/src/jobs/turborepo/turborepo_unit_tests.yml b/src/jobs/turborepo/turborepo_unit_tests.yml index b12da9fb..52fe1c35 100644 --- a/src/jobs/turborepo/turborepo_unit_tests.yml +++ b/src/jobs/turborepo/turborepo_unit_tests.yml @@ -17,6 +17,7 @@ steps: prefix: build-cache - run: command: | + yarn build --dry-run yarn build << parameters.command >> - turborepo_store_test_results: From 595e9b582278b606a1a7fb70696a8a8725073e17 Mon Sep 17 00:00:00 2001 From: Ben Teichman Date: Tue, 8 Jul 2025 11:25:49 -0400 Subject: [PATCH 09/25] chore: more debugging --- src/jobs/turborepo/turborepo_build.yml | 4 +++- src/jobs/turborepo/turborepo_dependency_tests.yml | 4 +++- src/jobs/turborepo/turborepo_integration_tests.yml | 5 +++-- src/jobs/turborepo/turborepo_types_tests.yml | 8 ++++---- src/jobs/turborepo/turborepo_unit_tests.yml | 5 +++-- 5 files changed, 16 insertions(+), 10 deletions(-) diff --git a/src/jobs/turborepo/turborepo_build.yml b/src/jobs/turborepo/turborepo_build.yml index 902a2716..c92408e3 100644 --- a/src/jobs/turborepo/turborepo_build.yml +++ b/src/jobs/turborepo/turborepo_build.yml @@ -6,7 +6,7 @@ parameters: command: description: Script to execute build type: string - default: yarn build + default: yarn build --summarize true executor: << parameters.executor >> steps: - checkout_clone @@ -17,5 +17,7 @@ steps: prefix: build-cache - run: command: << parameters.command >> + - store_artifacts: + path: .turbo/runs - turborepo_save_cache: prefix: build-cache diff --git a/src/jobs/turborepo/turborepo_dependency_tests.yml b/src/jobs/turborepo/turborepo_dependency_tests.yml index c39d9d06..752ed110 100644 --- a/src/jobs/turborepo/turborepo_dependency_tests.yml +++ b/src/jobs/turborepo/turborepo_dependency_tests.yml @@ -6,7 +6,7 @@ parameters: command: description: Script to check dependencies type: string - default: yarn test:dependencies + default: yarn test:dependencies --summarize true executor: << parameters.executor >> steps: - checkout_clone @@ -17,5 +17,7 @@ steps: prefix: dependencies-test-cache - run: command: << parameters.command >> + - store_artifacts: + path: .turbo/runs - turborepo_save_cache: prefix: dependencies-test-cache diff --git a/src/jobs/turborepo/turborepo_integration_tests.yml b/src/jobs/turborepo/turborepo_integration_tests.yml index c40620db..d96de6c1 100644 --- a/src/jobs/turborepo/turborepo_integration_tests.yml +++ b/src/jobs/turborepo/turborepo_integration_tests.yml @@ -20,9 +20,10 @@ steps: version: default - run: command: | - yarn build --dry-run - yarn build + yarn build --summarize true << parameters.command >> + - store_artifacts: + path: .turbo/runs - turborepo_store_test_results: report_name: ./integration.report.xml - turborepo_persist_coverage: diff --git a/src/jobs/turborepo/turborepo_types_tests.yml b/src/jobs/turborepo/turborepo_types_tests.yml index 6d97b346..09a2ffec 100644 --- a/src/jobs/turborepo/turborepo_types_tests.yml +++ b/src/jobs/turborepo/turborepo_types_tests.yml @@ -6,7 +6,7 @@ parameters: command: description: Script to check types type: string - default: yarn test:types + default: yarn test:types --summarize true executor: << parameters.executor >> steps: - checkout_clone @@ -17,8 +17,8 @@ steps: build_prefix: build-cache prefix: types-test-cache - run: - command: | - yarn build --dry-run - << parameters.command >> + command: << parameters.command >> + - store_artifacts: + path: .turbo/runs - turborepo_save_cache: prefix: types-test-cache diff --git a/src/jobs/turborepo/turborepo_unit_tests.yml b/src/jobs/turborepo/turborepo_unit_tests.yml index 52fe1c35..8fcfae30 100644 --- a/src/jobs/turborepo/turborepo_unit_tests.yml +++ b/src/jobs/turborepo/turborepo_unit_tests.yml @@ -17,9 +17,10 @@ steps: prefix: build-cache - run: command: | - yarn build --dry-run - yarn build + yarn build --summarize true << parameters.command >> + - store_artifacts: + path: .turbo/runs - turborepo_store_test_results: report_name: ./unit.report.xml - turborepo_persist_coverage: From f5bee8932e5ac25f9e719b744c3e5a68e90eb5be Mon Sep 17 00:00:00 2001 From: Ben Teichman Date: Wed, 9 Jul 2025 02:24:59 -0400 Subject: [PATCH 10/25] feat: encode hash --- .../turborepo/turborepo_generate_hashfile.yml | 22 ++++++++++++++++ .../turborepo/turborepo_restore_cache.yml | 26 ++++++++++++------- .../turborepo/turborepo_save_cache.yml | 3 ++- src/jobs/turborepo/turborepo_build.yml | 3 +++ .../turborepo/turborepo_dependency_tests.yml | 3 +++ .../turborepo/turborepo_integration_tests.yml | 3 +++ src/jobs/turborepo/turborepo_types_tests.yml | 8 +++++- src/jobs/turborepo/turborepo_unit_tests.yml | 3 +++ 8 files changed, 60 insertions(+), 11 deletions(-) create mode 100644 src/commands/turborepo/turborepo_generate_hashfile.yml diff --git a/src/commands/turborepo/turborepo_generate_hashfile.yml b/src/commands/turborepo/turborepo_generate_hashfile.yml new file mode 100644 index 00000000..a576e77f --- /dev/null +++ b/src/commands/turborepo/turborepo_generate_hashfile.yml @@ -0,0 +1,22 @@ +parameters: + prefix: + description: Cache prefix + type: string + command: + description: Script to execute turbo run command + type: string +steps: + - run: + name: Write build hashes to /tmp/hashfiles/<< parameter.prefix >>.hashfile.json + command: | + << parameters.command >> --dry-run json \ + | jq -c '. | \ + { \ + globalHash: .globalCacheInputs.hashOfExternalDependencies, \ + tasks: .tasks | map({ taskId, hash, externalHash: .hashOfExternalDependencies }) | sort_by(.taskId) \ + }' \ + > /tmp/hashfiles/<< parameter.prefix >>.hashfile.json + - store_artifacts: + working_dir: /tmp/hashfiles + path: << parameter.prefix >>.hashfile.json + destination: hashfiles diff --git a/src/commands/turborepo/turborepo_restore_cache.yml b/src/commands/turborepo/turborepo_restore_cache.yml index 121609b1..9bc6f941 100644 --- a/src/commands/turborepo/turborepo_restore_cache.yml +++ b/src/commands/turborepo/turborepo_restore_cache.yml @@ -2,24 +2,32 @@ parameters: prefix: description: Cache prefix type: string - build_prefix: + fallback_prefix: description: Build cache prefix to pull from as a fallback type: string default: '' steps: - when: - condition: << parameters.build_prefix >> + condition: << parameters.fallback_prefix >> steps: - restore_cache: + name: Restoring turborepo << parameters.prefix >> or << parameters.fallback_prefix >> cache keys: - - << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-{{ .Branch }} - - << parameters.build_prefix >>-{{ .Environment.CACHE_VERSION }}-{{ .Branch }} - - << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-master - - << parameters.build_prefix >>-{{ .Environment.CACHE_VERSION }}-master + - << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-{{ .Branch }}-{{ .Revision }}-{{ checksum "/tmp/hashfiles/<< parameters.prefix >>.hashfile.json" }} + - << parameters.fallback_prefix >>-{{ .Environment.CACHE_VERSION }}-{{ .Branch }}-{{ .Revision }}-{{ checksum "/tmp/hashfiles/<< parameters.fallback_prefix >>.hashfile.json" }} + - << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-{{ .Branch }}-{{ .Revision }}- + - << parameters.fallback_prefix >>-{{ .Environment.CACHE_VERSION }}-{{ .Branch }}-{{ .Revision }}- + - << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-{{ .Branch }}- + - << parameters.fallback_prefix >>-{{ .Environment.CACHE_VERSION }}-{{ .Branch }}- + - << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-master- + - << parameters.fallback_prefix >>-{{ .Environment.CACHE_VERSION }}-master- - unless: - condition: << parameters.build_prefix >> + condition: << parameters.fallback_prefix >> steps: - restore_cache: + name: Restoring turborepo << parameters.prefix >> cache keys: - - << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-{{ .Branch }} - - << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-master + - << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-{{ .Branch }}-{{ .Revision }}-{{ checksum "/tmp/hashfiles/<< parameters.prefix >>.hashfile.json" }} + - << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-{{ .Branch }}-{{ .Revision }}- + - << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-{{ .Branch }}- + - << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-master- diff --git a/src/commands/turborepo/turborepo_save_cache.yml b/src/commands/turborepo/turborepo_save_cache.yml index 1590fb76..799b6f5d 100644 --- a/src/commands/turborepo/turborepo_save_cache.yml +++ b/src/commands/turborepo/turborepo_save_cache.yml @@ -8,6 +8,7 @@ parameters: default: ./.turbo steps: - save_cache: - key: << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-{{ .Branch }} + name: Saving turborepo << parameters.prefix >> build cache + key: << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-{{ .Branch }}-{{ .Revision }}--{{ checksum "/tmp/hashfiles/<< parameters.prefix >>.hashfile.json" }} paths: - << parameters.path >> diff --git a/src/jobs/turborepo/turborepo_build.yml b/src/jobs/turborepo/turborepo_build.yml index c92408e3..bf5e54bf 100644 --- a/src/jobs/turborepo/turborepo_build.yml +++ b/src/jobs/turborepo/turborepo_build.yml @@ -13,6 +13,9 @@ steps: - attach_workspace: at: ~/voiceflow - turborepo_install + - turborepo_generate_hashfile: + prefix: build-cache + command: << parameters.command >> - turborepo_restore_cache: prefix: build-cache - run: diff --git a/src/jobs/turborepo/turborepo_dependency_tests.yml b/src/jobs/turborepo/turborepo_dependency_tests.yml index 752ed110..24f46d75 100644 --- a/src/jobs/turborepo/turborepo_dependency_tests.yml +++ b/src/jobs/turborepo/turborepo_dependency_tests.yml @@ -13,6 +13,9 @@ steps: - attach_workspace: at: ~/voiceflow - turborepo_install + - turborepo_generate_hashfile: + prefix: dependencies-test-cache + command: << parameters.command >> - turborepo_restore_cache: prefix: dependencies-test-cache - run: diff --git a/src/jobs/turborepo/turborepo_integration_tests.yml b/src/jobs/turborepo/turborepo_integration_tests.yml index d96de6c1..6bf20548 100644 --- a/src/jobs/turborepo/turborepo_integration_tests.yml +++ b/src/jobs/turborepo/turborepo_integration_tests.yml @@ -13,6 +13,9 @@ steps: - attach_workspace: at: ~/voiceflow - turborepo_install + - turborepo_generate_hashfile: + prefix: build-cache + command: yarn build - turborepo_restore_cache: prefix: build-cache # needed because integration tests use testcontainers which starts docker containers diff --git a/src/jobs/turborepo/turborepo_types_tests.yml b/src/jobs/turborepo/turborepo_types_tests.yml index 09a2ffec..21416306 100644 --- a/src/jobs/turborepo/turborepo_types_tests.yml +++ b/src/jobs/turborepo/turborepo_types_tests.yml @@ -13,8 +13,14 @@ steps: - attach_workspace: at: ~/voiceflow - turborepo_install + - turborepo_generate_hashfile: + prefix: build-cache + command: yarn build + - turborepo_generate_hashfile: + prefix: types-test-cache + command: << parameters.command >> - turborepo_restore_cache: - build_prefix: build-cache + fallback_prefix: build-cache prefix: types-test-cache - run: command: << parameters.command >> diff --git a/src/jobs/turborepo/turborepo_unit_tests.yml b/src/jobs/turborepo/turborepo_unit_tests.yml index 8fcfae30..fd6c1a44 100644 --- a/src/jobs/turborepo/turborepo_unit_tests.yml +++ b/src/jobs/turborepo/turborepo_unit_tests.yml @@ -13,6 +13,9 @@ steps: - attach_workspace: at: ~/voiceflow - turborepo_install + - turborepo_generate_hashfile: + prefix: build-cache + command: yarn build - turborepo_restore_cache: prefix: build-cache - run: From f12eec53188dc2ce3ae2811969f1f09075d69ee8 Mon Sep 17 00:00:00 2001 From: Ben Teichman Date: Wed, 9 Jul 2025 02:29:11 -0400 Subject: [PATCH 11/25] fix: use correct parameters variable --- src/commands/turborepo/turborepo_generate_hashfile.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/commands/turborepo/turborepo_generate_hashfile.yml b/src/commands/turborepo/turborepo_generate_hashfile.yml index a576e77f..bed12615 100644 --- a/src/commands/turborepo/turborepo_generate_hashfile.yml +++ b/src/commands/turborepo/turborepo_generate_hashfile.yml @@ -7,7 +7,7 @@ parameters: type: string steps: - run: - name: Write build hashes to /tmp/hashfiles/<< parameter.prefix >>.hashfile.json + name: Write build hashes to /tmp/hashfiles/<< parameters.prefix >>.hashfile.json command: | << parameters.command >> --dry-run json \ | jq -c '. | \ @@ -15,8 +15,8 @@ steps: globalHash: .globalCacheInputs.hashOfExternalDependencies, \ tasks: .tasks | map({ taskId, hash, externalHash: .hashOfExternalDependencies }) | sort_by(.taskId) \ }' \ - > /tmp/hashfiles/<< parameter.prefix >>.hashfile.json + > /tmp/hashfiles/<< parameters.prefix >>.hashfile.json - store_artifacts: working_dir: /tmp/hashfiles - path: << parameter.prefix >>.hashfile.json + path: << parameters.prefix >>.hashfile.json destination: hashfiles From 10e9abcdb12e59a3f7160cea66e13d6875585a6c Mon Sep 17 00:00:00 2001 From: Ben Teichman Date: Wed, 9 Jul 2025 02:49:48 -0400 Subject: [PATCH 12/25] fix: break up commands for debugging --- .../turborepo/turborepo_generate_hashfile.yml | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/commands/turborepo/turborepo_generate_hashfile.yml b/src/commands/turborepo/turborepo_generate_hashfile.yml index bed12615..143b6eae 100644 --- a/src/commands/turborepo/turborepo_generate_hashfile.yml +++ b/src/commands/turborepo/turborepo_generate_hashfile.yml @@ -9,13 +9,16 @@ steps: - run: name: Write build hashes to /tmp/hashfiles/<< parameters.prefix >>.hashfile.json command: | - << parameters.command >> --dry-run json \ - | jq -c '. | \ - { \ - globalHash: .globalCacheInputs.hashOfExternalDependencies, \ - tasks: .tasks | map({ taskId, hash, externalHash: .hashOfExternalDependencies }) | sort_by(.taskId) \ - }' \ - > /tmp/hashfiles/<< parameters.prefix >>.hashfile.json + report=$(<< parameters.command >> --dry-run json) + + echo "$report" + + hashfile=$(echo "$report" | | jq -c '. | { globalHash: .globalCacheInputs.hashOfExternalDependencies, tasks: .tasks | map({ taskId, hash, externalHash: .hashOfExternalDependencies }) | sort_by(.taskId) }') + + echo "$hashfile" + + mkdir -p /tmp/hashfiles + echo "$hashfile" > /tmp/hashfiles/<< parameters.prefix >>.hashfile.json - store_artifacts: working_dir: /tmp/hashfiles path: << parameters.prefix >>.hashfile.json From f6db27ef1c8a65c62823c6d3ba9e4ca3d372abf4 Mon Sep 17 00:00:00 2001 From: Ben Teichman Date: Wed, 9 Jul 2025 02:58:42 -0400 Subject: [PATCH 13/25] fix: syntax --- src/commands/turborepo/turborepo_generate_hashfile.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/commands/turborepo/turborepo_generate_hashfile.yml b/src/commands/turborepo/turborepo_generate_hashfile.yml index 143b6eae..1a39e4b6 100644 --- a/src/commands/turborepo/turborepo_generate_hashfile.yml +++ b/src/commands/turborepo/turborepo_generate_hashfile.yml @@ -13,7 +13,7 @@ steps: echo "$report" - hashfile=$(echo "$report" | | jq -c '. | { globalHash: .globalCacheInputs.hashOfExternalDependencies, tasks: .tasks | map({ taskId, hash, externalHash: .hashOfExternalDependencies }) | sort_by(.taskId) }') + hashfile=$(echo "$report" | jq -c '. | { globalHash: .globalCacheInputs.hashOfExternalDependencies, tasks: .tasks | map({ taskId, hash, externalHash: .hashOfExternalDependencies }) | sort_by(.taskId) }') echo "$hashfile" From 2016530a260c7302df0ff2f5b990a87d4a919be7 Mon Sep 17 00:00:00 2001 From: Ben Teichman Date: Wed, 9 Jul 2025 03:19:38 -0400 Subject: [PATCH 14/25] fix: storing artifacts --- src/commands/turborepo/turborepo_generate_hashfile.yml | 6 ++---- src/commands/turborepo/turborepo_restore_cache.yml | 4 ++-- src/commands/turborepo/turborepo_save_cache.yml | 2 +- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src/commands/turborepo/turborepo_generate_hashfile.yml b/src/commands/turborepo/turborepo_generate_hashfile.yml index 1a39e4b6..25f35755 100644 --- a/src/commands/turborepo/turborepo_generate_hashfile.yml +++ b/src/commands/turborepo/turborepo_generate_hashfile.yml @@ -7,7 +7,7 @@ parameters: type: string steps: - run: - name: Write build hashes to /tmp/hashfiles/<< parameters.prefix >>.hashfile.json + name: Write hashes to /tmp/hashfiles/<< parameters.prefix >>.hashfile.json command: | report=$(<< parameters.command >> --dry-run json) @@ -20,6 +20,4 @@ steps: mkdir -p /tmp/hashfiles echo "$hashfile" > /tmp/hashfiles/<< parameters.prefix >>.hashfile.json - store_artifacts: - working_dir: /tmp/hashfiles - path: << parameters.prefix >>.hashfile.json - destination: hashfiles + path: /tmp/hashfiles/<< parameters.prefix >>.hashfile.json diff --git a/src/commands/turborepo/turborepo_restore_cache.yml b/src/commands/turborepo/turborepo_restore_cache.yml index 9bc6f941..56e33777 100644 --- a/src/commands/turborepo/turborepo_restore_cache.yml +++ b/src/commands/turborepo/turborepo_restore_cache.yml @@ -11,7 +11,7 @@ steps: condition: << parameters.fallback_prefix >> steps: - restore_cache: - name: Restoring turborepo << parameters.prefix >> or << parameters.fallback_prefix >> cache + name: Restoring turborepo '<< parameters.prefix >>' or '<< parameters.fallback_prefix >>' cache keys: - << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-{{ .Branch }}-{{ .Revision }}-{{ checksum "/tmp/hashfiles/<< parameters.prefix >>.hashfile.json" }} - << parameters.fallback_prefix >>-{{ .Environment.CACHE_VERSION }}-{{ .Branch }}-{{ .Revision }}-{{ checksum "/tmp/hashfiles/<< parameters.fallback_prefix >>.hashfile.json" }} @@ -25,7 +25,7 @@ steps: condition: << parameters.fallback_prefix >> steps: - restore_cache: - name: Restoring turborepo << parameters.prefix >> cache + name: Restoring turborepo '<< parameters.prefix >>' cache keys: - << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-{{ .Branch }}-{{ .Revision }}-{{ checksum "/tmp/hashfiles/<< parameters.prefix >>.hashfile.json" }} - << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-{{ .Branch }}-{{ .Revision }}- diff --git a/src/commands/turborepo/turborepo_save_cache.yml b/src/commands/turborepo/turborepo_save_cache.yml index 799b6f5d..feb5be6c 100644 --- a/src/commands/turborepo/turborepo_save_cache.yml +++ b/src/commands/turborepo/turborepo_save_cache.yml @@ -8,7 +8,7 @@ parameters: default: ./.turbo steps: - save_cache: - name: Saving turborepo << parameters.prefix >> build cache + name: Saving turborepo '<< parameters.prefix >>' cache key: << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-{{ .Branch }}-{{ .Revision }}--{{ checksum "/tmp/hashfiles/<< parameters.prefix >>.hashfile.json" }} paths: - << parameters.path >> From 8508d4bf361a6d25812e2f8132616cf8caa4a78c Mon Sep 17 00:00:00 2001 From: Ben Teichman Date: Wed, 9 Jul 2025 03:35:47 -0400 Subject: [PATCH 15/25] chore: better caching --- .../turborepo/turborepo_generate_hashfile.yml | 3 ++- .../turborepo/turborepo_restore_cache.yml | 26 +++++++++---------- .../turborepo/turborepo_save_cache.yml | 9 +++++-- .../turborepo_store_test_results.yml | 2 ++ 4 files changed, 24 insertions(+), 16 deletions(-) diff --git a/src/commands/turborepo/turborepo_generate_hashfile.yml b/src/commands/turborepo/turborepo_generate_hashfile.yml index 25f35755..853ac1e9 100644 --- a/src/commands/turborepo/turborepo_generate_hashfile.yml +++ b/src/commands/turborepo/turborepo_generate_hashfile.yml @@ -7,7 +7,7 @@ parameters: type: string steps: - run: - name: Write hashes to /tmp/hashfiles/<< parameters.prefix >>.hashfile.json + name: Writing '<< parameters.command >>' hashes to '/tmp/hashfiles/<< parameters.prefix >>.hashfile.json' command: | report=$(<< parameters.command >> --dry-run json) @@ -20,4 +20,5 @@ steps: mkdir -p /tmp/hashfiles echo "$hashfile" > /tmp/hashfiles/<< parameters.prefix >>.hashfile.json - store_artifacts: + name: Storing '/tmp/hashfiles/<< parameters.prefix >>.hashfile.json' as artifact path: /tmp/hashfiles/<< parameters.prefix >>.hashfile.json diff --git a/src/commands/turborepo/turborepo_restore_cache.yml b/src/commands/turborepo/turborepo_restore_cache.yml index 56e33777..64aa2c81 100644 --- a/src/commands/turborepo/turborepo_restore_cache.yml +++ b/src/commands/turborepo/turborepo_restore_cache.yml @@ -11,23 +11,23 @@ steps: condition: << parameters.fallback_prefix >> steps: - restore_cache: - name: Restoring turborepo '<< parameters.prefix >>' or '<< parameters.fallback_prefix >>' cache + name: Restoring turborepo '<< parameters.prefix >>' cache, falling back to '<< parameters.fallback_prefix >>' cache keys: - - << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-{{ .Branch }}-{{ .Revision }}-{{ checksum "/tmp/hashfiles/<< parameters.prefix >>.hashfile.json" }} - - << parameters.fallback_prefix >>-{{ .Environment.CACHE_VERSION }}-{{ .Branch }}-{{ .Revision }}-{{ checksum "/tmp/hashfiles/<< parameters.fallback_prefix >>.hashfile.json" }} - - << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-{{ .Branch }}-{{ .Revision }}- - - << parameters.fallback_prefix >>-{{ .Environment.CACHE_VERSION }}-{{ .Branch }}-{{ .Revision }}- - - << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-{{ .Branch }}- - - << parameters.fallback_prefix >>-{{ .Environment.CACHE_VERSION }}-{{ .Branch }}- - - << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-master- - - << parameters.fallback_prefix >>-{{ .Environment.CACHE_VERSION }}-master- + - << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-branch-{{ .Branch }}-{{ checksum "/tmp/hashfiles/<< parameters.prefix >>.hashfile.json" }} + - << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-global-{{ checksum "/tmp/hashfiles/<< parameters.prefix >>.hashfile.json" }} + - << parameters.fallback_prefix >>-{{ .Environment.CACHE_VERSION }}-branch-{{ .Branch }}-{{ checksum "/tmp/hashfiles/<< parameters.fallback_prefix >>.hashfile.json" }} + - << parameters.fallback_prefix >>-{{ .Environment.CACHE_VERSION }}-global-{{ checksum "/tmp/hashfiles/<< parameters.fallback_prefix >>.hashfile.json" }} + - << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-branch-{{ .Branch }}- + - << parameters.fallback_prefix >>-{{ .Environment.CACHE_VERSION }}-branch-{{ .Branch }}- + - << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-branch-master- + - << parameters.fallback_prefix >>-{{ .Environment.CACHE_VERSION }}-branch-master- - unless: condition: << parameters.fallback_prefix >> steps: - restore_cache: name: Restoring turborepo '<< parameters.prefix >>' cache keys: - - << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-{{ .Branch }}-{{ .Revision }}-{{ checksum "/tmp/hashfiles/<< parameters.prefix >>.hashfile.json" }} - - << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-{{ .Branch }}-{{ .Revision }}- - - << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-{{ .Branch }}- - - << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-master- + - << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-branch-{{ .Branch }}-{{ checksum "/tmp/hashfiles/<< parameters.prefix >>.hashfile.json" }} + - << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-global-{{ checksum "/tmp/hashfiles/<< parameters.prefix >>.hashfile.json" }} + - << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-branch-{{ .Branch }}- + - << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-branch-master- diff --git a/src/commands/turborepo/turborepo_save_cache.yml b/src/commands/turborepo/turborepo_save_cache.yml index feb5be6c..0d87755e 100644 --- a/src/commands/turborepo/turborepo_save_cache.yml +++ b/src/commands/turborepo/turborepo_save_cache.yml @@ -8,7 +8,12 @@ parameters: default: ./.turbo steps: - save_cache: - name: Saving turborepo '<< parameters.prefix >>' cache - key: << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-{{ .Branch }}-{{ .Revision }}--{{ checksum "/tmp/hashfiles/<< parameters.prefix >>.hashfile.json" }} + name: Saving turborepo '<< parameters.prefix >>' global cache + key: << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-global-{{ checksum "/tmp/hashfiles/<< parameters.prefix >>.hashfile.json" }} + paths: + - << parameters.path >> + - save_cache: + name: Saving turborepo '<< parameters.prefix >>' branch cache + key: << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-branch-{{ .Branch }}-{{ checksum "/tmp/hashfiles/<< parameters.prefix >>.hashfile.json" }} paths: - << parameters.path >> diff --git a/src/commands/turborepo/turborepo_store_test_results.yml b/src/commands/turborepo/turborepo_store_test_results.yml index 46c8cb2a..b23f0fc2 100644 --- a/src/commands/turborepo/turborepo_store_test_results.yml +++ b/src/commands/turborepo/turborepo_store_test_results.yml @@ -4,6 +4,8 @@ parameters: type: string steps: - store_test_results: + name: Storing '<< parameters.report_name >>' results path: << parameters.report_name >> - store_artifacts: + name: Storing '<< parameters.report_name >>' as artifact path: << parameters.report_name >> From 5d7fb70f6f646aa5577259ef1861b87505c06177 Mon Sep 17 00:00:00 2001 From: Ben Teichman Date: Wed, 9 Jul 2025 10:38:43 -0400 Subject: [PATCH 16/25] chore: remove debug logging --- src/commands/turborepo/turborepo_generate_hashfile.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/commands/turborepo/turborepo_generate_hashfile.yml b/src/commands/turborepo/turborepo_generate_hashfile.yml index 853ac1e9..ad73bbad 100644 --- a/src/commands/turborepo/turborepo_generate_hashfile.yml +++ b/src/commands/turborepo/turborepo_generate_hashfile.yml @@ -13,12 +13,9 @@ steps: echo "$report" - hashfile=$(echo "$report" | jq -c '. | { globalHash: .globalCacheInputs.hashOfExternalDependencies, tasks: .tasks | map({ taskId, hash, externalHash: .hashOfExternalDependencies }) | sort_by(.taskId) }') - - echo "$hashfile" - - mkdir -p /tmp/hashfiles - echo "$hashfile" > /tmp/hashfiles/<< parameters.prefix >>.hashfile.json + echo "$report" \ + | jq -c '. | { globalHash: .globalCacheInputs.hashOfExternalDependencies, tasks: .tasks | map({ taskId, hash, externalHash: .hashOfExternalDependencies }) | sort_by(.taskId) }' \ + > /tmp/hashfiles/<< parameters.prefix >>.hashfile.json - store_artifacts: name: Storing '/tmp/hashfiles/<< parameters.prefix >>.hashfile.json' as artifact path: /tmp/hashfiles/<< parameters.prefix >>.hashfile.json From c75d5d04b403963ed896f8f458dacccf59878aa7 Mon Sep 17 00:00:00 2001 From: Ben Teichman Date: Wed, 9 Jul 2025 10:52:17 -0400 Subject: [PATCH 17/25] fix: create hashfiles folder --- src/commands/turborepo/turborepo_generate_hashfile.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/src/commands/turborepo/turborepo_generate_hashfile.yml b/src/commands/turborepo/turborepo_generate_hashfile.yml index ad73bbad..b4150d3f 100644 --- a/src/commands/turborepo/turborepo_generate_hashfile.yml +++ b/src/commands/turborepo/turborepo_generate_hashfile.yml @@ -13,6 +13,7 @@ steps: echo "$report" + mkdir -p /tmp/hashfiles echo "$report" \ | jq -c '. | { globalHash: .globalCacheInputs.hashOfExternalDependencies, tasks: .tasks | map({ taskId, hash, externalHash: .hashOfExternalDependencies }) | sort_by(.taskId) }' \ > /tmp/hashfiles/<< parameters.prefix >>.hashfile.json From 519a63f7717c1fcb6a2c10b3bdebb5c053da77b2 Mon Sep 17 00:00:00 2001 From: Ben Teichman Date: Wed, 9 Jul 2025 11:00:59 -0400 Subject: [PATCH 18/25] fix: restore build cache optionally for dependency tests --- src/jobs/turborepo/turborepo_dependency_tests.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/jobs/turborepo/turborepo_dependency_tests.yml b/src/jobs/turborepo/turborepo_dependency_tests.yml index 24f46d75..f242409f 100644 --- a/src/jobs/turborepo/turborepo_dependency_tests.yml +++ b/src/jobs/turborepo/turborepo_dependency_tests.yml @@ -13,10 +13,14 @@ steps: - attach_workspace: at: ~/voiceflow - turborepo_install + - turborepo_generate_hashfile: + prefix: build-cache + command: yarn build - turborepo_generate_hashfile: prefix: dependencies-test-cache command: << parameters.command >> - turborepo_restore_cache: + fallback_prefix: build-cache prefix: dependencies-test-cache - run: command: << parameters.command >> From fad9761639a7d18cca9f88b3f2e11d2c774e3f11 Mon Sep 17 00:00:00 2001 From: Ben Teichman Date: Wed, 9 Jul 2025 11:19:39 -0400 Subject: [PATCH 19/25] feat: new job --- src/commands/turborepo/turborepo_generate_hashfile.yml | 6 +++++- src/commands/turborepo/turborepo_store_summaries.yml | 8 ++++++++ src/jobs/turborepo/turborepo_build.yml | 3 +-- src/jobs/turborepo/turborepo_dependency_tests.yml | 3 +-- src/jobs/turborepo/turborepo_integration_tests.yml | 9 +++------ src/jobs/turborepo/turborepo_types_tests.yml | 3 +-- src/jobs/turborepo/turborepo_unit_tests.yml | 9 +++------ 7 files changed, 22 insertions(+), 19 deletions(-) create mode 100644 src/commands/turborepo/turborepo_store_summaries.yml diff --git a/src/commands/turborepo/turborepo_generate_hashfile.yml b/src/commands/turborepo/turborepo_generate_hashfile.yml index b4150d3f..708c9fa5 100644 --- a/src/commands/turborepo/turborepo_generate_hashfile.yml +++ b/src/commands/turborepo/turborepo_generate_hashfile.yml @@ -15,7 +15,11 @@ steps: mkdir -p /tmp/hashfiles echo "$report" \ - | jq -c '. | { globalHash: .globalCacheInputs.hashOfExternalDependencies, tasks: .tasks | map({ taskId, hash, externalHash: .hashOfExternalDependencies }) | sort_by(.taskId) }' \ + | jq -c '. | \ + { \ + globalHash: .globalCacheInputs.hashOfExternalDependencies, \ + tasks: .tasks | map({ taskId, hash, externalHash: .hashOfExternalDependencies }) | sort_by(.taskId) \ + }' \ > /tmp/hashfiles/<< parameters.prefix >>.hashfile.json - store_artifacts: name: Storing '/tmp/hashfiles/<< parameters.prefix >>.hashfile.json' as artifact diff --git a/src/commands/turborepo/turborepo_store_summaries.yml b/src/commands/turborepo/turborepo_store_summaries.yml new file mode 100644 index 00000000..7e4e193c --- /dev/null +++ b/src/commands/turborepo/turborepo_store_summaries.yml @@ -0,0 +1,8 @@ +parameters: + report_name: + description: Report name + type: string +steps: + - store_artifacts: + name: Storing turborepo task summaries as artifacts + path: .turbo/runs diff --git a/src/jobs/turborepo/turborepo_build.yml b/src/jobs/turborepo/turborepo_build.yml index bf5e54bf..28b1cfc3 100644 --- a/src/jobs/turborepo/turborepo_build.yml +++ b/src/jobs/turborepo/turborepo_build.yml @@ -20,7 +20,6 @@ steps: prefix: build-cache - run: command: << parameters.command >> - - store_artifacts: - path: .turbo/runs + - turborepo_store_summaries - turborepo_save_cache: prefix: build-cache diff --git a/src/jobs/turborepo/turborepo_dependency_tests.yml b/src/jobs/turborepo/turborepo_dependency_tests.yml index f242409f..726d8b2a 100644 --- a/src/jobs/turborepo/turborepo_dependency_tests.yml +++ b/src/jobs/turborepo/turborepo_dependency_tests.yml @@ -24,7 +24,6 @@ steps: prefix: dependencies-test-cache - run: command: << parameters.command >> - - store_artifacts: - path: .turbo/runs + - turborepo_store_summaries - turborepo_save_cache: prefix: dependencies-test-cache diff --git a/src/jobs/turborepo/turborepo_integration_tests.yml b/src/jobs/turborepo/turborepo_integration_tests.yml index 6bf20548..54841bf2 100644 --- a/src/jobs/turborepo/turborepo_integration_tests.yml +++ b/src/jobs/turborepo/turborepo_integration_tests.yml @@ -6,7 +6,7 @@ parameters: command: description: Script to execute tests type: string - default: yarn test:integration + default: yarn build --summarize true && yarn test:integration executor: << parameters.executor >> steps: - checkout_clone @@ -22,11 +22,8 @@ steps: - setup_remote_docker: version: default - run: - command: | - yarn build --summarize true - << parameters.command >> - - store_artifacts: - path: .turbo/runs + command: << parameters.command >> + - turborepo_store_summaries - turborepo_store_test_results: report_name: ./integration.report.xml - turborepo_persist_coverage: diff --git a/src/jobs/turborepo/turborepo_types_tests.yml b/src/jobs/turborepo/turborepo_types_tests.yml index 21416306..3f354036 100644 --- a/src/jobs/turborepo/turborepo_types_tests.yml +++ b/src/jobs/turborepo/turborepo_types_tests.yml @@ -24,7 +24,6 @@ steps: prefix: types-test-cache - run: command: << parameters.command >> - - store_artifacts: - path: .turbo/runs + - turborepo_store_summaries - turborepo_save_cache: prefix: types-test-cache diff --git a/src/jobs/turborepo/turborepo_unit_tests.yml b/src/jobs/turborepo/turborepo_unit_tests.yml index fd6c1a44..17b8cda5 100644 --- a/src/jobs/turborepo/turborepo_unit_tests.yml +++ b/src/jobs/turborepo/turborepo_unit_tests.yml @@ -6,7 +6,7 @@ parameters: command: description: Script to execute tests type: string - default: yarn test:unit + default: yarn build --summarize true && yarn test:unit executor: << parameters.executor >> steps: - checkout_clone @@ -19,11 +19,8 @@ steps: - turborepo_restore_cache: prefix: build-cache - run: - command: | - yarn build --summarize true - << parameters.command >> - - store_artifacts: - path: .turbo/runs + command: << parameters.command >> + - turborepo_store_summaries - turborepo_store_test_results: report_name: ./unit.report.xml - turborepo_persist_coverage: From b2d245edeef7812fa44627ae0bc86e3ff8c800a8 Mon Sep 17 00:00:00 2001 From: Ben Teichman Date: Wed, 9 Jul 2025 11:24:40 -0400 Subject: [PATCH 20/25] fix: remove unused param --- src/commands/turborepo/turborepo_store_summaries.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/commands/turborepo/turborepo_store_summaries.yml b/src/commands/turborepo/turborepo_store_summaries.yml index 7e4e193c..f8ddf266 100644 --- a/src/commands/turborepo/turborepo_store_summaries.yml +++ b/src/commands/turborepo/turborepo_store_summaries.yml @@ -1,7 +1,3 @@ -parameters: - report_name: - description: Report name - type: string steps: - store_artifacts: name: Storing turborepo task summaries as artifacts From a3963415313ae72334bc8b639bf0f980bf8b68bd Mon Sep 17 00:00:00 2001 From: Ben Teichman Date: Wed, 9 Jul 2025 11:28:52 -0400 Subject: [PATCH 21/25] fix: jq syntax --- src/commands/turborepo/turborepo_generate_hashfile.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/commands/turborepo/turborepo_generate_hashfile.yml b/src/commands/turborepo/turborepo_generate_hashfile.yml index 708c9fa5..b4150d3f 100644 --- a/src/commands/turborepo/turborepo_generate_hashfile.yml +++ b/src/commands/turborepo/turborepo_generate_hashfile.yml @@ -15,11 +15,7 @@ steps: mkdir -p /tmp/hashfiles echo "$report" \ - | jq -c '. | \ - { \ - globalHash: .globalCacheInputs.hashOfExternalDependencies, \ - tasks: .tasks | map({ taskId, hash, externalHash: .hashOfExternalDependencies }) | sort_by(.taskId) \ - }' \ + | jq -c '. | { globalHash: .globalCacheInputs.hashOfExternalDependencies, tasks: .tasks | map({ taskId, hash, externalHash: .hashOfExternalDependencies }) | sort_by(.taskId) }' \ > /tmp/hashfiles/<< parameters.prefix >>.hashfile.json - store_artifacts: name: Storing '/tmp/hashfiles/<< parameters.prefix >>.hashfile.json' as artifact From eec0f8b8665db02dc496d9f2dc32a792ed67ae36 Mon Sep 17 00:00:00 2001 From: Ben Teichman Date: Wed, 9 Jul 2025 12:08:57 -0400 Subject: [PATCH 22/25] feat: caching without hashfiles --- .../turborepo/turborepo_restore_cache.yml | 87 +++++++++++++++---- .../turborepo/turborepo_save_cache.yml | 39 ++++++--- src/jobs/turborepo/turborepo_lint_report.yml | 2 + 3 files changed, 100 insertions(+), 28 deletions(-) diff --git a/src/commands/turborepo/turborepo_restore_cache.yml b/src/commands/turborepo/turborepo_restore_cache.yml index 64aa2c81..d2288f90 100644 --- a/src/commands/turborepo/turborepo_restore_cache.yml +++ b/src/commands/turborepo/turborepo_restore_cache.yml @@ -1,3 +1,21 @@ +default: + - &branch_key_with_hashfile + << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-branch-{{ .Branch }}-{{ checksum "/tmp/hashfiles/<< parameters.prefix >>.hashfile.json" }} + - &branch_key_with_revision + << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-branch-{{ .Branch }}-{{ .Revision }} + - &global_key_with_hashfile + << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-global-{{ checksum "/tmp/hashfiles/<< parameters.prefix >>.hashfile.json" }} + - &global_key_with_revision + << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-global-{{ .Revision }} + - &branch_key + << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-branch-{{ .Branch }}- + - &master_branch_key + << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-branch-master- + - &fallback_branch_key + << parameters.fallback_prefix >>-{{ .Environment.CACHE_VERSION }}-branch-{{ .Branch }}- + - &fallback_master_branch_key + << parameters.fallback_prefix >>-{{ .Environment.CACHE_VERSION }}-branch-master- + parameters: prefix: description: Cache prefix @@ -6,28 +24,61 @@ parameters: description: Build cache prefix to pull from as a fallback type: string default: '' + affix_hashfile: + description: If true affixes the checksum of the relevant hashfile to the cache key + default: true steps: - when: condition: << parameters.fallback_prefix >> steps: - - restore_cache: - name: Restoring turborepo '<< parameters.prefix >>' cache, falling back to '<< parameters.fallback_prefix >>' cache - keys: - - << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-branch-{{ .Branch }}-{{ checksum "/tmp/hashfiles/<< parameters.prefix >>.hashfile.json" }} - - << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-global-{{ checksum "/tmp/hashfiles/<< parameters.prefix >>.hashfile.json" }} - - << parameters.fallback_prefix >>-{{ .Environment.CACHE_VERSION }}-branch-{{ .Branch }}-{{ checksum "/tmp/hashfiles/<< parameters.fallback_prefix >>.hashfile.json" }} - - << parameters.fallback_prefix >>-{{ .Environment.CACHE_VERSION }}-global-{{ checksum "/tmp/hashfiles/<< parameters.fallback_prefix >>.hashfile.json" }} - - << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-branch-{{ .Branch }}- - - << parameters.fallback_prefix >>-{{ .Environment.CACHE_VERSION }}-branch-{{ .Branch }}- - - << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-branch-master- - - << parameters.fallback_prefix >>-{{ .Environment.CACHE_VERSION }}-branch-master- + - when: + condition: << parameters.affix_hashfile >> + steps: + - restore_cache: + name: Restoring turborepo '<< parameters.prefix >>' cache, falling back to '<< parameters.fallback_prefix >>' cache + keys: + - *branch_key_with_hashfile + - *global_key_with_hashfile + - << parameters.fallback_prefix >>-{{ .Environment.CACHE_VERSION }}-branch-{{ .Branch }}-{{ checksum "/tmp/hashfiles/<< parameters.fallback_prefix >>.hashfile.json" }} + - << parameters.fallback_prefix >>-{{ .Environment.CACHE_VERSION }}-global-{{ checksum "/tmp/hashfiles/<< parameters.fallback_prefix >>.hashfile.json" }} + - *branch_key + - *fallback_branch_key + - *master_branch_key + - *fallback_master_branch_key + - unless: + condition: << parameters.affix_hashfile >> + steps: + - restore_cache: + name: Restoring turborepo '<< parameters.prefix >>' cache, falling back to '<< parameters.fallback_prefix >>' cache + keys: + - *branch_key_with_revision + - *global_key_with_revision + - << parameters.fallback_prefix >>-{{ .Environment.CACHE_VERSION }}-branch-{{ .Branch }}-{{ .Revision }} + - << parameters.fallback_prefix >>-{{ .Environment.CACHE_VERSION }}-global-{{ .Revision }} + - *branch_key + - *fallback_branch_key + - *master_branch_key + - *fallback_master_branch_key - unless: condition: << parameters.fallback_prefix >> steps: - - restore_cache: - name: Restoring turborepo '<< parameters.prefix >>' cache - keys: - - << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-branch-{{ .Branch }}-{{ checksum "/tmp/hashfiles/<< parameters.prefix >>.hashfile.json" }} - - << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-global-{{ checksum "/tmp/hashfiles/<< parameters.prefix >>.hashfile.json" }} - - << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-branch-{{ .Branch }}- - - << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-branch-master- + - when: + condition: << parameters.affix_hashfile >> + steps: + - restore_cache: + name: Restoring turborepo '<< parameters.prefix >>' cache + keys: + - *branch_key_with_hashfile + - *global_key_with_hashfile + - *branch_key + - *master_branch_key + - unless: + condition: << parameters.affix_hashfile >> + steps: + - restore_cache: + name: Restoring turborepo '<< parameters.prefix >>' cache + keys: + - *branch_key_with_revision + - *global_key_with_revision + - *branch_key + - *master_branch_key diff --git a/src/commands/turborepo/turborepo_save_cache.yml b/src/commands/turborepo/turborepo_save_cache.yml index 0d87755e..122be66f 100644 --- a/src/commands/turborepo/turborepo_save_cache.yml +++ b/src/commands/turborepo/turborepo_save_cache.yml @@ -6,14 +6,33 @@ parameters: description: Cached path type: string default: ./.turbo + affix_hashfile: + description: If true affixes the checksum of the relevant hashfile to the cache key + default: true steps: - - save_cache: - name: Saving turborepo '<< parameters.prefix >>' global cache - key: << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-global-{{ checksum "/tmp/hashfiles/<< parameters.prefix >>.hashfile.json" }} - paths: - - << parameters.path >> - - save_cache: - name: Saving turborepo '<< parameters.prefix >>' branch cache - key: << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-branch-{{ .Branch }}-{{ checksum "/tmp/hashfiles/<< parameters.prefix >>.hashfile.json" }} - paths: - - << parameters.path >> + - when: + condition: << parameters.affix_hashfile >> + steps: + - save_cache: + name: Saving turborepo '<< parameters.prefix >>' global cache + key: << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-global-{{ checksum "/tmp/hashfiles/<< parameters.prefix >>.hashfile.json" }} + paths: + - << parameters.path >> + - save_cache: + name: Saving turborepo '<< parameters.prefix >>' branch cache + key: << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-branch-{{ .Branch }}-{{ checksum "/tmp/hashfiles/<< parameters.prefix >>.hashfile.json" }} + paths: + - << parameters.path >> + - unless: + condition: << parameters.affix_hashfile >> + steps: + - save_cache: + name: Saving turborepo '<< parameters.prefix >>' global cache + key: << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-global-{{ .Revision }} + paths: + - << parameters.path >> + - save_cache: + name: Saving turborepo '<< parameters.prefix >>' branch cache + key: << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-branch-{{ .Branch }}-{{ .Revision }} + paths: + - << parameters.path >> diff --git a/src/jobs/turborepo/turborepo_lint_report.yml b/src/jobs/turborepo/turborepo_lint_report.yml index 0e00d82d..6ef24d97 100644 --- a/src/jobs/turborepo/turborepo_lint_report.yml +++ b/src/jobs/turborepo/turborepo_lint_report.yml @@ -15,11 +15,13 @@ steps: - turborepo_install - turborepo_restore_cache: prefix: eslint-cache + affix_hashfile: false - run: command: << parameters.command >> - turborepo_save_cache: prefix: eslint-cache path: ./.eslintcache + affix_hashfile: false - store_test_results: path: ./reports/lint.report.xml - store_artifacts: From 69c7d220161f21dac6c71d9331bc7c6de7bbe838 Mon Sep 17 00:00:00 2001 From: Ben Teichman Date: Wed, 9 Jul 2025 12:12:00 -0400 Subject: [PATCH 23/25] fix: add parameter type --- src/commands/turborepo/turborepo_restore_cache.yml | 1 + src/commands/turborepo/turborepo_save_cache.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/src/commands/turborepo/turborepo_restore_cache.yml b/src/commands/turborepo/turborepo_restore_cache.yml index d2288f90..0a05bbac 100644 --- a/src/commands/turborepo/turborepo_restore_cache.yml +++ b/src/commands/turborepo/turborepo_restore_cache.yml @@ -26,6 +26,7 @@ parameters: default: '' affix_hashfile: description: If true affixes the checksum of the relevant hashfile to the cache key + type: boolean default: true steps: - when: diff --git a/src/commands/turborepo/turborepo_save_cache.yml b/src/commands/turborepo/turborepo_save_cache.yml index 122be66f..5c15d504 100644 --- a/src/commands/turborepo/turborepo_save_cache.yml +++ b/src/commands/turborepo/turborepo_save_cache.yml @@ -8,6 +8,7 @@ parameters: default: ./.turbo affix_hashfile: description: If true affixes the checksum of the relevant hashfile to the cache key + type: boolean default: true steps: - when: From 4d428f48155356d1bb2851564202771d28bdeb6c Mon Sep 17 00:00:00 2001 From: Ben Teichman Date: Wed, 9 Jul 2025 12:15:26 -0400 Subject: [PATCH 24/25] fix: inline refs --- .../turborepo/turborepo_restore_cache.yml | 42 +++++++------------ 1 file changed, 16 insertions(+), 26 deletions(-) diff --git a/src/commands/turborepo/turborepo_restore_cache.yml b/src/commands/turborepo/turborepo_restore_cache.yml index 0a05bbac..e65fe2ef 100644 --- a/src/commands/turborepo/turborepo_restore_cache.yml +++ b/src/commands/turborepo/turborepo_restore_cache.yml @@ -1,21 +1,3 @@ -default: - - &branch_key_with_hashfile - << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-branch-{{ .Branch }}-{{ checksum "/tmp/hashfiles/<< parameters.prefix >>.hashfile.json" }} - - &branch_key_with_revision - << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-branch-{{ .Branch }}-{{ .Revision }} - - &global_key_with_hashfile - << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-global-{{ checksum "/tmp/hashfiles/<< parameters.prefix >>.hashfile.json" }} - - &global_key_with_revision - << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-global-{{ .Revision }} - - &branch_key - << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-branch-{{ .Branch }}- - - &master_branch_key - << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-branch-master- - - &fallback_branch_key - << parameters.fallback_prefix >>-{{ .Environment.CACHE_VERSION }}-branch-{{ .Branch }}- - - &fallback_master_branch_key - << parameters.fallback_prefix >>-{{ .Environment.CACHE_VERSION }}-branch-master- - parameters: prefix: description: Cache prefix @@ -38,22 +20,30 @@ steps: - restore_cache: name: Restoring turborepo '<< parameters.prefix >>' cache, falling back to '<< parameters.fallback_prefix >>' cache keys: - - *branch_key_with_hashfile - - *global_key_with_hashfile + - &branch_key_with_hashfile + << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-branch-{{ .Branch }}-{{ checksum "/tmp/hashfiles/<< parameters.prefix >>.hashfile.json" }} + - &global_key_with_hashfile + << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-global-{{ checksum "/tmp/hashfiles/<< parameters.prefix >>.hashfile.json" }} - << parameters.fallback_prefix >>-{{ .Environment.CACHE_VERSION }}-branch-{{ .Branch }}-{{ checksum "/tmp/hashfiles/<< parameters.fallback_prefix >>.hashfile.json" }} - << parameters.fallback_prefix >>-{{ .Environment.CACHE_VERSION }}-global-{{ checksum "/tmp/hashfiles/<< parameters.fallback_prefix >>.hashfile.json" }} - - *branch_key - - *fallback_branch_key - - *master_branch_key - - *fallback_master_branch_key + - &branch_key + << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-branch-{{ .Branch }}- + - &fallback_branch_key + << parameters.fallback_prefix >>-{{ .Environment.CACHE_VERSION }}-branch-{{ .Branch }}- + - &master_branch_key + << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-branch-master- + - &fallback_master_branch_key + << parameters.fallback_prefix >>-{{ .Environment.CACHE_VERSION }}-branch-master- - unless: condition: << parameters.affix_hashfile >> steps: - restore_cache: name: Restoring turborepo '<< parameters.prefix >>' cache, falling back to '<< parameters.fallback_prefix >>' cache keys: - - *branch_key_with_revision - - *global_key_with_revision + - &branch_key_with_revision + << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-branch-{{ .Branch }}-{{ .Revision }} + - &global_key_with_revision + << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-global-{{ .Revision }} - << parameters.fallback_prefix >>-{{ .Environment.CACHE_VERSION }}-branch-{{ .Branch }}-{{ .Revision }} - << parameters.fallback_prefix >>-{{ .Environment.CACHE_VERSION }}-global-{{ .Revision }} - *branch_key From f14fe793a3267aa2a8b7cf058403d85092383121 Mon Sep 17 00:00:00 2001 From: Ben Teichman Date: Wed, 9 Jul 2025 12:58:02 -0400 Subject: [PATCH 25/25] feat: turborepo schema validation with caching --- .../turborepo/turborepo_schema_validate.yml | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 src/jobs/turborepo/turborepo_schema_validate.yml diff --git a/src/jobs/turborepo/turborepo_schema_validate.yml b/src/jobs/turborepo/turborepo_schema_validate.yml new file mode 100644 index 00000000..2d69f1f6 --- /dev/null +++ b/src/jobs/turborepo/turborepo_schema_validate.yml @@ -0,0 +1,53 @@ +parameters: + executor: + description: Executor to run the command on + type: executor + default: node-executor-node-20 + schema_dir: + description: Directory to store schemas between jobs + type: string + default: "/tmp/schemas" + skip_tests: + description: Skip the tests but save the updated schemas + type: boolean + default: false + command: + description: Script to run schema validation + type: string + default: yarn schema:validate --summarize true +executor: << parameters.executor >> +steps: + - setup_remote_docker: + version: default + - checkout_clone + - attach_workspace: + at: ~/voiceflow + - turborepo_install + - turborepo_generate_hashfile: + prefix: build-cache + command: yarn build + - turborepo_generate_hashfile: + prefix: schema-validate-cache + command: << parameters.command >> + - turborepo_restore_cache: + fallback_prefix: build-cache + prefix: schema-validate-cache + - clone_repo: + step_name: Clone openapi-schemas repository + github_username: GITHUB_USERNAME + github_token: GITHUB_TOKEN + github_repo_name: openapi-schemas + path_to_clone: ~/schemas + - openapi_restore_schemas: + from: ~/schemas + to: ~/voiceflow + - run: + name: Validate Schemas + environment: + SKIP_ACCEPTANCE_TESTS: "<< parameters.skip_tests >>" + command: << parameters.command >> + - turborepo_store_summaries + - turborepo_save_cache: + prefix: schema-validate-cache + - openapi_persist_schemas: + workspace_dir: "<< parameters.schema_dir >>"