feat: turborepo ci commands and jobs (ENG-600)

src/commands/turborepo/turborepo_generate_hashfile.yml

@@ -0,0 +1,22 @@
+parameters:
+  prefix:
+    description: Cache prefix
+    type: string
+  command:
+    description: Script to execute turbo run command
+    type: string
+steps:
+  - run:
+      name: Writing '<< parameters.command >>' hashes to '/tmp/hashfiles/<< parameters.prefix >>.hashfile.json'
+      command: |
+        report=$(<< parameters.command >> --dry-run json)
+
+        echo "$report"
+
+        mkdir -p /tmp/hashfiles
+        echo "$report" \
+          | jq -c '. | { globalHash: .globalCacheInputs.hashOfExternalDependencies, tasks: .tasks | map({ taskId, hash, externalHash: .hashOfExternalDependencies }) | sort_by(.taskId) }' \
+          > /tmp/hashfiles/<< parameters.prefix >>.hashfile.json
+  - store_artifacts:
+      name: Storing '/tmp/hashfiles/<< parameters.prefix >>.hashfile.json' as artifact
+      path: /tmp/hashfiles/<< parameters.prefix >>.hashfile.json

src/commands/turborepo/turborepo_install.yml

@@ -0,0 +1,17 @@
+parameters:
+  cache_prefix:
+    description: Cache prefix
+    type: string
+    default: ''
+  command:
+    description: Script to install dependencies
+    type: string
+    default: yarn install --immutable
+steps:
+  - authenticate_npm
+  - vf_restore_cache:
+      cache_prefix: turbo-<< parameters.cache_prefix >>
+  - run:
+      command: << parameters.command >>
+  - vf_save_cache:
+      cache_prefix: turbo-<< parameters.cache_prefix >>

src/commands/turborepo/turborepo_persist_coverage.yml

@@ -0,0 +1,9 @@
+parameters:
+  coverage_dir:
+    description: Directory containing coverage information
+    type: string
+steps:
+  - persist_to_workspace:
+      root: "."
+      paths:
+        - << parameters.coverage_dir >>

src/commands/turborepo/turborepo_restore_cache.yml

@@ -0,0 +1,75 @@
+parameters:
+  prefix:
+    description: Cache prefix
+    type: string
+  fallback_prefix:
+    description: Build cache prefix to pull from as a fallback
+    type: string
+    default: ''
+  affix_hashfile:
+    description: If true affixes the checksum of the relevant hashfile to the cache key
+    type: boolean
+    default: true
+steps:
+  - when:
+      condition: << parameters.fallback_prefix >>
+      steps:
+        - when:
+            condition: << parameters.affix_hashfile >>
+            steps:
+              - restore_cache:
+                  name: Restoring turborepo '<< parameters.prefix >>' cache, falling back to '<< parameters.fallback_prefix >>' cache
+                  keys:
+                    - &branch_key_with_hashfile
+                      << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-branch-{{ .Branch }}-{{ checksum "/tmp/hashfiles/<< parameters.prefix >>.hashfile.json" }}
+                    - &global_key_with_hashfile
+                      << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-global-{{ checksum "/tmp/hashfiles/<< parameters.prefix >>.hashfile.json" }}
+                    - << parameters.fallback_prefix >>-{{ .Environment.CACHE_VERSION }}-branch-{{ .Branch }}-{{ checksum "/tmp/hashfiles/<< parameters.fallback_prefix >>.hashfile.json" }}
+                    - << parameters.fallback_prefix >>-{{ .Environment.CACHE_VERSION }}-global-{{ checksum "/tmp/hashfiles/<< parameters.fallback_prefix >>.hashfile.json" }}
+                    - &branch_key
+                      << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-branch-{{ .Branch }}-
+                    - &fallback_branch_key
+                      << parameters.fallback_prefix >>-{{ .Environment.CACHE_VERSION }}-branch-{{ .Branch }}-
+                    - &master_branch_key
+                      << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-branch-master-
+                    - &fallback_master_branch_key
+                      << parameters.fallback_prefix >>-{{ .Environment.CACHE_VERSION }}-branch-master-
+        - unless:
+            condition: << parameters.affix_hashfile >>
+            steps:
+              - restore_cache:
+                  name: Restoring turborepo '<< parameters.prefix >>' cache, falling back to '<< parameters.fallback_prefix >>' cache
+                  keys:
+                    - &branch_key_with_revision
+                      << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-branch-{{ .Branch }}-{{ .Revision }}
+                    - &global_key_with_revision
+                      << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-global-{{ .Revision }}
+                    - << parameters.fallback_prefix >>-{{ .Environment.CACHE_VERSION }}-branch-{{ .Branch }}-{{ .Revision }}
+                    - << parameters.fallback_prefix >>-{{ .Environment.CACHE_VERSION }}-global-{{ .Revision }}
+                    - *branch_key
+                    - *fallback_branch_key
+                    - *master_branch_key
+                    - *fallback_master_branch_key
+  - unless:
+      condition: << parameters.fallback_prefix >>
+      steps:
+        - when:
+            condition: << parameters.affix_hashfile >>
+            steps:
+              - restore_cache:
+                  name: Restoring turborepo '<< parameters.prefix >>' cache
+                  keys:
+                    - *branch_key_with_hashfile
+                    - *global_key_with_hashfile
+                    - *branch_key
+                    - *master_branch_key
+        - unless:
+            condition: << parameters.affix_hashfile >>
+            steps:
+              - restore_cache:
+                  name: Restoring turborepo '<< parameters.prefix >>' cache
+                  keys:
+                    - *branch_key_with_revision
+                    - *global_key_with_revision
+                    - *branch_key
+                    - *master_branch_key

src/commands/turborepo/turborepo_save_cache.yml

@@ -0,0 +1,39 @@
+parameters:
+  prefix:
+    description: Cache prefix
+    type: string
+  path:
+    description: Cached path
+    type: string
+    default: ./.turbo
+  affix_hashfile:
+    description: If true affixes the checksum of the relevant hashfile to the cache key
+    type: boolean
+    default: true
+steps:
+  - when:
+      condition: << parameters.affix_hashfile >>
+      steps:
+        - save_cache:
+            name: Saving turborepo '<< parameters.prefix >>' global cache
+            key: << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-global-{{ checksum "/tmp/hashfiles/<< parameters.prefix >>.hashfile.json" }}
+            paths:
+              - << parameters.path >>
+        - save_cache:
+            name: Saving turborepo '<< parameters.prefix >>' branch cache
+            key: << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-branch-{{ .Branch }}-{{ checksum "/tmp/hashfiles/<< parameters.prefix >>.hashfile.json" }}
+            paths:
+              - << parameters.path >>
+  - unless:
+      condition: << parameters.affix_hashfile >>
+      steps:
+        - save_cache:
+            name: Saving turborepo '<< parameters.prefix >>' global cache
+            key: << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-global-{{ .Revision }}
+            paths:
+              - << parameters.path >>
+        - save_cache:
+            name: Saving turborepo '<< parameters.prefix >>' branch cache
+            key: << parameters.prefix >>-{{ .Environment.CACHE_VERSION }}-branch-{{ .Branch }}-{{ .Revision }}
+            paths:
+              - << parameters.path >>

src/commands/turborepo/turborepo_store_summaries.yml

@@ -0,0 +1,4 @@
+steps:
+  - store_artifacts:
+      name: Storing turborepo task summaries as artifacts
+      path: .turbo/runs

src/commands/turborepo/turborepo_store_test_results.yml

@@ -0,0 +1,11 @@
+parameters:
+  report_name:
+    description: Report name
+    type: string
+steps:
+  - store_test_results:
+      name: Storing '<< parameters.report_name >>' results
+      path: << parameters.report_name >>
+  - store_artifacts:
+      name: Storing '<< parameters.report_name >>' as artifact
+      path: << parameters.report_name >>

src/commands/yarn/vf_save_cache.yml

@@ -14,4 +14,3 @@ steps:
         - << parameters.working_directory >>/.yarn/cache
         - << parameters.working_directory >>/.yarn/install-state.gz
         - << parameters.working_directory >>/node_modules
-        - ~/.cache/Cypress

src/jobs/turborepo/turborepo_build.yml

@@ -0,0 +1,25 @@
+parameters:
+  executor:
+    description: Executor to run the command on
+    type: executor
+    default: node-executor-node-20
+  command:
+    description: Script to execute build
+    type: string
+    default: yarn build --summarize true
+executor: << parameters.executor >>
+steps:
+  - checkout_clone
+  - attach_workspace:
+      at: ~/voiceflow
+  - turborepo_install
+  - turborepo_generate_hashfile:
+      prefix: build-cache
+      command: << parameters.command >>
+  - turborepo_restore_cache:
+      prefix: build-cache
+  - run:
+      command: << parameters.command >>
+  - turborepo_store_summaries
+  - turborepo_save_cache:
+      prefix: build-cache

src/jobs/turborepo/turborepo_dependency_tests.yml

@@ -0,0 +1,29 @@
+parameters:
+  executor:
+    description: Executor to run the command on
+    type: executor
+    default: node-executor-node-20
+  command:
+    description: Script to check dependencies
+    type: string
+    default: yarn test:dependencies --summarize true
+executor: << parameters.executor >>
+steps:
+  - checkout_clone
+  - attach_workspace:
+      at: ~/voiceflow
+  - turborepo_install
+  - turborepo_generate_hashfile:
+      prefix: build-cache
+      command: yarn build
+  - turborepo_generate_hashfile:
+      prefix: dependencies-test-cache
+      command: << parameters.command >>
+  - turborepo_restore_cache:
+      fallback_prefix: build-cache
+      prefix: dependencies-test-cache
+  - run:
+      command: << parameters.command >>
+  - turborepo_store_summaries
+  - turborepo_save_cache:
+      prefix: dependencies-test-cache

src/jobs/turborepo/turborepo_integration_tests.yml

@@ -0,0 +1,30 @@
+parameters:
+  executor:
+    description: Executor to run the command on
+    type: executor
+    default: node-executor-node-20
+  command:
+    description: Script to execute tests
+    type: string
+    default: yarn build --summarize true && yarn test:integration
+executor: << parameters.executor >>
+steps:
+  - checkout_clone
+  - attach_workspace:
+      at: ~/voiceflow
+  - turborepo_install
+  - turborepo_generate_hashfile:
+      prefix: build-cache
+      command: yarn build
+  - turborepo_restore_cache:
+      prefix: build-cache
+  # needed because integration tests use testcontainers which starts docker containers
+  - setup_remote_docker:
+      version: default
+  - run:
+      command: << parameters.command >>
+  - turborepo_store_summaries
+  - turborepo_store_test_results:
+      report_name: ./integration.report.xml
+  - turborepo_persist_coverage:
+      coverage_dir: ./sonar/integration-coverage

src/jobs/turborepo/turborepo_lint_dockerfile.yml

@@ -0,0 +1,18 @@
+parameters:
+  executor:
+    description: Executor to run the command on
+    type: executor
+    default: node-executor-node-20
+  command:
+    description: Script to lint dockerfiles
+    type: string
+    default: yarn lint:dockerfiles
+executor: << parameters.executor >>
+steps:
+  - checkout_clone
+  - attach_workspace:
+      at: ~/voiceflow
+  - turborepo_install
+  - install_latest_hadolint
+  - run:
+      command: << parameters.command >>

src/jobs/turborepo/turborepo_lint_report.yml

@@ -0,0 +1,28 @@
+parameters:
+  executor:
+    description: Executor to run the command on
+    type: executor
+    default: node-executor-node-20
+  command:
+    description: Script to lint files
+    type: string
+    default: yarn lint:report
+executor: << parameters.executor >>
+steps:
+  - checkout_clone
+  - attach_workspace:
+      at: ~/voiceflow
+  - turborepo_install
+  - turborepo_restore_cache:
+      prefix: eslint-cache
+      affix_hashfile: false
+  - run:
+      command: << parameters.command >>
+  - turborepo_save_cache:
+      prefix: eslint-cache
+      path: ./.eslintcache
+      affix_hashfile: false
+  - store_test_results:
+      path: ./reports/lint.report.xml
+  - store_artifacts:
+      path: ./reports/lint.report.xml

src/jobs/turborepo/turborepo_schema_validate.yml

@@ -0,0 +1,53 @@
+parameters:
+  executor:
+    description: Executor to run the command on
+    type: executor
+    default: node-executor-node-20
+  schema_dir:
+    description: Directory to store schemas between jobs
+    type: string
+    default: "/tmp/schemas"
+  skip_tests:
+    description: Skip the tests but save the updated schemas
+    type: boolean
+    default: false
+  command:
+    description: Script to run schema validation
+    type: string
+    default: yarn schema:validate --summarize true
+executor: << parameters.executor >>
+steps:
+  - setup_remote_docker:
+      version: default
+  - checkout_clone
+  - attach_workspace:
+      at: ~/voiceflow
+  - turborepo_install
+  - turborepo_generate_hashfile:
+      prefix: build-cache
+      command: yarn build
+  - turborepo_generate_hashfile:
+      prefix: schema-validate-cache
+      command: << parameters.command >>
+  - turborepo_restore_cache:
+      fallback_prefix: build-cache
+      prefix: schema-validate-cache
+  - clone_repo:
+      step_name: Clone openapi-schemas repository
+      github_username: GITHUB_USERNAME
+      github_token: GITHUB_TOKEN
+      github_repo_name: openapi-schemas
+      path_to_clone: ~/schemas
+  - openapi_restore_schemas:
+      from: ~/schemas
+      to: ~/voiceflow
+  - run:
+      name: Validate Schemas
+      environment:
+        SKIP_ACCEPTANCE_TESTS: "<< parameters.skip_tests >>"
+      command: << parameters.command >>
+  - turborepo_store_summaries
+  - turborepo_save_cache:
+      prefix: schema-validate-cache
+  - openapi_persist_schemas:
+      workspace_dir: "<< parameters.schema_dir >>"

src/jobs/turborepo/turborepo_sonarcloud_scan.yml

@@ -0,0 +1,12 @@
+parameters:
+  executor:
+    description: Executor to run the command on
+    type: executor
+    default: java-executor
+executor: << parameters.executor >>
+steps:
+  - checkout_clone
+  - attach_workspace:
+      at: ~/voiceflow
+  - sonarcloud/scan:
+      cache_version: 3