Tune CI for efficiency (#16631)

desertaxle · web-flow · commit e9890a92f948 · 2025-01-07T15:17:02.000-06:00
diff --git a/.github/workflows/benchmarks.yaml b/.github/workflows/benchmarks.yaml
@@ -27,10 +27,7 @@ concurrency:
 jobs:
   run-benchmarks:
     name: Benchmark
-    # Runs with ephemeral API and SQLite database right now
-
-    runs-on:
-      group: oss-larger-runners
+    runs-on: ubuntu-latest
     timeout-minutes: 20
 
     steps:
diff --git a/.github/workflows/codspeed-benchmarks.yaml b/.github/workflows/codspeed-benchmarks.yaml
@@ -28,10 +28,7 @@ concurrency:
 jobs:
   run-benchmarks:
     name: Benchmark
-    # Runs with ephemeral API and SQLite database right now
-
-    runs-on:
-      group: oss-larger-runners
+    runs-on: ubuntu-latest
     timeout-minutes: 20
 
     steps:
diff --git a/.github/workflows/markdown-tests.yaml b/.github/workflows/markdown-tests.yaml
@@ -52,8 +52,7 @@ concurrency:
 
 jobs:
   run-markdown-tests:
-    runs-on:
-      group: oss-larger-runners
+    runs-on: ubuntu-latest
     name: markdown:${{ matrix.python-version }}, ${{ matrix.database }}
     strategy:
       matrix:
diff --git a/.github/workflows/python-tests.yaml b/.github/workflows/python-tests.yaml
@@ -49,18 +49,30 @@ concurrency:
 
 jobs:
   run-tests:
-    runs-on:
-      group: oss-larger-runners
+    runs-on: ubuntu-latest
     name: ${{ matrix.test-type.name }} - python:${{ matrix.python-version }}, ${{ matrix.database }}
     strategy:
+      fail-fast: false
       matrix:
         test-type:
           - name: Server Tests
-            modules: tests/server/ tests/events/server
+            modules: tests/server/ tests/events/server --ignore=tests/server/database/ --ignore=tests/server/orchestration/
+          - name: Database and Orchestration Tests
+            modules: tests/server/database/ tests/server/orchestration/
           - name: Client Tests
-            modules: tests/ --ignore=tests/typesafety --ignore=tests/server/ --ignore=tests/events/server --ignore=tests/test_task_runners.py --ignore=tests/runner --ignore=tests/workers
-          - name: Runner and Worker Tests
-            modules: tests/test_task_runners.py tests/runner tests/workers
+            modules: >-
+              tests/
+              --ignore=tests/typesafety
+              --ignore=tests/server/
+              --ignore=tests/events/server
+              --ignore=tests/test_task_runners.py
+              --ignore=tests/runner
+              --ignore=tests/workers
+              --ignore=tests/cli/
+              --ignore=tests/test_settings.py
+              --ignore=tests/input/
+          - name: Runner, Worker, Settings, Input, and CLI Tests
+            modules: tests/test_task_runners.py tests/runner tests/workers tests/cli/ tests/test_settings.py tests/input/
         database:
           - "postgres:14"
           - "sqlite"
@@ -73,13 +85,21 @@ jobs:
           - database: "sqlite"
             test-type:
               name: Client Tests
-              modules: tests/ --ignore=tests/typesafety --ignore=tests/server/ --ignore=tests/events/server --ignore=tests/test_task_runners.py --ignore=tests/runner --ignore=tests/workers
+              modules: >-
+                tests/
+                --ignore=tests/typesafety
+                --ignore=tests/server/
+                --ignore=tests/events/server
+                --ignore=tests/test_task_runners.py
+                --ignore=tests/runner
+                --ignore=tests/workers
+                --ignore=tests/cli/
+                --ignore=tests/test_settings.py
+                --ignore=tests/input/
           - database: "sqlite"
             test-type:
-              name: Runner and Worker Tests
-              modules: tests/test_task_runners.py tests/runner tests/workers
-
-      fail-fast: true
+              name: Runner, Worker, Settings, Input, and CLI Tests
+              modules: tests/test_task_runners.py tests/runner tests/workers tests/cli/ tests/test_settings.py tests/input/
 
     timeout-minutes: 15
 
@@ -148,26 +168,8 @@ jobs:
           --publish 6379:6379
           redis:latest
 
-      - name: Set coverage file and artifact name
-        id: set_coverage_and_artifact_name
-        run: |
-          sanitized_test_type="${{ matrix.test-type.name }}"
-          sanitized_test_type="${sanitized_test_type// /_}"
-          sanitized_database="${{ matrix.database }}"
-          sanitized_database="${sanitized_database//:/\-}"
-          sanitized_python_version="${{ matrix.python-version }}"
-          export COVERAGE_FILE=".coverage.${sanitized_test_type}.${sanitized_python_version}.${sanitized_database}"
-          echo "COVERAGE_FILE=${COVERAGE_FILE}" >> $GITHUB_ENV
-          echo "artifact_name=coverage-data-${sanitized_test_type}-${{ matrix.python-version }}-${sanitized_database}" >> $GITHUB_OUTPUT
-
-      - name: Set coverage core
-        if: ${{ matrix.python-version == '3.12' }}
-        run: |
-          echo "COVERAGE_CORE=sysmon" >> $GITHUB_ENV
-
       - name: Run tests
         run: |
-          echo "Using COVERAGE_FILE=$COVERAGE_FILE"
           pytest ${{ matrix.test-type.modules }} \
           --numprocesses auto \
           --maxprocesses 6 \
@@ -176,17 +178,6 @@ jobs:
           --exclude-service kubernetes \
           --exclude-service docker \
           --durations 26 \
-          --cov=prefect \
-          --cov-config=setup.cfg \
-          --cov-report=''
-
-      - name: Upload coverage data
-        uses: actions/upload-artifact@v4
-        with:
-          name: ${{ steps.set_coverage_and_artifact_name.outputs.artifact_name }}
-          path: ${{ env.COVERAGE_FILE }}
-          include-hidden-files: true
-          retention-days: 1
 
       - name: Create and Upload failure flag
         if: ${{ failure() }}
@@ -213,8 +204,7 @@ jobs:
           || echo "Ignoring bad exit code"
 
   run-docker-tests:
-    runs-on:
-      group: oss-larger-runners
+    runs-on: ubuntu-latest
     name: docker, python:${{ matrix.python-version }}
     strategy:
       matrix:
@@ -226,7 +216,7 @@ jobs:
           - "3.11"
           - "3.12"
 
-      fail-fast: true
+      fail-fast: false
 
     timeout-minutes: 45
 
@@ -346,21 +336,6 @@ jobs:
           --publish 6379:6379
           redis:latest
 
-      - name: Set coverage file and artifact name
-        id: set_coverage_and_artifact_name
-        run: |
-          sanitized_database="${{ matrix.database }}"
-          sanitized_database="${sanitized_database//:/\-}"
-          sanitized_python_version="${{ matrix.python-version }}"
-          export COVERAGE_FILE=".coverage.${sanitized_python_version}.${sanitized_database}"
-          echo "COVERAGE_FILE=${COVERAGE_FILE}" >> $GITHUB_ENV
-          echo "artifact_name=coverage-data-docker-${{ matrix.python-version }}-${sanitized_database}" >> $GITHUB_OUTPUT
-
-      - name: Set coverage core
-        if: ${{ matrix.python-version == '3.12' }}
-        run: |
-          echo "COVERAGE_CORE=sysmon" >> $GITHUB_ENV
-
       - name: Run tests
         run: |
           echo "Using COVERAGE_FILE=$COVERAGE_FILE"
@@ -371,17 +346,6 @@ jobs:
           --disable-docker-image-builds \
           --only-service docker \
           --durations 26 \
-          --cov=prefect \
-          --cov-config=setup.cfg \
-          --cov-report=''
-
-      - name: Upload coverage data
-        uses: actions/upload-artifact@v4
-        with:
-          name: ${{ steps.set_coverage_and_artifact_name.outputs.artifact_name }}
-          path: ${{ env.COVERAGE_FILE }}
-          include-hidden-files: true
-          retention-days: 1
 
       - name: Create and Upload failure flag
         if: ${{ failure() }}
@@ -407,49 +371,6 @@ jobs:
           && docker container logs postgres \
           || echo "Ignoring bad exit code"
 
-  combine-coverage:
-    runs-on: ubuntu-latest
-    needs:
-      - run-tests
-      - run-docker-tests
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          persist-credentials: false
-
-      - name: Set up Python 3.12
-        uses: actions/setup-python@v5
-        id: setup_python
-        with:
-          python-version: "3.12"
-
-      - name: Download coverage data artifacts
-        uses: actions/download-artifact@v4
-        with:
-          pattern: coverage-data-*
-          merge-multiple: true
-
-      - name: Install coverage
-        run: pip install coverage
-
-      - name: Combine coverage data
-        run: coverage combine
-
-      - name: Generate HTML coverage report
-        run: coverage html
-
-      - name: Upload combined coverage report
-        uses: actions/upload-artifact@v4
-        with:
-          name: combined-coverage-report
-          path: htmlcov/
-
-      - name: Publish coverage markdown report
-        run: |
-          echo "## Coverage Report" >> $GITHUB_STEP_SUMMARY
-          echo "[Detailed Report](${{ steps.upload_combined_coverage_report.outputs.artifact_url }})" >> $GITHUB_STEP_SUMMARY
-          coverage report --format=markdown >> $GITHUB_STEP_SUMMARY
-
   run-typesafety-test:
     name: typesafety
     runs-on: ubuntu-latest
diff --git a/tests/cli/test_prompts.py b/tests/cli/test_prompts.py
@@ -83,7 +83,9 @@ async def test_find_all_flows_in_dir_tree(self, project_dir: Path):
         )
 
     async def test_find_all_flows_works_on_large_directory_structures(self):
-        flows = await search_for_flow_functions(str(prefect.__development_base_path__))
+        flows = await search_for_flow_functions(
+            str(prefect.__development_base_path__ / "tests")
+        )
         assert len(flows) > 500
 
     async def test_find_flow_functions_in_file_returns_empty_list_on_file_error(
diff --git a/tests/telemetry/test_instrumentation.py b/tests/telemetry/test_instrumentation.py
@@ -122,9 +122,16 @@ def test_meter_provider(telemetry_account_id: UUID, telemetry_workspace_id: UUID
     _, meter_provider, _ = setup_telemetry()
     assert isinstance(meter_provider, MeterProvider)
 
-    metric_reader = list(meter_provider._all_metric_readers)[0]
-    exporter = metric_reader._exporter
+    metric_reader = next(
+        (
+            reader
+            for reader in meter_provider._all_metric_readers
+            if isinstance(reader, PeriodicExportingMetricReader)
+        ),
+        None,
+    )
     assert isinstance(metric_reader, PeriodicExportingMetricReader)
+    exporter = metric_reader._exporter
     assert isinstance(exporter, OTLPMetricExporter)
 
     resource_attributes = {

Original file line number	Diff line number	Diff line change
`@@ -83,7 +83,9 @@ async def test_find_all_flows_in_dir_tree(self, project_dir: Path):`
`83`	`83`	`)`
`84`	`84`
`85`	`85`	`async def test_find_all_flows_works_on_large_directory_structures(self):`
`86`		`- flows = await search_for_flow_functions(str(prefect.__development_base_path__))`
	`86`	`+ flows = await search_for_flow_functions(`
	`87`	`+ str(prefect.__development_base_path__ / "tests")`
	`88`	`+ )`
`87`	`89`	`assert len(flows) > 500`
`88`	`90`
`89`	`91`	`async def test_find_flow_functions_in_file_returns_empty_list_on_file_error(`