basic e2e test for force telemetry verification (databricks#708)

* basic e2e test for force telemetry verification

Signed-off-by: Nikhil Suri <nikhil.suri@databricks.com>

* Added more integration test scenarios

Signed-off-by: Nikhil Suri <nikhil.suri@databricks.com>

* default on telemetry + logs to investigate failing test

Signed-off-by: Nikhil Suri <nikhil.suri@databricks.com>

* fixed linting issue

Signed-off-by: Nikhil Suri <nikhil.suri@databricks.com>

* added more logs to identify server side flag evaluation

Signed-off-by: Nikhil Suri <nikhil.suri@databricks.com>

* remove unused logs

Signed-off-by: Nikhil Suri <nikhil.suri@databricks.com>

* fix broken test case for default enable telemetry

Signed-off-by: Nikhil Suri <nikhil.suri@databricks.com>

* redcude test length and made more reusable code

Signed-off-by: Nikhil Suri <nikhil.suri@databricks.com>

* removed telemetry e2e to daily single run

Signed-off-by: Nikhil Suri <nikhil.suri@databricks.com>

---------

Signed-off-by: Nikhil Suri <nikhil.suri@databricks.com>

Author: nikhilsuri-db

.github/workflows/daily-telemetry-e2e.yml

@@ -0,0 +1,87 @@
+name: Daily Telemetry E2E Tests
+
+on:
+  schedule:
+    - cron: '0 0 * * 0'  # Run every Sunday at midnight UTC
+  
+  workflow_dispatch:  # Allow manual triggering
+    inputs:
+      test_pattern:
+        description: 'Test pattern to run (default: tests/e2e/test_telemetry_e2e.py)'
+        required: false
+        default: 'tests/e2e/test_telemetry_e2e.py'
+        type: string
+
+jobs:
+  telemetry-e2e-tests:
+    runs-on: ubuntu-latest
+    environment: azure-prod
+    
+    env:
+      DATABRICKS_SERVER_HOSTNAME: ${{ secrets.DATABRICKS_HOST }}
+      DATABRICKS_HTTP_PATH: ${{ secrets.TEST_PECO_WAREHOUSE_HTTP_PATH }}
+      DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }}
+      DATABRICKS_CATALOG: peco
+      DATABRICKS_USER: ${{ secrets.TEST_PECO_SP_ID }}
+    
+    steps:
+      #----------------------------------------------
+      #       check-out repo and set-up python
+      #----------------------------------------------
+      - name: Check out repository
+        uses: actions/checkout@v4
+      
+      - name: Set up python
+        id: setup-python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+      
+      #----------------------------------------------
+      #  -----  install & configure poetry  -----
+      #----------------------------------------------
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+          installer-parallel: true
+
+      #----------------------------------------------
+      #       load cached venv if cache exists
+      #----------------------------------------------
+      - name: Load cached venv
+        id: cached-poetry-dependencies
+        uses: actions/cache@v4
+        with:
+          path: .venv
+          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }}
+      
+      #----------------------------------------------
+      # install dependencies if cache does not exist
+      #----------------------------------------------
+      - name: Install dependencies
+        run: poetry install --no-interaction --all-extras
+      
+      #----------------------------------------------
+      #         run telemetry E2E tests
+      #----------------------------------------------
+      - name: Run telemetry E2E tests
+        run: |
+          TEST_PATTERN="${{ github.event.inputs.test_pattern || 'tests/e2e/test_telemetry_e2e.py' }}"
+          echo "Running tests: $TEST_PATTERN"
+          poetry run python -m pytest $TEST_PATTERN -v -s
+        
+      #----------------------------------------------
+      #       upload test results on failure
+      #----------------------------------------------
+      - name: Upload test results on failure
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: telemetry-test-results
+          path: |
+            .pytest_cache/
+            tests-unsafe.log
+          retention-days: 7
+

.github/workflows/integration.yml

@@ -54,5 +54,9 @@ jobs:
       #----------------------------------------------
       #              run test suite
       #----------------------------------------------
-      - name: Run e2e tests
-        run: poetry run python -m pytest tests/e2e -n auto
+      - name: Run e2e tests (excluding daily-only tests)
+        run: |
+          # Exclude telemetry E2E tests from PR runs (run daily instead)
+          poetry run python -m pytest tests/e2e \
+            --ignore=tests/e2e/test_telemetry_e2e.py \
+            -n auto