Migrate prefect-azure to prefect repository (#12787)

Author: desertaxle

.github/workflows/integration-pacakge-tests.yaml

@@ -0,0 +1,84 @@
+name: Integrations Packages Tests
+
+on:
+  pull_request:
+    paths:
+      - "src/integrations/*/**"
+  push:
+    branches:
+      - main
+    paths:
+      - "src/integrations/*/**"
+
+jobs:
+  prepare-matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+          fetch-depth: 0
+      - name: Set up Python
+        uses: actions/setup-python@v3
+        with:
+          python-version: "3.9"
+      - name: Generate matrix
+        id: set-matrix
+        run: |
+          if [[ $GITHUB_EVENT_NAME == 'pull_request' ]]; then
+            COMMIT_RANGE="${{ github.event.pull_request.base.sha }}..${{ github.event.pull_request.head.sha }}"
+          else
+            COMMIT_RANGE="${{ github.event.before }}..${{ github.event.after }}"
+          fi
+          python scripts/generate_integration_package_tests_matrix.py "$COMMIT_RANGE" > matrix.json
+          cat matrix.json
+          echo "matrix=$(cat matrix.json)" >> $GITHUB_OUTPUT
+
+  run-tests:
+    name: Run Tests for ${{ matrix.package }} on Python ${{ matrix.python-version }}
+    needs: prepare-matrix
+    runs-on: ubuntu-latest
+    strategy:
+      matrix: ${{fromJson(needs.prepare-matrix.outputs.matrix)}}
+      fail-fast: false
+    steps:
+      - name: Display current test matrix
+        run: echo '${{ toJSON(matrix) }}'
+
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        id: setup_python
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: UV Cache
+        # Manually cache the uv cache directory
+        # until setup-python supports it:
+        # https://github.com/actions/setup-python/issues/822
+        uses: actions/cache@v4
+        id: cache-uv
+        with:
+          path: ~/.cache/uv
+          key: uvcache-${{ runner.os }}-${{ steps.setup_python.outputs.python-version }}-${{ hashFiles(format('src/integrations/{0}/pyproject.toml', matrix.package)) }}
+
+      - name: Install dependencies
+        working-directory: src/integrations/${{ matrix.package }}
+        run: |
+          python -m pip install -U uv
+          uv pip install --upgrade --system -e .[dev]
+
+      - name: Run tests
+        env:
+          PREFECT_API_DATABASE_CONNECTION_URL: "sqlite+aiosqlite:///./orion-tests.db"
+        working-directory: src/integrations/${{ matrix.package }}
+        run: >
+          pytest tests
+          --numprocesses auto
+          --maxprocesses 6
+          --dist loadscope

.github/workflows/python-tests.yaml

@@ -9,7 +9,8 @@ on:
   pull_request:
     paths:
       - .github/workflows/python-tests.yaml
-      - "**/*.py"
+      - "src/prefect/**/*.py"
+      - "tests/**/*.py"
       - requirements.txt
       - requirements-client.txt
       - requirements-dev.txt
@@ -20,7 +21,8 @@ on:
       - main
     paths:
       - .github/workflows/python-tests.yaml
-      - "**/*.py"
+      - "src/prefect/**/*.py"
+      - "tests/**/*.py"
       - requirements.txt
       - requirements-client.txt
       - requirements-dev.txt

.gitignore

@@ -71,4 +71,7 @@ prefect.yaml
 !src/prefect/deployments/recipes/*/**
 
 # For development doc server if link
-libcairo.2.dylib
+libcairo.2.dylib
+
+# setuptools-scm generated files
+src/integrations/*/**/_version.py

scripts/generate_integration_package_tests_matrix.py

@@ -0,0 +1,46 @@
+import json
+import subprocess
+import sys
+from typing import Dict, List, Set
+
+PYTHON_VERSIONS = [
+    "3.8",
+    "3.9",
+    "3.10",
+    "3.11",
+    "3.12",
+]
+
+
+def get_changed_packages(commit_range: str) -> List[str]:
+    # Get the list of changed files in the specified commit range
+    result = subprocess.run(
+        ["git", "diff", "--name-only", commit_range],
+        capture_output=True,
+        text=True,
+        check=True,
+    )
+    changed_files = result.stdout.split()
+    # Filter for src/integrations directories
+    packages: Set[str] = set()
+    for file in changed_files:
+        parts = file.split("/")
+        if len(parts) > 1 and parts[0] == "src" and parts[1] == "integrations":
+            packages.add(parts[2])
+    return list(packages)
+
+
+def generate_matrix(packages: List[str], python_versions: List[str]) -> Dict:
+    matrix = {"include": []}
+    for package in packages:
+        for version in python_versions:
+            matrix["include"].append({"package": package, "python-version": version})
+    return matrix
+
+
+if __name__ == "__main__":
+    # Get the range of commits to check, or use a default range
+    commit_range = sys.argv[1] if len(sys.argv) > 1 else "HEAD~1..HEAD"
+    packages = get_changed_packages(commit_range=commit_range)
+    matrix = generate_matrix(packages, PYTHON_VERSIONS)
+    print(json.dumps(matrix))

src/integrations/prefect-azure/README.md

@@ -0,0 +1,144 @@
+# prefect-azure
+
+<p align="center">
+    <a href="https://pypi.python.org/pypi/prefect-azure/" alt="PyPI version">
+        <img alt="PyPI" src="https://img.shields.io/pypi/v/prefect-azure?color=26272B&labelColor=090422"></a>
+    <a href="https://pepy.tech/badge/prefect-azure/" alt="Downloads">
+        <img src="https://img.shields.io/pypi/dm/prefect-azure?color=26272B&labelColor=090422" /></a>
+</p>
+
+`prefect-azure` is a collection of Prefect integrations for orchestration workflows with Azure.
+
+## Getting Started
+
+### Installation
+
+Install `prefect-azure` with `pip`
+
+```bash
+pip install prefect-azure
+```
+
+To use Blob Storage:
+```bash
+pip install "prefect-azure[blob_storage]"
+```
+
+To use Cosmos DB:
+```bash
+pip install "prefect-azure[cosmos_db]"
+```
+
+To use ML Datastore:
+```bash
+pip install "prefect-azure[ml_datastore]"
+```
+
+## Examples
+
+### Download a blob
+
+```python
+from prefect import flow
+
+from prefect_azure import AzureBlobStorageCredentials
+from prefect_azure.blob_storage import blob_storage_download
+
+@flow
+def example_blob_storage_download_flow():
+    connection_string = "connection_string"
+    blob_storage_credentials = AzureBlobStorageCredentials(
+        connection_string=connection_string,
+    )
+    data = blob_storage_download(
+        blob="prefect.txt",
+        container="prefect",
+        azure_credentials=blob_storage_credentials,
+    )
+    return data
+
+example_blob_storage_download_flow()
+```
+
+Use `with_options` to customize options on any existing task or flow:
+```python
+custom_blob_storage_download_flow = example_blob_storage_download_flow.with_options(
+    name="My custom task name",
+    retries=2,
+    retry_delay_seconds=10,
+)
+```
+
+### Run a command on an Azure container instance
+
+```python
+from prefect import flow
+from prefect_azure import AzureContainerInstanceCredentials
+from prefect_azure.container_instance import AzureContainerInstanceJob
+
+
+@flow
+def container_instance_job_flow():
+    aci_credentials = AzureContainerInstanceCredentials.load("MY_BLOCK_NAME")
+    container_instance_job = AzureContainerInstanceJob(
+        aci_credentials=aci_credentials,
+        resource_group_name="azure_resource_group.example.name",
+        subscription_id="<MY_AZURE_SUBSCRIPTION_ID>",
+        command=["echo", "hello world"],
+    )
+    return container_instance_job.run()
+```
+
+### Use Azure Container Instance as infrastructure
+
+If we have `a_flow_module.py`:
+
+```python
+from prefect import flow, get_run_logger
+
+@flow
+def log_hello_flow(name="Marvin"):
+    logger = get_run_logger()
+    logger.info(f"{name} said hello!")
+
+if __name__ == "__main__":
+    log_hello_flow()
+```
+
+We can run that flow using an Azure Container Instance, but first create the infrastructure block:
+
+```python
+from prefect_azure import AzureContainerInstanceCredentials
+from prefect_azure.container_instance import AzureContainerInstanceJob
+
+container_instance_job = AzureContainerInstanceJob(
+    aci_credentials=AzureContainerInstanceCredentials.load("MY_BLOCK_NAME"),
+    resource_group_name="azure_resource_group.example.name",
+    subscription_id="<MY_AZURE_SUBSCRIPTION_ID>",
+)
+container_instance_job.save("aci-dev")
+```
+
+Then, create the deployment either on the UI or through the CLI:
+```bash
+prefect deployment build a_flow_module.py:log_hello_flow --name aci-dev -ib container-instance-job/aci-dev
+```
+
+Visit [Prefect Deployments](https://docs.prefect.io/tutorials/deployments/) for more information about deployments.
+
+## Azure Container Instance Worker
+The Azure Container Instance worker is an excellent way to run 
+your workflows on Azure. 
+
+To get started, create an Azure Container Instances typed work pool:
+```
+prefect work-pool create -t azure-container-instance my-aci-work-pool
+```
+
+Then, run a worker that pulls jobs from the work pool:
+```
+prefect worker start -n my-aci-worker -p my-aci-work-pool
+```
+
+The worker should automatically read the work pool's type and start an 
+Azure Container Instance worker.

src/integrations/prefect-azure/prefect_azure/__init__.py

@@ -0,0 +1,24 @@
+from .credentials import (  # noqa
+    AzureBlobStorageCredentials,
+    AzureCosmosDbCredentials,
+    AzureMlCredentials,
+    AzureContainerInstanceCredentials,
+)
+from .workers.container_instance import AzureContainerWorker  # noqa
+from .container_instance import AzureContainerInstanceJob  # noqa
+from .blob_storage import AzureBlobStorageContainer  # noqa
+
+__all__ = [
+    "AzureBlobStorageCredentials",
+    "AzureCosmosDbCredentials",
+    "AzureMlCredentials",
+    "AzureContainerInstanceCredentials",
+    "AzureContainerInstanceJob",
+    "AzureContainerWorker",
+    "AzureBlobStorageContainer",
+]
+
+try:
+    from ._version import version as __version__
+except ImportError:
+    __version__ = "unknown"