diff --git a/guides/fabric_foundry/README.md b/guides/fabric_foundry/README.md new file mode 100644 index 00000000..cea9b97b --- /dev/null +++ b/guides/fabric_foundry/README.md @@ -0,0 +1,130 @@ +# Fabric Foundry Integration + +This guide demonstrates how to automate the integration between Microsoft Fabric Data Agents and Azure AI Foundry. + +## Overview + +The automation script performs the following steps: + +1. **Upload Notebook**: Uploads the `create_fabric_data_agent.ipynb` notebook to a specified Fabric workspace +1. **Execute Notebook**: Triggers the notebook to run, which creates a Fabric Data Agent named "data_agent_automation_sample" +1. **Retrieve Artifact ID**: Gets the artifact ID of the newly created data agent +1. **Create Connection**: Creates a Connected Resource connection in AI Foundry pointing to the Fabric Data Agent +1. **Create Agent**: Creates an AI Foundry Agent that uses the Fabric Data Agent as a knowledge source via FabricTool + +## Prerequisites + +- Azure subscription with access to: + - Microsoft Fabric workspace + - Azure AI Foundry project (not Hub - must be a Project resource) + - AI model deployment in AI Foundry project (e.g., gpt-4o) +- Fabric workspace must have: + - NYC Taxi lakehouse named "NYCTaxi_756" with tables: year_2017, year_2018, year_2019, year_2020 + - Active capacity (required for notebook execution) +- Azure CLI or DefaultAzureCredential authentication configured +- Python 3.12 or higher +- uv package manager installed + +## Setup + +1. Install dependencies using uv: + +```bash +cd guides/fabric_foundry +uv sync +``` + +1. Create a `.env` file with required environment variables: + +```bash +# Fabric Configuration +FABRIC_WORKSPACE_ID=your-fabric-workspace-id + +# Azure AI Foundry Configuration +AI_FOUNDRY_SUBSCRIPTION_ID=your-subscription-id +AI_FOUNDRY_RESOURCE_GROUP=your-resource-group +AI_FOUNDRY_ACCOUNT_NAME=your-ai-foundry-account-name +AI_FOUNDRY_PROJECT_NAME=your-ai-foundry-project-name +AI_FOUNDRY_MODEL_DEPLOYMENT_NAME=gpt-4o +``` + +**Note**: Replace `AI_FOUNDRY_MODEL_DEPLOYMENT_NAME` with your actual model deployment name from the "Models + endpoints" tab in your AI Foundry project. + +## Usage + +Run the automation script: + +```bash +uv run python automate_fabric_foundry_integration.py +``` + +The script will: + +- Load configuration from `.env` file +- Authenticate using DefaultAzureCredential +- Upload and execute the notebook in Fabric (or reuse existing notebook) +- Wait for completion (up to 10 minutes) +- Create the Fabric connection in AI Foundry using ARM REST API +- Create an AI Foundry Agent with FabricTool using Azure AI Projects SDK +- Output the created resource IDs + +## Implementation Notes + +### Connection Creation + +- Uses Azure Resource Manager REST API (`Microsoft.CognitiveServices/accounts/projects/connections@2025-06-01`) +- Connection category: `CustomKeys` +- Authentication type: `CustomKeys` +- Stores workspace ID, artifact ID, and type in credentials + +### Agent Creation + +- Uses Azure AI Projects Python SDK (`azure-ai-projects>=1.0.0b1`) +- Uses Azure AI Agents SDK (`azure-ai-agents>=1.2.0b6`) for FabricTool +- FabricTool connects the agent to the Fabric Data Agent as a knowledge source +- Requires valid model deployment name from AI Foundry project + +## Files + +- `create_fabric_data_agent.ipynb`: Notebook that creates the Fabric Data Agent (must be run in Fabric) +- `automate_fabric_foundry_integration.py`: Main automation script +- `requirements.txt`: Python dependencies +- `.env`: Environment configuration (not committed to git) +- `README.md`: This file + +## Dependencies + +Key Python packages: + +- `azure-identity>=1.15.0` - Azure authentication +- `requests>=2.31.0` - HTTP requests for Fabric and ARM APIs +- `python-dotenv>=1.0.0` - Environment variable management +- `azure-ai-projects>=1.0.0b1` - AI Foundry Projects SDK +- `azure-ai-agents>=1.2.0b6` - AI Agents SDK with FabricTool support + +## Troubleshooting + +### CapacityNotActive Error + +If you get this error when triggering notebook execution, ensure your Fabric workspace has an active capacity assigned. + +### 404 Resource Not Found (Agent Creation) + +Verify that `AI_FOUNDRY_MODEL_DEPLOYMENT_NAME` matches exactly with a deployment name in your AI Foundry project's "Models + endpoints" tab. + +### Import Errors + +Make sure you're using the preview version of `azure-ai-agents` (1.2.0b6 or higher) which includes FabricTool: + +```bash +uv pip install --pre --upgrade azure-ai-agents +``` + +## Notes + +- The Fabric Data Agent SDK can only run inside Microsoft Fabric environments +- The automation uses the Fabric REST API to upload and trigger notebook execution +- The script waits up to 10 minutes for notebook completion with 10-second polling intervals +- Notebooks are reused if they already exist (prevents duplicate creation errors) +- All resources are created with simple configurations suitable for POC/demo purposes +- This is a proof-of-concept implementation - keep it simple! diff --git a/guides/fabric_foundry/automate_fabric_foundry_integration.py b/guides/fabric_foundry/automate_fabric_foundry_integration.py new file mode 100644 index 00000000..4453b251 --- /dev/null +++ b/guides/fabric_foundry/automate_fabric_foundry_integration.py @@ -0,0 +1,427 @@ +#!/usr/bin/env python3 +""" +Automate connecting AI Foundry to Microsoft Fabric via Fabric Data Agent. + +This script: +1. Uploads the create_fabric_data_agent notebook to a Fabric workspace +2. Triggers notebook execution to create the data agent +3. Retrieves the artifact ID of the created data agent +4. Creates a Connected Resource connection in AI Foundry to Fabric +5. Creates an AI Foundry Agent using this connection as a knowledge source +""" + +import os +import sys +import time +import base64 +from typing import Optional, Dict, Any +from azure.identity import DefaultAzureCredential +from azure.ai.projects import AIProjectClient +from azure.ai.projects.models import ConnectionType +import requests +from dotenv import load_dotenv + +# Load environment variables +load_dotenv() + +# Constants +FABRIC_DATA_AGENT_NAME = "data_agent_automation_sample" +AI_FOUNDRY_API_VERSION = "2025-06-01" +NOTEBOOK_POLL_INTERVAL = 10 +NOTEBOOK_TIMEOUT = 600 + + +class FabricFoundryIntegration: + """Handles integration between Microsoft Fabric and Azure AI Foundry.""" + + def __init__( + self, + fabric_workspace_id: str, + ai_foundry_subscription_id: str, + ai_foundry_resource_group: str, + ai_foundry_account_name: str, + ai_foundry_project_name: str, + ): + self.fabric_workspace_id = fabric_workspace_id + self.ai_foundry_subscription_id = ai_foundry_subscription_id + self.ai_foundry_resource_group = ai_foundry_resource_group + self.ai_foundry_account_name = ai_foundry_account_name + self.ai_foundry_project_name = ai_foundry_project_name + self.credential = DefaultAzureCredential() + self._fabric_token = None + self._arm_token = None + + # Initialize AI Project Client + project_scope = ( + f"https://{ai_foundry_account_name}.cognitiveservices.azure.com/" + ) + self.ai_client = AIProjectClient( + endpoint=project_scope, + credential=self.credential + ) + + # ============================================================================ + # Authentication Helpers + # ============================================================================ + + def _get_fabric_token(self) -> str: + """Get or refresh authentication token for Fabric APIs.""" + if not self._fabric_token: + token = self.credential.get_token("https://api.fabric.microsoft.com/.default") + self._fabric_token = token.token + return self._fabric_token + + def _get_arm_token(self) -> str: + """Get or refresh authentication token for Azure Resource Manager APIs.""" + if not self._arm_token: + token = self.credential.get_token("https://management.azure.com/.default") + self._arm_token = token.token + return self._arm_token + + def _get_fabric_headers(self) -> Dict[str, str]: + """Get headers for Fabric API requests.""" + return { + "Authorization": f"Bearer {self._get_fabric_token()}", + "Content-Type": "application/json", + } + + def _get_arm_headers(self) -> Dict[str, str]: + """Get headers for ARM API requests.""" + return { + "Authorization": f"Bearer {self._get_arm_token()}", + "Content-Type": "application/json", + } + + # ============================================================================ + # Fabric Notebook Operations + # ============================================================================ + + def _find_existing_notebook(self, notebook_name: str) -> Optional[str]: + """Check if notebook with given name already exists in workspace.""" + try: + list_url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.fabric_workspace_id}/items" + response = requests.get(list_url, headers=self._get_fabric_headers()) + response.raise_for_status() + items = response.json().get("value", []) + + for item in items: + if item.get("displayName") == notebook_name and item.get("type") == "Notebook": + return item["id"] + return None + except requests.exceptions.RequestException as e: + print(f"⚠ Warning: Could not check for existing notebook: {e}") + return None + + def _create_notebook(self, notebook_path: str, notebook_name: str) -> str: + """Create a new notebook in Fabric workspace.""" + with open(notebook_path, "rb") as f: + notebook_content = base64.b64encode(f.read()).decode("utf-8") + + create_url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.fabric_workspace_id}/items" + payload = { + "displayName": notebook_name, + "type": "Notebook", + "definition": { + "format": "ipynb", + "parts": [ + { + "path": "notebook-content.ipynb", + "payload": notebook_content, + "payloadType": "InlineBase64", + } + ], + }, + } + + response = requests.post(create_url, headers=self._get_fabric_headers(), json=payload) + if not response.ok: + raise Exception(f"Failed to create notebook: {response.status_code} - {response.text}") + + response_data = response.json() + if not response_data or "id" not in response_data: + raise ValueError("Response missing 'id' field") + + return response_data["id"] + + def upload_notebook(self, notebook_path: str, notebook_name: str) -> str: + """Upload notebook to Fabric workspace, reusing if it already exists.""" + # Check if notebook already exists + existing_id = self._find_existing_notebook(notebook_name) + if existing_id: + print(f"✓ Found existing notebook '{notebook_name}' with ID: {existing_id}") + return existing_id + + # Create new notebook + notebook_id = self._create_notebook(notebook_path, notebook_name) + print(f"✓ Uploaded notebook '{notebook_name}' with ID: {notebook_id}") + return notebook_id + + def trigger_notebook_run(self, notebook_id: str) -> str: + """Trigger notebook execution and return job ID.""" + run_url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.fabric_workspace_id}/notebooks/{notebook_id}/jobs/instances?jobType=RunNotebook" + + response = requests.post(run_url, headers=self._get_fabric_headers()) + if not response.ok: + raise Exception(f"Failed to trigger notebook: {response.status_code} - {response.text}") + + # Handle 202 Accepted response + if response.status_code == 202: + location = response.headers.get("Location", "") + if location: + job_id = location.split("/")[-1] + print(f"✓ Triggered notebook run with job ID: {job_id}") + return job_id + + # Fallback: query for most recent job + print(f"✓ Notebook run triggered (202 Accepted), retrieving job ID...") + time.sleep(2) + jobs_url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.fabric_workspace_id}/notebooks/{notebook_id}/jobs/instances" + jobs_response = requests.get(jobs_url, headers=self._get_fabric_headers()) + jobs_response.raise_for_status() + jobs = jobs_response.json().get("value", []) + if jobs: + job_id = jobs[0]["id"] + print(f"✓ Found running job with ID: {job_id}") + return job_id + raise ValueError("Notebook run triggered but no job ID found") + + # Normal response with body + response_data = response.json() + job_id = response_data["id"] + print(f"✓ Triggered notebook run with job ID: {job_id}") + return job_id + + def wait_for_notebook_completion(self, notebook_id: str, job_id: str) -> bool: + """Wait for notebook execution to complete.""" + status_url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.fabric_workspace_id}/notebooks/{notebook_id}/jobs/instances/{job_id}" + + start_time = time.time() + while time.time() - start_time < NOTEBOOK_TIMEOUT: + try: + response = requests.get(status_url, headers=self._get_fabric_headers()) + response.raise_for_status() + status = response.json()["status"] + + if status == "Completed": + print("✓ Notebook execution completed successfully") + return True + elif status in ["Failed", "Cancelled"]: + print(f"✗ Notebook execution {status}") + return False + + print(f" Notebook status: {status}, waiting...") + time.sleep(NOTEBOOK_POLL_INTERVAL) + except requests.exceptions.RequestException as e: + print(f"⚠ Warning: Error checking notebook status: {e}") + time.sleep(NOTEBOOK_POLL_INTERVAL) + + print("✗ Timeout waiting for notebook completion") + return False + + # ============================================================================ + # Fabric Data Agent Operations + # ============================================================================ + + def get_data_agent_artifact_id(self, data_agent_name: str) -> Optional[str]: + """Retrieve the artifact ID of the created Fabric Data Agent.""" + try: + list_url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.fabric_workspace_id}/items" + response = requests.get(list_url, headers=self._get_fabric_headers()) + response.raise_for_status() + items = response.json().get("value", []) + + for item in items: + if item.get("displayName") == data_agent_name: + item_id = item["id"] + + # Get full item details + get_item_url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.fabric_workspace_id}/items/{item_id}" + item_response = requests.get(get_item_url, headers=self._get_fabric_headers()) + item_response.raise_for_status() + item_details = item_response.json() + + artifact_id = item_details["id"] + print(f"✓ Found data agent '{data_agent_name}' with artifact ID: {artifact_id}") + return artifact_id + + print(f"✗ Data agent '{data_agent_name}' not found") + return None + except requests.exceptions.RequestException as e: + print(f"✗ Error retrieving data agent: {e}") + return None + + # ============================================================================ + # AI Foundry Operations + # ============================================================================ + + def _build_connection_url(self, connection_name: str) -> str: + """Build the ARM URL for AI Foundry Project connection.""" + return ( + f"https://management.azure.com/subscriptions/{self.ai_foundry_subscription_id}" + f"/resourceGroups/{self.ai_foundry_resource_group}" + f"/providers/Microsoft.CognitiveServices/accounts/{self.ai_foundry_account_name}" + f"/projects/{self.ai_foundry_project_name}" + f"/connections/{connection_name}" + f"?api-version={AI_FOUNDRY_API_VERSION}" + ) + + # ============================================================================ + # Fabric Data Agent Operations + + def create_foundry_to_fabric_connection(self, data_agent_artifact_id: str) -> str: + """Create a Connected Resource connection in AI Foundry Project to Fabric.""" + connection_name = "fabric-data-agent-connection" + connection_url = self._build_connection_url(connection_name) + + connection_payload = { + "properties": { + "category": "CustomKeys", + "authType": "CustomKeys", + "credentials": { + "keys": { + "workspace-id": self.fabric_workspace_id, + "artifact-id": data_agent_artifact_id, + "type": "fabric_dataagent" + } + }, + } + } + + response = requests.put(connection_url, headers=self._get_arm_headers(), json=connection_payload) + if not response.ok: + raise Exception(f"Failed to create connection: {response.status_code} - {response.text}") + + print(f"✓ Created Fabric connection: {connection_name}") + + # Return full connection resource ID for agent creation + connection_id = ( + f"/subscriptions/{self.ai_foundry_subscription_id}" + f"/resourceGroups/{self.ai_foundry_resource_group}" + f"/providers/Microsoft.CognitiveServices/accounts/{self.ai_foundry_account_name}" + f"/projects/{self.ai_foundry_project_name}" + f"/connections/{connection_name}" + ) + return connection_id + + def create_ai_foundry_agent(self, connection_id: str, model_deployment_name: str) -> str: + """Create an AI Foundry Agent using the Fabric connection as knowledge source.""" + from azure.ai.agents.models import FabricTool + + print("Available model deployments in the project:") + deployments = self.ai_client.deployments.list() # .deployments is the property for model deployments + for d in deployments: + print(f"Name: {d.name}, Model: {d.modelName}, Version: {d.modelVersion}, Type: {d.type}") + + # Initialize an Agent Fabric tool and add the connection id + fabric = FabricTool(connection_id=connection_id) + + # Create an Agent with the Fabric tool + agent = self.ai_client.agents.create_agent( + model=model_deployment_name, + name="fabric-data-agent", + instructions="You are a helpful assistant with access to NYC taxi data through Fabric Data Agent. Help users answer questions about taxi ridership.", + tools=fabric.definitions + ) + + print(f"✓ Created AI Foundry Agent with ID: {agent.id}") + return agent.id + + +# ============================================================================ +# Main Execution +# ============================================================================ + +def validate_environment() -> Dict[str, str]: + """Validate and return required environment variables.""" + required_vars = { + "FABRIC_WORKSPACE_ID": os.getenv("FABRIC_WORKSPACE_ID", ""), + "AI_FOUNDRY_SUBSCRIPTION_ID": os.getenv("AI_FOUNDRY_SUBSCRIPTION_ID", ""), + "AI_FOUNDRY_RESOURCE_GROUP": os.getenv("AI_FOUNDRY_RESOURCE_GROUP", ""), + "AI_FOUNDRY_ACCOUNT_NAME": os.getenv("AI_FOUNDRY_ACCOUNT_NAME", ""), + "AI_FOUNDRY_PROJECT_NAME": os.getenv("AI_FOUNDRY_PROJECT_NAME", ""), + "AI_FOUNDRY_MODEL_DEPLOYMENT_NAME": os.getenv("AI_FOUNDRY_MODEL_DEPLOYMENT_NAME", "gpt-4o"), + } + + missing_vars = [key for key, value in required_vars.items() if not value] + if missing_vars: + print("✗ Error: Required environment variables not set:") + for var in missing_vars: + print(f" - {var}") + sys.exit(1) + + return required_vars + + +def main(): + """Main execution flow.""" + print("Starting Fabric-Foundry integration automation...") + print("=" * 60) + + # Validate environment + env_vars = validate_environment() + + # Initialize integration + try: + integration = FabricFoundryIntegration( + fabric_workspace_id=env_vars["FABRIC_WORKSPACE_ID"], + ai_foundry_subscription_id=env_vars["AI_FOUNDRY_SUBSCRIPTION_ID"], + ai_foundry_resource_group=env_vars["AI_FOUNDRY_RESOURCE_GROUP"], + ai_foundry_account_name=env_vars["AI_FOUNDRY_ACCOUNT_NAME"], + ai_foundry_project_name=env_vars["AI_FOUNDRY_PROJECT_NAME"], + ) + except Exception as e: + print(f"✗ Error initializing integration: {e}") + sys.exit(1) + + try: + # Step 1: Upload notebook + print("\n[Step 1/6] Uploading notebook...") + notebook_path = "create_fabric_data_agent.ipynb" + notebook_id = integration.upload_notebook(notebook_path, "create_fabric_data_agent") + + # Step 2: Trigger notebook run + print("\n[Step 2/6] Triggering notebook execution...") + job_id = integration.trigger_notebook_run(notebook_id) + + # Step 3: Wait for completion + print("\n[Step 3/6] Waiting for notebook completion...") + if not integration.wait_for_notebook_completion(notebook_id, job_id): + print("✗ Notebook execution failed, aborting") + sys.exit(1) + + # Step 4: Retrieve data agent artifact ID + print("\n[Step 4/6] Retrieving data agent artifact ID...") + data_agent_artifact_id = integration.get_data_agent_artifact_id(FABRIC_DATA_AGENT_NAME) + if not data_agent_artifact_id: + print("✗ Could not retrieve data agent artifact ID, aborting") + sys.exit(1) + + # Step 5: Create Fabric connection in AI Foundry Project + print("\n[Step 5/6] Creating AI Foundry connection") + connection_id = integration.create_foundry_to_fabric_connection(data_agent_artifact_id) + + # Step 6: Create AI Foundry Agent + print("\n[Step 6/6] Creating AI Foundry agent") + agent_id = integration.create_ai_foundry_agent( + connection_id, + env_vars["AI_FOUNDRY_MODEL_DEPLOYMENT_NAME"] + ) + + # Success summary + print("\n" + "=" * 60) + print("✓ Integration complete!") + print(f" - Data Agent Artifact ID: {data_agent_artifact_id}") + print(f" - Connection ID: {connection_id}") + print(f" - Agent ID: {agent_id}") + print("=" * 60) + + except KeyboardInterrupt: + print("\n\n✗ Process interrupted by user") + sys.exit(1) + except Exception as e: + print(f"\n✗ Unexpected error: {e}") + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/guides/fabric_foundry/create_fabric_data_agent.ipynb b/guides/fabric_foundry/create_fabric_data_agent.ipynb new file mode 100644 index 00000000..0bcf3515 --- /dev/null +++ b/guides/fabric_foundry/create_fabric_data_agent.ipynb @@ -0,0 +1,77 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "2d685538", + "metadata": {}, + "source": [ + "# Create Fabric Data Agent\n", + "Uses NYC taxi data. \n", + "Note: Fabric data agent sdk cannot be run outside of Microsoft Fabric." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5e4cee85", + "metadata": {}, + "outputs": [], + "source": [ + "%pip install fabric-data-agent-sdk" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5fb5751b", + "metadata": {}, + "outputs": [], + "source": [ + "from fabric.dataagent.client import (\n", + " FabricDataAgentManagement,\n", + " create_data_agent,\n", + " delete_data_agent,\n", + ")\n", + "\n", + "# Hardcoded for now\n", + "data_agent_name = \"data_agent_automation_sample\"\n", + "lakehouse_name = \"NYCTaxi_756\"\n", + "\n", + "# create DataAgent\n", + "data_agent = create_data_agent(data_agent_name)\n", + "\n", + "data_agent.update_configuration(\n", + " instructions=\"You are a helpful assistant, help users with their questions\",\n", + ")\n", + "\n", + "# datasource type could be: lakehouse, kqldatabase, warehouse or semanticmodel\n", + "data_agent.add_datasource(lakehouse_name, type=\"lakehouse\")\n", + "\n", + "# Publish\n", + "data_agent.publish()\n", + "\n", + "datasource = data_agent.get_datasources()[0]\n", + "\n", + "# Add data sources\n", + "datasource.select(\"dbo\", \"year_2017\")\n", + "datasource.select(\"dbo\", \"year_2018\")\n", + "datasource.select(\"dbo\", \"year_2019\")\n", + "datasource.select(\"dbo\", \"year_2020\")\n", + "\n", + "# Add additoinal instructions to the datasource\n", + "ds_notes = \"When asked about taxi ridership information use tables in dbo with year_ prefix to retrieve detailed information\"\n", + "datasource.update_configuration(instructions=ds_notes)\n", + "\n", + "# Get configuration\n", + "datasource.get_configuration()" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/guides/fabric_foundry/requirements.txt b/guides/fabric_foundry/requirements.txt new file mode 100644 index 00000000..d9e6b379 --- /dev/null +++ b/guides/fabric_foundry/requirements.txt @@ -0,0 +1,5 @@ +azure-identity>=1.15.0 +requests>=2.31.0 +python-dotenv>=1.0.0 +azure-ai-projects>=1.0.0b1 +azure-ai-agents>=1.2.0b6