From 52ae9c9f4dadbab24903495cbcf2d93565c17be3 Mon Sep 17 00:00:00 2001 From: Lace Lofranco Date: Mon, 10 Nov 2025 22:54:23 +1000 Subject: [PATCH 01/11] feat: fix bug in connection creation --- guides/fabric_foundry/README.md | 77 +++++ .../automate_fabric_foundry_integration.py | 283 ++++++++++++++++++ .../create_fabric_data_agent.ipynb | 77 +++++ guides/fabric_foundry/requirements.txt | 2 + 4 files changed, 439 insertions(+) create mode 100644 guides/fabric_foundry/README.md create mode 100644 guides/fabric_foundry/automate_fabric_foundry_integration.py create mode 100644 guides/fabric_foundry/create_fabric_data_agent.ipynb create mode 100644 guides/fabric_foundry/requirements.txt diff --git a/guides/fabric_foundry/README.md b/guides/fabric_foundry/README.md new file mode 100644 index 00000000..e777af1d --- /dev/null +++ b/guides/fabric_foundry/README.md @@ -0,0 +1,77 @@ +# Fabric Foundry Integration + +This guide demonstrates how to automate the integration between Microsoft Fabric Data Agents and Azure AI Foundry. + +## Overview + +The automation script performs the following steps: + +1. **Upload Notebook**: Uploads the `create_fabric_data_agent.ipynb` notebook to a specified Fabric workspace +1. **Execute Notebook**: Triggers the notebook to run, which creates a Fabric Data Agent named "data_agent_automation_sample" +1. **Retrieve Artifact ID**: Gets the artifact ID of the newly created data agent +1. **Create Connection**: Creates a Connected Resource connection in AI Foundry pointing to the Fabric Data Agent +1. **Create Agent**: Creates an AI Foundry Agent that uses the Fabric Data Agent as a knowledge source + +## Prerequisites + +- Azure subscription with access to: + - Microsoft Fabric workspace + - Azure AI Foundry project (not Hub - must be a Project resource) +- Fabric workspace must have: + - NYC Taxi lakehouse named "NYCTaxi_756" with tables: year_2017, year_2018, year_2019, year_2020 +- Azure CLI or DefaultAzureCredential authentication configured +- Python 3.8 or higher + +## Setup + +1. Install dependencies: + +```bash +pip install -r requirements.txt +``` + +1. Set required environment variables: + +```bash +export FABRIC_WORKSPACE_ID="your-fabric-workspace-id" +export AI_FOUNDRY_SUBSCRIPTION_ID="your-subscription-id" +export AI_FOUNDRY_RESOURCE_GROUP="your-resource-group" +export AI_FOUNDRY_ACCOUNT_NAME="your-ai-foundry-account-name" +export AI_FOUNDRY_PROJECT_NAME="your-ai-foundry-project-name" +``` + +## Usage + +Run the automation script: + +```bash +python automate_fabric_foundry_integration.py +``` + +The script will: + +- Authenticate using DefaultAzureCredential +- Upload and execute the notebook in Fabric +- Wait for completion +- Create the necessary connections and agent in AI Foundry Project using ARM REST APIs +- Output the created resource IDs + +## Implementation Notes + +- Uses Azure Resource Manager REST API (`Microsoft.CognitiveServices/accounts/projects/connections@2025-06-01`) for creating connections +- Uses Azure Resource Manager REST API for creating agents in AI Foundry Projects +- No Python SDK dependency for AI Foundry - direct REST API calls for maximum compatibility + +## Files + +- `create_fabric_data_agent.ipynb`: Notebook that creates the Fabric Data Agent (must be run in Fabric) +- `automate_fabric_foundry_integration.py`: Main automation script +- `requirements.txt`: Python dependencies +- `README.md`: This file + +## Notes + +- The Fabric Data Agent SDK can only run inside Microsoft Fabric environments +- The automation uses the Fabric REST API to upload and trigger notebook execution +- The script waits up to 10 minutes for notebook completion +- All resources are created with simple configurations suitable for POC/demo purposes diff --git a/guides/fabric_foundry/automate_fabric_foundry_integration.py b/guides/fabric_foundry/automate_fabric_foundry_integration.py new file mode 100644 index 00000000..53b61ba1 --- /dev/null +++ b/guides/fabric_foundry/automate_fabric_foundry_integration.py @@ -0,0 +1,283 @@ +#!/usr/bin/env python3 +""" +Automate connecting AI Foundry to Microsoft Fabric via Fabric Data Agent. + +This script: +1. Uploads the create_fabric_data_agent notebook to a Fabric workspace +2. Triggers notebook execution to create the data agent +3. Retrieves the artifact ID of the created data agent +4. Creates a Connected Resource connection in AI Foundry to Fabric +5. Creates an AI Foundry Agent using this connection as a knowledge source +""" + +import os +import time +from typing import Optional +from azure.identity import DefaultAzureCredential +import requests + + +class FabricFoundryIntegration: + def __init__( + self, + fabric_workspace_id: str, + ai_foundry_subscription_id: str, + ai_foundry_resource_group: str, + ai_foundry_account_name: str, + ai_foundry_project_name: str, + ): + self.fabric_workspace_id = fabric_workspace_id + self.ai_foundry_subscription_id = ai_foundry_subscription_id + self.ai_foundry_resource_group = ai_foundry_resource_group + self.ai_foundry_account_name = ai_foundry_account_name + self.ai_foundry_project_name = ai_foundry_project_name + self.credential = DefaultAzureCredential() + self.fabric_token = None + + def get_fabric_token(self) -> str: + """Get authentication token for Fabric APIs.""" + token = self.credential.get_token("https://api.fabric.microsoft.com/.default") + return token.token + + def upload_notebook(self, notebook_path: str, notebook_name: str) -> str: + """Upload notebook to Fabric workspace and return artifact ID.""" + self.fabric_token = self.get_fabric_token() + headers = { + "Authorization": f"Bearer {self.fabric_token}", + "Content-Type": "application/json", + } + + with open(notebook_path, "r") as f: + notebook_content = f.read() + + # Create notebook in Fabric workspace + create_url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.fabric_workspace_id}/notebooks" + payload = { + "displayName": notebook_name, + "definition": { + "format": "ipynb", + "parts": [ + { + "path": "notebook-content.ipynb", + "payload": notebook_content, + "payloadType": "InlineBase64", + } + ], + }, + } + + response = requests.post(create_url, headers=headers, json=payload) + response.raise_for_status() + notebook_id = response.json()["id"] + print(f"✓ Uploaded notebook '{notebook_name}' with ID: {notebook_id}") + return notebook_id + + def trigger_notebook_run(self, notebook_id: str) -> str: + """Trigger notebook execution and return job ID.""" + headers = { + "Authorization": f"Bearer {self.fabric_token}", + "Content-Type": "application/json", + } + + run_url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.fabric_workspace_id}/notebooks/{notebook_id}/jobs/instances?jobType=RunNotebook" + + response = requests.post(run_url, headers=headers) + response.raise_for_status() + job_id = response.json()["id"] + print(f"✓ Triggered notebook run with job ID: {job_id}") + return job_id + + def wait_for_notebook_completion(self, notebook_id: str, job_id: str, timeout: int = 600) -> bool: + """Wait for notebook execution to complete.""" + headers = {"Authorization": f"Bearer {self.fabric_token}"} + status_url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.fabric_workspace_id}/notebooks/{notebook_id}/jobs/instances/{job_id}" + + start_time = time.time() + while time.time() - start_time < timeout: + response = requests.get(status_url, headers=headers) + response.raise_for_status() + status = response.json()["status"] + + if status == "Completed": + print("✓ Notebook execution completed successfully") + return True + elif status in ["Failed", "Cancelled"]: + print(f"✗ Notebook execution {status}") + return False + + print(f" Notebook status: {status}, waiting...") + time.sleep(10) + + print("✗ Timeout waiting for notebook completion") + return False + + def get_data_agent_artifact_id(self, data_agent_name: str = "data_agent_automation_sample") -> Optional[str]: + """Retrieve the artifact ID of the created Fabric Data Agent.""" + headers = {"Authorization": f"Bearer {self.fabric_token}"} + + # List all DataAgent artifacts in the workspace + list_url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.fabric_workspace_id}/dataAgents" + + response = requests.get(list_url, headers=headers) + response.raise_for_status() + data_agents = response.json().get("value", []) + + for agent in data_agents: + if agent.get("displayName") == data_agent_name: + artifact_id = agent["id"] + print(f"✓ Found data agent '{data_agent_name}' with artifact ID: {artifact_id}") + return artifact_id + + print(f"✗ Data agent '{data_agent_name}' not found") + return None + + def create_fabric_connection(self, data_agent_artifact_id: str, ai_foundry_account_name: str) -> str: + """Create a Connected Resource connection in AI Foundry Project to Fabric.""" + connection_name = "fabric-data-agent-connection" + + # Get ARM token + arm_token = self.credential.get_token("https://management.azure.com/.default") + headers = { + "Authorization": f"Bearer {arm_token.token}", + "Content-Type": "application/json", + } + + # Construct AI Foundry Project connection endpoint + connection_url = ( + f"https://management.azure.com/subscriptions/{self.ai_foundry_subscription_id}" + f"/resourceGroups/{self.ai_foundry_resource_group}" + f"/providers/Microsoft.CognitiveServices/accounts/{self.ai_foundry_account_name}" + f"/projects/{self.ai_foundry_project_name}" + f"/connections/{connection_name}" + f"?api-version=2025-06-01" + ) + + # Connection payload for Fabric Data Agent + connection_payload = { + "properties": { + "category": "FabricDataAgent", + "target": f"https://api.fabric.microsoft.com/v1/workspaces/{self.fabric_workspace_id}/dataAgents/{data_agent_artifact_id}", + "authType": "AAD", + "metadata": { + "workspaceId": self.fabric_workspace_id, + "artifactId": data_agent_artifact_id, + "apiType": "FabricDataAgent", + }, + } + } + + response = requests.put(connection_url, headers=headers, json=connection_payload) + response.raise_for_status() + print(f"✓ Created Fabric connection: {connection_name}") + return connection_name + + def create_ai_foundry_agent(self, connection_name: str) -> str: + """Create an AI Foundry Agent using the Fabric connection as knowledge source.""" + + # Get ARM token + arm_token = self.credential.get_token("https://management.azure.com/.default") + headers = { + "Authorization": f"Bearer {arm_token.token}", + "Content-Type": "application/json", + } + + agent_name = "fabric-data-agent" + + # Construct AI Foundry Project agent endpoint + agent_url = ( + f"https://management.azure.com/subscriptions/{self.ai_foundry_subscription_id}" + f"/resourceGroups/{self.ai_foundry_resource_group}" + f"/providers/Microsoft.CognitiveServices/accounts/{self.ai_foundry_account_name}" + f"/projects/{self.ai_foundry_project_name}" + f"/agents/{agent_name}" + f"?api-version=2025-06-01" + ) + + # Agent payload with Fabric Data Agent connection + agent_payload = { + "properties": { + "model": "gpt-4o", + "instructions": "You are a helpful assistant with access to NYC taxi data through Fabric Data Agent. Help users answer questions about taxi ridership.", + "tools": [{"type": "code_interpreter"}], + "toolResources": { + "codeInterpreter": { + "dataSources": [ + { + "type": "fabric_data_agent", + "connectionId": connection_name, + } + ] + } + }, + } + } + + response = requests.put(agent_url, headers=headers, json=agent_payload) + response.raise_for_status() + agent_id = response.json().get("id", agent_name) + print(f"✓ Created AI Foundry Agent with ID: {agent_id}") + return agent_id + + +def main(): + """Main execution flow.""" + # Configuration - these should be provided as environment variables or arguments + FABRIC_WORKSPACE_ID = os.getenv("FABRIC_WORKSPACE_ID", "") + AI_FOUNDRY_SUBSCRIPTION_ID = os.getenv("AI_FOUNDRY_SUBSCRIPTION_ID", "") + AI_FOUNDRY_RESOURCE_GROUP = os.getenv("AI_FOUNDRY_RESOURCE_GROUP", "") + AI_FOUNDRY_ACCOUNT_NAME = os.getenv("AI_FOUNDRY_ACCOUNT_NAME", "") + AI_FOUNDRY_PROJECT_NAME = os.getenv("AI_FOUNDRY_PROJECT_NAME", "") + + if not all([FABRIC_WORKSPACE_ID, AI_FOUNDRY_SUBSCRIPTION_ID, AI_FOUNDRY_RESOURCE_GROUP, AI_FOUNDRY_ACCOUNT_NAME, AI_FOUNDRY_PROJECT_NAME]): + print("Error: Required environment variables not set:") + print(" - FABRIC_WORKSPACE_ID") + print(" - AI_FOUNDRY_SUBSCRIPTION_ID") + print(" - AI_FOUNDRY_RESOURCE_GROUP") + print(" - AI_FOUNDRY_ACCOUNT_NAME") + print(" - AI_FOUNDRY_PROJECT_NAME") + return + + integration = FabricFoundryIntegration( + fabric_workspace_id=FABRIC_WORKSPACE_ID, + ai_foundry_subscription_id=AI_FOUNDRY_SUBSCRIPTION_ID, + ai_foundry_resource_group=AI_FOUNDRY_RESOURCE_GROUP, + ai_foundry_account_name=AI_FOUNDRY_ACCOUNT_NAME, + ai_foundry_project_name=AI_FOUNDRY_PROJECT_NAME, + ) + + print("Starting Fabric-Foundry integration automation...") + print("=" * 60) + + # Step 1: Upload notebook + notebook_path = "create_fabric_data_agent.ipynb" + notebook_id = integration.upload_notebook(notebook_path, "create_fabric_data_agent") + + # Step 2: Trigger notebook run + job_id = integration.trigger_notebook_run(notebook_id) + + # Wait for completion + if not integration.wait_for_notebook_completion(notebook_id, job_id): + print("✗ Notebook execution failed, aborting") + return + + # Step 3: Retrieve data agent artifact ID + data_agent_artifact_id = integration.get_data_agent_artifact_id() + if not data_agent_artifact_id: + print("✗ Could not retrieve data agent artifact ID, aborting") + return + + # Step 4: Create Fabric connection in AI Foundry Project + connection_name = integration.create_fabric_connection(data_agent_artifact_id, AI_FOUNDRY_ACCOUNT_NAME) + + # Step 5: Create AI Foundry Agent + agent_id = integration.create_ai_foundry_agent(connection_name) + + print("=" * 60) + print("✓ Integration complete!") + print(f" - Data Agent Artifact ID: {data_agent_artifact_id}") + print(f" - Connection Name: {connection_name}") + print(f" - Agent ID: {agent_id}") + + +if __name__ == "__main__": + main() diff --git a/guides/fabric_foundry/create_fabric_data_agent.ipynb b/guides/fabric_foundry/create_fabric_data_agent.ipynb new file mode 100644 index 00000000..0bcf3515 --- /dev/null +++ b/guides/fabric_foundry/create_fabric_data_agent.ipynb @@ -0,0 +1,77 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "2d685538", + "metadata": {}, + "source": [ + "# Create Fabric Data Agent\n", + "Uses NYC taxi data. \n", + "Note: Fabric data agent sdk cannot be run outside of Microsoft Fabric." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5e4cee85", + "metadata": {}, + "outputs": [], + "source": [ + "%pip install fabric-data-agent-sdk" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5fb5751b", + "metadata": {}, + "outputs": [], + "source": [ + "from fabric.dataagent.client import (\n", + " FabricDataAgentManagement,\n", + " create_data_agent,\n", + " delete_data_agent,\n", + ")\n", + "\n", + "# Hardcoded for now\n", + "data_agent_name = \"data_agent_automation_sample\"\n", + "lakehouse_name = \"NYCTaxi_756\"\n", + "\n", + "# create DataAgent\n", + "data_agent = create_data_agent(data_agent_name)\n", + "\n", + "data_agent.update_configuration(\n", + " instructions=\"You are a helpful assistant, help users with their questions\",\n", + ")\n", + "\n", + "# datasource type could be: lakehouse, kqldatabase, warehouse or semanticmodel\n", + "data_agent.add_datasource(lakehouse_name, type=\"lakehouse\")\n", + "\n", + "# Publish\n", + "data_agent.publish()\n", + "\n", + "datasource = data_agent.get_datasources()[0]\n", + "\n", + "# Add data sources\n", + "datasource.select(\"dbo\", \"year_2017\")\n", + "datasource.select(\"dbo\", \"year_2018\")\n", + "datasource.select(\"dbo\", \"year_2019\")\n", + "datasource.select(\"dbo\", \"year_2020\")\n", + "\n", + "# Add additoinal instructions to the datasource\n", + "ds_notes = \"When asked about taxi ridership information use tables in dbo with year_ prefix to retrieve detailed information\"\n", + "datasource.update_configuration(instructions=ds_notes)\n", + "\n", + "# Get configuration\n", + "datasource.get_configuration()" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/guides/fabric_foundry/requirements.txt b/guides/fabric_foundry/requirements.txt new file mode 100644 index 00000000..b43aa2ae --- /dev/null +++ b/guides/fabric_foundry/requirements.txt @@ -0,0 +1,2 @@ +azure-identity>=1.15.0 +requests>=2.31.0 From 14f944fdcc5c0f2a4aca6a72020d4e85f42a545f Mon Sep 17 00:00:00 2001 From: Lace Lofranco Date: Mon, 10 Nov 2025 23:13:52 +1000 Subject: [PATCH 02/11] feat: fix bug in connection creation --- .../automate_fabric_foundry_integration.py | 37 ++++++++++++------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/guides/fabric_foundry/automate_fabric_foundry_integration.py b/guides/fabric_foundry/automate_fabric_foundry_integration.py index 53b61ba1..43a1d69d 100644 --- a/guides/fabric_foundry/automate_fabric_foundry_integration.py +++ b/guides/fabric_foundry/automate_fabric_foundry_integration.py @@ -115,23 +115,32 @@ def get_data_agent_artifact_id(self, data_agent_name: str = "data_agent_automati """Retrieve the artifact ID of the created Fabric Data Agent.""" headers = {"Authorization": f"Bearer {self.fabric_token}"} - # List all DataAgent artifacts in the workspace - list_url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.fabric_workspace_id}/dataAgents" + # List all items in the workspace + list_url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.fabric_workspace_id}/items" response = requests.get(list_url, headers=headers) response.raise_for_status() - data_agents = response.json().get("value", []) + items = response.json().get("value", []) - for agent in data_agents: - if agent.get("displayName") == data_agent_name: - artifact_id = agent["id"] + # Find the DataAgent item by name and type + for item in items: + if item.get("displayName") == data_agent_name: # and item.get("type") == "DataAgent" + item_id = item["id"] + + # Get the specific item to retrieve full details + get_item_url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.fabric_workspace_id}/items/{item_id}" + item_response = requests.get(get_item_url, headers=headers) + item_response.raise_for_status() + item_details = item_response.json() + + artifact_id = item_details["id"] print(f"✓ Found data agent '{data_agent_name}' with artifact ID: {artifact_id}") return artifact_id print(f"✗ Data agent '{data_agent_name}' not found") return None - def create_fabric_connection(self, data_agent_artifact_id: str, ai_foundry_account_name: str) -> str: + def create_fabric_connection(self, data_agent_artifact_id: str) -> str: """Create a Connected Resource connection in AI Foundry Project to Fabric.""" connection_name = "fabric-data-agent-connection" @@ -157,12 +166,14 @@ def create_fabric_connection(self, data_agent_artifact_id: str, ai_foundry_accou "properties": { "category": "FabricDataAgent", "target": f"https://api.fabric.microsoft.com/v1/workspaces/{self.fabric_workspace_id}/dataAgents/{data_agent_artifact_id}", - "authType": "AAD", - "metadata": { - "workspaceId": self.fabric_workspace_id, - "artifactId": data_agent_artifact_id, - "apiType": "FabricDataAgent", - }, + "authType": "CustomKeys", + "credentials": { + "keys": { + "workspace-id": self.fabric_workspace_id, + "artifact-id": data_agent_artifact_id, + "type": "fabric_dataagent" + } + } } } From bac60c186aca122b7577e2f17d503cefd44fa3fa Mon Sep 17 00:00:00 2001 From: Lace Lofranco Date: Mon, 10 Nov 2025 23:16:10 +1000 Subject: [PATCH 03/11] feat: bug fixing --- .../fabric_foundry/automate_fabric_foundry_integration.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/guides/fabric_foundry/automate_fabric_foundry_integration.py b/guides/fabric_foundry/automate_fabric_foundry_integration.py index 43a1d69d..bf1285ce 100644 --- a/guides/fabric_foundry/automate_fabric_foundry_integration.py +++ b/guides/fabric_foundry/automate_fabric_foundry_integration.py @@ -12,6 +12,7 @@ import os import time +import base64 from typing import Optional from azure.identity import DefaultAzureCredential import requests @@ -47,8 +48,8 @@ def upload_notebook(self, notebook_path: str, notebook_name: str) -> str: "Content-Type": "application/json", } - with open(notebook_path, "r") as f: - notebook_content = f.read() + with open(notebook_path, "rb") as f: + notebook_content = base64.b64encode(f.read()).decode("utf-8") # Create notebook in Fabric workspace create_url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.fabric_workspace_id}/notebooks" @@ -278,7 +279,7 @@ def main(): return # Step 4: Create Fabric connection in AI Foundry Project - connection_name = integration.create_fabric_connection(data_agent_artifact_id, AI_FOUNDRY_ACCOUNT_NAME) + connection_name = integration.create_fabric_connection(data_agent_artifact_id) # Step 5: Create AI Foundry Agent agent_id = integration.create_ai_foundry_agent(connection_name) From ff5b9e76173fd1440ecc21e2215c7842536f0c28 Mon Sep 17 00:00:00 2001 From: Lace Lofranco Date: Mon, 10 Nov 2025 23:25:19 +1000 Subject: [PATCH 04/11] feat: bug fixing --- guides/fabric_foundry/automate_fabric_foundry_integration.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/guides/fabric_foundry/automate_fabric_foundry_integration.py b/guides/fabric_foundry/automate_fabric_foundry_integration.py index bf1285ce..16102525 100644 --- a/guides/fabric_foundry/automate_fabric_foundry_integration.py +++ b/guides/fabric_foundry/automate_fabric_foundry_integration.py @@ -17,6 +17,7 @@ from azure.identity import DefaultAzureCredential import requests +FABRIC_DATA_AGENT_NAME = "data_agent_automation_sample" class FabricFoundryIntegration: def __init__( @@ -112,7 +113,7 @@ def wait_for_notebook_completion(self, notebook_id: str, job_id: str, timeout: i print("✗ Timeout waiting for notebook completion") return False - def get_data_agent_artifact_id(self, data_agent_name: str = "data_agent_automation_sample") -> Optional[str]: + def get_data_agent_artifact_id(self, data_agent_name: str) -> Optional[str]: """Retrieve the artifact ID of the created Fabric Data Agent.""" headers = {"Authorization": f"Bearer {self.fabric_token}"} @@ -273,7 +274,7 @@ def main(): return # Step 3: Retrieve data agent artifact ID - data_agent_artifact_id = integration.get_data_agent_artifact_id() + data_agent_artifact_id = integration.get_data_agent_artifact_id(FABRIC_DATA_AGENT_NAME) if not data_agent_artifact_id: print("✗ Could not retrieve data agent artifact ID, aborting") return From fb306c8b653a4932f7602dc09e8c7c593830c914 Mon Sep 17 00:00:00 2001 From: Lace Lofranco Date: Tue, 11 Nov 2025 12:13:27 +1000 Subject: [PATCH 05/11] feat: bug fixing --- .../automate_fabric_foundry_integration.py | 67 +++++++++++++++++-- guides/fabric_foundry/requirements.txt | 1 + 2 files changed, 63 insertions(+), 5 deletions(-) diff --git a/guides/fabric_foundry/automate_fabric_foundry_integration.py b/guides/fabric_foundry/automate_fabric_foundry_integration.py index 16102525..713b1ab7 100644 --- a/guides/fabric_foundry/automate_fabric_foundry_integration.py +++ b/guides/fabric_foundry/automate_fabric_foundry_integration.py @@ -14,9 +14,14 @@ import time import base64 from typing import Optional +from pathlib import Path from azure.identity import DefaultAzureCredential import requests +# Load environment variables from .env file +from dotenv import load_dotenv +load_dotenv() + FABRIC_DATA_AGENT_NAME = "data_agent_automation_sample" class FabricFoundryIntegration: @@ -49,13 +54,27 @@ def upload_notebook(self, notebook_path: str, notebook_name: str) -> str: "Content-Type": "application/json", } + # Check if notebook already exists + list_url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.fabric_workspace_id}/items" + list_response = requests.get(list_url, headers=headers) + list_response.raise_for_status() + items = list_response.json().get("value", []) + + for item in items: + if item.get("displayName") == notebook_name and item.get("type") == "Notebook": + notebook_id = item["id"] + print(f"✓ Found existing notebook '{notebook_name}' with ID: {notebook_id}") + return notebook_id + + # Notebook doesn't exist, create it with open(notebook_path, "rb") as f: notebook_content = base64.b64encode(f.read()).decode("utf-8") - # Create notebook in Fabric workspace - create_url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.fabric_workspace_id}/notebooks" + # Create notebook item in Fabric workspace + create_url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.fabric_workspace_id}/items" payload = { "displayName": notebook_name, + "type": "Notebook", "definition": { "format": "ipynb", "parts": [ @@ -69,8 +88,17 @@ def upload_notebook(self, notebook_path: str, notebook_name: str) -> str: } response = requests.post(create_url, headers=headers, json=payload) + if not response.ok: + print(f"✗ Error creating notebook: {response.status_code}") + print(f" Response: {response.text}") response.raise_for_status() - notebook_id = response.json()["id"] + + response_data = response.json() + if not response_data or "id" not in response_data: + print(f"✗ Unexpected response format: {response.text}") + raise ValueError("Response missing 'id' field") + + notebook_id = response_data["id"] print(f"✓ Uploaded notebook '{notebook_name}' with ID: {notebook_id}") return notebook_id @@ -84,8 +112,37 @@ def trigger_notebook_run(self, notebook_id: str) -> str: run_url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.fabric_workspace_id}/notebooks/{notebook_id}/jobs/instances?jobType=RunNotebook" response = requests.post(run_url, headers=headers) + if not response.ok: + print(f"✗ Error triggering notebook: {response.status_code}") + print(f" Response: {response.text}") response.raise_for_status() - job_id = response.json()["id"] + + # For 202 Accepted, the job ID might be in Location header + if response.status_code == 202: + location = response.headers.get("Location", "") + if location: + # Extract job ID from Location header + job_id = location.split("/")[-1] + print(f"✓ Triggered notebook run with job ID: {job_id}") + return job_id + else: + print(f"✓ Notebook run triggered (202 Accepted), checking for running jobs...") + # Get the most recent job + time.sleep(2) # Give it a moment to register + jobs_url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.fabric_workspace_id}/notebooks/{notebook_id}/jobs/instances" + jobs_response = requests.get(jobs_url, headers=headers) + jobs_response.raise_for_status() + jobs = jobs_response.json().get("value", []) + if jobs: + job_id = jobs[0]["id"] + print(f"✓ Found running job with ID: {job_id}") + return job_id + else: + raise ValueError("Notebook run triggered but no job ID found") + + # Normal response with body + response_data = response.json() + job_id = response_data["id"] print(f"✓ Triggered notebook run with job ID: {job_id}") return job_id @@ -126,7 +183,7 @@ def get_data_agent_artifact_id(self, data_agent_name: str) -> Optional[str]: # Find the DataAgent item by name and type for item in items: - if item.get("displayName") == data_agent_name: # and item.get("type") == "DataAgent" + if item.get("displayName") == data_agent_name: # and item.get("type") == "aiskills" item_id = item["id"] # Get the specific item to retrieve full details diff --git a/guides/fabric_foundry/requirements.txt b/guides/fabric_foundry/requirements.txt index b43aa2ae..c3dd1e9a 100644 --- a/guides/fabric_foundry/requirements.txt +++ b/guides/fabric_foundry/requirements.txt @@ -1,2 +1,3 @@ azure-identity>=1.15.0 requests>=2.31.0 +python-dotenv>=1.0.0 From 9e84064664088b33b1bd00b301fffa47188eb464 Mon Sep 17 00:00:00 2001 From: Lace Lofranco Date: Tue, 11 Nov 2025 14:03:04 +1000 Subject: [PATCH 06/11] feat: bug fixing --- .../automate_fabric_foundry_integration.py | 456 ++++++++++-------- 1 file changed, 261 insertions(+), 195 deletions(-) diff --git a/guides/fabric_foundry/automate_fabric_foundry_integration.py b/guides/fabric_foundry/automate_fabric_foundry_integration.py index 713b1ab7..44219080 100644 --- a/guides/fabric_foundry/automate_fabric_foundry_integration.py +++ b/guides/fabric_foundry/automate_fabric_foundry_integration.py @@ -11,20 +11,27 @@ """ import os +import sys import time import base64 -from typing import Optional -from pathlib import Path +from typing import Optional, Dict, Any from azure.identity import DefaultAzureCredential import requests - -# Load environment variables from .env file from dotenv import load_dotenv + +# Load environment variables load_dotenv() +# Constants FABRIC_DATA_AGENT_NAME = "data_agent_automation_sample" +AI_FOUNDRY_API_VERSION = "2025-06-01" +NOTEBOOK_POLL_INTERVAL = 10 +NOTEBOOK_TIMEOUT = 600 + class FabricFoundryIntegration: + """Handles integration between Microsoft Fabric and Azure AI Foundry.""" + def __init__( self, fabric_workspace_id: str, @@ -39,38 +46,66 @@ def __init__( self.ai_foundry_account_name = ai_foundry_account_name self.ai_foundry_project_name = ai_foundry_project_name self.credential = DefaultAzureCredential() - self.fabric_token = None - - def get_fabric_token(self) -> str: - """Get authentication token for Fabric APIs.""" - token = self.credential.get_token("https://api.fabric.microsoft.com/.default") - return token.token + self._fabric_token = None + self._arm_token = None + + # ============================================================================ + # Authentication Helpers + # ============================================================================ + + def _get_fabric_token(self) -> str: + """Get or refresh authentication token for Fabric APIs.""" + if not self._fabric_token: + token = self.credential.get_token("https://api.fabric.microsoft.com/.default") + self._fabric_token = token.token + return self._fabric_token + + def _get_arm_token(self) -> str: + """Get or refresh authentication token for Azure Resource Manager APIs.""" + if not self._arm_token: + token = self.credential.get_token("https://management.azure.com/.default") + self._arm_token = token.token + return self._arm_token + + def _get_fabric_headers(self) -> Dict[str, str]: + """Get headers for Fabric API requests.""" + return { + "Authorization": f"Bearer {self._get_fabric_token()}", + "Content-Type": "application/json", + } - def upload_notebook(self, notebook_path: str, notebook_name: str) -> str: - """Upload notebook to Fabric workspace and return artifact ID.""" - self.fabric_token = self.get_fabric_token() - headers = { - "Authorization": f"Bearer {self.fabric_token}", + def _get_arm_headers(self) -> Dict[str, str]: + """Get headers for ARM API requests.""" + return { + "Authorization": f"Bearer {self._get_arm_token()}", "Content-Type": "application/json", } - # Check if notebook already exists - list_url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.fabric_workspace_id}/items" - list_response = requests.get(list_url, headers=headers) - list_response.raise_for_status() - items = list_response.json().get("value", []) - - for item in items: - if item.get("displayName") == notebook_name and item.get("type") == "Notebook": - notebook_id = item["id"] - print(f"✓ Found existing notebook '{notebook_name}' with ID: {notebook_id}") - return notebook_id - - # Notebook doesn't exist, create it + # ============================================================================ + # Fabric Notebook Operations + # ============================================================================ + + def _find_existing_notebook(self, notebook_name: str) -> Optional[str]: + """Check if notebook with given name already exists in workspace.""" + try: + list_url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.fabric_workspace_id}/items" + response = requests.get(list_url, headers=self._get_fabric_headers()) + response.raise_for_status() + items = response.json().get("value", []) + + for item in items: + if item.get("displayName") == notebook_name and item.get("type") == "Notebook": + return item["id"] + return None + except requests.exceptions.RequestException as e: + print(f"⚠ Warning: Could not check for existing notebook: {e}") + return None + + def _create_notebook(self, notebook_path: str, notebook_name: str) -> str: + """Create a new notebook in Fabric workspace.""" with open(notebook_path, "rb") as f: notebook_content = base64.b64encode(f.read()).decode("utf-8") - # Create notebook item in Fabric workspace create_url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.fabric_workspace_id}/items" payload = { "displayName": notebook_name, @@ -87,58 +122,57 @@ def upload_notebook(self, notebook_path: str, notebook_name: str) -> str: }, } - response = requests.post(create_url, headers=headers, json=payload) + response = requests.post(create_url, headers=self._get_fabric_headers(), json=payload) if not response.ok: - print(f"✗ Error creating notebook: {response.status_code}") - print(f" Response: {response.text}") - response.raise_for_status() + raise Exception(f"Failed to create notebook: {response.status_code} - {response.text}") response_data = response.json() if not response_data or "id" not in response_data: - print(f"✗ Unexpected response format: {response.text}") raise ValueError("Response missing 'id' field") - notebook_id = response_data["id"] + return response_data["id"] + + def upload_notebook(self, notebook_path: str, notebook_name: str) -> str: + """Upload notebook to Fabric workspace, reusing if it already exists.""" + # Check if notebook already exists + existing_id = self._find_existing_notebook(notebook_name) + if existing_id: + print(f"✓ Found existing notebook '{notebook_name}' with ID: {existing_id}") + return existing_id + + # Create new notebook + notebook_id = self._create_notebook(notebook_path, notebook_name) print(f"✓ Uploaded notebook '{notebook_name}' with ID: {notebook_id}") return notebook_id def trigger_notebook_run(self, notebook_id: str) -> str: """Trigger notebook execution and return job ID.""" - headers = { - "Authorization": f"Bearer {self.fabric_token}", - "Content-Type": "application/json", - } - run_url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.fabric_workspace_id}/notebooks/{notebook_id}/jobs/instances?jobType=RunNotebook" - response = requests.post(run_url, headers=headers) + response = requests.post(run_url, headers=self._get_fabric_headers()) if not response.ok: - print(f"✗ Error triggering notebook: {response.status_code}") - print(f" Response: {response.text}") - response.raise_for_status() + raise Exception(f"Failed to trigger notebook: {response.status_code} - {response.text}") - # For 202 Accepted, the job ID might be in Location header + # Handle 202 Accepted response if response.status_code == 202: location = response.headers.get("Location", "") if location: - # Extract job ID from Location header job_id = location.split("/")[-1] print(f"✓ Triggered notebook run with job ID: {job_id}") return job_id - else: - print(f"✓ Notebook run triggered (202 Accepted), checking for running jobs...") - # Get the most recent job - time.sleep(2) # Give it a moment to register - jobs_url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.fabric_workspace_id}/notebooks/{notebook_id}/jobs/instances" - jobs_response = requests.get(jobs_url, headers=headers) - jobs_response.raise_for_status() - jobs = jobs_response.json().get("value", []) - if jobs: - job_id = jobs[0]["id"] - print(f"✓ Found running job with ID: {job_id}") - return job_id - else: - raise ValueError("Notebook run triggered but no job ID found") + + # Fallback: query for most recent job + print(f"✓ Notebook run triggered (202 Accepted), retrieving job ID...") + time.sleep(2) + jobs_url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.fabric_workspace_id}/notebooks/{notebook_id}/jobs/instances" + jobs_response = requests.get(jobs_url, headers=self._get_fabric_headers()) + jobs_response.raise_for_status() + jobs = jobs_response.json().get("value", []) + if jobs: + job_id = jobs[0]["id"] + print(f"✓ Found running job with ID: {job_id}") + return job_id + raise ValueError("Notebook run triggered but no job ID found") # Normal response with body response_data = response.json() @@ -146,81 +180,96 @@ def trigger_notebook_run(self, notebook_id: str) -> str: print(f"✓ Triggered notebook run with job ID: {job_id}") return job_id - def wait_for_notebook_completion(self, notebook_id: str, job_id: str, timeout: int = 600) -> bool: + def wait_for_notebook_completion(self, notebook_id: str, job_id: str) -> bool: """Wait for notebook execution to complete.""" - headers = {"Authorization": f"Bearer {self.fabric_token}"} status_url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.fabric_workspace_id}/notebooks/{notebook_id}/jobs/instances/{job_id}" start_time = time.time() - while time.time() - start_time < timeout: - response = requests.get(status_url, headers=headers) - response.raise_for_status() - status = response.json()["status"] - - if status == "Completed": - print("✓ Notebook execution completed successfully") - return True - elif status in ["Failed", "Cancelled"]: - print(f"✗ Notebook execution {status}") - return False - - print(f" Notebook status: {status}, waiting...") - time.sleep(10) + while time.time() - start_time < NOTEBOOK_TIMEOUT: + try: + response = requests.get(status_url, headers=self._get_fabric_headers()) + response.raise_for_status() + status = response.json()["status"] + + if status == "Completed": + print("✓ Notebook execution completed successfully") + return True + elif status in ["Failed", "Cancelled"]: + print(f"✗ Notebook execution {status}") + return False + + print(f" Notebook status: {status}, waiting...") + time.sleep(NOTEBOOK_POLL_INTERVAL) + except requests.exceptions.RequestException as e: + print(f"⚠ Warning: Error checking notebook status: {e}") + time.sleep(NOTEBOOK_POLL_INTERVAL) print("✗ Timeout waiting for notebook completion") return False + # ============================================================================ + # Fabric Data Agent Operations + # ============================================================================ + def get_data_agent_artifact_id(self, data_agent_name: str) -> Optional[str]: """Retrieve the artifact ID of the created Fabric Data Agent.""" - headers = {"Authorization": f"Bearer {self.fabric_token}"} - - # List all items in the workspace - list_url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.fabric_workspace_id}/items" - - response = requests.get(list_url, headers=headers) - response.raise_for_status() - items = response.json().get("value", []) - - # Find the DataAgent item by name and type - for item in items: - if item.get("displayName") == data_agent_name: # and item.get("type") == "aiskills" - item_id = item["id"] - - # Get the specific item to retrieve full details - get_item_url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.fabric_workspace_id}/items/{item_id}" - item_response = requests.get(get_item_url, headers=headers) - item_response.raise_for_status() - item_details = item_response.json() - - artifact_id = item_details["id"] - print(f"✓ Found data agent '{data_agent_name}' with artifact ID: {artifact_id}") - return artifact_id - - print(f"✗ Data agent '{data_agent_name}' not found") - return None - - def create_fabric_connection(self, data_agent_artifact_id: str) -> str: - """Create a Connected Resource connection in AI Foundry Project to Fabric.""" - connection_name = "fabric-data-agent-connection" - - # Get ARM token - arm_token = self.credential.get_token("https://management.azure.com/.default") - headers = { - "Authorization": f"Bearer {arm_token.token}", - "Content-Type": "application/json", - } - - # Construct AI Foundry Project connection endpoint - connection_url = ( + try: + list_url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.fabric_workspace_id}/items" + response = requests.get(list_url, headers=self._get_fabric_headers()) + response.raise_for_status() + items = response.json().get("value", []) + + for item in items: + if item.get("displayName") == data_agent_name: + item_id = item["id"] + + # Get full item details + get_item_url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.fabric_workspace_id}/items/{item_id}" + item_response = requests.get(get_item_url, headers=self._get_fabric_headers()) + item_response.raise_for_status() + item_details = item_response.json() + + artifact_id = item_details["id"] + print(f"✓ Found data agent '{data_agent_name}' with artifact ID: {artifact_id}") + return artifact_id + + print(f"✗ Data agent '{data_agent_name}' not found") + return None + except requests.exceptions.RequestException as e: + print(f"✗ Error retrieving data agent: {e}") + return None + + # ============================================================================ + # AI Foundry Operations + # ============================================================================ + + def _build_connection_url(self, connection_name: str) -> str: + """Build the ARM URL for AI Foundry Project connection.""" + return ( f"https://management.azure.com/subscriptions/{self.ai_foundry_subscription_id}" f"/resourceGroups/{self.ai_foundry_resource_group}" f"/providers/Microsoft.CognitiveServices/accounts/{self.ai_foundry_account_name}" f"/projects/{self.ai_foundry_project_name}" f"/connections/{connection_name}" - f"?api-version=2025-06-01" + f"?api-version={AI_FOUNDRY_API_VERSION}" ) - # Connection payload for Fabric Data Agent + def _build_agent_url(self, agent_name: str) -> str: + """Build the ARM URL for AI Foundry Project agent.""" + return ( + f"https://management.azure.com/subscriptions/{self.ai_foundry_subscription_id}" + f"/resourceGroups/{self.ai_foundry_resource_group}" + f"/providers/Microsoft.CognitiveServices/accounts/{self.ai_foundry_account_name}" + f"/projects/{self.ai_foundry_project_name}" + f"/agents/{agent_name}" + f"?api-version={AI_FOUNDRY_API_VERSION}" + ) + + def create_fabric_connection(self, data_agent_artifact_id: str) -> str: + """Create a Connected Resource connection in AI Foundry Project to Fabric.""" + connection_name = "fabric-data-agent-connection" + connection_url = self._build_connection_url(connection_name) + connection_payload = { "properties": { "category": "FabricDataAgent", @@ -230,40 +279,24 @@ def create_fabric_connection(self, data_agent_artifact_id: str) -> str: "keys": { "workspace-id": self.fabric_workspace_id, "artifact-id": data_agent_artifact_id, - "type": "fabric_dataagent" + "type": "fabric_dataagent", } - } + }, } } - response = requests.put(connection_url, headers=headers, json=connection_payload) - response.raise_for_status() + response = requests.put(connection_url, headers=self._get_arm_headers(), json=connection_payload) + if not response.ok: + raise Exception(f"Failed to create connection: {response.status_code} - {response.text}") + print(f"✓ Created Fabric connection: {connection_name}") return connection_name def create_ai_foundry_agent(self, connection_name: str) -> str: """Create an AI Foundry Agent using the Fabric connection as knowledge source.""" - - # Get ARM token - arm_token = self.credential.get_token("https://management.azure.com/.default") - headers = { - "Authorization": f"Bearer {arm_token.token}", - "Content-Type": "application/json", - } - agent_name = "fabric-data-agent" + agent_url = self._build_agent_url(agent_name) - # Construct AI Foundry Project agent endpoint - agent_url = ( - f"https://management.azure.com/subscriptions/{self.ai_foundry_subscription_id}" - f"/resourceGroups/{self.ai_foundry_resource_group}" - f"/providers/Microsoft.CognitiveServices/accounts/{self.ai_foundry_account_name}" - f"/projects/{self.ai_foundry_project_name}" - f"/agents/{agent_name}" - f"?api-version=2025-06-01" - ) - - # Agent payload with Fabric Data Agent connection agent_payload = { "properties": { "model": "gpt-4o", @@ -282,71 +315,104 @@ def create_ai_foundry_agent(self, connection_name: str) -> str: } } - response = requests.put(agent_url, headers=headers, json=agent_payload) - response.raise_for_status() + response = requests.put(agent_url, headers=self._get_arm_headers(), json=agent_payload) + if not response.ok: + raise Exception(f"Failed to create agent: {response.status_code} - {response.text}") + agent_id = response.json().get("id", agent_name) print(f"✓ Created AI Foundry Agent with ID: {agent_id}") return agent_id -def main(): - """Main execution flow.""" - # Configuration - these should be provided as environment variables or arguments - FABRIC_WORKSPACE_ID = os.getenv("FABRIC_WORKSPACE_ID", "") - AI_FOUNDRY_SUBSCRIPTION_ID = os.getenv("AI_FOUNDRY_SUBSCRIPTION_ID", "") - AI_FOUNDRY_RESOURCE_GROUP = os.getenv("AI_FOUNDRY_RESOURCE_GROUP", "") - AI_FOUNDRY_ACCOUNT_NAME = os.getenv("AI_FOUNDRY_ACCOUNT_NAME", "") - AI_FOUNDRY_PROJECT_NAME = os.getenv("AI_FOUNDRY_PROJECT_NAME", "") - - if not all([FABRIC_WORKSPACE_ID, AI_FOUNDRY_SUBSCRIPTION_ID, AI_FOUNDRY_RESOURCE_GROUP, AI_FOUNDRY_ACCOUNT_NAME, AI_FOUNDRY_PROJECT_NAME]): - print("Error: Required environment variables not set:") - print(" - FABRIC_WORKSPACE_ID") - print(" - AI_FOUNDRY_SUBSCRIPTION_ID") - print(" - AI_FOUNDRY_RESOURCE_GROUP") - print(" - AI_FOUNDRY_ACCOUNT_NAME") - print(" - AI_FOUNDRY_PROJECT_NAME") - return - - integration = FabricFoundryIntegration( - fabric_workspace_id=FABRIC_WORKSPACE_ID, - ai_foundry_subscription_id=AI_FOUNDRY_SUBSCRIPTION_ID, - ai_foundry_resource_group=AI_FOUNDRY_RESOURCE_GROUP, - ai_foundry_account_name=AI_FOUNDRY_ACCOUNT_NAME, - ai_foundry_project_name=AI_FOUNDRY_PROJECT_NAME, - ) +# ============================================================================ +# Main Execution +# ============================================================================ - print("Starting Fabric-Foundry integration automation...") - print("=" * 60) - - # Step 1: Upload notebook - notebook_path = "create_fabric_data_agent.ipynb" - notebook_id = integration.upload_notebook(notebook_path, "create_fabric_data_agent") - - # Step 2: Trigger notebook run - job_id = integration.trigger_notebook_run(notebook_id) +def validate_environment() -> Dict[str, str]: + """Validate and return required environment variables.""" + required_vars = { + "FABRIC_WORKSPACE_ID": os.getenv("FABRIC_WORKSPACE_ID", ""), + "AI_FOUNDRY_SUBSCRIPTION_ID": os.getenv("AI_FOUNDRY_SUBSCRIPTION_ID", ""), + "AI_FOUNDRY_RESOURCE_GROUP": os.getenv("AI_FOUNDRY_RESOURCE_GROUP", ""), + "AI_FOUNDRY_ACCOUNT_NAME": os.getenv("AI_FOUNDRY_ACCOUNT_NAME", ""), + "AI_FOUNDRY_PROJECT_NAME": os.getenv("AI_FOUNDRY_PROJECT_NAME", ""), + } - # Wait for completion - if not integration.wait_for_notebook_completion(notebook_id, job_id): - print("✗ Notebook execution failed, aborting") - return + missing_vars = [key for key, value in required_vars.items() if not value] + if missing_vars: + print("✗ Error: Required environment variables not set:") + for var in missing_vars: + print(f" - {var}") + sys.exit(1) - # Step 3: Retrieve data agent artifact ID - data_agent_artifact_id = integration.get_data_agent_artifact_id(FABRIC_DATA_AGENT_NAME) - if not data_agent_artifact_id: - print("✗ Could not retrieve data agent artifact ID, aborting") - return + return required_vars - # Step 4: Create Fabric connection in AI Foundry Project - connection_name = integration.create_fabric_connection(data_agent_artifact_id) - - # Step 5: Create AI Foundry Agent - agent_id = integration.create_ai_foundry_agent(connection_name) +def main(): + """Main execution flow.""" + print("Starting Fabric-Foundry integration automation...") print("=" * 60) - print("✓ Integration complete!") - print(f" - Data Agent Artifact ID: {data_agent_artifact_id}") - print(f" - Connection Name: {connection_name}") - print(f" - Agent ID: {agent_id}") + + # Validate environment + env_vars = validate_environment() + + # Initialize integration + try: + integration = FabricFoundryIntegration( + fabric_workspace_id=env_vars["FABRIC_WORKSPACE_ID"], + ai_foundry_subscription_id=env_vars["AI_FOUNDRY_SUBSCRIPTION_ID"], + ai_foundry_resource_group=env_vars["AI_FOUNDRY_RESOURCE_GROUP"], + ai_foundry_account_name=env_vars["AI_FOUNDRY_ACCOUNT_NAME"], + ai_foundry_project_name=env_vars["AI_FOUNDRY_PROJECT_NAME"], + ) + except Exception as e: + print(f"✗ Error initializing integration: {e}") + sys.exit(1) + + try: + # Step 1: Upload notebook + print("\n[Step 1/5] Uploading notebook...") + notebook_path = "create_fabric_data_agent.ipynb" + notebook_id = integration.upload_notebook(notebook_path, "create_fabric_data_agent") + + # Step 2: Trigger notebook run + print("\n[Step 2/5] Triggering notebook execution...") + job_id = integration.trigger_notebook_run(notebook_id) + + # Step 3: Wait for completion + print("\n[Step 3/5] Waiting for notebook completion...") + if not integration.wait_for_notebook_completion(notebook_id, job_id): + print("✗ Notebook execution failed, aborting") + sys.exit(1) + + # Step 4: Retrieve data agent artifact ID + print("\n[Step 4/5] Retrieving data agent artifact ID...") + data_agent_artifact_id = integration.get_data_agent_artifact_id(FABRIC_DATA_AGENT_NAME) + if not data_agent_artifact_id: + print("✗ Could not retrieve data agent artifact ID, aborting") + sys.exit(1) + + # Step 5: Create Fabric connection in AI Foundry Project + print("\n[Step 5/5] Creating AI Foundry connection and agent...") + connection_name = integration.create_fabric_connection(data_agent_artifact_id) + + # Step 6: Create AI Foundry Agent + agent_id = integration.create_ai_foundry_agent(connection_name) + + # Success summary + print("\n" + "=" * 60) + print("✓ Integration complete!") + print(f" - Data Agent Artifact ID: {data_agent_artifact_id}") + print(f" - Connection Name: {connection_name}") + print(f" - Agent ID: {agent_id}") + print("=" * 60) + + except KeyboardInterrupt: + print("\n\n✗ Process interrupted by user") + sys.exit(1) + except Exception as e: + print(f"\n✗ Unexpected error: {e}") + sys.exit(1) if __name__ == "__main__": From ee356a1e216ed95f3333de86a3a41e5fc5c691cc Mon Sep 17 00:00:00 2001 From: Lace Lofranco Date: Tue, 11 Nov 2025 14:25:31 +1000 Subject: [PATCH 07/11] feat: bug fixing --- .../automate_fabric_foundry_integration.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/guides/fabric_foundry/automate_fabric_foundry_integration.py b/guides/fabric_foundry/automate_fabric_foundry_integration.py index 44219080..43e92a59 100644 --- a/guides/fabric_foundry/automate_fabric_foundry_integration.py +++ b/guides/fabric_foundry/automate_fabric_foundry_integration.py @@ -265,21 +265,20 @@ def _build_agent_url(self, agent_name: str) -> str: f"?api-version={AI_FOUNDRY_API_VERSION}" ) - def create_fabric_connection(self, data_agent_artifact_id: str) -> str: + def create_foundry_to_fabric_connection(self, data_agent_artifact_id: str) -> str: """Create a Connected Resource connection in AI Foundry Project to Fabric.""" connection_name = "fabric-data-agent-connection" connection_url = self._build_connection_url(connection_name) connection_payload = { "properties": { - "category": "FabricDataAgent", - "target": f"https://api.fabric.microsoft.com/v1/workspaces/{self.fabric_workspace_id}/dataAgents/{data_agent_artifact_id}", + "category": "CustomKeys", "authType": "CustomKeys", "credentials": { "keys": { "workspace-id": self.fabric_workspace_id, "artifact-id": data_agent_artifact_id, - "type": "fabric_dataagent", + "type": "fabric_dataagent" } }, } @@ -394,7 +393,7 @@ def main(): # Step 5: Create Fabric connection in AI Foundry Project print("\n[Step 5/5] Creating AI Foundry connection and agent...") - connection_name = integration.create_fabric_connection(data_agent_artifact_id) + connection_name = integration.create_foundry_to_fabric_connection(data_agent_artifact_id) # Step 6: Create AI Foundry Agent agent_id = integration.create_ai_foundry_agent(connection_name) From 91fde3f84849582e47e157cd5300a2886cad8149 Mon Sep 17 00:00:00 2001 From: Lace Lofranco Date: Tue, 11 Nov 2025 15:13:01 +1000 Subject: [PATCH 08/11] fix: bugs --- .../automate_fabric_foundry_integration.py | 97 ++++++++++--------- guides/fabric_foundry/requirements.txt | 2 + 2 files changed, 54 insertions(+), 45 deletions(-) diff --git a/guides/fabric_foundry/automate_fabric_foundry_integration.py b/guides/fabric_foundry/automate_fabric_foundry_integration.py index 43e92a59..39df7d9a 100644 --- a/guides/fabric_foundry/automate_fabric_foundry_integration.py +++ b/guides/fabric_foundry/automate_fabric_foundry_integration.py @@ -16,6 +16,8 @@ import base64 from typing import Optional, Dict, Any from azure.identity import DefaultAzureCredential +from azure.ai.projects import AIProjectClient +from azure.ai.projects.models import ConnectionType import requests from dotenv import load_dotenv @@ -49,6 +51,18 @@ def __init__( self._fabric_token = None self._arm_token = None + # Initialize AI Project Client + project_scope = ( + f"https://{ai_foundry_account_name}.cognitiveservices.azure.com/" + ) + self.ai_client = AIProjectClient( + endpoint=project_scope, + credential=self.credential, + subscription_id=ai_foundry_subscription_id, + resource_group_name=ai_foundry_resource_group, + project_name=ai_foundry_project_name + ) + # ============================================================================ # Authentication Helpers # ============================================================================ @@ -254,16 +268,8 @@ def _build_connection_url(self, connection_name: str) -> str: f"?api-version={AI_FOUNDRY_API_VERSION}" ) - def _build_agent_url(self, agent_name: str) -> str: - """Build the ARM URL for AI Foundry Project agent.""" - return ( - f"https://management.azure.com/subscriptions/{self.ai_foundry_subscription_id}" - f"/resourceGroups/{self.ai_foundry_resource_group}" - f"/providers/Microsoft.CognitiveServices/accounts/{self.ai_foundry_account_name}" - f"/projects/{self.ai_foundry_project_name}" - f"/agents/{agent_name}" - f"?api-version={AI_FOUNDRY_API_VERSION}" - ) + # ============================================================================ + # Fabric Data Agent Operations def create_foundry_to_fabric_connection(self, data_agent_artifact_id: str) -> str: """Create a Connected Resource connection in AI Foundry Project to Fabric.""" @@ -289,38 +295,34 @@ def create_foundry_to_fabric_connection(self, data_agent_artifact_id: str) -> st raise Exception(f"Failed to create connection: {response.status_code} - {response.text}") print(f"✓ Created Fabric connection: {connection_name}") - return connection_name - def create_ai_foundry_agent(self, connection_name: str) -> str: + # Return full connection resource ID for agent creation + connection_id = ( + f"/subscriptions/{self.ai_foundry_subscription_id}" + f"/resourceGroups/{self.ai_foundry_resource_group}" + f"/providers/Microsoft.CognitiveServices/accounts/{self.ai_foundry_account_name}" + f"/projects/{self.ai_foundry_project_name}" + f"/connections/{connection_name}" + ) + return connection_id + + def create_ai_foundry_agent(self, connection_id: str, model_deployment_name: str) -> str: """Create an AI Foundry Agent using the Fabric connection as knowledge source.""" - agent_name = "fabric-data-agent" - agent_url = self._build_agent_url(agent_name) + from azure.ai.agents.models import FabricTool - agent_payload = { - "properties": { - "model": "gpt-4o", - "instructions": "You are a helpful assistant with access to NYC taxi data through Fabric Data Agent. Help users answer questions about taxi ridership.", - "tools": [{"type": "code_interpreter"}], - "toolResources": { - "codeInterpreter": { - "dataSources": [ - { - "type": "fabric_data_agent", - "connectionId": connection_name, - } - ] - } - }, - } - } + # Initialize an Agent Fabric tool and add the connection id + fabric = FabricTool(connection_id=connection_id) - response = requests.put(agent_url, headers=self._get_arm_headers(), json=agent_payload) - if not response.ok: - raise Exception(f"Failed to create agent: {response.status_code} - {response.text}") + # Create an Agent with the Fabric tool + agent = self.ai_client.agents.create_agent( + model=model_deployment_name, + name="fabric-data-agent", + instructions="You are a helpful assistant with access to NYC taxi data through Fabric Data Agent. Help users answer questions about taxi ridership.", + tools=fabric.definitions + ) - agent_id = response.json().get("id", agent_name) - print(f"✓ Created AI Foundry Agent with ID: {agent_id}") - return agent_id + print(f"✓ Created AI Foundry Agent with ID: {agent.id}") + return agent.id # ============================================================================ @@ -335,6 +337,7 @@ def validate_environment() -> Dict[str, str]: "AI_FOUNDRY_RESOURCE_GROUP": os.getenv("AI_FOUNDRY_RESOURCE_GROUP", ""), "AI_FOUNDRY_ACCOUNT_NAME": os.getenv("AI_FOUNDRY_ACCOUNT_NAME", ""), "AI_FOUNDRY_PROJECT_NAME": os.getenv("AI_FOUNDRY_PROJECT_NAME", ""), + "AI_FOUNDRY_MODEL_DEPLOYMENT_NAME": os.getenv("AI_FOUNDRY_MODEL_DEPLOYMENT_NAME", "gpt-4o"), } missing_vars = [key for key, value in required_vars.items() if not value] @@ -370,39 +373,43 @@ def main(): try: # Step 1: Upload notebook - print("\n[Step 1/5] Uploading notebook...") + print("\n[Step 1/6] Uploading notebook...") notebook_path = "create_fabric_data_agent.ipynb" notebook_id = integration.upload_notebook(notebook_path, "create_fabric_data_agent") # Step 2: Trigger notebook run - print("\n[Step 2/5] Triggering notebook execution...") + print("\n[Step 2/6] Triggering notebook execution...") job_id = integration.trigger_notebook_run(notebook_id) # Step 3: Wait for completion - print("\n[Step 3/5] Waiting for notebook completion...") + print("\n[Step 3/6] Waiting for notebook completion...") if not integration.wait_for_notebook_completion(notebook_id, job_id): print("✗ Notebook execution failed, aborting") sys.exit(1) # Step 4: Retrieve data agent artifact ID - print("\n[Step 4/5] Retrieving data agent artifact ID...") + print("\n[Step 4/6] Retrieving data agent artifact ID...") data_agent_artifact_id = integration.get_data_agent_artifact_id(FABRIC_DATA_AGENT_NAME) if not data_agent_artifact_id: print("✗ Could not retrieve data agent artifact ID, aborting") sys.exit(1) # Step 5: Create Fabric connection in AI Foundry Project - print("\n[Step 5/5] Creating AI Foundry connection and agent...") - connection_name = integration.create_foundry_to_fabric_connection(data_agent_artifact_id) + print("\n[Step 5/6] Creating AI Foundry connection") + connection_id = integration.create_foundry_to_fabric_connection(data_agent_artifact_id) # Step 6: Create AI Foundry Agent - agent_id = integration.create_ai_foundry_agent(connection_name) + print("\n[Step 6/6] Creating AI Foundry agent") + agent_id = integration.create_ai_foundry_agent( + connection_id, + env_vars["AI_FOUNDRY_MODEL_DEPLOYMENT_NAME"] + ) # Success summary print("\n" + "=" * 60) print("✓ Integration complete!") print(f" - Data Agent Artifact ID: {data_agent_artifact_id}") - print(f" - Connection Name: {connection_name}") + print(f" - Connection ID: {connection_id}") print(f" - Agent ID: {agent_id}") print("=" * 60) diff --git a/guides/fabric_foundry/requirements.txt b/guides/fabric_foundry/requirements.txt index c3dd1e9a..d9e6b379 100644 --- a/guides/fabric_foundry/requirements.txt +++ b/guides/fabric_foundry/requirements.txt @@ -1,3 +1,5 @@ azure-identity>=1.15.0 requests>=2.31.0 python-dotenv>=1.0.0 +azure-ai-projects>=1.0.0b1 +azure-ai-agents>=1.2.0b6 From def6ae05804992e5501c45cf91988f6c23608024 Mon Sep 17 00:00:00 2001 From: Lace Lofranco Date: Tue, 11 Nov 2025 16:14:16 +1000 Subject: [PATCH 09/11] fix: bugs --- .../automate_fabric_foundry_integration.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/guides/fabric_foundry/automate_fabric_foundry_integration.py b/guides/fabric_foundry/automate_fabric_foundry_integration.py index 39df7d9a..4453b251 100644 --- a/guides/fabric_foundry/automate_fabric_foundry_integration.py +++ b/guides/fabric_foundry/automate_fabric_foundry_integration.py @@ -57,10 +57,7 @@ def __init__( ) self.ai_client = AIProjectClient( endpoint=project_scope, - credential=self.credential, - subscription_id=ai_foundry_subscription_id, - resource_group_name=ai_foundry_resource_group, - project_name=ai_foundry_project_name + credential=self.credential ) # ============================================================================ @@ -310,6 +307,11 @@ def create_ai_foundry_agent(self, connection_id: str, model_deployment_name: str """Create an AI Foundry Agent using the Fabric connection as knowledge source.""" from azure.ai.agents.models import FabricTool + print("Available model deployments in the project:") + deployments = self.ai_client.deployments.list() # .deployments is the property for model deployments + for d in deployments: + print(f"Name: {d.name}, Model: {d.modelName}, Version: {d.modelVersion}, Type: {d.type}") + # Initialize an Agent Fabric tool and add the connection id fabric = FabricTool(connection_id=connection_id) From 243ed80612e55c2c3d7c3e4cdf036245befda005 Mon Sep 17 00:00:00 2001 From: Lace Lofranco Date: Tue, 11 Nov 2025 16:18:04 +1000 Subject: [PATCH 10/11] feat: bug fixing --- guides/fabric_foundry/README.md | 97 +++++++++++++++++++++++++++------ 1 file changed, 79 insertions(+), 18 deletions(-) diff --git a/guides/fabric_foundry/README.md b/guides/fabric_foundry/README.md index e777af1d..2aea889c 100644 --- a/guides/fabric_foundry/README.md +++ b/guides/fabric_foundry/README.md @@ -10,68 +10,129 @@ The automation script performs the following steps: 1. **Execute Notebook**: Triggers the notebook to run, which creates a Fabric Data Agent named "data_agent_automation_sample" 1. **Retrieve Artifact ID**: Gets the artifact ID of the newly created data agent 1. **Create Connection**: Creates a Connected Resource connection in AI Foundry pointing to the Fabric Data Agent -1. **Create Agent**: Creates an AI Foundry Agent that uses the Fabric Data Agent as a knowledge source +1. **Create Agent**: Creates an AI Foundry Agent that uses the Fabric Data Agent as a knowledge source via FabricTool ## Prerequisites - Azure subscription with access to: - Microsoft Fabric workspace - Azure AI Foundry project (not Hub - must be a Project resource) + - AI model deployment in AI Foundry project (e.g., gpt-4o) - Fabric workspace must have: - NYC Taxi lakehouse named "NYCTaxi_756" with tables: year_2017, year_2018, year_2019, year_2020 + - Active capacity (required for notebook execution) - Azure CLI or DefaultAzureCredential authentication configured -- Python 3.8 or higher +- Python 3.12 or higher +- uv package manager installed ## Setup -1. Install dependencies: +1. Install dependencies using uv: ```bash -pip install -r requirements.txt +cd guides/fabric_foundry +uv sync ``` -1. Set required environment variables: +1. Create a `.env` file with required environment variables: ```bash -export FABRIC_WORKSPACE_ID="your-fabric-workspace-id" -export AI_FOUNDRY_SUBSCRIPTION_ID="your-subscription-id" -export AI_FOUNDRY_RESOURCE_GROUP="your-resource-group" -export AI_FOUNDRY_ACCOUNT_NAME="your-ai-foundry-account-name" -export AI_FOUNDRY_PROJECT_NAME="your-ai-foundry-project-name" +# Fabric Configuration +FABRIC_WORKSPACE_ID=your-fabric-workspace-id + +# Azure AI Foundry Configuration +AI_FOUNDRY_SUBSCRIPTION_ID=your-subscription-id +AI_FOUNDRY_RESOURCE_GROUP=your-resource-group +AI_FOUNDRY_ACCOUNT_NAME=your-ai-foundry-account-name +AI_FOUNDRY_PROJECT_NAME=your-ai-foundry-project-name +AI_FOUNDRY_MODEL_DEPLOYMENT_NAME=gpt-4o ``` +**Note**: Replace `AI_FOUNDRY_MODEL_DEPLOYMENT_NAME` with your actual model deployment name from the "Models + endpoints" tab in your AI Foundry project. + ## Usage Run the automation script: ```bash -python automate_fabric_foundry_integration.py +uv run python automate_fabric_foundry_integration.py ``` The script will: +- Load configuration from `.env` file - Authenticate using DefaultAzureCredential -- Upload and execute the notebook in Fabric -- Wait for completion -- Create the necessary connections and agent in AI Foundry Project using ARM REST APIs +- Upload and execute the notebook in Fabric (or reuse existing notebook) +- Wait for completion (up to 10 minutes) +- Create the Fabric connection in AI Foundry using ARM REST API +- Create an AI Foundry Agent with FabricTool using Azure AI Projects SDK - Output the created resource IDs +## Debugging + +To debug the script in VS Code: + +1. The `.vscode/launch.json` is configured to use the uv virtual environment +1. Set breakpoints in the script +1. Press F5 or use "Run and Debug" to start debugging + ## Implementation Notes -- Uses Azure Resource Manager REST API (`Microsoft.CognitiveServices/accounts/projects/connections@2025-06-01`) for creating connections -- Uses Azure Resource Manager REST API for creating agents in AI Foundry Projects -- No Python SDK dependency for AI Foundry - direct REST API calls for maximum compatibility +### Connection Creation + +- Uses Azure Resource Manager REST API (`Microsoft.CognitiveServices/accounts/projects/connections@2025-06-01`) +- Connection category: `CustomKeys` +- Authentication type: `CustomKeys` +- Stores workspace ID, artifact ID, and type in credentials + +### Agent Creation + +- Uses Azure AI Projects Python SDK (`azure-ai-projects>=1.0.0b1`) +- Uses Azure AI Agents SDK (`azure-ai-agents>=1.2.0b6`) for FabricTool +- FabricTool connects the agent to the Fabric Data Agent as a knowledge source +- Requires valid model deployment name from AI Foundry project ## Files - `create_fabric_data_agent.ipynb`: Notebook that creates the Fabric Data Agent (must be run in Fabric) - `automate_fabric_foundry_integration.py`: Main automation script - `requirements.txt`: Python dependencies +- `.env`: Environment configuration (not committed to git) - `README.md`: This file +## Dependencies + +Key Python packages: + +- `azure-identity>=1.15.0` - Azure authentication +- `requests>=2.31.0` - HTTP requests for Fabric and ARM APIs +- `python-dotenv>=1.0.0` - Environment variable management +- `azure-ai-projects>=1.0.0b1` - AI Foundry Projects SDK +- `azure-ai-agents>=1.2.0b6` - AI Agents SDK with FabricTool support + +## Troubleshooting + +### CapacityNotActive Error + +If you get this error when triggering notebook execution, ensure your Fabric workspace has an active capacity assigned. + +### 404 Resource Not Found (Agent Creation) + +Verify that `AI_FOUNDRY_MODEL_DEPLOYMENT_NAME` matches exactly with a deployment name in your AI Foundry project's "Models + endpoints" tab. + +### Import Errors + +Make sure you're using the preview version of `azure-ai-agents` (1.2.0b6 or higher) which includes FabricTool: + +```bash +uv pip install --pre --upgrade azure-ai-agents +``` + ## Notes - The Fabric Data Agent SDK can only run inside Microsoft Fabric environments - The automation uses the Fabric REST API to upload and trigger notebook execution -- The script waits up to 10 minutes for notebook completion +- The script waits up to 10 minutes for notebook completion with 10-second polling intervals +- Notebooks are reused if they already exist (prevents duplicate creation errors) - All resources are created with simple configurations suitable for POC/demo purposes +- This is a proof-of-concept implementation - keep it simple! From dcdc61c1ce14140b83f1fd127860377057a25f3e Mon Sep 17 00:00:00 2001 From: Lace Lofranco Date: Tue, 11 Nov 2025 16:18:37 +1000 Subject: [PATCH 11/11] feat: bug fixing --- guides/fabric_foundry/README.md | 8 -------- 1 file changed, 8 deletions(-) diff --git a/guides/fabric_foundry/README.md b/guides/fabric_foundry/README.md index 2aea889c..cea9b97b 100644 --- a/guides/fabric_foundry/README.md +++ b/guides/fabric_foundry/README.md @@ -68,14 +68,6 @@ The script will: - Create an AI Foundry Agent with FabricTool using Azure AI Projects SDK - Output the created resource IDs -## Debugging - -To debug the script in VS Code: - -1. The `.vscode/launch.json` is configured to use the uv virtual environment -1. Set breakpoints in the script -1. Press F5 or use "Run and Debug" to start debugging - ## Implementation Notes ### Connection Creation