From 64a98327124abf313bd4e31c4a14032a59fd4518 Mon Sep 17 00:00:00 2001 From: Tyron Stewart Date: Wed, 17 Sep 2025 16:12:17 +1000 Subject: [PATCH] Add new ADF community template: CSV Landing with Databricks Submission guidelines met and Test import successful --- .../CSV Landing with Databricks.json | 117 ++++++++++++++++++ .../CSV Landing with Databricks/manifest.json | 41 ++++++ 2 files changed, 158 insertions(+) create mode 100644 community templates/CSV Landing with Databricks/CSV Landing with Databricks.json create mode 100644 community templates/CSV Landing with Databricks/manifest.json diff --git a/community templates/CSV Landing with Databricks/CSV Landing with Databricks.json b/community templates/CSV Landing with Databricks/CSV Landing with Databricks.json new file mode 100644 index 00000000..ea8ec65f --- /dev/null +++ b/community templates/CSV Landing with Databricks/CSV Landing with Databricks.json @@ -0,0 +1,117 @@ +{ + "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "factoryName": { + "type": "string", + "metadata": "Data Factory name" + }, + "": { + "type": "string" + } + }, + "variables": { + "factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]" + }, + "resources": [ + { + "name": "[concat(parameters('factoryName'), '/CSV Landing with Databricks')]", + "type": "Microsoft.DataFactory/factories/pipelines", + "apiVersion": "2018-06-01", + "properties": { + "description": "Save CSV file to Azure Data Lake Storage (ADLS Gen2) and Create Bronze Layer Delta Table in Unity Catalog", + "activities": [ + { + "name": "Source to Landing", + "description": "Save CSV files from Microsoft Learn GitHub Repo to Azure Data Lake Storage (ADLS Gen2)", + "type": "DatabricksNotebook", + "dependsOn": [], + "policy": { + "timeout": "0.12:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "notebookPath": "/Users///Process Data", + "baseParameters": { + "destination": { + "value": "@pipeline().parameters.landing", + "type": "Expression" + }, + "source": { + "value": "@pipeline().parameters.csv_source", + "type": "Expression" + } + } + }, + "linkedServiceName": { + "referenceName": "[parameters('')]", + "type": "LinkedServiceReference" + } + }, + { + "name": "Landing to Bronze", + "description": "Create Bronze Layer Delta Table", + "type": "DatabricksNotebook", + "dependsOn": [ + { + "activity": "Source to Landing", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "0.12:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "notebookPath": "/Users///Landing to Bronze", + "baseParameters": { + "source": { + "value": "@pipeline().parameters.landing", + "type": "Expression" + }, + "destination": { + "value": "@pipeline().parameters.bronze", + "type": "Expression" + }, + "batch_id": { + "value": "@pipeline().RunId", + "type": "Expression" + } + } + }, + "linkedServiceName": { + "referenceName": "[parameters('')]", + "type": "LinkedServiceReference" + } + } + ], + "parameters": { + "landing": { + "type": "string", + "defaultValue": "abfss://@.dfs.core.windows.net/00_landing/products/" + }, + "bronze": { + "type": "string", + "defaultValue": "abfss://@.dfs.core.windows.net/01_bronze/products/" + }, + "csv_source": { + "type": "string", + "defaultValue": "https://raw.githubusercontent.com/MicrosoftLearning/mslearn-databricks/main/data/products.csv" + } + }, + "annotations": [] + }, + "dependsOn": [] + } + ] +} \ No newline at end of file diff --git a/community templates/CSV Landing with Databricks/manifest.json b/community templates/CSV Landing with Databricks/manifest.json new file mode 100644 index 00000000..f32f8aac --- /dev/null +++ b/community templates/CSV Landing with Databricks/manifest.json @@ -0,0 +1,41 @@ +{ + "name": "CSV Landing with Databricks", + "description": "Orchestrate Databricks Notebooks for CSV file -> to Azure Data Lake Storage -> Unity Catalog Delta Table", + "image": "NotebookSource to LandingNotebookLanding to Bronze", + "icons": [ + "DatabricksNotebook", + "DatabricksNotebook" + ], + "requires": { + "linkedservices": { + "AzureDatabricksYes": { + "supportTypes": [ + "AzureDatabricks" + ] + } + } + }, + "documentation": "https://github.com/ty-dataloom/ADF-Databricks?tab=readme-ov-file#adf-databricks", + "contributorType": "Community", + "author": "DATA LOOM", + "annotations": [ + "ADLS Gen2", + "Databricks", + "Medallion", + "Bronze", + "CSV" + ], + "services": [ + "Azure Data Lake Storage", + "Azure Databricks", + "Azure Storage" + ], + "categories": [ + "Azure Pipeline", + "Data flow", + "Data Movement", + "Integration", + "Copy", + "Analytics" + ] +} \ No newline at end of file