diff --git a/knowledge_base/dashboard_deploy/.gitignore b/knowledge_base/dashboard_deploy/.gitignore new file mode 100644 index 0000000..de811f1 --- /dev/null +++ b/knowledge_base/dashboard_deploy/.gitignore @@ -0,0 +1,2 @@ + +.databricks diff --git a/knowledge_base/dashboard_deploy/README.md b/knowledge_base/dashboard_deploy/README.md new file mode 100644 index 0000000..6068b03 --- /dev/null +++ b/knowledge_base/dashboard_deploy/README.md @@ -0,0 +1,59 @@ +# Dashboard Deployment to higher environment + +This example demonstrates how to deploy the existing dashboard from dev to target Databricks workspace, using dab bundle +it also allows you to change the catalog and schema names of dashboard json dataset query from dev to target +specifies access to the groups on the target dashboard + +For more information about AI/BI dashboards, please refer to the [documentation](https://docs.databricks.com/dashboards/index.html). + +## Prerequisites + +* Databricks CLI v0.232.0 or above +* Databricks configuration file (`~/.databricks.cfg`) with a profile named dev and target workspace configured +* The catalog, schema and tables mentioned in dashboard datasets exists in target workspace + + +## Usage + +Invoke the shell script dashboard_deploy.sh to deploy the dashboard from dev to target Databricks workspace. + +Example +``` sh + # 'dev-dashboard-id' 'target-warehouse 'dev to target catalog/schema mapping' 'access 'target dashboard name' + # name' group' + +sh dashboard_deploy.sh 01efbbade84416dc984927ca794fd768 'Shared Endpoint' '{"samples": "shared", "nyctaxi": "ru_nyctaxi"}' test_group client_dashboard +``` +Output + +Clean up the temporary files if any +******************************************** +============================================ +Running script with the provided arguments... +Get the Dev Dashboard metadata json +******************************************** +============================================ +modify Query in Dashboard metadata json of catalog and schema names of target workspace +Namespace(file_path='dev_dashboard.json', replacements='{"samples": "shared", "nyctaxi": "ru_nyctaxi"}', target_file='./src/dashboard.json') +{"samples": "shared", "nyctaxi": "ru_nyctaxi"} +******************************************** +============================================ +Validate DAB bundle dashboard to the target workspace +Name: client_dashboard +Target: target +Workspace: +User: rushabh.mehta@databricks.com +Path: /Workspace/Users/rushabh.mehta@databricks.com/.bundle/client_dashboard/target + +Validation OK! +******************************************** +============================================ +Deploy the modified dashboard to the target workspace +Uploading bundle files to /Workspace/Users/rushabh.mehta@databricks.com/.bundle/client_dashboard/target/files... +Deploying resources... +Updating deployment state... +Deployment complete! +******************************************** +============================================ +Optional-Clean up the temporary files + diff --git a/knowledge_base/dashboard_deploy/dashboard_deploy.sh b/knowledge_base/dashboard_deploy/dashboard_deploy.sh new file mode 100644 index 0000000..c43ef35 --- /dev/null +++ b/knowledge_base/dashboard_deploy/dashboard_deploy.sh @@ -0,0 +1,68 @@ +#!/bin/bash + +# Check if the correct number of arguments is provided +if [ "$#" -ne 5 ]; then + echo "Usage: $0 " + exit 1 +fi + +# Assign arguments to variables +dashboard_id=$1 +target_warehouse_name=$2 +replacements=$3 +access_group=$4 +dashboard_name=$5 + +# Function to check the result of the last command and exit if it failed +check_result() { + if [ $1 -ne 0 ]; then + echo "Error: $2" + exit 1 + fi +} + +clean_up() { + rm -rf .databricks + rm -f dev_dashboard.json + rm -f ./src/dashboard.json +} + +# Print the received arguments +echo "Dashboard ID: $dashboard_id" +echo "Target Warehouse Name: $target_warehouse_name" +echo "Replacements: $replacements" +echo "dashboard_name: $dashboard_name" + +echo "============================================" +echo "Clean up the temporary files if any" +clean_up +check_result $? "Failed to clean up the temporary files" +echo "********************************************" +echo "============================================" +echo "Running script with the provided arguments..." +echo "Get the Dev Dashboard metadata json" +databricks lakeview get $dashboard_id --output json --profile dev >> dev_dashboard.json +check_result $? "Failed to get the dashboard metadata json" +echo "********************************************" +echo "============================================" +echo "modify Query in Dashboard metadata json of catalog and schema names of target workspace" +python3 dashboard_json_query_modify.py --file_path dev_dashboard.json --replacements "$replacements" --target_file ./src/dashboard.json +check_result $? "Failed to modify the dashboard metadata json" +echo "********************************************" +echo "============================================" +echo "Validate DAB bundle dashboard to the target workspace" +databricks bundle validate --var "warehouse_name=${target_warehouse_name}" --var "access_group=${access_group}" \ +--var "dashboard_name=${dashboard_name}" --profile target +check_result $? "Failed to validate the dashboard bundle" +echo "********************************************" +echo "============================================" +echo "Deploy the modified dashboard to the target workspace" +databricks bundle deploy --var "warehouse_name=${target_warehouse_name}" --var "access_group=${access_group}" \ +--var "dashboard_name=${dashboard_name}" --profile target +check_result $? "Failed to deploy the dashboard bundle" +echo "********************************************" +echo "============================================" +echo "Optional-Clean up the temporary files" +clean_up +check_result $? "Failed to clean up the temporary files" +echo "********************************************" \ No newline at end of file diff --git a/knowledge_base/dashboard_deploy/dashboard_json_query_modify.py b/knowledge_base/dashboard_deploy/dashboard_json_query_modify.py new file mode 100644 index 0000000..288e077 --- /dev/null +++ b/knowledge_base/dashboard_deploy/dashboard_json_query_modify.py @@ -0,0 +1,39 @@ +import json +import argparse +def replace_keys_in_query(query_string, replacements): + for key, value in replacements.items(): + query_string = query_string.replace(key, value) + return query_string + +def query_in_datasets(data, replacements): + if isinstance(data, dict): + if 'datasets' in data: + for dataset in data['datasets']: + if 'query' in dataset: + dataset['query'] = replace_keys_in_query(dataset['query'], replacements) + + return data + +def parse_json_file(file_path, replacements, target_file_path): + with open(file_path, 'r') as file: + data = json.load(file) + + dashboard_json = json.loads(data['serialized_dashboard']) + modified_data = query_in_datasets(dashboard_json, replacements) + + with open(target_file_path, 'w') as file: + json.dump(modified_data, file, indent=4) + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='Parse JSON file') + + parser.add_argument('--file_path', required=True, help='Path to the Dashboard JSON file') + parser.add_argument('--replacements', required=True, help='Replacements in JSON format') + parser.add_argument('--target_file', required=True, help='Dashboard json modified file save location') + + + args = parser.parse_args() + print(args) + print(args.replacements) + + parse_json_file(args.file_path, json.loads(f'{args.replacements}'), args.target_file) \ No newline at end of file diff --git a/knowledge_base/dashboard_deploy/databricks.yml b/knowledge_base/dashboard_deploy/databricks.yml new file mode 100644 index 0000000..e27a2c4 --- /dev/null +++ b/knowledge_base/dashboard_deploy/databricks.yml @@ -0,0 +1,28 @@ +bundle: + name: ${var.dashboard_name} + +include: + - resources/*.yml + +variables: + # The "warehouse_id" variable is used to reference the warehouse used by the dashboard. + warehouse_id: + lookup: + # Replace this with the name of your SQL warehouse. + warehouse: ${var.warehouse_name} + dev_workspace: + default: "host" + target_workspace: + default: "target_workspace" + dashboard_name: + default: "def" + access_group: + default: "def group" + warehouse_name: + default: "warehouse" + + +targets: + target: + default: true + diff --git a/knowledge_base/dashboard_deploy/resources/dashboard.yml b/knowledge_base/dashboard_deploy/resources/dashboard.yml new file mode 100644 index 0000000..91c154c --- /dev/null +++ b/knowledge_base/dashboard_deploy/resources/dashboard.yml @@ -0,0 +1,25 @@ +resources: + dashboards: + dashboard: + display_name: ${var.dashboard_name} + file_path: ../src/dashboard.json + warehouse_id: ${var.warehouse_id} + + # If "embed_credentials" is set to true, the bundle deployment identity's credentials + # are used to execute queries for all dashboard viewers. + # If it is set to false, each viewer's own credentials will be used instead. + # + # It defaults to false. + # + # embed_credentials: false + + # The "parent_path" field can be configured to place the dashboard in a + # non-standard folder in the workspace. + # + # It defaults to "${workspace.resource_path}", which is located + # under the bundle deployment root. + # + # parent_path: ${workspace.resource_path} + permissions: + - level: CAN_READ + group_name: ${var.access_group} diff --git a/knowledge_base/dashboard_deploy/src/temp b/knowledge_base/dashboard_deploy/src/temp new file mode 100644 index 0000000..e69de29