From 75ae401e5450bb7eeb3efe9812e96fee1f9836f0 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Fri, 21 Jun 2024 14:30:40 +0200 Subject: [PATCH 1/2] Example to demonstrate a job picking up a `requirements.txt` --- .../job_with_requirements_txt/.gitignore | 1 + .../job_with_requirements_txt/README.md | 34 +++++++++++++++++++ .../job_with_requirements_txt/databricks.yml | 13 +++++++ .../requirements.txt | 1 + .../resources/job_with_requirements_txt.yml | 22 ++++++++++++ .../job_with_requirements_txt/src/main.py | 3 ++ 6 files changed, 74 insertions(+) create mode 100644 knowledge_base/job_with_requirements_txt/.gitignore create mode 100644 knowledge_base/job_with_requirements_txt/README.md create mode 100644 knowledge_base/job_with_requirements_txt/databricks.yml create mode 100644 knowledge_base/job_with_requirements_txt/requirements.txt create mode 100644 knowledge_base/job_with_requirements_txt/resources/job_with_requirements_txt.yml create mode 100644 knowledge_base/job_with_requirements_txt/src/main.py diff --git a/knowledge_base/job_with_requirements_txt/.gitignore b/knowledge_base/job_with_requirements_txt/.gitignore new file mode 100644 index 0000000..15bcc6d --- /dev/null +++ b/knowledge_base/job_with_requirements_txt/.gitignore @@ -0,0 +1 @@ +.databricks diff --git a/knowledge_base/job_with_requirements_txt/README.md b/knowledge_base/job_with_requirements_txt/README.md new file mode 100644 index 0000000..f1f4893 --- /dev/null +++ b/knowledge_base/job_with_requirements_txt/README.md @@ -0,0 +1,34 @@ +# Job that uses `requirements.txt` + +This example demonstrates how to make a job pick up a `requirements.txt` dependency file. + +## Prerequisites + +* Databricks CLI v0.222.0 (unreleased) or above + +## Usage + +Update the `host` field under `workspace` in `databricks.yml` to the Databricks workspace you wish to deploy to. + +Run `databricks bundle deploy` to deploy the job. + +Run `databricks bundle run job_with_requirements_txt` to run the job. + +Example output: + +``` +$ databricks bundle run job_with_requirements_txt +Run URL: https://... + +2024-06-21 13:38:12 "[dev pieter_noordhuis] Example job that uses a requirements.txt file" TERMINATED SUCCESS + _____________ +| Hello, world! | + ============= + \ + \ + ^__^ + (oo)\_______ + (__)\ )\/\ + ||----w | + || || +``` diff --git a/knowledge_base/job_with_requirements_txt/databricks.yml b/knowledge_base/job_with_requirements_txt/databricks.yml new file mode 100644 index 0000000..06ee7ed --- /dev/null +++ b/knowledge_base/job_with_requirements_txt/databricks.yml @@ -0,0 +1,13 @@ +bundle: + name: job_with_requirements_txt + +include: + - resources/*.yml + +workspace: + host: https://myworkspace.cloud.databricks.com + +targets: + dev: + default: true + mode: development diff --git a/knowledge_base/job_with_requirements_txt/requirements.txt b/knowledge_base/job_with_requirements_txt/requirements.txt new file mode 100644 index 0000000..bc5e683 --- /dev/null +++ b/knowledge_base/job_with_requirements_txt/requirements.txt @@ -0,0 +1 @@ +cowsaw diff --git a/knowledge_base/job_with_requirements_txt/resources/job_with_requirements_txt.yml b/knowledge_base/job_with_requirements_txt/resources/job_with_requirements_txt.yml new file mode 100644 index 0000000..9905309 --- /dev/null +++ b/knowledge_base/job_with_requirements_txt/resources/job_with_requirements_txt.yml @@ -0,0 +1,22 @@ +resources: + jobs: + job_with_requirements_txt: + name: Example job that uses a requirements.txt file + + tasks: + - task_key: task + job_cluster_key: default + spark_python_task: + python_file: ../src/main.py + libraries: + - requirements: /Workspace/${workspace.file_path}/requirements.txt + + job_clusters: + - job_cluster_key: default + new_cluster: + spark_version: 14.3.x-scala2.12 + node_type_id: i3.xlarge + num_workers: 0 + spark_conf: + spark.master: local[*, 4] + spark.databricks.cluster.profile: singleNode diff --git a/knowledge_base/job_with_requirements_txt/src/main.py b/knowledge_base/job_with_requirements_txt/src/main.py new file mode 100644 index 0000000..753eb3d --- /dev/null +++ b/knowledge_base/job_with_requirements_txt/src/main.py @@ -0,0 +1,3 @@ +from cowsay import cow + +cow("Hello, world!") From 8ff0133e2b4f1a3d85fabf6f0e202a67f15edc41 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Fri, 21 Jun 2024 14:58:08 +0200 Subject: [PATCH 2/2] Fixes --- knowledge_base/job_with_requirements_txt/README.md | 2 +- knowledge_base/job_with_requirements_txt/requirements.txt | 2 +- .../resources/job_with_requirements_txt.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/knowledge_base/job_with_requirements_txt/README.md b/knowledge_base/job_with_requirements_txt/README.md index f1f4893..4da57f7 100644 --- a/knowledge_base/job_with_requirements_txt/README.md +++ b/knowledge_base/job_with_requirements_txt/README.md @@ -20,7 +20,7 @@ Example output: $ databricks bundle run job_with_requirements_txt Run URL: https://... -2024-06-21 13:38:12 "[dev pieter_noordhuis] Example job that uses a requirements.txt file" TERMINATED SUCCESS +2024-06-21 14:48:09 "[dev pieter_noordhuis] Example job that uses a requirements.txt file" TERMINATED SUCCESS _____________ | Hello, world! | ============= diff --git a/knowledge_base/job_with_requirements_txt/requirements.txt b/knowledge_base/job_with_requirements_txt/requirements.txt index bc5e683..c6b9ffd 100644 --- a/knowledge_base/job_with_requirements_txt/requirements.txt +++ b/knowledge_base/job_with_requirements_txt/requirements.txt @@ -1 +1 @@ -cowsaw +cowsay diff --git a/knowledge_base/job_with_requirements_txt/resources/job_with_requirements_txt.yml b/knowledge_base/job_with_requirements_txt/resources/job_with_requirements_txt.yml index 9905309..365259c 100644 --- a/knowledge_base/job_with_requirements_txt/resources/job_with_requirements_txt.yml +++ b/knowledge_base/job_with_requirements_txt/resources/job_with_requirements_txt.yml @@ -14,7 +14,7 @@ resources: job_clusters: - job_cluster_key: default new_cluster: - spark_version: 14.3.x-scala2.12 + spark_version: 15.1.x-scala2.12 node_type_id: i3.xlarge num_workers: 0 spark_conf: