From 466cb907e50bf54fef0d82dde6b4865bb0156ad5 Mon Sep 17 00:00:00 2001 From: chris-s-friedman Date: Fri, 9 Jan 2026 09:47:32 -0500 Subject: [PATCH 01/35] =?UTF-8?q?=E2=9C=A8=20New=20models=20and=20dags=20f?= =?UTF-8?q?or=20example=20study?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/kids_first/example_study.py | 40 +++++++++++++++++ .../_kf_example_study__models.yml | 45 +++++++++++++++++++ .../example_study/my_first_dbt_model.sql | 27 +++++++++++ .../example_study/my_second_dbt_model.sql | 6 +++ .../example_study/my_third_dbt_model.sql | 7 +++ dbt_project/seeds/_seeds.yml | 12 +++++ dbt_project/seeds/alphabet_grouping.csv | 27 +++++++++++ 7 files changed, 164 insertions(+) create mode 100644 dags/kids_first/example_study.py create mode 100644 dbt_project/models/kids_first/example_study/_kf_example_study__models.yml create mode 100644 dbt_project/models/kids_first/example_study/my_first_dbt_model.sql create mode 100644 dbt_project/models/kids_first/example_study/my_second_dbt_model.sql create mode 100644 dbt_project/models/kids_first/example_study/my_third_dbt_model.sql create mode 100644 dbt_project/seeds/alphabet_grouping.csv diff --git a/dags/kids_first/example_study.py b/dags/kids_first/example_study.py new file mode 100644 index 0000000..e849569 --- /dev/null +++ b/dags/kids_first/example_study.py @@ -0,0 +1,40 @@ +import os +from datetime import datetime, timedelta + +from cosmos import ( + DbtDag, + ProjectConfig, + ProfileConfig, + ExecutionConfig, + RenderConfig, +) + +profile_config = ProfileConfig( + profile_name="d3b_study_transform", + target_name="dev", + profiles_yml_filepath="/opt/airflow/dbt/profiles.yml", +) + +access_dag = DbtDag( + project_config=ProjectConfig( + "/opt/airflow/dbt/deidentified_etl", + # todo uncomment the line below and remove the corresponding, + # `install_deps` operator arg + # install_dbt_deps=True, #this is a feature coming in v1.9 + ), + profile_config=profile_config, + execution_config=ExecutionConfig( + dbt_executable_path=f"{os.environ['AIRFLOW_HOME']}/dbt_venv/bin/dbt", + ), + render_config=RenderConfig(select=["config.meta.study:kf_example_study"]), + operator_args={ + "install_deps": True, + }, + # normal dag parameters + schedule="@daily", + start_date=datetime(2026, 1, 1), + catchup=False, + dag_id="kf_example_study", + tags=["POC", "Kids First"], + # default_args={"retries": 2}, +) diff --git a/dbt_project/models/kids_first/example_study/_kf_example_study__models.yml b/dbt_project/models/kids_first/example_study/_kf_example_study__models.yml new file mode 100644 index 0000000..3320f5d --- /dev/null +++ b/dbt_project/models/kids_first/example_study/_kf_example_study__models.yml @@ -0,0 +1,45 @@ +--- +version: 2 + +models: + - name: my_first_dbt_model + description: "A starter dbt model" + config: + meta: + study: kf_example_study + columns: + - name: id + description: "The primary key for this table" + data_tests: + - unique + - not_null + + - name: my_second_dbt_model + description: "A starter dbt model" + config: + meta: + study: kf_example_study + columns: + - name: id + description: "The primary key for this table" + data_tests: + - unique + - not_null + + - name: my_third_dbt_model + description: "a model that maps values in alphabet_groups to + `my_second_dbt_model`" + config: + meta: + study: kf_example_study + columns: + - name: id + description: "The primary key for this table" + data_tests: + #- unique + - not_null + - name: letter + description: "The letter in the alphabet" + data_tests: + - unique + - not_null diff --git a/dbt_project/models/kids_first/example_study/my_first_dbt_model.sql b/dbt_project/models/kids_first/example_study/my_first_dbt_model.sql new file mode 100644 index 0000000..5b51873 --- /dev/null +++ b/dbt_project/models/kids_first/example_study/my_first_dbt_model.sql @@ -0,0 +1,27 @@ + +/* + Welcome to your first dbt model! + Did you know that you can also configure models directly within SQL files? + This will override configurations stated in dbt_project.yml + + Try changing "table" to "view" below +*/ + +{{ config(materialized='table', schema='kf_example') }} + +with source_data as ( + + select 1 as id + union all + select null as id + +) + +select * +from source_data + +/* + Uncomment the line below to remove records with null `id` values +*/ + +where id is not null diff --git a/dbt_project/models/kids_first/example_study/my_second_dbt_model.sql b/dbt_project/models/kids_first/example_study/my_second_dbt_model.sql new file mode 100644 index 0000000..3fdf3f1 --- /dev/null +++ b/dbt_project/models/kids_first/example_study/my_second_dbt_model.sql @@ -0,0 +1,6 @@ +{{ config(materialized='table', schema='kf_example') }} +-- Use the `ref` function to select from other models + +select * +from {{ ref('my_first_dbt_model') }} +where id = 1 diff --git a/dbt_project/models/kids_first/example_study/my_third_dbt_model.sql b/dbt_project/models/kids_first/example_study/my_third_dbt_model.sql new file mode 100644 index 0000000..f4006d1 --- /dev/null +++ b/dbt_project/models/kids_first/example_study/my_third_dbt_model.sql @@ -0,0 +1,7 @@ +{{ config(materialized='table', schema='kf_example') }} + +select + id, + letter +from {{ ref('my_second_dbt_model') }} +left join {{ ref('alphabet_grouping') }} on id = letter_grouping \ No newline at end of file diff --git a/dbt_project/seeds/_seeds.yml b/dbt_project/seeds/_seeds.yml index e69de29..3ea41e7 100644 --- a/dbt_project/seeds/_seeds.yml +++ b/dbt_project/seeds/_seeds.yml @@ -0,0 +1,12 @@ +seeds: + - name: alphabet_grouping + config: + schema: kf + description: | + An example seed file that groups letters of the alphabet. This seed is + used by the kids first example study models. + columns: + - name: letter + description: A letter of the alphabet + - name: letter_grouping + description: The grouping that the letter belongs to \ No newline at end of file diff --git a/dbt_project/seeds/alphabet_grouping.csv b/dbt_project/seeds/alphabet_grouping.csv new file mode 100644 index 0000000..5dd554f --- /dev/null +++ b/dbt_project/seeds/alphabet_grouping.csv @@ -0,0 +1,27 @@ +letter,letter_grouping +a,1 +b,1 +c,2 +d,2 +e,3 +f,3 +g,3 +h,4 +i,4 +j,4 +k,5 +l,5 +m,6 +n,6 +o,7 +p,7 +q,7 +r,8 +s,8 +t,8 +u,9 +v,9 +w,9 +x,10 +y,10 +z,10 \ No newline at end of file From 2029fda523f46a2452717b50f81a3d29b5de3339 Mon Sep 17 00:00:00 2001 From: chris-s-friedman Date: Fri, 9 Jan 2026 13:39:23 -0500 Subject: [PATCH 02/35] =?UTF-8?q?=F0=9F=93=9D=20Add=20documentation=20on?= =?UTF-8?q?=20how=20to=20run=20dbt=20in=20airflow?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/kids_first/example_study.py | 23 ++-- dbt_project/profiles.yml | 1 + docs/guides/running-models-in-airflow.md | 144 +++++++++++++++++++++++ 3 files changed, 154 insertions(+), 14 deletions(-) create mode 100644 docs/guides/running-models-in-airflow.md diff --git a/dags/kids_first/example_study.py b/dags/kids_first/example_study.py index e849569..b4af55d 100644 --- a/dags/kids_first/example_study.py +++ b/dags/kids_first/example_study.py @@ -1,5 +1,5 @@ import os -from datetime import datetime, timedelta +from datetime import datetime from cosmos import ( DbtDag, @@ -10,31 +10,26 @@ ) profile_config = ProfileConfig( - profile_name="d3b_study_transform", - target_name="dev", - profiles_yml_filepath="/opt/airflow/dbt/profiles.yml", + profile_name=os.environ["DBT_PROFILE_NAME"], + profiles_yml_filepath=os.environ["DBT_PROFILES_YML_PATH"], + target_name="prd", ) -access_dag = DbtDag( +example_study_dag = DbtDag( project_config=ProjectConfig( - "/opt/airflow/dbt/deidentified_etl", - # todo uncomment the line below and remove the corresponding, - # `install_deps` operator arg - # install_dbt_deps=True, #this is a feature coming in v1.9 + os.environ["DBT_PROJECT_DIR"], + install_dbt_deps=True, ), profile_config=profile_config, execution_config=ExecutionConfig( - dbt_executable_path=f"{os.environ['AIRFLOW_HOME']}/dbt_venv/bin/dbt", + dbt_executable_path=os.environ["DBT_EXECUTABLE_PATH"], ), render_config=RenderConfig(select=["config.meta.study:kf_example_study"]), - operator_args={ - "install_deps": True, - }, # normal dag parameters schedule="@daily", start_date=datetime(2026, 1, 1), catchup=False, dag_id="kf_example_study", tags=["POC", "Kids First"], - # default_args={"retries": 2}, + default_args={"retries": 2}, ) diff --git a/dbt_project/profiles.yml b/dbt_project/profiles.yml index ee7c9a7..d2e0871 100644 --- a/dbt_project/profiles.yml +++ b/dbt_project/profiles.yml @@ -20,3 +20,4 @@ include_dbt_sandbox: threads: 4 type: postgres user: "{{ env_var('INCLUDEWAREHOUSE_USERNAME') }}" + diff --git a/docs/guides/running-models-in-airflow.md b/docs/guides/running-models-in-airflow.md new file mode 100644 index 0000000..b90f632 --- /dev/null +++ b/docs/guides/running-models-in-airflow.md @@ -0,0 +1,144 @@ +# How to Run dbt in Airflow + +This guide discusses how dbt models may be run in airflow and the file +configuration that is needed to run those models. + +## How Airflow Runs dbt Models + +The airflow instance that is used to orchestrate data pipelines is hosted in +AWS using Managed Workflow for Apache Airflow (MWAA). MWAA provides a simple way +for the airflow instance - as well as its workers to be provisioned and +dispatched. As of the writing of this guide, the MWAA instance used by these +workflows is utilizing Airflow version 3.0.6 and python version 3.12. + +To run dbt models, we utilize [`cosmos`](https://astronomer.github.io/astronomer-cosmos/index.html). +`cosmos` is a python library written for Airflow that has operators that use dbt +and convert sets of dbt models within a dbt project into DAGs. While an older +approach may have specified running dbt models within a bash command within an +airflow bash operator, using the `cosmos` `DbtDag` operator allows individual +models and those models' tests to be materialized as individual tasks. + +For a dbt model - or set of models - to be picked up by airflow and run, those +models need to be referenced by a DAG. + +### Getting files into MWAA + +When pull requests are merged into main branch in this repository, a GitHub +action is triggered to mirror the contents of this repository into the s3 bucket +that MWAA uses to store models and DAGs. + +## How to write a DAG file + +A DAG that runs dbt models has three important parts that are needed for those +models to run: + +1. imports +2. profile configuration +3. the model DAG + +### Example DAG + +An example DAG is below. Its components will be described in the following +sections. The purpose of this DAG is to run models in the study +`kf_example_study` every day. + +```python +import os +from datetime import datetime + +from cosmos import ( + DbtDag, + ProjectConfig, + ProfileConfig, + ExecutionConfig, + RenderConfig, +) + +profile_config = ProfileConfig( + profile_name=os.environ["DBT_PROFILE_NAME"], + profiles_yml_filepath=os.environ["DBT_PROFILES_YML_PATH"], + target_name="prd", +) + +example_study_dag = DbtDag( + project_config=ProjectConfig( + "/opt/airflow/dbt/deidentified_etl", + install_dbt_deps=True, + ), + profile_config=profile_config, + execution_config=ExecutionConfig( + dbt_executable_path=os.environ["DBT_EXECUTABLE_PATH"], + ), + render_config=RenderConfig(select=["config.meta.study:kf_example_study"]), + # normal dag parameters + schedule="@daily", + start_date=datetime(2026, 1, 1), + catchup=False, + dag_id="kf_example_study", + tags=["POC", "Kids First"], + default_args={"retries": 2}, +) +``` + +### Imports + +This section has all of the packages and modules that are needed for the DAG +file to run. An important set of items that need to be imported are the cosmos +imports that have both the `DbtDag` operator and the appropriate cosmos +`Config`s. Information about different configuration modules used by cosmos +is [here](https://astronomer.github.io/astronomer-cosmos/configuration/index.html). + +Minimally for cosmos, the below is needed: + +```python +from cosmos import ( + DbtDag, + ProjectConfig, + ProfileConfig, + ExecutionConfig, + RenderConfig, +) +``` + +In the example, the `import os` is used to extract environment variables on the +machine used to run dbt and `import datetime` is used to identify when the +DAG should start being run. + +### `profile_config` + +The `profile_config` is used by cosmos to identify the profile to be used by dbt +commands. The values for `profile_name` and `profiles_yml_filepath` should use +the indicated environment variables. Acceptable values for `target_name` are +`qa` and `prd` + +### the model DAG + +The model DAG(s) python object can be named whatever makes the most sense for +the DAG + +[`project_config`](https://astronomer.github.io/astronomer-cosmos/configuration/project-config.html), +[`profile_config`](https://astronomer.github.io/astronomer-cosmos/profiles/index.html), +and [`execution_config`](https://astronomer.github.io/astronomer-cosmos/configuration/execution-config.html) +will look identical between DAGs and should follow the model of this example. +More information about these configurations is linked. + +[`render_config`](https://astronomer.github.io/astronomer-cosmos/configuration/render-config.html) +is used to identify what dbt model(s) should be run and follow dbt's `--select` +behavior. More information about using `RenderConfig` to select models can be +found [here](https://astronomer.github.io/astronomer-cosmos/configuration/selecting-excluding.html). + +The rest of the DAG's parameters are familiar to other airflow operators. + +* `schedule` indicates how often a DAG should be run +* `dag_id` is the identifier for the DAG within airflow and should follow the + form `{program}_{study}`. +* `tags` is used to tag the DAG and should minimally have a tag for the program + the DAG corresponds to. + +## Required components of models + +To be picked up by airflow, all that is required for models is that they have +the corresponding filtering criteria used in the `select` of the `RenderConfig`. + +In this example, the `meta` `config` property `study` was created and used to +filter models. From 51a73fb2a04359b6900b537e7b5b4357d4a29505 Mon Sep 17 00:00:00 2001 From: chris-s-friedman Date: Fri, 9 Jan 2026 13:44:03 -0500 Subject: [PATCH 03/35] =?UTF-8?q?=F0=9F=93=8C=20Pin=20dbt=20to=20version?= =?UTF-8?q?=201.10?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dbt_project/dbt_project.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dbt_project/dbt_project.yml b/dbt_project/dbt_project.yml index 8acfff1..aaa4b51 100644 --- a/dbt_project/dbt_project.yml +++ b/dbt_project/dbt_project.yml @@ -19,6 +19,8 @@ seed-paths: ["seeds"] macro-paths: ["macros"] snapshot-paths: ["snapshots"] +require-dbt-version: [">=1.10.0", "<1.11.0"] + target-path: "target" # directory which will store compiled SQL files clean-targets: # directories to be removed by `dbt clean` - "target" From ffd836c455c45762d535cdd28e3c99d30ae51594 Mon Sep 17 00:00:00 2001 From: chris-s-friedman Date: Fri, 9 Jan 2026 13:47:28 -0500 Subject: [PATCH 04/35] =?UTF-8?q?=E2=9C=A8=20Add=20qa=20and=20prd=20profil?= =?UTF-8?q?es?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dbt_project/profiles.yml | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/dbt_project/profiles.yml b/dbt_project/profiles.yml index d2e0871..a4d580e 100644 --- a/dbt_project/profiles.yml +++ b/dbt_project/profiles.yml @@ -1,6 +1,6 @@ --- include_dbt_sandbox: - target: ci + target: dev outputs: ci: type: postgres @@ -20,4 +20,22 @@ include_dbt_sandbox: threads: 4 type: postgres user: "{{ env_var('INCLUDEWAREHOUSE_USERNAME') }}" + qa: + type: postgres + host: "{{ env_var('INCLUDEWAREHOUSE_HOST') }}" + user: "{{ env_var('INCLUDEWAREHOUSE_SCV_USERNAME') }}" + password: "{{ env_var('INCLUDEWAREHOUSE_SCV_PASSWORD') }}" + port: 5432 + dbname: postgres + schema: qa + threads: 4 + prd: + type: postgres + host: "{{ env_var('INCLUDEWAREHOUSE_HOST') }}" + user: "{{ env_var('INCLUDEWAREHOUSE_SCV_USERNAME') }}" + password: "{{ env_var('INCLUDEWAREHOUSE_SCV_PASSWORD') }}" + port: 5432 + dbname: postgres + schema: prd + threads: 4 From bc4bee639cf18f3cae6ad2984d2628956995adfd Mon Sep 17 00:00:00 2001 From: chris-s-friedman Date: Mon, 12 Jan 2026 14:23:20 -0500 Subject: [PATCH 05/35] =?UTF-8?q?=F0=9F=90=9B=20Fix=20issues=20with=20dbt?= =?UTF-8?q?=20project=20structure=20and=20with=20dag?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/kids_first/example_study.py | 3 --- dbt_project/dbt_project.yml | 4 ++++ 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/dags/kids_first/example_study.py b/dags/kids_first/example_study.py index b4af55d..b2a2160 100644 --- a/dags/kids_first/example_study.py +++ b/dags/kids_first/example_study.py @@ -27,9 +27,6 @@ render_config=RenderConfig(select=["config.meta.study:kf_example_study"]), # normal dag parameters schedule="@daily", - start_date=datetime(2026, 1, 1), - catchup=False, dag_id="kf_example_study", tags=["POC", "Kids First"], - default_args={"retries": 2}, ) diff --git a/dbt_project/dbt_project.yml b/dbt_project/dbt_project.yml index aaa4b51..21259bd 100644 --- a/dbt_project/dbt_project.yml +++ b/dbt_project/dbt_project.yml @@ -19,6 +19,10 @@ seed-paths: ["seeds"] macro-paths: ["macros"] snapshot-paths: ["snapshots"] +# Note that this the dbt version here should align with the dbt version +# specified in requirements.txt and with the dbt version installed in the Airflow +# environment. That installation is managed here: +# https://github.com/d3b-center/aws-infra-d3b-accounts/blob/main/mwaa/terraform/include/us-east-1/prd/startup_script.sh require-dbt-version: [">=1.10.0", "<1.11.0"] target-path: "target" # directory which will store compiled SQL files From 1228b9f902c4763f25e1e7bcbc03cc0607a2ab30 Mon Sep 17 00:00:00 2001 From: chris-s-friedman Date: Tue, 13 Jan 2026 08:50:55 -0500 Subject: [PATCH 06/35] =?UTF-8?q?=F0=9F=93=8C=20Unpin=20dbt=20version?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit move this to a seperate PR --- dbt_project/dbt_project.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/dbt_project/dbt_project.yml b/dbt_project/dbt_project.yml index 21259bd..8acfff1 100644 --- a/dbt_project/dbt_project.yml +++ b/dbt_project/dbt_project.yml @@ -19,12 +19,6 @@ seed-paths: ["seeds"] macro-paths: ["macros"] snapshot-paths: ["snapshots"] -# Note that this the dbt version here should align with the dbt version -# specified in requirements.txt and with the dbt version installed in the Airflow -# environment. That installation is managed here: -# https://github.com/d3b-center/aws-infra-d3b-accounts/blob/main/mwaa/terraform/include/us-east-1/prd/startup_script.sh -require-dbt-version: [">=1.10.0", "<1.11.0"] - target-path: "target" # directory which will store compiled SQL files clean-targets: # directories to be removed by `dbt clean` - "target" From 9f7b96c27df6bda37847f651d7b3b057f340795f Mon Sep 17 00:00:00 2001 From: chris-s-friedman Date: Tue, 13 Jan 2026 09:25:58 -0500 Subject: [PATCH 07/35] =?UTF-8?q?=F0=9F=9A=A7=20Test=20the=20dbt=20dags?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/export-to-airflow.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/export-to-airflow.yml b/.github/workflows/export-to-airflow.yml index 17eec17..3153517 100644 --- a/.github/workflows/export-to-airflow.yml +++ b/.github/workflows/export-to-airflow.yml @@ -3,6 +3,8 @@ name: Export Repo to Airflow on: push: branches: [ "main" ] + pull_request: + branches: [ "main" ] permissions: contents: read From 3e0439bafe95a38b08aa5fcd78b04df47f869089 Mon Sep 17 00:00:00 2001 From: chris-s-friedman Date: Tue, 13 Jan 2026 11:13:57 -0500 Subject: [PATCH 08/35] =?UTF-8?q?=F0=9F=9A=A7=20Test=20that=20dbt=20is=20w?= =?UTF-8?q?here=20it=20should=20be?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/dbt_bash.py | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 dags/dbt_bash.py diff --git a/dags/dbt_bash.py b/dags/dbt_bash.py new file mode 100644 index 0000000..53b32e8 --- /dev/null +++ b/dags/dbt_bash.py @@ -0,0 +1,46 @@ +from datetime import datetime, timedelta +import os + +# The DAG object; we'll need this to instantiate a DAG +from airflow.models.dag import DAG + +# Operators; we need this to operate! +from airflow.operators.bash import BashOperator + +with DAG( + "dbt_bash_status", + # These args will get passed on to each operator + # You can override them on a per-task basis during operator initialization + default_args={ + "depends_on_past": False, + "email": ["airflow@example.com"], + "email_on_failure": False, + "email_on_retry": False, + "retries": 1, + "retry_delay": timedelta(minutes=5), + # 'queue': 'bash_queue', + # 'pool': 'backfill', + # 'priority_weight': 10, + # 'end_date': datetime(2016, 1, 1), + # 'wait_for_downstream': False, + # 'sla': timedelta(hours=2), + # 'execution_timeout': timedelta(seconds=300), + # 'on_failure_callback': some_function, # or list of functions + # 'on_success_callback': some_other_function, # or list of functions + # 'on_retry_callback': another_function, # or list of functions + # 'sla_miss_callback': yet_another_function, # or list of functions + # 'on_skipped_callback': another_function, #or list of functions + # 'trigger_rule': 'all_success' + }, + description="Provide dbt information", + schedule=timedelta(days=1), + start_date=datetime(2021, 1, 1), + catchup=False, + tags=["POC"], +) as dag: + + # t1, t2 and t3 are examples of tasks created by instantiating operators + t1 = BashOperator( + task_id="dbt_version", + bash_command=f"{os.environ['AIRFLOW_HOME']}/dbt_venv/bin/dbt --version", + ) From c43f5581f62d67f457cbd8d65170673011ec1a18 Mon Sep 17 00:00:00 2001 From: chris-s-friedman Date: Tue, 13 Jan 2026 11:26:31 -0500 Subject: [PATCH 09/35] =?UTF-8?q?=F0=9F=9A=A7=20Test=20putting=20example?= =?UTF-8?q?=20study=20in=20parent=20dags=20folder?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/{kids_first => }/example_study.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename dags/{kids_first => }/example_study.py (100%) diff --git a/dags/kids_first/example_study.py b/dags/example_study.py similarity index 100% rename from dags/kids_first/example_study.py rename to dags/example_study.py From b1bb042c75070e0c634d98d633b0feccc0f2da11 Mon Sep 17 00:00:00 2001 From: chris-s-friedman Date: Tue, 13 Jan 2026 11:40:54 -0500 Subject: [PATCH 10/35] =?UTF-8?q?=F0=9F=9A=A7=20Testing=20nested=20dags?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/{ => kids_first}/dbt_bash.py | 0 dags/{ => kids_first}/example_study.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename dags/{ => kids_first}/dbt_bash.py (100%) rename dags/{ => kids_first}/example_study.py (100%) diff --git a/dags/dbt_bash.py b/dags/kids_first/dbt_bash.py similarity index 100% rename from dags/dbt_bash.py rename to dags/kids_first/dbt_bash.py diff --git a/dags/example_study.py b/dags/kids_first/example_study.py similarity index 100% rename from dags/example_study.py rename to dags/kids_first/example_study.py From 49f932674a4d621533974f3756376c8172b09717 Mon Sep 17 00:00:00 2001 From: chris-s-friedman Date: Tue, 13 Jan 2026 13:08:27 -0500 Subject: [PATCH 11/35] =?UTF-8?q?=F0=9F=90=9B=20Change=20environ=20calls?= =?UTF-8?q?=20to=20get?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/kids_first/example_study.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dags/kids_first/example_study.py b/dags/kids_first/example_study.py index b2a2160..f437f0c 100644 --- a/dags/kids_first/example_study.py +++ b/dags/kids_first/example_study.py @@ -10,19 +10,19 @@ ) profile_config = ProfileConfig( - profile_name=os.environ["DBT_PROFILE_NAME"], - profiles_yml_filepath=os.environ["DBT_PROFILES_YML_PATH"], + profile_name=os.environ.get("DBT_PROFILE_NAME", "default"), + profiles_yml_filepath=os.environ.get("DBT_PROFILES_YML_PATH", "default"), target_name="prd", ) example_study_dag = DbtDag( project_config=ProjectConfig( - os.environ["DBT_PROJECT_DIR"], + os.environ.get("DBT_PROJECT_DIR", "default"), install_dbt_deps=True, ), profile_config=profile_config, execution_config=ExecutionConfig( - dbt_executable_path=os.environ["DBT_EXECUTABLE_PATH"], + dbt_executable_path=os.environ.get("DBT_EXECUTABLE_PATH", "default"), ), render_config=RenderConfig(select=["config.meta.study:kf_example_study"]), # normal dag parameters From c838266f4eb53e13e13ab68024d7939d7c315fe8 Mon Sep 17 00:00:00 2001 From: chris-s-friedman Date: Wed, 14 Jan 2026 13:21:24 -0500 Subject: [PATCH 12/35] =?UTF-8?q?=E2=9E=95=20Use=20airflow=20variable=20in?= =?UTF-8?q?stead=20of=20os.environ?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/kids_first/example_study.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/dags/kids_first/example_study.py b/dags/kids_first/example_study.py index f437f0c..6472545 100644 --- a/dags/kids_first/example_study.py +++ b/dags/kids_first/example_study.py @@ -1,4 +1,5 @@ import os +from airflow.sdk import Variable from datetime import datetime from cosmos import ( @@ -10,19 +11,19 @@ ) profile_config = ProfileConfig( - profile_name=os.environ.get("DBT_PROFILE_NAME", "default"), - profiles_yml_filepath=os.environ.get("DBT_PROFILES_YML_PATH", "default"), + profile_name=Variable.get("DBT_PROFILE_NAME"), + profiles_yml_filepath=Variable.get("DBT_PROFILES_YML_PATH"), target_name="prd", ) example_study_dag = DbtDag( project_config=ProjectConfig( - os.environ.get("DBT_PROJECT_DIR", "default"), + Variable.get("DBT_PROJECT_DIR"), install_dbt_deps=True, ), profile_config=profile_config, execution_config=ExecutionConfig( - dbt_executable_path=os.environ.get("DBT_EXECUTABLE_PATH", "default"), + dbt_executable_path=Variable.get("DBT_EXECUTABLE_PATH"), ), render_config=RenderConfig(select=["config.meta.study:kf_example_study"]), # normal dag parameters From 396b33f208af176b6b954f4fa46682b2441fa1ae Mon Sep 17 00:00:00 2001 From: chris-s-friedman Date: Wed, 14 Jan 2026 13:36:37 -0500 Subject: [PATCH 13/35] =?UTF-8?q?=F0=9F=9A=A7=20Testing=20listing=20files?= =?UTF-8?q?=20where=20I=20think=20dbt=20project=20dir=20is?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/kids_first/dbt_bash.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dags/kids_first/dbt_bash.py b/dags/kids_first/dbt_bash.py index 53b32e8..3f4aa23 100644 --- a/dags/kids_first/dbt_bash.py +++ b/dags/kids_first/dbt_bash.py @@ -1,4 +1,5 @@ from datetime import datetime, timedelta +from airflow.sdk import Variable import os # The DAG object; we'll need this to instantiate a DAG @@ -44,3 +45,6 @@ task_id="dbt_version", bash_command=f"{os.environ['AIRFLOW_HOME']}/dbt_venv/bin/dbt --version", ) + t2 = BashOperator( + task_id="list_files_in_dbt_project", + bash_command="ls -la " + Variable.get("DBT_PROJECT_DIR"), From 0bb28557862a1221adba2fa0af50175fa574f404 Mon Sep 17 00:00:00 2001 From: chris-s-friedman Date: Wed, 14 Jan 2026 13:39:32 -0500 Subject: [PATCH 14/35] =?UTF-8?q?=E2=9C=8F=EF=B8=8F=20Need=20a=20closing?= =?UTF-8?q?=20parens?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/kids_first/dbt_bash.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dags/kids_first/dbt_bash.py b/dags/kids_first/dbt_bash.py index 3f4aa23..1fd3959 100644 --- a/dags/kids_first/dbt_bash.py +++ b/dags/kids_first/dbt_bash.py @@ -48,3 +48,4 @@ t2 = BashOperator( task_id="list_files_in_dbt_project", bash_command="ls -la " + Variable.get("DBT_PROJECT_DIR"), + ) From ef6281dab4813697493fb5256037726b111fe3ac Mon Sep 17 00:00:00 2001 From: chris-s-friedman Date: Wed, 14 Jan 2026 13:43:13 -0500 Subject: [PATCH 15/35] =?UTF-8?q?=F0=9F=9A=A7=20Still=20trying=20to=20find?= =?UTF-8?q?=20my=20dbt=20project?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/kids_first/dbt_bash.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dags/kids_first/dbt_bash.py b/dags/kids_first/dbt_bash.py index 1fd3959..4c673f0 100644 --- a/dags/kids_first/dbt_bash.py +++ b/dags/kids_first/dbt_bash.py @@ -47,5 +47,5 @@ ) t2 = BashOperator( task_id="list_files_in_dbt_project", - bash_command="ls -la " + Variable.get("DBT_PROJECT_DIR"), + bash_command="ls -la usr/local/airflow/dags", ) From bbbd2b27338d3e5de3d926a46139770d78018aab Mon Sep 17 00:00:00 2001 From: chris-s-friedman Date: Wed, 14 Jan 2026 13:49:53 -0500 Subject: [PATCH 16/35] =?UTF-8?q?=F0=9F=94=A5=20Remove=20ls=20in=20dags=20?= =?UTF-8?q?directory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/kids_first/dbt_bash.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/dags/kids_first/dbt_bash.py b/dags/kids_first/dbt_bash.py index 4c673f0..5c34ed0 100644 --- a/dags/kids_first/dbt_bash.py +++ b/dags/kids_first/dbt_bash.py @@ -45,7 +45,3 @@ task_id="dbt_version", bash_command=f"{os.environ['AIRFLOW_HOME']}/dbt_venv/bin/dbt --version", ) - t2 = BashOperator( - task_id="list_files_in_dbt_project", - bash_command="ls -la usr/local/airflow/dags", - ) From 59d960d274bf188ce91bfe56e21b16ff3c93d607 Mon Sep 17 00:00:00 2001 From: chris-s-friedman Date: Fri, 16 Jan 2026 08:36:19 -0500 Subject: [PATCH 17/35] =?UTF-8?q?=F0=9F=9A=A7=20Check=20for=20secret=20var?= =?UTF-8?q?iables?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/kids_first/dbt_bash.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/dags/kids_first/dbt_bash.py b/dags/kids_first/dbt_bash.py index 5c34ed0..d680282 100644 --- a/dags/kids_first/dbt_bash.py +++ b/dags/kids_first/dbt_bash.py @@ -7,6 +7,7 @@ # Operators; we need this to operate! from airflow.operators.bash import BashOperator +from airflow.operators.python import PythonOperator with DAG( "dbt_bash_status", @@ -45,3 +46,15 @@ task_id="dbt_version", bash_command=f"{os.environ['AIRFLOW_HOME']}/dbt_venv/bin/dbt --version", ) + + def check_for_warehouse_host_var(): + warehouse_host = Variable.get("INCLUDEWAREHOUSE_HOST", default_var=None) + if warehouse_host: + print("WAREHOUSE_HOST exists!") + else: + print("WAREHOUSE_HOST variable is not set.") + + t2 = PythonOperator( + task_id="check_for_warehouse_host_var", + python_callable=check_for_warehouse_host_var, + ) From aabeeaf79c91e48973528bf3951cf434339542a6 Mon Sep 17 00:00:00 2001 From: chris-s-friedman Date: Fri, 16 Jan 2026 09:21:11 -0500 Subject: [PATCH 18/35] =?UTF-8?q?=F0=9F=90=9B=20Correctly=20namearg?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/kids_first/dbt_bash.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dags/kids_first/dbt_bash.py b/dags/kids_first/dbt_bash.py index d680282..98b3eb3 100644 --- a/dags/kids_first/dbt_bash.py +++ b/dags/kids_first/dbt_bash.py @@ -48,7 +48,7 @@ ) def check_for_warehouse_host_var(): - warehouse_host = Variable.get("INCLUDEWAREHOUSE_HOST", default_var=None) + warehouse_host = Variable.get("INCLUDEWAREHOUSE_HOST", default=None) if warehouse_host: print("WAREHOUSE_HOST exists!") else: From f7ed140d03d07a269805544b2ca9d3bf39a6991b Mon Sep 17 00:00:00 2001 From: chris-s-friedman Date: Fri, 16 Jan 2026 15:08:06 -0500 Subject: [PATCH 19/35] =?UTF-8?q?=F0=9F=9A=A7=20Change=20cosmos=20to=20use?= =?UTF-8?q?=20connectio?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/kids_first/dbt_bash.py | 4 ++-- dags/kids_first/example_study.py | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/dags/kids_first/dbt_bash.py b/dags/kids_first/dbt_bash.py index 98b3eb3..db14d3a 100644 --- a/dags/kids_first/dbt_bash.py +++ b/dags/kids_first/dbt_bash.py @@ -50,9 +50,9 @@ def check_for_warehouse_host_var(): warehouse_host = Variable.get("INCLUDEWAREHOUSE_HOST", default=None) if warehouse_host: - print("WAREHOUSE_HOST exists!") + print("INCLUDEWAREHOUSE_HOST exists!") else: - print("WAREHOUSE_HOST variable is not set.") + print("INCLUDEWAREHOUSE_HOST variable is not set.") t2 = PythonOperator( task_id="check_for_warehouse_host_var", diff --git a/dags/kids_first/example_study.py b/dags/kids_first/example_study.py index 6472545..762bded 100644 --- a/dags/kids_first/example_study.py +++ b/dags/kids_first/example_study.py @@ -9,11 +9,14 @@ ExecutionConfig, RenderConfig, ) +from cosmos.profiles import PostgresUserPasswordProfileMapping profile_config = ProfileConfig( profile_name=Variable.get("DBT_PROFILE_NAME"), - profiles_yml_filepath=Variable.get("DBT_PROFILES_YML_PATH"), target_name="prd", + profile_mapping=PostgresUserPasswordProfileMapping( + conn_id="postgres_dev_svc" + ), ) example_study_dag = DbtDag( From a2a6e5cb4194c9181c174f70da95c7d3451c4ca4 Mon Sep 17 00:00:00 2001 From: chris-s-friedman Date: Fri, 16 Jan 2026 15:10:21 -0500 Subject: [PATCH 20/35] =?UTF-8?q?=F0=9F=90=9B=20Must=20specify=20schema?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/kids_first/example_study.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dags/kids_first/example_study.py b/dags/kids_first/example_study.py index 762bded..19ba1cb 100644 --- a/dags/kids_first/example_study.py +++ b/dags/kids_first/example_study.py @@ -15,7 +15,8 @@ profile_name=Variable.get("DBT_PROFILE_NAME"), target_name="prd", profile_mapping=PostgresUserPasswordProfileMapping( - conn_id="postgres_dev_svc" + conn_id="postgres_dev_svc", + profile_args={"schema": "prd"}, ), ) From 400898dbf3e0e5157b791b68ee912dbf6ce3d4aa Mon Sep 17 00:00:00 2001 From: chris-s-friedman Date: Fri, 16 Jan 2026 15:24:23 -0500 Subject: [PATCH 21/35] =?UTF-8?q?=E2=9C=8F=EF=B8=8F=20Update=20docs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/kids_first/example_study.py | 2 -- docs/guides/running-models-in-airflow.md | 32 +++++++++++++----------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/dags/kids_first/example_study.py b/dags/kids_first/example_study.py index 19ba1cb..ded91f3 100644 --- a/dags/kids_first/example_study.py +++ b/dags/kids_first/example_study.py @@ -1,6 +1,4 @@ -import os from airflow.sdk import Variable -from datetime import datetime from cosmos import ( DbtDag, diff --git a/docs/guides/running-models-in-airflow.md b/docs/guides/running-models-in-airflow.md index b90f632..8168cc4 100644 --- a/docs/guides/running-models-in-airflow.md +++ b/docs/guides/running-models-in-airflow.md @@ -43,8 +43,7 @@ sections. The purpose of this DAG is to run models in the study `kf_example_study` every day. ```python -import os -from datetime import datetime +from airflow.sdk import Variable from cosmos import ( DbtDag, @@ -53,30 +52,31 @@ from cosmos import ( ExecutionConfig, RenderConfig, ) +from cosmos.profiles import PostgresUserPasswordProfileMapping profile_config = ProfileConfig( - profile_name=os.environ["DBT_PROFILE_NAME"], - profiles_yml_filepath=os.environ["DBT_PROFILES_YML_PATH"], + profile_name=Variable.get("DBT_PROFILE_NAME"), target_name="prd", + profile_mapping=PostgresUserPasswordProfileMapping( + conn_id="postgres_dev_svc", + profile_args={"schema": "prd"}, + ), ) example_study_dag = DbtDag( project_config=ProjectConfig( - "/opt/airflow/dbt/deidentified_etl", + Variable.get("DBT_PROJECT_DIR"), install_dbt_deps=True, ), profile_config=profile_config, execution_config=ExecutionConfig( - dbt_executable_path=os.environ["DBT_EXECUTABLE_PATH"], + dbt_executable_path=Variable.get("DBT_EXECUTABLE_PATH"), ), render_config=RenderConfig(select=["config.meta.study:kf_example_study"]), # normal dag parameters schedule="@daily", - start_date=datetime(2026, 1, 1), - catchup=False, dag_id="kf_example_study", tags=["POC", "Kids First"], - default_args={"retries": 2}, ) ``` @@ -98,18 +98,20 @@ from cosmos import ( ExecutionConfig, RenderConfig, ) +from cosmos.profiles import PostgresUserPasswordProfileMapping ``` -In the example, the `import os` is used to extract environment variables on the -machine used to run dbt and `import datetime` is used to identify when the -DAG should start being run. +In the example, the `from airflow.sdk import Variable` is used to extract +environment variables in the airflow environment used by commands in the DAG. ### `profile_config` The `profile_config` is used by cosmos to identify the profile to be used by dbt -commands. The values for `profile_name` and `profiles_yml_filepath` should use -the indicated environment variables. Acceptable values for `target_name` are -`qa` and `prd` +commands. The value for `profile_name` should use the indicated variables. Acceptable values for `target_name` are `qa` and `prd`. + +The `profile_mapping` takes advantage of an airflow connection object to connect +to the warehouse. Pass it the indicated connection id and make sure to set the +`schema` appropriately for the `target_name` ### the model DAG From 72720182a4dafdc8d69d15edf3b794d613ad7788 Mon Sep 17 00:00:00 2001 From: chris-s-friedman Date: Fri, 16 Jan 2026 15:25:34 -0500 Subject: [PATCH 22/35] =?UTF-8?q?=F0=9F=94=A7=20Don't=20run=20the=20action?= =?UTF-8?q?=20on=20merge?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/export-to-airflow.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/export-to-airflow.yml b/.github/workflows/export-to-airflow.yml index 3153517..17eec17 100644 --- a/.github/workflows/export-to-airflow.yml +++ b/.github/workflows/export-to-airflow.yml @@ -3,8 +3,6 @@ name: Export Repo to Airflow on: push: branches: [ "main" ] - pull_request: - branches: [ "main" ] permissions: contents: read From 63f835985089d9811e2ec4b03f4950d835c53446 Mon Sep 17 00:00:00 2001 From: chris-s-friedman Date: Wed, 21 Jan 2026 09:24:29 -0500 Subject: [PATCH 23/35] =?UTF-8?q?=F0=9F=94=A7=20Useprd=20connection?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/kids_first/example_study.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dags/kids_first/example_study.py b/dags/kids_first/example_study.py index ded91f3..f0c69ec 100644 --- a/dags/kids_first/example_study.py +++ b/dags/kids_first/example_study.py @@ -13,7 +13,7 @@ profile_name=Variable.get("DBT_PROFILE_NAME"), target_name="prd", profile_mapping=PostgresUserPasswordProfileMapping( - conn_id="postgres_dev_svc", + conn_id="postgres_prd_svc", profile_args={"schema": "prd"}, ), ) From c15906376ab5493d0698fd3318a17e06f8c94666 Mon Sep 17 00:00:00 2001 From: Christopher Friedman Date: Wed, 21 Jan 2026 09:27:51 -0500 Subject: [PATCH 24/35] Update docs/guides/running-models-in-airflow.md Co-authored-by: Amanda Warkow --- docs/guides/running-models-in-airflow.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/guides/running-models-in-airflow.md b/docs/guides/running-models-in-airflow.md index 8168cc4..2924ac8 100644 --- a/docs/guides/running-models-in-airflow.md +++ b/docs/guides/running-models-in-airflow.md @@ -1,6 +1,6 @@ # How to Run dbt in Airflow -This guide discusses how dbt models may be run in airflow and the file +This guide discusses how dbt models may be run in Airflow and the file configuration that is needed to run those models. ## How Airflow Runs dbt Models From 4685c51bfd553e4fa9be5572a0122024a76baa6d Mon Sep 17 00:00:00 2001 From: Christopher Friedman Date: Wed, 21 Jan 2026 09:27:58 -0500 Subject: [PATCH 25/35] Update docs/guides/running-models-in-airflow.md Co-authored-by: Amanda Warkow --- docs/guides/running-models-in-airflow.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/guides/running-models-in-airflow.md b/docs/guides/running-models-in-airflow.md index 2924ac8..12f145f 100644 --- a/docs/guides/running-models-in-airflow.md +++ b/docs/guides/running-models-in-airflow.md @@ -5,7 +5,7 @@ configuration that is needed to run those models. ## How Airflow Runs dbt Models -The airflow instance that is used to orchestrate data pipelines is hosted in +The Airflow instance that is used to orchestrate data pipelines is hosted in AWS using Managed Workflow for Apache Airflow (MWAA). MWAA provides a simple way for the airflow instance - as well as its workers to be provisioned and dispatched. As of the writing of this guide, the MWAA instance used by these From 87dcc830f3a37c2d2f861d6ca53e2342397ae13e Mon Sep 17 00:00:00 2001 From: Christopher Friedman Date: Wed, 21 Jan 2026 09:28:09 -0500 Subject: [PATCH 26/35] Update docs/guides/running-models-in-airflow.md Co-authored-by: Amanda Warkow --- docs/guides/running-models-in-airflow.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/guides/running-models-in-airflow.md b/docs/guides/running-models-in-airflow.md index 12f145f..8149076 100644 --- a/docs/guides/running-models-in-airflow.md +++ b/docs/guides/running-models-in-airflow.md @@ -14,7 +14,7 @@ workflows is utilizing Airflow version 3.0.6 and python version 3.12. To run dbt models, we utilize [`cosmos`](https://astronomer.github.io/astronomer-cosmos/index.html). `cosmos` is a python library written for Airflow that has operators that use dbt and convert sets of dbt models within a dbt project into DAGs. While an older -approach may have specified running dbt models within a bash command within an +approach may have specified running dbt models via a bash command within an airflow bash operator, using the `cosmos` `DbtDag` operator allows individual models and those models' tests to be materialized as individual tasks. From 678a3f850e0c8e8e1c95624b380ae3217fc7d479 Mon Sep 17 00:00:00 2001 From: Christopher Friedman Date: Wed, 21 Jan 2026 09:28:15 -0500 Subject: [PATCH 27/35] Update docs/guides/running-models-in-airflow.md Co-authored-by: Amanda Warkow --- docs/guides/running-models-in-airflow.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/guides/running-models-in-airflow.md b/docs/guides/running-models-in-airflow.md index 8149076..f1d948e 100644 --- a/docs/guides/running-models-in-airflow.md +++ b/docs/guides/running-models-in-airflow.md @@ -7,7 +7,7 @@ configuration that is needed to run those models. The Airflow instance that is used to orchestrate data pipelines is hosted in AWS using Managed Workflow for Apache Airflow (MWAA). MWAA provides a simple way -for the airflow instance - as well as its workers to be provisioned and +for the Airflow instance - as well as its workers to be provisioned and dispatched. As of the writing of this guide, the MWAA instance used by these workflows is utilizing Airflow version 3.0.6 and python version 3.12. From 63328665cbdc48fa02a8a24f6fcb9d43ca0ad451 Mon Sep 17 00:00:00 2001 From: Christopher Friedman Date: Wed, 21 Jan 2026 09:28:28 -0500 Subject: [PATCH 28/35] Update docs/guides/running-models-in-airflow.md Co-authored-by: Amanda Warkow --- docs/guides/running-models-in-airflow.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/guides/running-models-in-airflow.md b/docs/guides/running-models-in-airflow.md index f1d948e..ce05dee 100644 --- a/docs/guides/running-models-in-airflow.md +++ b/docs/guides/running-models-in-airflow.md @@ -18,7 +18,7 @@ approach may have specified running dbt models via a bash command within an airflow bash operator, using the `cosmos` `DbtDag` operator allows individual models and those models' tests to be materialized as individual tasks. -For a dbt model - or set of models - to be picked up by airflow and run, those +For a dbt model - or set of models - to be picked up by Airflow and run, those models need to be referenced by a DAG. ### Getting files into MWAA From e4ec61b9dca239d6b617346ba0a50722e3cd412b Mon Sep 17 00:00:00 2001 From: Christopher Friedman Date: Wed, 21 Jan 2026 09:28:37 -0500 Subject: [PATCH 29/35] Update docs/guides/running-models-in-airflow.md Co-authored-by: Amanda Warkow --- docs/guides/running-models-in-airflow.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/guides/running-models-in-airflow.md b/docs/guides/running-models-in-airflow.md index ce05dee..4dee690 100644 --- a/docs/guides/running-models-in-airflow.md +++ b/docs/guides/running-models-in-airflow.md @@ -15,7 +15,7 @@ To run dbt models, we utilize [`cosmos`](https://astronomer.github.io/astronomer `cosmos` is a python library written for Airflow that has operators that use dbt and convert sets of dbt models within a dbt project into DAGs. While an older approach may have specified running dbt models via a bash command within an -airflow bash operator, using the `cosmos` `DbtDag` operator allows individual +Airflow bash operator, using the `cosmos` `DbtDag` operator allows individual models and those models' tests to be materialized as individual tasks. For a dbt model - or set of models - to be picked up by Airflow and run, those From 2880530fca890c86d24a932eaadaa46353dea56b Mon Sep 17 00:00:00 2001 From: Christopher Friedman Date: Wed, 21 Jan 2026 09:28:44 -0500 Subject: [PATCH 30/35] Update docs/guides/running-models-in-airflow.md Co-authored-by: Amanda Warkow --- docs/guides/running-models-in-airflow.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/guides/running-models-in-airflow.md b/docs/guides/running-models-in-airflow.md index 4dee690..40574c2 100644 --- a/docs/guides/running-models-in-airflow.md +++ b/docs/guides/running-models-in-airflow.md @@ -16,7 +16,7 @@ To run dbt models, we utilize [`cosmos`](https://astronomer.github.io/astronomer and convert sets of dbt models within a dbt project into DAGs. While an older approach may have specified running dbt models via a bash command within an Airflow bash operator, using the `cosmos` `DbtDag` operator allows individual -models and those models' tests to be materialized as individual tasks. +models and their corresponding tests to be materialized as individual tasks. For a dbt model - or set of models - to be picked up by Airflow and run, those models need to be referenced by a DAG. From f6ffd5f39913e0f5c45dfbfcb0a74aa9483a65e9 Mon Sep 17 00:00:00 2001 From: chris-s-friedman Date: Wed, 21 Jan 2026 09:29:53 -0500 Subject: [PATCH 31/35] =?UTF-8?q?=F0=9F=8E=A8=20Remove=20commented=20out?= =?UTF-8?q?=20code?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../kids_first/example_study/_kf_example_study__models.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/dbt_project/models/kids_first/example_study/_kf_example_study__models.yml b/dbt_project/models/kids_first/example_study/_kf_example_study__models.yml index 3320f5d..51661a6 100644 --- a/dbt_project/models/kids_first/example_study/_kf_example_study__models.yml +++ b/dbt_project/models/kids_first/example_study/_kf_example_study__models.yml @@ -36,7 +36,6 @@ models: - name: id description: "The primary key for this table" data_tests: - #- unique - not_null - name: letter description: "The letter in the alphabet" From 53232abb53c7f7163bb4156825b77ddcf7ac3780 Mon Sep 17 00:00:00 2001 From: chris-s-friedman Date: Wed, 21 Jan 2026 09:30:40 -0500 Subject: [PATCH 32/35] =?UTF-8?q?=E2=9C=8F=EF=B8=8F=20Point=20to=20prd=20c?= =?UTF-8?q?onnection?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/guides/running-models-in-airflow.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/guides/running-models-in-airflow.md b/docs/guides/running-models-in-airflow.md index 8168cc4..d6b5d0d 100644 --- a/docs/guides/running-models-in-airflow.md +++ b/docs/guides/running-models-in-airflow.md @@ -58,7 +58,7 @@ profile_config = ProfileConfig( profile_name=Variable.get("DBT_PROFILE_NAME"), target_name="prd", profile_mapping=PostgresUserPasswordProfileMapping( - conn_id="postgres_dev_svc", + conn_id="postgres_prd_svc", profile_args={"schema": "prd"}, ), ) From d837b7cd47ad44574748bd15d376a9fae82c2a2b Mon Sep 17 00:00:00 2001 From: chris-s-friedman Date: Wed, 21 Jan 2026 09:31:29 -0500 Subject: [PATCH 33/35] =?UTF-8?q?=F0=9F=9A=A7=20Try=20to=20run=20in=20prd?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/export-to-airflow.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/export-to-airflow.yml b/.github/workflows/export-to-airflow.yml index 17eec17..3153517 100644 --- a/.github/workflows/export-to-airflow.yml +++ b/.github/workflows/export-to-airflow.yml @@ -3,6 +3,8 @@ name: Export Repo to Airflow on: push: branches: [ "main" ] + pull_request: + branches: [ "main" ] permissions: contents: read From ddd0fd0648a56f7f0a1dc5ade54fca734ca0f134 Mon Sep 17 00:00:00 2001 From: chris-s-friedman Date: Wed, 21 Jan 2026 09:32:38 -0500 Subject: [PATCH 34/35] =?UTF-8?q?=F0=9F=93=9D=20Add=20note=20about=20targe?= =?UTF-8?q?t?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/kids_first/example_study.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dags/kids_first/example_study.py b/dags/kids_first/example_study.py index f0c69ec..d6af732 100644 --- a/dags/kids_first/example_study.py +++ b/dags/kids_first/example_study.py @@ -10,6 +10,7 @@ from cosmos.profiles import PostgresUserPasswordProfileMapping profile_config = ProfileConfig( + # make sure target_name and profile_mapping align profile_name=Variable.get("DBT_PROFILE_NAME"), target_name="prd", profile_mapping=PostgresUserPasswordProfileMapping( From c43eed5f948738829a411581f573845f79f01942 Mon Sep 17 00:00:00 2001 From: chris-s-friedman Date: Wed, 21 Jan 2026 09:55:47 -0500 Subject: [PATCH 35/35] =?UTF-8?q?=F0=9F=94=A5=20Dont=20push=20to=20prd=20d?= =?UTF-8?q?uring=20pr?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/export-to-airflow.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/export-to-airflow.yml b/.github/workflows/export-to-airflow.yml index 3153517..17eec17 100644 --- a/.github/workflows/export-to-airflow.yml +++ b/.github/workflows/export-to-airflow.yml @@ -3,8 +3,6 @@ name: Export Repo to Airflow on: push: branches: [ "main" ] - pull_request: - branches: [ "main" ] permissions: contents: read