Skip to content

Commit acd99b4

Browse files
github-actions[bot]swtwskp-pekalagithub-actions
authored
Release 0.24.0 (#53)
* Remove ephemeral tasks from DAG (#48) * Remove ephemeral tasks * Simplify test contraction * Add ephemeral tests * Project dependencies (#52) * Remove ephemeral tasks from DAG (#48) * Remove ephemeral tasks * Simplify test contraction * Add ephemeral tests * Refactoring towards builder parameters * Refactoring towards builder parameters * Airflow project dependencies * Airflow project dependencies - tests * Airflow project dependencies - tests * Airflow project dependencies - tests * Airflow project dependencies - tests * Airflow project dependencies - tests * Airflow project dependencies - tests * Airflow project dependencies - tests * Airflow project dependencies - tests * Airflow project dependencies - tests Co-authored-by: Andrzej Swatowski <33041023+swtwsk@users.noreply.github.com> Co-authored-by: p-pekala <piotr.pekala@getindata.com> * FIX #53 - Bump version and CHANGELOG for release 0.24.0 Co-authored-by: Andrzej Swatowski <33041023+swtwsk@users.noreply.github.com> Co-authored-by: Piotr Pękala <piotr_pekala@wp.pl> Co-authored-by: p-pekala <piotr.pekala@getindata.com> Co-authored-by: github-actions <github-actions@github.com>
1 parent d01423b commit acd99b4

27 files changed

+603
-465
lines changed

‎.pre-commit-config.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ repos:
1212
args: ["--profile", "black", "--filter-files"]
1313

1414
- repo: https://github.com/psf/black
15-
rev: 21.12b0
15+
rev: 22.3.0
1616
hooks:
1717
- id: black
1818

‎CHANGELOG.md

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,10 @@
22

33
## [Unreleased]
44

5+
## [0.24.0] - 2022-04-22
6+
7+
- Dependencies between project in Airflow
8+
59
## [0.23.0] - 2022-03-22
610

711
## [0.22.0] - 2022-03-21
@@ -108,10 +112,14 @@ This version brings compatibility with `dbt 1.0`.
108112

109113
- Initial implementation of `dbt_airflow_manifest_parser` library.
110114

111-
[Unreleased]: https://github.com/getindata/dbt-airflow-factory/compare/0.23.0...HEAD
115+
[Unreleased]: https://github.com/getindata/dbt-airflow-factory/compare/0.24.0...HEAD
116+
117+
[0.24.0]: https://github.com/getindata/dbt-airflow-factory/compare/0.23.0...0.24.0
112118

113119
[0.23.0]: https://github.com/getindata/dbt-airflow-factory/compare/0.22.0...0.23.0
114120

121+
[0.22.0]: https://github.com/getindata/dbt-airflow-factory/compare/0.21.0...0.22.0
122+
115123
[0.21.0]: https://github.com/getindata/dbt-airflow-factory/compare/0.20.1...0.21.0
116124

117125
[0.20.1]: https://github.com/getindata/dbt-airflow-factory/compare/0.20.0...0.20.1

‎dbt_airflow_factory/__init__.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
version = "0.23.0"
1+
version = "0.24.0"

‎dbt_airflow_factory/airflow_dag_factory.py

Lines changed: 20 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -13,10 +13,10 @@
1313

1414
from pytimeparse import parse
1515

16-
from dbt_airflow_factory.builder import DbtAirflowTasksBuilder
1716
from dbt_airflow_factory.builder_factory import DbtAirflowTasksBuilderFactory
1817
from dbt_airflow_factory.config_utils import read_config
1918
from dbt_airflow_factory.notifications.handler import NotificationHandlersFactory
19+
from dbt_airflow_factory.tasks_builder.builder import DbtAirflowTasksBuilder
2020

2121

2222
class AirflowDagFactory:
@@ -54,16 +54,12 @@ def __init__(
5454
execution_env_config_file_name: str = "execution_env.yml",
5555
airflow_config_file_name: str = "airflow.yml",
5656
):
57+
self._notifications_handlers_builder = NotificationHandlersFactory()
58+
self.airflow_config = self._read_config(dag_path, env, airflow_config_file_name)
5759
self._builder = DbtAirflowTasksBuilderFactory(
58-
dag_path,
59-
env,
60-
dbt_config_file_name,
61-
execution_env_config_file_name,
60+
dag_path, env, self.airflow_config, dbt_config_file_name, execution_env_config_file_name
6261
).create()
63-
self._notifications_handlers_builder = NotificationHandlersFactory()
6462
self.dag_path = dag_path
65-
self.env = env
66-
self.airflow_config_file_name = airflow_config_file_name
6763

6864
def create(self) -> DAG:
6965
"""
@@ -72,51 +68,45 @@ def create(self) -> DAG:
7268
:return: Generated DAG.
7369
:rtype: airflow.models.dag.DAG
7470
"""
75-
config = self.read_config()
76-
with DAG(default_args=config["default_args"], **config["dag"]) as dag:
77-
self.create_tasks(config)
71+
with DAG(
72+
default_args=self.airflow_config["default_args"], **self.airflow_config["dag"]
73+
) as dag:
74+
self.create_tasks()
7875
return dag
7976

80-
def create_tasks(self, config: dict) -> None:
77+
def create_tasks(self) -> None:
8178
"""
8279
Parse ``manifest.json`` and create tasks based on the data contained there.
83-
84-
:param config: Dictionary representing ``airflow.yml``.
85-
:type config: dict
8680
"""
87-
start = self._create_starting_task(config)
81+
start = self._create_starting_task()
8882
end = DummyOperator(task_id="end")
89-
tasks = self._builder.parse_manifest_into_tasks(
90-
self._manifest_file_path(config),
91-
config.get("use_task_group", False),
92-
config.get("show_ephemeral_models", True),
93-
)
83+
tasks = self._builder.parse_manifest_into_tasks(self._manifest_file_path())
9484
for starting_task in tasks.get_starting_tasks():
9585
start >> starting_task.get_start_task()
9686
for ending_task in tasks.get_ending_tasks():
9787
ending_task.get_end_task() >> end
9888

99-
def _create_starting_task(self, config: dict) -> BaseOperator:
100-
if config.get("seed_task", True):
89+
def _create_starting_task(self) -> BaseOperator:
90+
if self.airflow_config.get("seed_task", True):
10191
return self._builder.create_seed_task()
10292
else:
10393
return DummyOperator(task_id="start")
10494

105-
def _manifest_file_path(self, config: dict) -> str:
106-
file_dir = config.get("manifest_dir_path", self.dag_path)
107-
return os.path.join(file_dir, config.get("manifest_file_name", "manifest.json"))
95+
def _manifest_file_path(self) -> str:
96+
file_dir = self.airflow_config.get("manifest_dir_path", self.dag_path)
97+
return os.path.join(
98+
file_dir, self.airflow_config.get("manifest_file_name", "manifest.json")
99+
)
108100

109-
def read_config(self) -> dict:
101+
def _read_config(self, dag_path: str, env: str, airflow_config_file_name: str) -> dict:
110102
"""
111103
Read ``airflow.yml`` from ``config`` directory into a dictionary.
112104
113105
:return: Dictionary representing ``airflow.yml``.
114106
:rtype: dict
115107
:raises KeyError: No ``default_args`` key in ``airflow.yml``.
116108
"""
117-
config = read_config(
118-
self.dag_path, self.env, self.airflow_config_file_name, replace_jinja=True
119-
)
109+
config = read_config(dag_path, env, airflow_config_file_name, replace_jinja=True)
120110
if "retry_delay" in config["default_args"]:
121111
config["default_args"]["retry_delay"] = parse(config["default_args"]["retry_delay"])
122112
if "failure_handlers" in config:

0 commit comments

Comments
 (0)