Merge pull request #39 from getindata/release-0.18.1

swtwsk · web-flow · commit a506a3c0fff4 · 2022-01-18T11:02:58.000+01:00
Release 0.18.1
diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
@@ -32,6 +32,16 @@ jobs:
           pip freeze
           pipdeptree
           pre-commit run --all-files
+
       - name: Test with tox
         run: |
-          tox -e py37
+          tox -e py37
+
+      - name: Report coverage
+        uses: paambaati/codeclimate-action@v2.7.5
+        env:
+          CC_TEST_REPORTER_ID: ${{ secrets.CODE_CLIMATE }}
+        with:
+          coverageCommand: coverage xml
+          debug: true
+          coverageLocations: coverage.xml:coverage.py
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,6 +2,13 @@
 
 ## [Unreleased]
 
+## [0.18.1] - 2022-01-18
+
+### Fixed
+
+-   Jinja's `FileSystemLoader` gets `str` instead of `pathlib.Path` to fix types incompatibility for `Jinja < 2.11.0`.
+-   Use `get_start_task()` and `get_end_task()` in `AirflowDagFactory.create_tasks(config)` to prevent ephemeral ending tasks from throwing.
+
 ## [0.18.0] - 2022-01-14
 
 ### Added
@@ -77,7 +84,9 @@
 
 -   Initial implementation of `dbt_airflow_manifest_parser` library.
 
-[Unreleased]: https://github.com/getindata/dbt-airflow-factory/compare/0.18.0...HEAD
+[Unreleased]: https://github.com/getindata/dbt-airflow-factory/compare/0.18.1...HEAD
+
+[0.18.1]: https://github.com/getindata/dbt-airflow-factory/compare/0.18.0...0.18.1
 
 [0.18.0]: https://github.com/getindata/dbt-airflow-factory/compare/0.17.0...0.18.0
 
diff --git a/README.md b/README.md
@@ -1,7 +1,18 @@
 # DBT Airflow Factory
 
+[![Python Version](https://img.shields.io/badge/python-3.7%20%7C%203.8%20%7C%203.9-blue.svg)](https://github.com/getindata/dbt-airflow-factory)
+[![PyPI Version](https://badge.fury.io/py/dbt-airflow-factory.svg)](https://pypi.org/project/dbt-airflow-factory/)
+[![Downloads](https://pepy.tech/badge/dbt-airflow-factory)](https://pepy.tech/project/dbt-airflow-factory)
+[![Maintainability](https://api.codeclimate.com/v1/badges/47fd3570c858b6c166ad/maintainability)](https://codeclimate.com/github/getindata/dbt-airflow-factory/maintainability)
+[![Test Coverage](https://api.codeclimate.com/v1/badges/47fd3570c858b6c166ad/test_coverage)](https://codeclimate.com/github/getindata/dbt-airflow-factory/test_coverage)
+[![Documentation Status](https://readthedocs.org/projects/dbt-airflow-factory/badge/?version=latest)](https://dbt-airflow-factory.readthedocs.io/en/latest/?badge=latest)
+
 Library to convert DBT manifest metadata to Airflow tasks
 
+## Documentation
+
+Read the full documentation at [https://dbt-airflow-factory.readthedocs.io/](https://dbt-airflow-factory.readthedocs.io/en/latest/index.html)
+
 ## Installation
 
 Use the package manager [pip][pip] to install the library:
diff --git a/dbt_airflow_factory/__init__.py b/dbt_airflow_factory/__init__.py
@@ -1 +1 @@
-version = "0.18.0"
+version = "0.18.1"
diff --git a/dbt_airflow_factory/airflow_dag_factory.py b/dbt_airflow_factory/airflow_dag_factory.py
@@ -85,9 +85,9 @@ def create_tasks(self, config: dict) -> None:
             self._manifest_file_path(config), config.get("use_task_group") or False
         )
         for starting_task in tasks.get_starting_tasks():
-            start >> starting_task.run_airflow_task
+            start >> starting_task.get_start_task()
         for ending_task in tasks.get_ending_tasks():
-            ending_task.test_airflow_task >> end
+            ending_task.get_end_task() >> end
 
     def _create_starting_task(self, config: dict) -> BaseOperator:
         if config.get("seed_task", True):
diff --git a/dbt_airflow_factory/config_utils.py b/dbt_airflow_factory/config_utils.py
@@ -106,7 +106,7 @@ def get(item: str) -> Any:
             return Variable.get(item)
 
     file_path = pathlib.Path(file_path)
-    jinja_loader = FileSystemLoader(file_path.parent)
+    jinja_loader = FileSystemLoader(str(file_path.parent))
     jinja_env = NativeEnvironment(loader=jinja_loader)
 
     return jinja_env.get_template(file_path.name).render(
diff --git a/dbt_airflow_manifest_parser/config_utils.py b/dbt_airflow_manifest_parser/config_utils.py
diff --git a/docs/changelog.md b/docs/changelog.md
@@ -0,0 +1,2 @@
+```{include} ../CHANGELOG.md
+```
diff --git a/docs/changelog.rst b/docs/changelog.rst
diff --git a/docs/index.rst b/docs/index.rst
@@ -1,6 +1,26 @@
 ``dbt-airflow-factory``
 =======================
 
+.. image:: https://img.shields.io/badge/python-3.7%20%7C%203.8%20%7C%203.9-blue.svg
+   :target: https://github.com/getindata/dbt-airflow-factory
+   :alt: Python Version
+
+.. image:: https://badge.fury.io/py/dbt-airflow-factory.svg
+   :target: https://pypi.org/project/dbt-airflow-factory/
+   :alt: PyPI Version
+
+.. image:: https://pepy.tech/badge/dbt-airflow-factory
+   :target: https://pepy.tech/project/dbt-airflow-factory
+   :alt: Downloads
+
+.. image:: https://api.codeclimate.com/v1/badges/47fd3570c858b6c166ad/maintainability
+   :target: https://codeclimate.com/github/getindata/dbt-airflow-factory/maintainability
+   :alt: Maintainability
+
+.. image:: https://api.codeclimate.com/v1/badges/47fd3570c858b6c166ad/test_coverage
+   :target: https://codeclimate.com/github/getindata/dbt-airflow-factory/test_coverage
+   :alt: Test Coverage
+
 
 Installation
 ------------
diff --git a/docs/requirements.txt b/docs/requirements.txt
@@ -1,4 +1,5 @@
 sphinx-rtd-theme==1.0.0
 sphinx-click>=3.0,<3.1
 myst-parser>=0.16, <0.17
-docutils>=0.17,<0.18
+docutils<0.17
+apache-airflow[kubernetes]>=2.2.0,<3.0.0
diff --git a/setup.cfg b/setup.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.18.0
+current_version = 0.18.1
 
 [bumpversion:file:setup.py]
 
diff --git a/setup.py b/setup.py
@@ -23,13 +23,13 @@
         "sphinx-rtd-theme==1.0.0",
         "sphinx-click>=3.0,<3.1",
         "myst-parser>=0.16, <0.17",
-        "docutils>=0.17,<0.18",
+        "docutils<0.17",
     ],
 }
 
 setup(
     name="dbt-airflow-factory",
-    version="0.18.0",
+    version="0.18.1",
     description="Library to convert DBT manifest metadata to Airflow tasks",
     long_description=README,
     long_description_content_type="text/markdown",
diff --git a/tests/manifest_ephemeral.json b/tests/manifest_ephemeral.json
@@ -2,7 +2,9 @@
   "nodes": {
     "model.dbt_test.model1": {
       "depends_on": {
-        "nodes": ["source.dbt_test.source1"]
+        "nodes": [
+          "source.dbt_test.source1"
+        ]
       },
       "config": {
         "materialized": "table"
@@ -100,6 +102,16 @@
       "config": {
         "materialized": "ephemeral"
       }
+    },
+    "model.dbt_test.model11": {
+      "depends_on": {
+        "nodes": [
+          "model.dbt_test.model10"
+        ]
+      },
+      "config": {
+        "materialized": "ephemeral"
+      }
     }
   },
   "sources": {
diff --git a/tests/test_ephemeral_operator.py b/tests/test_ephemeral_operator.py
@@ -19,7 +19,7 @@ def test_ephemeral_dag_factory():
     dag = factory.create()
 
     # then
-    assert len(dag.tasks) == 15
+    assert len(dag.tasks) == 16
 
     task_group_names = [
         el
@@ -39,6 +39,7 @@ def test_ephemeral_dag_factory():
             "model8",
             "model9",
             "model10",
+            "model11",
         ]
     ]
     assert set(dag.task_ids) == set(
@@ -102,3 +103,11 @@ def test_ephemeral_tasks():
         "model10__ephemeral"
         in tasks.get_task("model.dbt_test.model9").run_airflow_task.downstream_task_ids
     )
+    assert (
+        "model11__ephemeral"
+        in tasks.get_task("model.dbt_test.model10").run_airflow_task.downstream_task_ids
+    )
+    assert (
+        "model10__ephemeral"
+        in tasks.get_task("model.dbt_test.model11").run_airflow_task.upstream_task_ids
+    )

Original file line number	Diff line number	Diff line change
`@@ -1 +1 @@`
`1`		`-version = "0.18.0"`
	`1`	`+version = "0.18.1"`