From e1d34e70ac47772fc068ac6556d5bdb35d3273b6 Mon Sep 17 00:00:00 2001 From: Bruno Antonellini Date: Tue, 31 Dec 2024 11:11:14 -0300 Subject: [PATCH 1/2] DCV-3128 changes to setup_template --- dbt_coves/tasks/setup/main.py | 38 ++++++++++++++++++++++++++++++++-- dbt_coves/tasks/setup/utils.py | 5 ++++- 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/dbt_coves/tasks/setup/main.py b/dbt_coves/tasks/setup/main.py index f12cd961..41667395 100644 --- a/dbt_coves/tasks/setup/main.py +++ b/dbt_coves/tasks/setup/main.py @@ -3,8 +3,10 @@ import copier import questionary +import requests from rich.console import Console +from dbt_coves import __dbt_major_version__, __dbt_minor_version__, __dbt_patch_version__ from dbt_coves.tasks.base import NonDbtBaseTask from dbt_coves.utils.tracking import trackable @@ -19,6 +21,11 @@ "dbt-coves' config and/or templates": "setup_dbt_coves", } +THIRD_PARTY_PRECOMMIT_REPOS = { + "dbt_checkpoint": "https://api.github.com/repos/dbt-checkpoint/dbt-checkpoint/tags", + "yamllint": "https://api.github.com/repos/adrienverge/yamllint/tags", +} + console = Console() @@ -72,8 +79,21 @@ def get_config_value(self, key): def run(self) -> int: self.repo_path = os.environ.get("DATACOVES__REPO_PATH", Path().resolve()) self.copier_context = {"no_prompt": self.get_config_value("no_prompt")} + self.copier_context[ + "dbt_core_version" + ] = f"{__dbt_major_version__}.{__dbt_minor_version__}.{__dbt_patch_version__}" + self.copier_context[ + "dbt_adapter_version" + ] = f"{__dbt_major_version__}.{__dbt_minor_version__}" return self.setup_datacoves() + def _get_latest_repo_tag(self, repo_url): + res = requests.get(f"{repo_url}") + if res.status_code != 200: + return None + tags = res.json() + return tags[0].get("name") + def setup_datacoves(self): choices = questionary.checkbox( "What services would you like to set up?", @@ -123,7 +143,9 @@ def setup_datacoves(self): dbt_adapter = os.environ.get("DATACOVES__DBT_ADAPTER") if dbt_adapter: - self.copier_context["dbt_adapter"] = dbt_adapter + self.copier_context["datacoves_dbt_adapter"] = dbt_adapter + else: + self.copier_context["datacoves_dbt_adapter"] = False # sample DAG data if "setup_airflow_dag" in services: @@ -137,11 +159,23 @@ def setup_datacoves(self): yml_dags_path = os.environ.get("DATACOVES__AIRFLOW_DAGS_YML_PATH") if not yml_dags_path: self.copier_context["yml_dags_confirm_path"] = True - self.copier_context["tentative_yml_dags_path"] = "orchestrate/dag_yml_definitions" + self.copier_context["tentative_yml_dags_path"] = "orchestrate/dags_yml_definitions" else: self.copier_context["yml_dags_path"] = yml_dags_path for service in services: self.copier_context[service] = True + + if "setup_precommit" in services: + self.copier_context["dbt_checkpoint_version"] = self._get_latest_repo_tag( + THIRD_PARTY_PRECOMMIT_REPOS["dbt_checkpoint"] + ) + self.copier_context["yamllint_version"] = self._get_latest_repo_tag( + THIRD_PARTY_PRECOMMIT_REPOS["yamllint"] + ) + self.copier_context["sqlfluff_version"] = os.environ.get( + "DATACOVES__SQLFLUFF_VERSION", "3.1.1" + ) + copier.run_auto( src_path=self.get_config_value("template_url"), dst_path=self.repo_path, diff --git a/dbt_coves/tasks/setup/utils.py b/dbt_coves/tasks/setup/utils.py index 3f7c5839..90d1e05f 100644 --- a/dbt_coves/tasks/setup/utils.py +++ b/dbt_coves/tasks/setup/utils.py @@ -48,7 +48,10 @@ def get_git_root(path=None): def get_dbt_projects(path=os.getcwd()): dbt_projects = [] for file in Path(path).rglob("dbt_project.yml"): - if "dbt_packages" not in str(file): + file_str = str(file) + if all( + folder not in file_str for folder in ["dbt_packages", "compiled", "target", "macros"] + ): project_name = open_yaml(file)["name"] project_path = str(file.relative_to(path).parent) dbt_projects.append({"path": project_path, "name": project_name}) From 25553ab1c55860d575873db4b4b7871023b3bee1 Mon Sep 17 00:00:00 2001 From: Bruno Antonellini Date: Tue, 31 Dec 2024 13:23:03 -0300 Subject: [PATCH 2/2] Add datacoves major-minor var --- dbt_coves/tasks/setup/main.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dbt_coves/tasks/setup/main.py b/dbt_coves/tasks/setup/main.py index 41667395..de8adcea 100644 --- a/dbt_coves/tasks/setup/main.py +++ b/dbt_coves/tasks/setup/main.py @@ -176,6 +176,11 @@ def setup_datacoves(self): "DATACOVES__SQLFLUFF_VERSION", "3.1.1" ) + if "setup_ci_cd" in services: + self.copier_context["datacoves_env_version"] = os.environ.get( + "DATACOVES__VERSION_MAJOR_MINOR__ENV", "3" + ) + copier.run_auto( src_path=self.get_config_value("template_url"), dst_path=self.repo_path,