From c2ad137588de12aeb7b4b3179f066c18cc0a7214 Mon Sep 17 00:00:00 2001 From: Noel Gomez Date: Sun, 5 Oct 2025 13:47:09 -0700 Subject: [PATCH 01/21] add titan --- .../create_snowflake_objects.py | 0 secure/{ => permifrost}/databases.yml | 0 secure/{ => permifrost}/permifrost.yml | 0 secure/{ => permifrost}/permifrost_pii.yml | 0 secure/{ => permifrost}/roles.yml | 0 secure/{ => permifrost}/run_permifrost.sh | 0 secure/{ => permifrost}/run_permifrost_pii.sh | 0 secure/{ => permifrost}/users.yml | 0 secure/{ => permifrost}/users_pii.yml | 0 secure/{ => permifrost}/warehouses.yml | 0 secure/titan/.env.sample | 5 ++ secure/titan/README.md | 16 ++++ secure/titan/a_TODO/roles__base.yml | 24 ++++++ secure/titan/a_TODO/roles__functional.yml | 44 +++++++++++ secure/titan/apply.sh | 25 +++++++ secure/titan/plan.sh | 25 +++++++ secure/titan/resources/databases.yml | 74 +++++++++++++++++++ .../resources/object_templates/database.yml | 18 +++++ .../resources/object_templates/schema.yml | 18 +++++ .../resources/object_templates/warehouses.yml | 24 ++++++ secure/titan/resources/roles__functional.yml | 48 ++++++++++++ secure/titan/resources/schemas.yml | 19 +++++ secure/titan/resources/stages.yml | 7 ++ secure/titan/resources/users.yml | 46 ++++++++++++ secure/titan/resources/warehouses.yml | 29 ++++++++ 25 files changed, 422 insertions(+) rename secure/{ => permifrost}/create_snowflake_objects.py (100%) rename secure/{ => permifrost}/databases.yml (100%) rename secure/{ => permifrost}/permifrost.yml (100%) rename secure/{ => permifrost}/permifrost_pii.yml (100%) rename secure/{ => permifrost}/roles.yml (100%) rename secure/{ => permifrost}/run_permifrost.sh (100%) rename secure/{ => permifrost}/run_permifrost_pii.sh (100%) rename secure/{ => permifrost}/users.yml (100%) rename secure/{ => permifrost}/users_pii.yml (100%) rename secure/{ => permifrost}/warehouses.yml (100%) create mode 100644 secure/titan/.env.sample create mode 100644 secure/titan/README.md create mode 100644 secure/titan/a_TODO/roles__base.yml create mode 100644 secure/titan/a_TODO/roles__functional.yml create mode 100755 secure/titan/apply.sh create mode 100755 secure/titan/plan.sh create mode 100644 secure/titan/resources/databases.yml create mode 100644 secure/titan/resources/object_templates/database.yml create mode 100644 secure/titan/resources/object_templates/schema.yml create mode 100644 secure/titan/resources/object_templates/warehouses.yml create mode 100644 secure/titan/resources/roles__functional.yml create mode 100644 secure/titan/resources/schemas.yml create mode 100644 secure/titan/resources/stages.yml create mode 100644 secure/titan/resources/users.yml create mode 100644 secure/titan/resources/warehouses.yml diff --git a/secure/create_snowflake_objects.py b/secure/permifrost/create_snowflake_objects.py similarity index 100% rename from secure/create_snowflake_objects.py rename to secure/permifrost/create_snowflake_objects.py diff --git a/secure/databases.yml b/secure/permifrost/databases.yml similarity index 100% rename from secure/databases.yml rename to secure/permifrost/databases.yml diff --git a/secure/permifrost.yml b/secure/permifrost/permifrost.yml similarity index 100% rename from secure/permifrost.yml rename to secure/permifrost/permifrost.yml diff --git a/secure/permifrost_pii.yml b/secure/permifrost/permifrost_pii.yml similarity index 100% rename from secure/permifrost_pii.yml rename to secure/permifrost/permifrost_pii.yml diff --git a/secure/roles.yml b/secure/permifrost/roles.yml similarity index 100% rename from secure/roles.yml rename to secure/permifrost/roles.yml diff --git a/secure/run_permifrost.sh b/secure/permifrost/run_permifrost.sh similarity index 100% rename from secure/run_permifrost.sh rename to secure/permifrost/run_permifrost.sh diff --git a/secure/run_permifrost_pii.sh b/secure/permifrost/run_permifrost_pii.sh similarity index 100% rename from secure/run_permifrost_pii.sh rename to secure/permifrost/run_permifrost_pii.sh diff --git a/secure/users.yml b/secure/permifrost/users.yml similarity index 100% rename from secure/users.yml rename to secure/permifrost/users.yml diff --git a/secure/users_pii.yml b/secure/permifrost/users_pii.yml similarity index 100% rename from secure/users_pii.yml rename to secure/permifrost/users_pii.yml diff --git a/secure/warehouses.yml b/secure/permifrost/warehouses.yml similarity index 100% rename from secure/warehouses.yml rename to secure/permifrost/warehouses.yml diff --git a/secure/titan/.env.sample b/secure/titan/.env.sample new file mode 100644 index 000000000..dbd2d7eb6 --- /dev/null +++ b/secure/titan/.env.sample @@ -0,0 +1,5 @@ +SNOWFLAKE_ACCOUNT= +SNOWFLAKE_USER= +SNOWFLAKE_ROLE=securityadmin +SNOWFLAKE_PRIVATE_KEY_PATH=/config/.ssl/dev-private.pem +SNOWFLAKE_AUTHENTICATOR=SNOWFLAKE_JWT diff --git a/secure/titan/README.md b/secure/titan/README.md new file mode 100644 index 000000000..11e078306 --- /dev/null +++ b/secure/titan/README.md @@ -0,0 +1,16 @@ +# Securing Snowflake Objects + +Using Titan allows us to create Snowflake objects and to also apply permissions to them + +## Running Plan + +First run `plan.sh` to see the changes that will be applied to Snowflake + +## Running Apple + +First run `apply.sh` will apply all changes to Snowflake + +### Titan Docs +https://titan-core.gitbook.io/titan-core + +https://github.com/datacoves/titan/tree/rc/docs diff --git a/secure/titan/a_TODO/roles__base.yml b/secure/titan/a_TODO/roles__base.yml new file mode 100644 index 000000000..bea2a39ac --- /dev/null +++ b/secure/titan/a_TODO/roles__base.yml @@ -0,0 +1,24 @@ +roles: + - name: z_create__schema__db__raw + - name: z_create__account__database + - name: z_tables_views__r + - name: z_db__tst_chiquita__r # Used to grant access to TST databases + +grants: + # System role grants (this will be recreated on every run) + - priv: "CREATE SCHEMA" + on: "database RAW" + to: z_create__schema__db__raw + + - priv: "CREATE DATABASE" + on: "ACCOUNT" + to: z_create__account__database + + - for_each: var.schemas + priv: "SELECT" + on: + - all tables in schema {{ each.value.name }} + - all views in schema {{ each.value.name }} + - "future tables in schema {{ each.value.name }}" + - "future views in schema {{ each.value.name }}" + to: z_tables_views__r diff --git a/secure/titan/a_TODO/roles__functional.yml b/secure/titan/a_TODO/roles__functional.yml new file mode 100644 index 000000000..78ee83909 --- /dev/null +++ b/secure/titan/a_TODO/roles__functional.yml @@ -0,0 +1,44 @@ +roles: + - name: loader + - name: transformer_dbt + - name: analyst + - name: z_base__analyst + +role_grants: + - to_role: z_base__analyst + roles: + - z_db__raw + - z_db__prd_chiquita + + - z_schema__AKS_DATACOVES + - z_schema__AKS_DATACOVES_CDC + - z_schema__AKS_DATACOVES_CDC_STAGING + - z_schema__AKS_GRAFANA + - z_schema__AKS_GRAFANA_STAGING + - z_schema__DATACOVES + - z_schema__L1__GRAFANA + - z_schema__DBT_TEST__AUDIT + - z_schema__SEEDS + - z_schema__L3_MARTS + + - z_tables_views__r + + - z_wh__wh_transforming + + - to_role: SYSADMIN + roles: + - z_create__schema__db__raw + + - to_role: analyst + roles: + - z_base__analyst + + - to_role: loader + roles: + - z_wh__wh_transforming + + - to_role: transformer_dbt + roles: + - z_create__account__database + - z_tables_views__r + - z_wh__wh_transforming diff --git a/secure/titan/apply.sh b/secure/titan/apply.sh new file mode 100755 index 000000000..061ddfbec --- /dev/null +++ b/secure/titan/apply.sh @@ -0,0 +1,25 @@ +#!/bin/bash +secure_path=/config/workspace/secure/titan +cd $secure_path + +if [ -f .env ]; then + echo "File .env found." +else + echo "File .env does not exist. Please create a .env file with the following variables:" + echo "" + echo "SNOWFLAKE_ACCOUNT=" + echo "SNOWFLAKE_USER=" + echo "SNOWFLAKE_ROLE=" + echo "SNOWFLAKE_PRIVATE_KEY_PATH=" + echo "SNOWFLAKE_AUTHENTICATOR=SNOWFLAKE_JWT" + echo "" + exit 1 +fi + +export $(cat .env | xargs) + +uvx --from titan-core@git+https://github.com/datacoves/titan.git@rc \ + --refresh \ + titan apply \ + --config resources/ \ + --sync_resources role,grant,role_grant diff --git a/secure/titan/plan.sh b/secure/titan/plan.sh new file mode 100755 index 000000000..689b2521e --- /dev/null +++ b/secure/titan/plan.sh @@ -0,0 +1,25 @@ +#!/bin/bash +secure_path=/config/workspace/secure/titan +cd $secure_path + +if [ -f .env ]; then + echo "File .env found." +else + echo "File .env does not exist. Please create a .env file with the following variables:" + echo "" + echo "SNOWFLAKE_ACCOUNT=" + echo "SNOWFLAKE_USER=" + echo "SNOWFLAKE_ROLE=" + echo "SNOWFLAKE_PRIVATE_KEY_PATH=" + echo "SNOWFLAKE_AUTHENTICATOR=SNOWFLAKE_JWT" + echo "" + exit 1 +fi + +export $(cat .env | xargs) + +uvx --from titan-core@git+https://github.com/datacoves/titan.git@rc \ + --refresh \ + titan plan \ + --config resources/ \ + --sync_resources role,grant,role_grant diff --git a/secure/titan/resources/databases.yml b/secure/titan/resources/databases.yml new file mode 100644 index 000000000..8365c09e5 --- /dev/null +++ b/secure/titan/resources/databases.yml @@ -0,0 +1,74 @@ +vars: + - name: databases + type: list + default: + - name: raw + owner: loader + max_data_extension_time_in_days: 10 + - name: balboa + owner: transformer_dbt + max_data_extension_time_in_days: 30 + - name: balboa_apps + owner: transformer_dbt + max_data_extension_time_in_days: 10 + - name: balboa_dev + owner: transformer_dbt + max_data_extension_time_in_days: 5 + - name: great_bay + owner: transformer_dbt + max_data_extension_time_in_days: 10 + - name: sushi_db + owner: transformer_dbt + max_data_extension_time_in_days: 10 + + +# - balboa: +# shared: false +# schemas: +# - L1_ACCOUNT_USAGE +# - L1_COUNTRY_DATA +# - L1_COUNTRY_GEO +# - L1_COVID19_EPIDEMIOLOGICAL_DATA +# - L1_GOOGLE_ANALYTICS_4 +# - L1_LOANS +# - L1_OBSERVE +# - L1_US_POPULATION +# - L1_USGS__EARTHQUAKE_DATA + +# - L2_COUNTRY_DEMOGRAPHICS +# - L2_COVID_OBSERVATIONS +# - L2_SNOWFLAKE_USAGE + +# - L3_COVID_ANALYTICS +# - L3_EARTHQUAKE_ANALYTICS +# - L3_LOAN_ANALYTICS + +# - DBT_TEST__AUDIT + +# - SEEDS + +# - MASKING_POLICIES + +# - balboa_apps: +# shared: false +# schemas: +# - RESOURCES + +# - balboa_dev: +# shared: false +# # - convexa_dev: +# # shared: false +# # Snowflake database contains metadata describing usage +# - snowflake: +# shared: true + +# - covid19_epidemiological_data: +# shared: true + +# - great_bay: +# shared: false +# schemas: +# - COVE_MARKETING + +# - sushi_db: +# shared: false diff --git a/secure/titan/resources/object_templates/database.yml b/secure/titan/resources/object_templates/database.yml new file mode 100644 index 000000000..c62a5d899 --- /dev/null +++ b/secure/titan/resources/object_templates/database.yml @@ -0,0 +1,18 @@ +# Databases +databases: + - for_each: var.databases + name: "{{ each.value.name }}" + owner: "{{ each.value.owner }}" + max_data_extension_time_in_days: "{{ each.value.max_data_extension_time_in_days }}" + +# Database roles +roles: + - for_each: var.databases + name: "z_db__{{ each.value.name }}" + +# Database grants +grants: + - for_each: var.databases + priv: USAGE + on: database "{{ each.value.name }}" + to: "z_db__{{ each.value.name }}" diff --git a/secure/titan/resources/object_templates/schema.yml b/secure/titan/resources/object_templates/schema.yml new file mode 100644 index 000000000..c6f160dd7 --- /dev/null +++ b/secure/titan/resources/object_templates/schema.yml @@ -0,0 +1,18 @@ +# Schemas +schemas: + - for_each: var.schemas + name: "{{ each.value.name.split('.')[1] }}" + database: "{{ each.value.name.split('.')[0] }}" + owner: "{{ each.value.get('owner', parent.owner) }}" + +# Schema roles +roles: + - for_each: var.schemas + name: "z_schema__{{ each.value.name.split('.')[1] }}" + +# Schemas grants +grants: + - for_each: var.schemas + priv: USAGE + on: "schema {{ each.value.name }}" + to: "z_schema__{{ each.value.name.split('.')[1] }}" diff --git a/secure/titan/resources/object_templates/warehouses.yml b/secure/titan/resources/object_templates/warehouses.yml new file mode 100644 index 000000000..b1b6f958d --- /dev/null +++ b/secure/titan/resources/object_templates/warehouses.yml @@ -0,0 +1,24 @@ +# Warehouses +warehouses: + - for_each: var.warehouses + name: "{{ each.value.name }}" + warehouse_size: "{{ each.value.size }}" + auto_suspend: "{{ each.value.auto_suspend }}" + auto_resume: true + initially_suspended: true + statement_timeout_in_seconds: "{{ each.value.statement_timeout_in_seconds | default(3600) }}" + +# Warehouse roles +roles: + - for_each: var.warehouses + name: "z_wh__{{ each.value.name }}" + + +# Warehouse grants +grants: + - for_each: var.warehouses + priv: + - USAGE + - MONITOR + on: warehouse {{ each.value.name }} + to: z_wh__{{ each.value.name }} diff --git a/secure/titan/resources/roles__functional.yml b/secure/titan/resources/roles__functional.yml new file mode 100644 index 000000000..e57835c8c --- /dev/null +++ b/secure/titan/resources/roles__functional.yml @@ -0,0 +1,48 @@ +roles: + - name: loader + - name: transformer_dbt + - name: analyst + - name: z_base__analyst + +role_grants: + - to_role: z_base__analyst + roles: + - z_db__raw + - z_db__balboa + - z_db__balboa_apps + - z_db__balboa_dev + - z_db__great_bay + - z_db__sushi_db + +# - z_schema__AKS_DATACOVES +# - z_schema__AKS_DATACOVES_CDC +# - z_schema__AKS_DATACOVES_CDC_STAGING +# - z_schema__AKS_GRAFANA +# - z_schema__AKS_GRAFANA_STAGING +# - z_schema__DATACOVES +# - z_schema__L1__GRAFANA +# - z_schema__DBT_TEST__AUDIT +# - z_schema__SEEDS +# - z_schema__L3_MARTS + +# - z_tables_views__r + +# - z_wh__wh_transforming + +# - to_role: SYSADMIN +# roles: +# - z_create__schema__db__raw + + - to_role: analyst + roles: + - z_base__analyst + +# - to_role: loader +# roles: +# - z_wh__wh_transforming + +# - to_role: transformer_dbt +# roles: +# - z_create__account__database +# - z_tables_views__r +# - z_wh__wh_transforming diff --git a/secure/titan/resources/schemas.yml b/secure/titan/resources/schemas.yml new file mode 100644 index 000000000..199bd0794 --- /dev/null +++ b/secure/titan/resources/schemas.yml @@ -0,0 +1,19 @@ +# Schemas +vars: + - name: schemas + type: list + default: + - name: RAW.SNAPSHOTS + owner: transformer_dbt + - name: RAW.DBT_ARTIFACTS + owner: transformer_dbt + + + # - name: PRD_CHIQUITA.L1_STAGING + # - name: PRD_CHIQUITA.L1_STAGING__SEEDS + # - name: PRD_CHIQUITA.L2_CORE + # - name: PRD_CHIQUITA.L3_MARTS + # - name: PRD_CHIQUITA.DBT_TEST__AUDIT + # - name: PRD_CHIQUITA.SEEDS + + # - name: PRD_DBT_GENERATED.DBT_ARTIFACTS diff --git a/secure/titan/resources/stages.yml b/secure/titan/resources/stages.yml new file mode 100644 index 000000000..3258ef1a2 --- /dev/null +++ b/secure/titan/resources/stages.yml @@ -0,0 +1,7 @@ +stages: + - name: raw.dbt_artifacts.artifacts + type: internal + owner: transformer_dbt + directory: + enable: true + comment: Used to store dbt artifacts diff --git a/secure/titan/resources/users.yml b/secure/titan/resources/users.yml new file mode 100644 index 000000000..abd73c22f --- /dev/null +++ b/secure/titan/resources/users.yml @@ -0,0 +1,46 @@ +# Users should not be created on titan since passwords / keypairs can be lost on every run +# users: +# - name: svc_orchestrate +# owner: SECURITYADMIN +# type: SERVICE +# - name: svc_ci +# owner: SECURITYADMIN +# type: SERVICE +# - name: svc_looker +# owner: SECURITYADMIN +# type: SERVICE + +role_grants: + - to_user: gomezn + roles: + - analyst + - loader + - transformer_dbt + - accountadmin + - securityadmin + + # - to_user: alejandro + # roles: + # - analyst + # - loader + # - transformer_dbt + # - accountadmin + # - securityadmin + + # - to_user: sebastian + # roles: + # - analyst + # - loader + # - transformer_dbt + # - accountadmin + # - securityadmin + + # - to_user: svc_ci + # roles: + # - analyst + # - loader + # - transformer_dbt + + # - to_user: svc_looker + # roles: + # - analyst diff --git a/secure/titan/resources/warehouses.yml b/secure/titan/resources/warehouses.yml new file mode 100644 index 000000000..999a83a94 --- /dev/null +++ b/secure/titan/resources/warehouses.yml @@ -0,0 +1,29 @@ +vars: + - name: warehouses + type: list + default: + - name: wh_admin + size: x-small + auto_suspend: 60 # seconds 0 means it never suspends + statement_timeout_in_seconds: 7200 # seconds, defaults to 3600 if not defined + - name: wh_catalog + size: x-small + auto_suspend: 60 + - name: wh_integration + size: x-small + auto_suspend: 60 + - name: wh_loading + size: x-small + auto_suspend: 60 + - name: wh_orchestrating + size: x-small + auto_suspend: 60 + - name: wh_transforming + size: x-small + auto_suspend: 60 + - name: wh_transforming_dynamic_tables + size: x-small + auto_suspend: 60 + - name: wh_transforming_sqlmesh + size: x-small + auto_suspend: 60 From db6abd304f4b4bf5670c461046120c60f7fe6797 Mon Sep 17 00:00:00 2001 From: Sebastian Sassi Date: Fri, 10 Oct 2025 16:56:31 -0700 Subject: [PATCH 02/21] Add role grants. --- secure/titan/resources/roles__functional.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/secure/titan/resources/roles__functional.yml b/secure/titan/resources/roles__functional.yml index e57835c8c..cc8eadc8e 100644 --- a/secure/titan/resources/roles__functional.yml +++ b/secure/titan/resources/roles__functional.yml @@ -13,7 +13,11 @@ role_grants: - z_db__balboa_dev - z_db__great_bay - z_db__sushi_db - + - priv: "USAGE" + on: + - "all schema in database RAW" + - "future schemas in database RAW" + to: z_schemas__raw # - z_schema__AKS_DATACOVES # - z_schema__AKS_DATACOVES_CDC # - z_schema__AKS_DATACOVES_CDC_STAGING From 11cd4eb0abec23caac8cf691383a45e2a5f1b002 Mon Sep 17 00:00:00 2001 From: Noel Gomez Date: Sat, 18 Oct 2025 13:57:31 -0700 Subject: [PATCH 03/21] Titan role updates --- secure/titan/a_TODO/roles__base.yml | 24 ------- secure/titan/resources/databases.yml | 27 ++++++++ secure/titan/resources/roles__base.yml | 30 +++++++++ secure/titan/resources/roles__functional.yml | 52 +++++++-------- secure/titan/resources/schemas.yml | 32 ++++++++- secure/titan/resources/users.yml | 68 +++++++++++--------- 6 files changed, 152 insertions(+), 81 deletions(-) delete mode 100644 secure/titan/a_TODO/roles__base.yml create mode 100644 secure/titan/resources/roles__base.yml diff --git a/secure/titan/a_TODO/roles__base.yml b/secure/titan/a_TODO/roles__base.yml deleted file mode 100644 index bea2a39ac..000000000 --- a/secure/titan/a_TODO/roles__base.yml +++ /dev/null @@ -1,24 +0,0 @@ -roles: - - name: z_create__schema__db__raw - - name: z_create__account__database - - name: z_tables_views__r - - name: z_db__tst_chiquita__r # Used to grant access to TST databases - -grants: - # System role grants (this will be recreated on every run) - - priv: "CREATE SCHEMA" - on: "database RAW" - to: z_create__schema__db__raw - - - priv: "CREATE DATABASE" - on: "ACCOUNT" - to: z_create__account__database - - - for_each: var.schemas - priv: "SELECT" - on: - - all tables in schema {{ each.value.name }} - - all views in schema {{ each.value.name }} - - "future tables in schema {{ each.value.name }}" - - "future views in schema {{ each.value.name }}" - to: z_tables_views__r diff --git a/secure/titan/resources/databases.yml b/secure/titan/resources/databases.yml index 8365c09e5..f7e382e3b 100644 --- a/secure/titan/resources/databases.yml +++ b/secure/titan/resources/databases.yml @@ -21,6 +21,33 @@ vars: owner: transformer_dbt max_data_extension_time_in_days: 10 +roles: + # - name: z_db__covid19_epidemiological_data + # - name: z_schemas__db__covid19_epidemiological_data + - name: z_db__snowflake + # - name: z_schemas__db__snowflake + +grants: + # - priv: "USAGE" + # on: "database covid19_epidemiological_data" + # to: z_db__covid19_epidemiological_data + + # - priv: "USAGE" + # on: + # - "all schemas in database covid19_epidemiological_data" + # - "future schemas in database covid19_epidemiological_data" + # to: z_schemas__db__covid19_epidemiological_data + + # - priv: "IMPORTED PRIVILEGES" + # on: "database snowflake" + # to: z_db__snowflake + + # - priv: "USAGE" + # on: + # - "all schemas in database snowflake" + # - "future schemas in database snowflake" + # to: z_schemas__db__snowflake + # - balboa: # shared: false diff --git a/secure/titan/resources/roles__base.yml b/secure/titan/resources/roles__base.yml new file mode 100644 index 000000000..71d1551b4 --- /dev/null +++ b/secure/titan/resources/roles__base.yml @@ -0,0 +1,30 @@ +roles: + - name: z_account__create_database + - name: z_db__tst_balboa # Used to grant access to TST databases + - name: z_db__balboa_dev__create_schema + - name: z_schemas__db__raw + - name: z_tables_views__select + +grants: + - priv: "CREATE DATABASE" + on: "ACCOUNT" + to: z_account__create_database + + - priv: "CREATE SCHEMA" + on: "database balboa_dev" + to: z_db__balboa_dev__create_schema + + - priv: "USAGE" + on: + - "all schemas in database raw" + - "future schemas in database raw" + to: z_schemas__db__raw + + - for_each: var.schemas + priv: "SELECT" + on: + - all tables in schema {{ each.value.name }} + - all views in schema {{ each.value.name }} + - "future tables in schema {{ each.value.name }}" + - "future views in schema {{ each.value.name }}" + to: z_tables_views__select diff --git a/secure/titan/resources/roles__functional.yml b/secure/titan/resources/roles__functional.yml index e57835c8c..7988077ac 100644 --- a/secure/titan/resources/roles__functional.yml +++ b/secure/titan/resources/roles__functional.yml @@ -1,48 +1,48 @@ roles: + - name: z_base__analyst + + - name: analyst - name: loader - name: transformer_dbt - - name: analyst - - name: z_base__analyst role_grants: - to_role: z_base__analyst roles: - z_db__raw + - z_schemas__db__raw + - z_db__balboa - z_db__balboa_apps + - z_db__balboa_dev + - z_db__balboa_dev__create_schema + - z_db__great_bay - z_db__sushi_db -# - z_schema__AKS_DATACOVES -# - z_schema__AKS_DATACOVES_CDC -# - z_schema__AKS_DATACOVES_CDC_STAGING -# - z_schema__AKS_GRAFANA -# - z_schema__AKS_GRAFANA_STAGING -# - z_schema__DATACOVES -# - z_schema__L1__GRAFANA -# - z_schema__DBT_TEST__AUDIT -# - z_schema__SEEDS -# - z_schema__L3_MARTS - -# - z_tables_views__r + # - z_db__covid19_epidemiological_data + # - z_schemas__db__covid19_epidemiological_data -# - z_wh__wh_transforming + - z_db__snowflake + # - z_schemas__db__snowflake -# - to_role: SYSADMIN -# roles: -# - z_create__schema__db__raw + - z_wh__wh_transforming + - z_wh__wh_transforming_dynamic_tables - to_role: analyst roles: - z_base__analyst + - z_tables_views__select -# - to_role: loader -# roles: -# - z_wh__wh_transforming + - to_role: loader + roles: + - z_wh__wh_loading + + - to_role: transformer_dbt + roles: + - z_account__create_database + # - z_db__covid19_epidemiological_data -# - to_role: transformer_dbt -# roles: -# - z_create__account__database -# - z_tables_views__r -# - z_wh__wh_transforming + - z_wh__wh_transforming + - z_wh__wh_transforming_dynamic_tables + - z_tables_views__select diff --git a/secure/titan/resources/schemas.yml b/secure/titan/resources/schemas.yml index 199bd0794..6c81ea008 100644 --- a/secure/titan/resources/schemas.yml +++ b/secure/titan/resources/schemas.yml @@ -3,10 +3,38 @@ vars: - name: schemas type: list default: - - name: RAW.SNAPSHOTS - owner: transformer_dbt + - name: RAW.AB_LOANS + - name: RAW.AIRBYTE_INTERNAL + - name: RAW.COUNTRY_GEO - name: RAW.DBT_ARTIFACTS owner: transformer_dbt + - name: RAW.GOOGLE_ANALYTICS_4 + - name: RAW.LOANS + - name: RAW.RAW + - name: RAW.SNAPSHOTS + owner: transformer_dbt + - name: RAW.USGS__EARTHQUAKE_DATA + - name: RAW.US_POPULATION + + - name: BALBOA.DBT_TEST__AUDIT + - name: BALBOA.L1_ACCOUNT_USAGE + - name: BALBOA.L1_COUNTRY_DATA + - name: BALBOA.L1_COUNTRY_GEO + - name: BALBOA.L1_COVID19_EPIDEMIOLOGICAL_DATA + - name: BALBOA.L1_GOOGLE_ANALYTICS_4 + - name: BALBOA.L1_LOANS + - name: BALBOA.L1_OBSERVE + - name: BALBOA.L1_USGS__EARTHQUAKE_DATA + - name: BALBOA.L1_US_POPULATION + - name: BALBOA.L1_US_POPULATIONS + - name: BALBOA.L2_COUNTRY_DEMOGRAPHICS + - name: BALBOA.L2_COVID_OBSERVATIONS + - name: BALBOA.L2_SNOWFLAKE_USAGE + - name: BALBOA.L3_COVID_ANALYTICS + - name: BALBOA.L3_EARTHQUAKE_ANALYTICS + - name: BALBOA.L3_LOAN_ANALYTICS + - name: BALBOA.MASKING_POLICIES + - name: BALBOA.SEEDS # - name: PRD_CHIQUITA.L1_STAGING diff --git a/secure/titan/resources/users.yml b/secure/titan/resources/users.yml index abd73c22f..789c5367b 100644 --- a/secure/titan/resources/users.yml +++ b/secure/titan/resources/users.yml @@ -1,14 +1,23 @@ -# Users should not be created on titan since passwords / keypairs can be lost on every run +# Users should not be created on titan since keypairs can be lost on every run # users: -# - name: svc_orchestrate +# - name: svc_airbyte # owner: SECURITYADMIN # type: SERVICE -# - name: svc_ci +# - name: svc_airflow # owner: SECURITYADMIN # type: SERVICE -# - name: svc_looker +# - name: svc_balboa_ci # owner: SECURITYADMIN # type: SERVICE +# - name: svc_fivetran +# owner: SECURITYADMIN +# type: SERVICE + +# create or replace user svc_fivetran +# type = 'service' +# default_warehouse = 'wh_loading' +# rsa_public_key = '' + role_grants: - to_user: gomezn @@ -19,28 +28,29 @@ role_grants: - accountadmin - securityadmin - # - to_user: alejandro - # roles: - # - analyst - # - loader - # - transformer_dbt - # - accountadmin - # - securityadmin - - # - to_user: sebastian - # roles: - # - analyst - # - loader - # - transformer_dbt - # - accountadmin - # - securityadmin - - # - to_user: svc_ci - # roles: - # - analyst - # - loader - # - transformer_dbt - - # - to_user: svc_looker - # roles: - # - analyst + - to_user: sebastian + roles: + - analyst + - loader + - transformer_dbt + - accountadmin + - securityadmin + + + + - to_user: svc_airbyte + roles: + - loader + + - to_user: svc_airflow + roles: + - loader + - transformer_dbt + + - to_user: svc_balboa_ci + roles: + - transformer_dbt + + - to_user: svc_fivetran + roles: + - loader From bb9f62522ea1e1f9c6e9dd2f101c24e6b4fee0f2 Mon Sep 17 00:00:00 2001 From: Noel Gomez Date: Sat, 18 Oct 2025 14:42:04 -0700 Subject: [PATCH 04/21] update model for country geo due to source schema change --- .../models/L1_inlets/country_geo/stg_country_polygons.sql | 4 ---- 1 file changed, 4 deletions(-) diff --git a/transform/models/L1_inlets/country_geo/stg_country_polygons.sql b/transform/models/L1_inlets/country_geo/stg_country_polygons.sql index ae6a659b0..3a49168c5 100644 --- a/transform/models/L1_inlets/country_geo/stg_country_polygons.sql +++ b/transform/models/L1_inlets/country_geo/stg_country_polygons.sql @@ -11,10 +11,8 @@ final as ( name as country_name, iso3166_1_alpha_2 as country_code_2, iso3166_1_alpha_3 as country_code_3, - type as feature_type, geometry_type, try_to_geography(geometry) as geography, - features as raw_geojson from raw_source ) @@ -23,11 +21,9 @@ select country_name, country_code_2, country_code_3, - feature_type, geometry_type, geography, case when geography is not null then st_area(geography) end as area_m2, case when geography is not null then st_perimeter(geography) end as perimeter_m, case when geography is not null then st_centroid(geography) end as centroid, - raw_geojson from final From a29c1ef937485d07c3ad500803c01a5a6eb64781 Mon Sep 17 00:00:00 2001 From: Noel Gomez Date: Sat, 18 Oct 2025 14:42:44 -0700 Subject: [PATCH 05/21] fix linting issue --- transform/models/L1_inlets/country_geo/stg_country_polygons.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transform/models/L1_inlets/country_geo/stg_country_polygons.sql b/transform/models/L1_inlets/country_geo/stg_country_polygons.sql index 3a49168c5..ceda61ab0 100644 --- a/transform/models/L1_inlets/country_geo/stg_country_polygons.sql +++ b/transform/models/L1_inlets/country_geo/stg_country_polygons.sql @@ -12,7 +12,7 @@ final as ( iso3166_1_alpha_2 as country_code_2, iso3166_1_alpha_3 as country_code_3, geometry_type, - try_to_geography(geometry) as geography, + try_to_geography(geometry) as geography from raw_source ) From d29e7041afa42c2ec2dfcd22d19342a677ec8c67 Mon Sep 17 00:00:00 2001 From: Noel Gomez Date: Sat, 18 Oct 2025 14:43:38 -0700 Subject: [PATCH 06/21] fix linting issue --- transform/models/L1_inlets/country_geo/stg_country_polygons.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transform/models/L1_inlets/country_geo/stg_country_polygons.sql b/transform/models/L1_inlets/country_geo/stg_country_polygons.sql index ceda61ab0..c05aab4f1 100644 --- a/transform/models/L1_inlets/country_geo/stg_country_polygons.sql +++ b/transform/models/L1_inlets/country_geo/stg_country_polygons.sql @@ -25,5 +25,5 @@ select geography, case when geography is not null then st_area(geography) end as area_m2, case when geography is not null then st_perimeter(geography) end as perimeter_m, - case when geography is not null then st_centroid(geography) end as centroid, + case when geography is not null then st_centroid(geography) end as centroid from final From c499ce927d684562f6c10a80a77cd1836523f574 Mon Sep 17 00:00:00 2001 From: Noel Gomez Date: Mon, 20 Oct 2025 15:47:38 -0700 Subject: [PATCH 07/21] update roles for covid and snowflake dbs --- secure/titan/resources/databases.yml | 14 +++++++------- secure/titan/resources/roles__functional.yml | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/secure/titan/resources/databases.yml b/secure/titan/resources/databases.yml index f7e382e3b..4784ee62b 100644 --- a/secure/titan/resources/databases.yml +++ b/secure/titan/resources/databases.yml @@ -22,15 +22,15 @@ vars: max_data_extension_time_in_days: 10 roles: - # - name: z_db__covid19_epidemiological_data + - name: z_db__covid19_epidemiological_data # - name: z_schemas__db__covid19_epidemiological_data - name: z_db__snowflake # - name: z_schemas__db__snowflake grants: - # - priv: "USAGE" - # on: "database covid19_epidemiological_data" - # to: z_db__covid19_epidemiological_data + - priv: "USAGE" + on: "database covid19_epidemiological_data" + to: z_db__covid19_epidemiological_data # - priv: "USAGE" # on: @@ -38,9 +38,9 @@ grants: # - "future schemas in database covid19_epidemiological_data" # to: z_schemas__db__covid19_epidemiological_data - # - priv: "IMPORTED PRIVILEGES" - # on: "database snowflake" - # to: z_db__snowflake + - priv: "IMPORTED PRIVILEGES" + on: "database snowflake" + to: z_db__snowflake # - priv: "USAGE" # on: diff --git a/secure/titan/resources/roles__functional.yml b/secure/titan/resources/roles__functional.yml index 7988077ac..65af51be9 100644 --- a/secure/titan/resources/roles__functional.yml +++ b/secure/titan/resources/roles__functional.yml @@ -20,7 +20,7 @@ role_grants: - z_db__great_bay - z_db__sushi_db - # - z_db__covid19_epidemiological_data + - z_db__covid19_epidemiological_data # - z_schemas__db__covid19_epidemiological_data - z_db__snowflake From bcf33da62624b518f6f4004604aaddceb86db1bd Mon Sep 17 00:00:00 2001 From: Noel Gomez Date: Tue, 21 Oct 2025 09:55:52 -0700 Subject: [PATCH 08/21] add more titan roles --- secure/titan/a_TODO/roles__functional.yml | 44 ------------- secure/titan/resources/databases.yml | 68 +------------------- secure/titan/resources/roles__functional.yml | 33 +++++++++- secure/titan/resources/schemas.yml | 18 +++--- transform/dbt_project.yml | 4 +- 5 files changed, 42 insertions(+), 125 deletions(-) delete mode 100644 secure/titan/a_TODO/roles__functional.yml diff --git a/secure/titan/a_TODO/roles__functional.yml b/secure/titan/a_TODO/roles__functional.yml deleted file mode 100644 index 78ee83909..000000000 --- a/secure/titan/a_TODO/roles__functional.yml +++ /dev/null @@ -1,44 +0,0 @@ -roles: - - name: loader - - name: transformer_dbt - - name: analyst - - name: z_base__analyst - -role_grants: - - to_role: z_base__analyst - roles: - - z_db__raw - - z_db__prd_chiquita - - - z_schema__AKS_DATACOVES - - z_schema__AKS_DATACOVES_CDC - - z_schema__AKS_DATACOVES_CDC_STAGING - - z_schema__AKS_GRAFANA - - z_schema__AKS_GRAFANA_STAGING - - z_schema__DATACOVES - - z_schema__L1__GRAFANA - - z_schema__DBT_TEST__AUDIT - - z_schema__SEEDS - - z_schema__L3_MARTS - - - z_tables_views__r - - - z_wh__wh_transforming - - - to_role: SYSADMIN - roles: - - z_create__schema__db__raw - - - to_role: analyst - roles: - - z_base__analyst - - - to_role: loader - roles: - - z_wh__wh_transforming - - - to_role: transformer_dbt - roles: - - z_create__account__database - - z_tables_views__r - - z_wh__wh_transforming diff --git a/secure/titan/resources/databases.yml b/secure/titan/resources/databases.yml index 4784ee62b..4f2cf7df9 100644 --- a/secure/titan/resources/databases.yml +++ b/secure/titan/resources/databases.yml @@ -23,79 +23,13 @@ vars: roles: - name: z_db__covid19_epidemiological_data - # - name: z_schemas__db__covid19_epidemiological_data - name: z_db__snowflake - # - name: z_schemas__db__snowflake grants: - - priv: "USAGE" + - priv: "IMPORTED PRIVILEGES" on: "database covid19_epidemiological_data" to: z_db__covid19_epidemiological_data - # - priv: "USAGE" - # on: - # - "all schemas in database covid19_epidemiological_data" - # - "future schemas in database covid19_epidemiological_data" - # to: z_schemas__db__covid19_epidemiological_data - - priv: "IMPORTED PRIVILEGES" on: "database snowflake" to: z_db__snowflake - - # - priv: "USAGE" - # on: - # - "all schemas in database snowflake" - # - "future schemas in database snowflake" - # to: z_schemas__db__snowflake - - -# - balboa: -# shared: false -# schemas: -# - L1_ACCOUNT_USAGE -# - L1_COUNTRY_DATA -# - L1_COUNTRY_GEO -# - L1_COVID19_EPIDEMIOLOGICAL_DATA -# - L1_GOOGLE_ANALYTICS_4 -# - L1_LOANS -# - L1_OBSERVE -# - L1_US_POPULATION -# - L1_USGS__EARTHQUAKE_DATA - -# - L2_COUNTRY_DEMOGRAPHICS -# - L2_COVID_OBSERVATIONS -# - L2_SNOWFLAKE_USAGE - -# - L3_COVID_ANALYTICS -# - L3_EARTHQUAKE_ANALYTICS -# - L3_LOAN_ANALYTICS - -# - DBT_TEST__AUDIT - -# - SEEDS - -# - MASKING_POLICIES - -# - balboa_apps: -# shared: false -# schemas: -# - RESOURCES - -# - balboa_dev: -# shared: false -# # - convexa_dev: -# # shared: false -# # Snowflake database contains metadata describing usage -# - snowflake: -# shared: true - -# - covid19_epidemiological_data: -# shared: true - -# - great_bay: -# shared: false -# schemas: -# - COVE_MARKETING - -# - sushi_db: -# shared: false diff --git a/secure/titan/resources/roles__functional.yml b/secure/titan/resources/roles__functional.yml index 65af51be9..d6d71a4e2 100644 --- a/secure/titan/resources/roles__functional.yml +++ b/secure/titan/resources/roles__functional.yml @@ -8,24 +8,52 @@ roles: role_grants: - to_role: z_base__analyst roles: + # RAW - z_db__raw - z_schemas__db__raw + # BALBOA - z_db__balboa + - z_schema__dbt_test__audit + - z_schema__l1_account_usage + - z_schema__l1_country_data + - z_schema__l1_country_geo + - z_schema__l1_covid19_epidemiological_data + - z_schema__l1_google_analytics_4 + - z_schema__l1_loans + - z_schema__l1_observe + - z_schema__l1_usgs__earthquake_data + - z_schema__l1_us_population + + - z_schema__l2_country_demographics + - z_schema__l2_covid_observations + - z_schema__l2_snowflake_usage + + - z_schema__l3_covid_analytics + - z_schema__l3_earthquake_analytics + - z_schema__l3_loan_analytics + + - z_schema__masking_policies + - z_schema__seeds + + # BALBOA_APPS - z_db__balboa_apps + - z_schema__resources + # BALBOA_DEV - z_db__balboa_dev - z_db__balboa_dev__create_schema + # GREAT_BAY - z_db__great_bay - z_db__sushi_db + # Shared DBs - z_db__covid19_epidemiological_data - # - z_schemas__db__covid19_epidemiological_data - z_db__snowflake - # - z_schemas__db__snowflake + # WAREHOUSES - z_wh__wh_transforming - z_wh__wh_transforming_dynamic_tables @@ -41,7 +69,6 @@ role_grants: - to_role: transformer_dbt roles: - z_account__create_database - # - z_db__covid19_epidemiological_data - z_wh__wh_transforming - z_wh__wh_transforming_dynamic_tables diff --git a/secure/titan/resources/schemas.yml b/secure/titan/resources/schemas.yml index 6c81ea008..8693d8841 100644 --- a/secure/titan/resources/schemas.yml +++ b/secure/titan/resources/schemas.yml @@ -3,6 +3,7 @@ vars: - name: schemas type: list default: + # RAW DB - name: RAW.AB_LOANS - name: RAW.AIRBYTE_INTERNAL - name: RAW.COUNTRY_GEO @@ -16,7 +17,9 @@ vars: - name: RAW.USGS__EARTHQUAKE_DATA - name: RAW.US_POPULATION + # BALBOA DB - name: BALBOA.DBT_TEST__AUDIT + - name: BALBOA.L1_ACCOUNT_USAGE - name: BALBOA.L1_COUNTRY_DATA - name: BALBOA.L1_COUNTRY_GEO @@ -26,22 +29,19 @@ vars: - name: BALBOA.L1_OBSERVE - name: BALBOA.L1_USGS__EARTHQUAKE_DATA - name: BALBOA.L1_US_POPULATION - - name: BALBOA.L1_US_POPULATIONS + - name: BALBOA.L2_COUNTRY_DEMOGRAPHICS - name: BALBOA.L2_COVID_OBSERVATIONS - name: BALBOA.L2_SNOWFLAKE_USAGE + - name: BALBOA.L3_COVID_ANALYTICS - name: BALBOA.L3_EARTHQUAKE_ANALYTICS - name: BALBOA.L3_LOAN_ANALYTICS - name: BALBOA.MASKING_POLICIES - name: BALBOA.SEEDS + # BALBOA_APPS DB + - name: BALBOA_APPS.RESOURCES - # - name: PRD_CHIQUITA.L1_STAGING - # - name: PRD_CHIQUITA.L1_STAGING__SEEDS - # - name: PRD_CHIQUITA.L2_CORE - # - name: PRD_CHIQUITA.L3_MARTS - # - name: PRD_CHIQUITA.DBT_TEST__AUDIT - # - name: PRD_CHIQUITA.SEEDS - - # - name: PRD_DBT_GENERATED.DBT_ARTIFACTS + # GREAT_BAY DB + - name: GREAT_BAY.COVE_MARKETING diff --git a/transform/dbt_project.yml b/transform/dbt_project.yml index 65300484a..b0705dce1 100644 --- a/transform/dbt_project.yml +++ b/transform/dbt_project.yml @@ -106,8 +106,8 @@ models: loan_analytics: +schema: L3_LOAN_ANALYTICS # cannot persist docs on dynamic tables - +persist_docs: - relation: false + # +persist_docs: + # relation: false +materialized: table # L4_shares: From 26e976138632eb04074fa44adbf7a28697ec8989 Mon Sep 17 00:00:00 2001 From: Noel Gomez Date: Tue, 21 Oct 2025 13:26:39 -0700 Subject: [PATCH 09/21] fix macro and script to get artifacts to not fail on first run and add stage permissions --- automate/dbt/get_artifacts.sh | 16 ++++++-- secure/titan/resources/roles__functional.yml | 11 ++++- secure/titan/resources/stages.yml | 17 ++++++++ .../tooling/ci-cd/get_last_artifacts.sql | 40 ++++++++++++++++--- 4 files changed, 75 insertions(+), 9 deletions(-) diff --git a/automate/dbt/get_artifacts.sh b/automate/dbt/get_artifacts.sh index 2806c72f1..5996d37fb 100755 --- a/automate/dbt/get_artifacts.sh +++ b/automate/dbt/get_artifacts.sh @@ -8,13 +8,21 @@ cd $DATACOVES__DBT_HOME mkdir -p logs dbt run-operation get_last_artifacts -LINES_IN_MANIFEST="$(grep -c '^' logs/manifest.json)" + +# Check if manifest.json exists before trying to grep it +if [ -f "logs/manifest.json" ]; then + LINES_IN_MANIFEST="$(grep -c '^' logs/manifest.json)" +else + LINES_IN_MANIFEST=0 +fi if [ $LINES_IN_MANIFEST -eq 0 ] then echo "Manifest not found in Snowflake stage, contact the Snowflake administrator to load a updated manifest to snowflake." # This is used by github actions - echo "manifest_found=false" >> $GITHUB_OUTPUT + if [ -n "$GITHUB_OUTPUT" ]; then + echo "manifest_found=false" >> $GITHUB_OUTPUT + fi # This is used by Jenkins # echo "false" > temp_MANIFEST_FOUND.txt @@ -22,7 +30,9 @@ else echo "Updated manifest from production" # This is used by github actions - echo "manifest_found=true" >> $GITHUB_OUTPUT + if [ -n "$GITHUB_OUTPUT" ]; then + echo "manifest_found=true" >> $GITHUB_OUTPUT + fi # This is used by Jenkins # echo "true" > temp_MANIFEST_FOUND.txt diff --git a/secure/titan/resources/roles__functional.yml b/secure/titan/resources/roles__functional.yml index d6d71a4e2..16066c80f 100644 --- a/secure/titan/resources/roles__functional.yml +++ b/secure/titan/resources/roles__functional.yml @@ -50,9 +50,11 @@ role_grants: # Shared DBs - z_db__covid19_epidemiological_data - - z_db__snowflake + # STAGES + - z_stage__raw__dbt_artifacts__artifacts__read + # WAREHOUSES - z_wh__wh_transforming - z_wh__wh_transforming_dynamic_tables @@ -70,6 +72,13 @@ role_grants: roles: - z_account__create_database + # RAW + - z_db__raw + - z_schemas__db__raw + - z_wh__wh_transforming - z_wh__wh_transforming_dynamic_tables + + - z_stage__raw__dbt_artifacts__artifacts__write + - z_tables_views__select diff --git a/secure/titan/resources/stages.yml b/secure/titan/resources/stages.yml index 3258ef1a2..7c208d8f0 100644 --- a/secure/titan/resources/stages.yml +++ b/secure/titan/resources/stages.yml @@ -5,3 +5,20 @@ stages: directory: enable: true comment: Used to store dbt artifacts + +roles: + - name: z_stage__raw__dbt_artifacts__artifacts__read + - name: z_stage__raw__dbt_artifacts__artifacts__write + +grants: + - priv: "READ" + on: "stage raw.dbt_artifacts.artifacts" + to: z_stage__raw__dbt_artifacts__artifacts__read + + - priv: "READ" + on: "stage raw.dbt_artifacts.artifacts" + to: z_stage__raw__dbt_artifacts__artifacts__write + + - priv: "WRITE" + on: "stage raw.dbt_artifacts.artifacts" + to: z_stage__raw__dbt_artifacts__artifacts__write diff --git a/transform/macros/tooling/ci-cd/get_last_artifacts.sql b/transform/macros/tooling/ci-cd/get_last_artifacts.sql index 80331a228..da94aaf17 100644 --- a/transform/macros/tooling/ci-cd/get_last_artifacts.sql +++ b/transform/macros/tooling/ci-cd/get_last_artifacts.sql @@ -20,11 +20,41 @@ {% set artifacts_destination = "file://" + logs_dir %} - {% set get_query %} - get @{{ stage }}/manifest.json {{ artifacts_destination }}; - get @{{ stage }}/catalog.json {{ artifacts_destination }}; - {% endset %} + {# Check if files exist by looking at the LIST results #} + {% set manifest_found = false %} + {% set catalog_found = false %} + + {% for row in results.rows %} + {% if 'manifest.json' in row[0] %} + {% set manifest_found = true %} + {% endif %} + {% if 'catalog.json' in row[0] %} + {% set catalog_found = true %} + {% endif %} + {% endfor %} + + {# Only attempt to get files that exist #} + {% if manifest_found or catalog_found %} + {% set get_commands = [] %} + {% if manifest_found %} + {% set _ = get_commands.append("get @" ~ stage ~ "/manifest.json " ~ artifacts_destination ~ ";") %} + {% endif %} + {% if catalog_found %} + {% set _ = get_commands.append("get @" ~ stage ~ "/catalog.json " ~ artifacts_destination ~ ";") %} + {% endif %} + + {% set get_query = get_commands | join('\n ') %} + {% set results = run_query(get_query) %} - {% set results = run_query(get_query) %} + {{ print("✓ Successfully retrieved available artifact files") }} + {% if not manifest_found %} + {{ print("manifest.json not found in stage") }} + {% endif %} + {% if not catalog_found %} + {{ print("catalog.json not found in stage") }} + {% endif %} + {% else %} + {{ print("File not found - no artifact files (manifest.json or catalog.json) found in stage " ~ stage) }} + {% endif %} {% endmacro %} From e9d99034bb3c4b121af655605404df7f1ebfab40 Mon Sep 17 00:00:00 2001 From: Noel Gomez Date: Tue, 21 Oct 2025 14:36:30 -0700 Subject: [PATCH 10/21] change workflow 20 to only run on PR to main --- .github/workflows/20_release_dbt_checks.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/20_release_dbt_checks.yml b/.github/workflows/20_release_dbt_checks.yml index a3d7af160..7a4139636 100644 --- a/.github/workflows/20_release_dbt_checks.yml +++ b/.github/workflows/20_release_dbt_checks.yml @@ -3,8 +3,8 @@ name: "20: 🏎️ dbt Release Branch Validations" on: # yamllint disable-line rule:truthy pull_request: types: [opened, synchronize, labeled, unlabeled] - # No branch restrictions for validate-branch job to run on all PRs - # dbt job will be conditionally run based on target branch + branches: + - main paths: - transform/**/* @@ -112,7 +112,7 @@ jobs: run: automate/git/branch_validator.py dbt: - if: github.base_ref == 'main' && github.event.action != 'unlabeled' && (github.event.action != 'labeled' || github.event.label.name == 'full-refresh') + if: github.event.action != 'unlabeled' && (github.event.action != 'labeled' || github.event.label.name == 'full-refresh') name: Pull Request dbt Tests runs-on: ubuntu-latest needs: [validate-branch] From 964f1b2818a774dc72d49b909986218ddafd556e Mon Sep 17 00:00:00 2001 From: Noel Gomez Date: Tue, 21 Oct 2025 14:44:35 -0700 Subject: [PATCH 11/21] update CI workflow --- .github/workflows/10_feature_dbt_checks.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/10_feature_dbt_checks.yml b/.github/workflows/10_feature_dbt_checks.yml index 0f0c2d923..751907581 100644 --- a/.github/workflows/10_feature_dbt_checks.yml +++ b/.github/workflows/10_feature_dbt_checks.yml @@ -99,6 +99,7 @@ jobs: run: "../automate/dbt/get_artifacts.sh" - name: Clone incremental models that are directly or indirectly affected by the change + if: ${{ steps.prod_manifest.outputs.manifest_found == 'true' }} run: | dbt clone -s state:modified+,config.materialized:incremental,state:old --state logs dbt clone -s state:modified+,config.materialized:snapshot,state:old --state logs @@ -110,8 +111,14 @@ jobs: - name: Run Observe Model run: "dbt build --fail-fast -s L1_inlets.observe" - # There is an issue with --empty and dynamic tables so need to exclude them + # There is an issue with --empty and dynamic tables so need to run them by themselves + - name: Governance run of dynamic tables + if: ${{ steps.prod_manifest.outputs.manifest_found == 'false' }} + run: "dbt build --fail-fast -s config.materialized:dynamic_table -s test_failures" + + # There is an issue with --empty and dynamic tables so need to run them by themselves - name: Governance run of dynamic tables + if: ${{ steps.prod_manifest.outputs.manifest_found == 'true' }} run: "dbt build --fail-fast -s config.materialized:dynamic_table -s test_failures --defer --state logs" # There is an issue with --empty and dynamic tables so need to exclude them From 10c25482c329ff733284e064dafe3268a3f91180 Mon Sep 17 00:00:00 2001 From: Noel Gomez Date: Tue, 21 Oct 2025 14:58:27 -0700 Subject: [PATCH 12/21] fix selector in workflow --- .github/workflows/10_feature_dbt_checks.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/10_feature_dbt_checks.yml b/.github/workflows/10_feature_dbt_checks.yml index 751907581..d4f31aad9 100644 --- a/.github/workflows/10_feature_dbt_checks.yml +++ b/.github/workflows/10_feature_dbt_checks.yml @@ -114,12 +114,12 @@ jobs: # There is an issue with --empty and dynamic tables so need to run them by themselves - name: Governance run of dynamic tables if: ${{ steps.prod_manifest.outputs.manifest_found == 'false' }} - run: "dbt build --fail-fast -s config.materialized:dynamic_table -s test_failures" + run: "dbt build --fail-fast -s config.materialized:dynamic_table -s stg_test_failures" # There is an issue with --empty and dynamic tables so need to run them by themselves - name: Governance run of dynamic tables if: ${{ steps.prod_manifest.outputs.manifest_found == 'true' }} - run: "dbt build --fail-fast -s config.materialized:dynamic_table -s test_failures --defer --state logs" + run: "dbt build --fail-fast -s config.materialized:dynamic_table -s stg_test_failures --defer --state logs" # There is an issue with --empty and dynamic tables so need to exclude them - name: Governance run of dbt with EMPTY models using slim mode From 694fe727d0ee5f128c79f84d27b441773533b21a Mon Sep 17 00:00:00 2001 From: Noel Gomez Date: Tue, 21 Oct 2025 16:13:36 -0700 Subject: [PATCH 13/21] grant tranformer access to additional warehouses and dbs --- secure/titan/resources/roles__functional.yml | 6 ++++++ secure/titan/resources/stages.yml | 8 +++----- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/secure/titan/resources/roles__functional.yml b/secure/titan/resources/roles__functional.yml index 16066c80f..356a275b6 100644 --- a/secure/titan/resources/roles__functional.yml +++ b/secure/titan/resources/roles__functional.yml @@ -76,6 +76,12 @@ role_grants: - z_db__raw - z_schemas__db__raw + # Shared DBs + - z_db__covid19_epidemiological_data + - z_db__snowflake + + - z_wh__wh_orchestrating + - z_wh__wh_integration - z_wh__wh_transforming - z_wh__wh_transforming_dynamic_tables diff --git a/secure/titan/resources/stages.yml b/secure/titan/resources/stages.yml index 7c208d8f0..1f2a8b9fb 100644 --- a/secure/titan/resources/stages.yml +++ b/secure/titan/resources/stages.yml @@ -15,10 +15,8 @@ grants: on: "stage raw.dbt_artifacts.artifacts" to: z_stage__raw__dbt_artifacts__artifacts__read - - priv: "READ" - on: "stage raw.dbt_artifacts.artifacts" - to: z_stage__raw__dbt_artifacts__artifacts__write - - - priv: "WRITE" + - priv: + - READ + - WRITE on: "stage raw.dbt_artifacts.artifacts" to: z_stage__raw__dbt_artifacts__artifacts__write From f469b873c81ce07ab940b13291ae1914eb4c9761 Mon Sep 17 00:00:00 2001 From: Noel Gomez Date: Tue, 21 Oct 2025 16:19:18 -0700 Subject: [PATCH 14/21] include seeds when running dynamic models for the first time --- .github/workflows/10_feature_dbt_checks.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/10_feature_dbt_checks.yml b/.github/workflows/10_feature_dbt_checks.yml index d4f31aad9..9601330e5 100644 --- a/.github/workflows/10_feature_dbt_checks.yml +++ b/.github/workflows/10_feature_dbt_checks.yml @@ -114,12 +114,12 @@ jobs: # There is an issue with --empty and dynamic tables so need to run them by themselves - name: Governance run of dynamic tables if: ${{ steps.prod_manifest.outputs.manifest_found == 'false' }} - run: "dbt build --fail-fast -s config.materialized:dynamic_table -s stg_test_failures" + run: "dbt build --fail-fast -s config.materialized:dynamic_table, stg_test_failures, resource_type:seed" # There is an issue with --empty and dynamic tables so need to run them by themselves - name: Governance run of dynamic tables if: ${{ steps.prod_manifest.outputs.manifest_found == 'true' }} - run: "dbt build --fail-fast -s config.materialized:dynamic_table -s stg_test_failures --defer --state logs" + run: "dbt build --fail-fast -s config.materialized:dynamic_table, stg_test_failures --defer --state logs" # There is an issue with --empty and dynamic tables so need to exclude them - name: Governance run of dbt with EMPTY models using slim mode From c2f633c8b9df8d2025f8a5d2290def7b7cda298f Mon Sep 17 00:00:00 2001 From: Noel Gomez Date: Tue, 21 Oct 2025 16:30:12 -0700 Subject: [PATCH 15/21] account for not having catalog.json --- .github/workflows/10_feature_dbt_checks.yml | 5 ++++ automate/dbt/get_artifacts.sh | 27 ++++++++++++++++++++- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/.github/workflows/10_feature_dbt_checks.yml b/.github/workflows/10_feature_dbt_checks.yml index 9601330e5..6c7d49ade 100644 --- a/.github/workflows/10_feature_dbt_checks.yml +++ b/.github/workflows/10_feature_dbt_checks.yml @@ -132,8 +132,13 @@ jobs: run: "dbt build --fail-fast --empty --exclude config.materialized:dynamic_table ${{ env.FULL_REFRESH_FLAG }}" - name: Generate Docs Combining Prod and branch catalog.json + if: ${{ steps.prod_manifest.outputs.catalog_found == 'true' }} run: "dbt-coves generate docs --merge-deferred --state logs" + - name: Generate Docs Combining Prod and branch catalog.json + if: ${{ steps.prod_manifest.outputs.catalog_found == 'false' }} + run: "dbt docs generate" + - name: Run governance checks run: "pre-commit run --from-ref origin/${{ github.event.pull_request.base.ref }} --to-ref HEAD" diff --git a/automate/dbt/get_artifacts.sh b/automate/dbt/get_artifacts.sh index 5996d37fb..0e2568087 100755 --- a/automate/dbt/get_artifacts.sh +++ b/automate/dbt/get_artifacts.sh @@ -18,7 +18,7 @@ fi if [ $LINES_IN_MANIFEST -eq 0 ] then - echo "Manifest not found in Snowflake stage, contact the Snowflake administrator to load a updated manifest to snowflake." + echo "Manifest not found in Snowflake stage, contact the Snowflake administrator to load a updated manifest.json to snowflake." # This is used by github actions if [ -n "$GITHUB_OUTPUT" ]; then echo "manifest_found=false" >> $GITHUB_OUTPUT @@ -37,3 +37,28 @@ else # This is used by Jenkins # echo "true" > temp_MANIFEST_FOUND.txt fi + + +# Check if catalog.json exists before trying to grep it +if [ -f "logs/catalog.json" ]; then + LINES_IN_CATALOG="$(grep -c '^' logs/catalog.json)" +else + LINES_IN_CATALOG=0 +fi + + +if [ $LINES_IN_CATALOG -eq 0 ] +then + echo "Catalog not found in Snowflake stage, contact the Snowflake administrator to load a updated catalog.json to snowflake." + # This is used by github actions + if [ -n "$GITHUB_OUTPUT" ]; then + echo "catalog_found=false" >> $GITHUB_OUTPUT + fi +else + echo "Updated catalog from production" + + # This is used by github actions + if [ -n "$GITHUB_OUTPUT" ]; then + echo "catalog_found=true" >> $GITHUB_OUTPUT + fi +fi From de2bccff3e69d05ae74bcb12144749fe4ef90273 Mon Sep 17 00:00:00 2001 From: Noel Gomez Date: Tue, 21 Oct 2025 16:58:45 -0700 Subject: [PATCH 16/21] remove extra cols --- .../models/L1_inlets/country_geo/stg_country_polygons.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/transform/models/L1_inlets/country_geo/stg_country_polygons.yml b/transform/models/L1_inlets/country_geo/stg_country_polygons.yml index 87edd254c..d8453a0f7 100644 --- a/transform/models/L1_inlets/country_geo/stg_country_polygons.yml +++ b/transform/models/L1_inlets/country_geo/stg_country_polygons.yml @@ -14,8 +14,6 @@ models: description: 'The two-letter ISO 3166-1 alpha-2 code of the country.' - name: country_code_3 description: 'The three-letter ISO 3166-1 alpha-3 code of the country.' - - name: feature_type - description: 'The type of geographical feature the polygon represents, such as a country or territory.' - name: geometry_type description: 'The geometric shape type used for the polygon, such as Polygon or Multipolygon.' - name: geography @@ -26,5 +24,3 @@ models: description: 'The perimeter of the polygon in meters, indicating the boundary length of the country’s representation.' - name: centroid description: 'The centroid point of the polygon, typically used to determine the geographic center or balancing point of the shape.' - - name: raw_geojson - description: 'The raw GeoJSON data used to create and describe the country’s polygon features, suitable for mapping applications.' From 1419d5ebbb1d08b587e54c90a5b2dc9324e4ad47 Mon Sep 17 00:00:00 2001 From: Noel Gomez Date: Tue, 21 Oct 2025 17:09:53 -0700 Subject: [PATCH 17/21] update db roles --- .github/workflows/10_feature_dbt_checks.yml | 2 +- secure/titan/resources/databases.yml | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/10_feature_dbt_checks.yml b/.github/workflows/10_feature_dbt_checks.yml index 6c7d49ade..2cb250e79 100644 --- a/.github/workflows/10_feature_dbt_checks.yml +++ b/.github/workflows/10_feature_dbt_checks.yml @@ -135,7 +135,7 @@ jobs: if: ${{ steps.prod_manifest.outputs.catalog_found == 'true' }} run: "dbt-coves generate docs --merge-deferred --state logs" - - name: Generate Docs Combining Prod and branch catalog.json + - name: Generate dbt Docs if: ${{ steps.prod_manifest.outputs.catalog_found == 'false' }} run: "dbt docs generate" diff --git a/secure/titan/resources/databases.yml b/secure/titan/resources/databases.yml index 4f2cf7df9..5c0e0891c 100644 --- a/secure/titan/resources/databases.yml +++ b/secure/titan/resources/databases.yml @@ -22,6 +22,8 @@ vars: max_data_extension_time_in_days: 10 roles: + # Used to grant access to test db + - name: z_db_balboa_tst - name: z_db__covid19_epidemiological_data - name: z_db__snowflake From fa30c2cbe534a30cc4a607d044db635e744b0b8e Mon Sep 17 00:00:00 2001 From: Noel Gomez Date: Tue, 21 Oct 2025 17:24:28 -0700 Subject: [PATCH 18/21] fix dbt selector --- .github/workflows/10_feature_dbt_checks.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/10_feature_dbt_checks.yml b/.github/workflows/10_feature_dbt_checks.yml index 2cb250e79..92d52c0ba 100644 --- a/.github/workflows/10_feature_dbt_checks.yml +++ b/.github/workflows/10_feature_dbt_checks.yml @@ -114,12 +114,12 @@ jobs: # There is an issue with --empty and dynamic tables so need to run them by themselves - name: Governance run of dynamic tables if: ${{ steps.prod_manifest.outputs.manifest_found == 'false' }} - run: "dbt build --fail-fast -s config.materialized:dynamic_table, stg_test_failures, resource_type:seed" + run: "dbt build --fail-fast -s config.materialized:dynamic_table stg_test_failures resource_type:seed" # There is an issue with --empty and dynamic tables so need to run them by themselves - name: Governance run of dynamic tables if: ${{ steps.prod_manifest.outputs.manifest_found == 'true' }} - run: "dbt build --fail-fast -s config.materialized:dynamic_table, stg_test_failures --defer --state logs" + run: "dbt build --fail-fast -s config.materialized:dynamic_table stg_test_failures --defer --state logs" # There is an issue with --empty and dynamic tables so need to exclude them - name: Governance run of dbt with EMPTY models using slim mode From d8927a13e4667d4a757ba51d69eba8ac9afa1f65 Mon Sep 17 00:00:00 2001 From: Noel Gomez Date: Tue, 21 Oct 2025 17:33:13 -0700 Subject: [PATCH 19/21] update roles and grant permissions macro --- secure/titan/resources/databases.yml | 2 +- .../tooling/ci-cd/grant_access_to_pr_database.sql | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/secure/titan/resources/databases.yml b/secure/titan/resources/databases.yml index 5c0e0891c..34d98412c 100644 --- a/secure/titan/resources/databases.yml +++ b/secure/titan/resources/databases.yml @@ -23,7 +23,7 @@ vars: roles: # Used to grant access to test db - - name: z_db_balboa_tst + - name: z_db__balboa_tst - name: z_db__covid19_epidemiological_data - name: z_db__snowflake diff --git a/transform/macros/tooling/ci-cd/grant_access_to_pr_database.sql b/transform/macros/tooling/ci-cd/grant_access_to_pr_database.sql index 273753da6..40bbf7bd8 100644 --- a/transform/macros/tooling/ci-cd/grant_access_to_pr_database.sql +++ b/transform/macros/tooling/ci-cd/grant_access_to_pr_database.sql @@ -1,11 +1,11 @@ {# This macro grants access to a test database #} {# - To run: + To run: dbt run-operation grant_access_to_pr_database #} {%- macro grant_access_to_pr_database() -%} - {% set db_role_name = 'z_db_balboa_tst' %} + {% set db_role_name = 'z_db__balboa_tst' %} {% set db_name = target.database %} {% set apply_db_grants_sql %} @@ -15,7 +15,7 @@ {% do run_query(apply_db_grants_sql) %} {% set schemas_list %} - select schema_name + select schema_name from {{ db_name }}.information_schema.schemata where schema_name not in ('INFORMATION_SCHEMA','PUBLIC','DBT_TEST__AUDIT') {% endset %} @@ -24,9 +24,9 @@ {% for schema in schemas %} {% set apply_schema_grants_sql %} - grant usage on schema {{db_name}}.{{ schema[0] }} to z_schema_{{schema[0]}}; - grant select on all tables in schema {{db_name}}.{{ schema[0] }} to role z_tables_views_general; - grant select on all views in schema {{db_name}}.{{ schema[0] }} to role z_tables_views_general; + grant usage on schema {{db_name}}.{{ schema[0] }} to z_schema__{{schema[0]}}; + grant select on all tables in schema {{db_name}}.{{ schema[0] }} to role z_tables_views__select; + grant select on all views in schema {{db_name}}.{{ schema[0] }} to role z_tables_views__select; {% endset %} {% do run_query(apply_schema_grants_sql) %} From 7be0a99a3af5aafdced9ebb18dc30bedaa74a92d Mon Sep 17 00:00:00 2001 From: Sebastian Sassi Date: Thu, 23 Oct 2025 14:20:40 -0300 Subject: [PATCH 20/21] Remove quotes on db names. --- secure/titan/resources/object_templates/database.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/secure/titan/resources/object_templates/database.yml b/secure/titan/resources/object_templates/database.yml index c62a5d899..a7de7d541 100644 --- a/secure/titan/resources/object_templates/database.yml +++ b/secure/titan/resources/object_templates/database.yml @@ -14,5 +14,5 @@ roles: grants: - for_each: var.databases priv: USAGE - on: database "{{ each.value.name }}" + on: database {{ each.value.name }} to: "z_db__{{ each.value.name }}" From a83965eac2e45d96d320b21addc1ebbadf6a0942 Mon Sep 17 00:00:00 2001 From: Noel Gomez Date: Mon, 27 Oct 2025 10:55:10 -0700 Subject: [PATCH 21/21] make quoting consistent --- secure/titan/resources/object_templates/database.yml | 2 +- secure/titan/resources/object_templates/warehouses.yml | 4 ++-- secure/titan/resources/roles__base.yml | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/secure/titan/resources/object_templates/database.yml b/secure/titan/resources/object_templates/database.yml index a7de7d541..ee69e4cbe 100644 --- a/secure/titan/resources/object_templates/database.yml +++ b/secure/titan/resources/object_templates/database.yml @@ -14,5 +14,5 @@ roles: grants: - for_each: var.databases priv: USAGE - on: database {{ each.value.name }} + on: "database {{ each.value.name }}" to: "z_db__{{ each.value.name }}" diff --git a/secure/titan/resources/object_templates/warehouses.yml b/secure/titan/resources/object_templates/warehouses.yml index b1b6f958d..cb6c46870 100644 --- a/secure/titan/resources/object_templates/warehouses.yml +++ b/secure/titan/resources/object_templates/warehouses.yml @@ -20,5 +20,5 @@ grants: priv: - USAGE - MONITOR - on: warehouse {{ each.value.name }} - to: z_wh__{{ each.value.name }} + on: "warehouse {{ each.value.name }}" + to: "z_wh__{{ each.value.name }}" diff --git a/secure/titan/resources/roles__base.yml b/secure/titan/resources/roles__base.yml index 71d1551b4..050f52471 100644 --- a/secure/titan/resources/roles__base.yml +++ b/secure/titan/resources/roles__base.yml @@ -23,8 +23,8 @@ grants: - for_each: var.schemas priv: "SELECT" on: - - all tables in schema {{ each.value.name }} - - all views in schema {{ each.value.name }} + - "all tables in schema {{ each.value.name }}" + - "all views in schema {{ each.value.name }}" - "future tables in schema {{ each.value.name }}" - "future views in schema {{ each.value.name }}" to: z_tables_views__select