diff --git a/.github/workflows/10_feature_dbt_checks.yml b/.github/workflows/10_feature_dbt_checks.yml index 0f0c2d923..92d52c0ba 100644 --- a/.github/workflows/10_feature_dbt_checks.yml +++ b/.github/workflows/10_feature_dbt_checks.yml @@ -99,6 +99,7 @@ jobs: run: "../automate/dbt/get_artifacts.sh" - name: Clone incremental models that are directly or indirectly affected by the change + if: ${{ steps.prod_manifest.outputs.manifest_found == 'true' }} run: | dbt clone -s state:modified+,config.materialized:incremental,state:old --state logs dbt clone -s state:modified+,config.materialized:snapshot,state:old --state logs @@ -110,9 +111,15 @@ jobs: - name: Run Observe Model run: "dbt build --fail-fast -s L1_inlets.observe" - # There is an issue with --empty and dynamic tables so need to exclude them + # There is an issue with --empty and dynamic tables so need to run them by themselves - name: Governance run of dynamic tables - run: "dbt build --fail-fast -s config.materialized:dynamic_table -s test_failures --defer --state logs" + if: ${{ steps.prod_manifest.outputs.manifest_found == 'false' }} + run: "dbt build --fail-fast -s config.materialized:dynamic_table stg_test_failures resource_type:seed" + + # There is an issue with --empty and dynamic tables so need to run them by themselves + - name: Governance run of dynamic tables + if: ${{ steps.prod_manifest.outputs.manifest_found == 'true' }} + run: "dbt build --fail-fast -s config.materialized:dynamic_table stg_test_failures --defer --state logs" # There is an issue with --empty and dynamic tables so need to exclude them - name: Governance run of dbt with EMPTY models using slim mode @@ -125,8 +132,13 @@ jobs: run: "dbt build --fail-fast --empty --exclude config.materialized:dynamic_table ${{ env.FULL_REFRESH_FLAG }}" - name: Generate Docs Combining Prod and branch catalog.json + if: ${{ steps.prod_manifest.outputs.catalog_found == 'true' }} run: "dbt-coves generate docs --merge-deferred --state logs" + - name: Generate dbt Docs + if: ${{ steps.prod_manifest.outputs.catalog_found == 'false' }} + run: "dbt docs generate" + - name: Run governance checks run: "pre-commit run --from-ref origin/${{ github.event.pull_request.base.ref }} --to-ref HEAD" diff --git a/.github/workflows/20_release_dbt_checks.yml b/.github/workflows/20_release_dbt_checks.yml index a3d7af160..7a4139636 100644 --- a/.github/workflows/20_release_dbt_checks.yml +++ b/.github/workflows/20_release_dbt_checks.yml @@ -3,8 +3,8 @@ name: "20: 🏎️ dbt Release Branch Validations" on: # yamllint disable-line rule:truthy pull_request: types: [opened, synchronize, labeled, unlabeled] - # No branch restrictions for validate-branch job to run on all PRs - # dbt job will be conditionally run based on target branch + branches: + - main paths: - transform/**/* @@ -112,7 +112,7 @@ jobs: run: automate/git/branch_validator.py dbt: - if: github.base_ref == 'main' && github.event.action != 'unlabeled' && (github.event.action != 'labeled' || github.event.label.name == 'full-refresh') + if: github.event.action != 'unlabeled' && (github.event.action != 'labeled' || github.event.label.name == 'full-refresh') name: Pull Request dbt Tests runs-on: ubuntu-latest needs: [validate-branch] diff --git a/automate/dbt/get_artifacts.sh b/automate/dbt/get_artifacts.sh index 2806c72f1..0e2568087 100755 --- a/automate/dbt/get_artifacts.sh +++ b/automate/dbt/get_artifacts.sh @@ -8,13 +8,21 @@ cd $DATACOVES__DBT_HOME mkdir -p logs dbt run-operation get_last_artifacts -LINES_IN_MANIFEST="$(grep -c '^' logs/manifest.json)" + +# Check if manifest.json exists before trying to grep it +if [ -f "logs/manifest.json" ]; then + LINES_IN_MANIFEST="$(grep -c '^' logs/manifest.json)" +else + LINES_IN_MANIFEST=0 +fi if [ $LINES_IN_MANIFEST -eq 0 ] then - echo "Manifest not found in Snowflake stage, contact the Snowflake administrator to load a updated manifest to snowflake." + echo "Manifest not found in Snowflake stage, contact the Snowflake administrator to load a updated manifest.json to snowflake." # This is used by github actions - echo "manifest_found=false" >> $GITHUB_OUTPUT + if [ -n "$GITHUB_OUTPUT" ]; then + echo "manifest_found=false" >> $GITHUB_OUTPUT + fi # This is used by Jenkins # echo "false" > temp_MANIFEST_FOUND.txt @@ -22,8 +30,35 @@ else echo "Updated manifest from production" # This is used by github actions - echo "manifest_found=true" >> $GITHUB_OUTPUT + if [ -n "$GITHUB_OUTPUT" ]; then + echo "manifest_found=true" >> $GITHUB_OUTPUT + fi # This is used by Jenkins # echo "true" > temp_MANIFEST_FOUND.txt fi + + +# Check if catalog.json exists before trying to grep it +if [ -f "logs/catalog.json" ]; then + LINES_IN_CATALOG="$(grep -c '^' logs/catalog.json)" +else + LINES_IN_CATALOG=0 +fi + + +if [ $LINES_IN_CATALOG -eq 0 ] +then + echo "Catalog not found in Snowflake stage, contact the Snowflake administrator to load a updated catalog.json to snowflake." + # This is used by github actions + if [ -n "$GITHUB_OUTPUT" ]; then + echo "catalog_found=false" >> $GITHUB_OUTPUT + fi +else + echo "Updated catalog from production" + + # This is used by github actions + if [ -n "$GITHUB_OUTPUT" ]; then + echo "catalog_found=true" >> $GITHUB_OUTPUT + fi +fi diff --git a/secure/create_snowflake_objects.py b/secure/permifrost/create_snowflake_objects.py similarity index 100% rename from secure/create_snowflake_objects.py rename to secure/permifrost/create_snowflake_objects.py diff --git a/secure/databases.yml b/secure/permifrost/databases.yml similarity index 100% rename from secure/databases.yml rename to secure/permifrost/databases.yml diff --git a/secure/permifrost.yml b/secure/permifrost/permifrost.yml similarity index 100% rename from secure/permifrost.yml rename to secure/permifrost/permifrost.yml diff --git a/secure/permifrost_pii.yml b/secure/permifrost/permifrost_pii.yml similarity index 100% rename from secure/permifrost_pii.yml rename to secure/permifrost/permifrost_pii.yml diff --git a/secure/roles.yml b/secure/permifrost/roles.yml similarity index 100% rename from secure/roles.yml rename to secure/permifrost/roles.yml diff --git a/secure/run_permifrost.sh b/secure/permifrost/run_permifrost.sh similarity index 100% rename from secure/run_permifrost.sh rename to secure/permifrost/run_permifrost.sh diff --git a/secure/run_permifrost_pii.sh b/secure/permifrost/run_permifrost_pii.sh similarity index 100% rename from secure/run_permifrost_pii.sh rename to secure/permifrost/run_permifrost_pii.sh diff --git a/secure/users.yml b/secure/permifrost/users.yml similarity index 100% rename from secure/users.yml rename to secure/permifrost/users.yml diff --git a/secure/users_pii.yml b/secure/permifrost/users_pii.yml similarity index 100% rename from secure/users_pii.yml rename to secure/permifrost/users_pii.yml diff --git a/secure/warehouses.yml b/secure/permifrost/warehouses.yml similarity index 100% rename from secure/warehouses.yml rename to secure/permifrost/warehouses.yml diff --git a/secure/titan/.env.sample b/secure/titan/.env.sample new file mode 100644 index 000000000..dbd2d7eb6 --- /dev/null +++ b/secure/titan/.env.sample @@ -0,0 +1,5 @@ +SNOWFLAKE_ACCOUNT= +SNOWFLAKE_USER= +SNOWFLAKE_ROLE=securityadmin +SNOWFLAKE_PRIVATE_KEY_PATH=/config/.ssl/dev-private.pem +SNOWFLAKE_AUTHENTICATOR=SNOWFLAKE_JWT diff --git a/secure/titan/README.md b/secure/titan/README.md new file mode 100644 index 000000000..11e078306 --- /dev/null +++ b/secure/titan/README.md @@ -0,0 +1,16 @@ +# Securing Snowflake Objects + +Using Titan allows us to create Snowflake objects and to also apply permissions to them + +## Running Plan + +First run `plan.sh` to see the changes that will be applied to Snowflake + +## Running Apple + +First run `apply.sh` will apply all changes to Snowflake + +### Titan Docs +https://titan-core.gitbook.io/titan-core + +https://github.com/datacoves/titan/tree/rc/docs diff --git a/secure/titan/apply.sh b/secure/titan/apply.sh new file mode 100755 index 000000000..061ddfbec --- /dev/null +++ b/secure/titan/apply.sh @@ -0,0 +1,25 @@ +#!/bin/bash +secure_path=/config/workspace/secure/titan +cd $secure_path + +if [ -f .env ]; then + echo "File .env found." +else + echo "File .env does not exist. Please create a .env file with the following variables:" + echo "" + echo "SNOWFLAKE_ACCOUNT=" + echo "SNOWFLAKE_USER=" + echo "SNOWFLAKE_ROLE=" + echo "SNOWFLAKE_PRIVATE_KEY_PATH=" + echo "SNOWFLAKE_AUTHENTICATOR=SNOWFLAKE_JWT" + echo "" + exit 1 +fi + +export $(cat .env | xargs) + +uvx --from titan-core@git+https://github.com/datacoves/titan.git@rc \ + --refresh \ + titan apply \ + --config resources/ \ + --sync_resources role,grant,role_grant diff --git a/secure/titan/plan.sh b/secure/titan/plan.sh new file mode 100755 index 000000000..689b2521e --- /dev/null +++ b/secure/titan/plan.sh @@ -0,0 +1,25 @@ +#!/bin/bash +secure_path=/config/workspace/secure/titan +cd $secure_path + +if [ -f .env ]; then + echo "File .env found." +else + echo "File .env does not exist. Please create a .env file with the following variables:" + echo "" + echo "SNOWFLAKE_ACCOUNT=" + echo "SNOWFLAKE_USER=" + echo "SNOWFLAKE_ROLE=" + echo "SNOWFLAKE_PRIVATE_KEY_PATH=" + echo "SNOWFLAKE_AUTHENTICATOR=SNOWFLAKE_JWT" + echo "" + exit 1 +fi + +export $(cat .env | xargs) + +uvx --from titan-core@git+https://github.com/datacoves/titan.git@rc \ + --refresh \ + titan plan \ + --config resources/ \ + --sync_resources role,grant,role_grant diff --git a/secure/titan/resources/databases.yml b/secure/titan/resources/databases.yml new file mode 100644 index 000000000..34d98412c --- /dev/null +++ b/secure/titan/resources/databases.yml @@ -0,0 +1,37 @@ +vars: + - name: databases + type: list + default: + - name: raw + owner: loader + max_data_extension_time_in_days: 10 + - name: balboa + owner: transformer_dbt + max_data_extension_time_in_days: 30 + - name: balboa_apps + owner: transformer_dbt + max_data_extension_time_in_days: 10 + - name: balboa_dev + owner: transformer_dbt + max_data_extension_time_in_days: 5 + - name: great_bay + owner: transformer_dbt + max_data_extension_time_in_days: 10 + - name: sushi_db + owner: transformer_dbt + max_data_extension_time_in_days: 10 + +roles: + # Used to grant access to test db + - name: z_db__balboa_tst + - name: z_db__covid19_epidemiological_data + - name: z_db__snowflake + +grants: + - priv: "IMPORTED PRIVILEGES" + on: "database covid19_epidemiological_data" + to: z_db__covid19_epidemiological_data + + - priv: "IMPORTED PRIVILEGES" + on: "database snowflake" + to: z_db__snowflake diff --git a/secure/titan/resources/object_templates/database.yml b/secure/titan/resources/object_templates/database.yml new file mode 100644 index 000000000..ee69e4cbe --- /dev/null +++ b/secure/titan/resources/object_templates/database.yml @@ -0,0 +1,18 @@ +# Databases +databases: + - for_each: var.databases + name: "{{ each.value.name }}" + owner: "{{ each.value.owner }}" + max_data_extension_time_in_days: "{{ each.value.max_data_extension_time_in_days }}" + +# Database roles +roles: + - for_each: var.databases + name: "z_db__{{ each.value.name }}" + +# Database grants +grants: + - for_each: var.databases + priv: USAGE + on: "database {{ each.value.name }}" + to: "z_db__{{ each.value.name }}" diff --git a/secure/titan/resources/object_templates/schema.yml b/secure/titan/resources/object_templates/schema.yml new file mode 100644 index 000000000..c6f160dd7 --- /dev/null +++ b/secure/titan/resources/object_templates/schema.yml @@ -0,0 +1,18 @@ +# Schemas +schemas: + - for_each: var.schemas + name: "{{ each.value.name.split('.')[1] }}" + database: "{{ each.value.name.split('.')[0] }}" + owner: "{{ each.value.get('owner', parent.owner) }}" + +# Schema roles +roles: + - for_each: var.schemas + name: "z_schema__{{ each.value.name.split('.')[1] }}" + +# Schemas grants +grants: + - for_each: var.schemas + priv: USAGE + on: "schema {{ each.value.name }}" + to: "z_schema__{{ each.value.name.split('.')[1] }}" diff --git a/secure/titan/resources/object_templates/warehouses.yml b/secure/titan/resources/object_templates/warehouses.yml new file mode 100644 index 000000000..cb6c46870 --- /dev/null +++ b/secure/titan/resources/object_templates/warehouses.yml @@ -0,0 +1,24 @@ +# Warehouses +warehouses: + - for_each: var.warehouses + name: "{{ each.value.name }}" + warehouse_size: "{{ each.value.size }}" + auto_suspend: "{{ each.value.auto_suspend }}" + auto_resume: true + initially_suspended: true + statement_timeout_in_seconds: "{{ each.value.statement_timeout_in_seconds | default(3600) }}" + +# Warehouse roles +roles: + - for_each: var.warehouses + name: "z_wh__{{ each.value.name }}" + + +# Warehouse grants +grants: + - for_each: var.warehouses + priv: + - USAGE + - MONITOR + on: "warehouse {{ each.value.name }}" + to: "z_wh__{{ each.value.name }}" diff --git a/secure/titan/resources/roles__base.yml b/secure/titan/resources/roles__base.yml new file mode 100644 index 000000000..050f52471 --- /dev/null +++ b/secure/titan/resources/roles__base.yml @@ -0,0 +1,30 @@ +roles: + - name: z_account__create_database + - name: z_db__tst_balboa # Used to grant access to TST databases + - name: z_db__balboa_dev__create_schema + - name: z_schemas__db__raw + - name: z_tables_views__select + +grants: + - priv: "CREATE DATABASE" + on: "ACCOUNT" + to: z_account__create_database + + - priv: "CREATE SCHEMA" + on: "database balboa_dev" + to: z_db__balboa_dev__create_schema + + - priv: "USAGE" + on: + - "all schemas in database raw" + - "future schemas in database raw" + to: z_schemas__db__raw + + - for_each: var.schemas + priv: "SELECT" + on: + - "all tables in schema {{ each.value.name }}" + - "all views in schema {{ each.value.name }}" + - "future tables in schema {{ each.value.name }}" + - "future views in schema {{ each.value.name }}" + to: z_tables_views__select diff --git a/secure/titan/resources/roles__functional.yml b/secure/titan/resources/roles__functional.yml new file mode 100644 index 000000000..356a275b6 --- /dev/null +++ b/secure/titan/resources/roles__functional.yml @@ -0,0 +1,90 @@ +roles: + - name: z_base__analyst + + - name: analyst + - name: loader + - name: transformer_dbt + +role_grants: + - to_role: z_base__analyst + roles: + # RAW + - z_db__raw + - z_schemas__db__raw + + # BALBOA + - z_db__balboa + - z_schema__dbt_test__audit + - z_schema__l1_account_usage + - z_schema__l1_country_data + - z_schema__l1_country_geo + - z_schema__l1_covid19_epidemiological_data + - z_schema__l1_google_analytics_4 + - z_schema__l1_loans + - z_schema__l1_observe + - z_schema__l1_usgs__earthquake_data + - z_schema__l1_us_population + + - z_schema__l2_country_demographics + - z_schema__l2_covid_observations + - z_schema__l2_snowflake_usage + + - z_schema__l3_covid_analytics + - z_schema__l3_earthquake_analytics + - z_schema__l3_loan_analytics + + - z_schema__masking_policies + - z_schema__seeds + + # BALBOA_APPS + - z_db__balboa_apps + - z_schema__resources + + # BALBOA_DEV + - z_db__balboa_dev + - z_db__balboa_dev__create_schema + + # GREAT_BAY + - z_db__great_bay + - z_db__sushi_db + + # Shared DBs + - z_db__covid19_epidemiological_data + - z_db__snowflake + + # STAGES + - z_stage__raw__dbt_artifacts__artifacts__read + + # WAREHOUSES + - z_wh__wh_transforming + - z_wh__wh_transforming_dynamic_tables + + - to_role: analyst + roles: + - z_base__analyst + - z_tables_views__select + + - to_role: loader + roles: + - z_wh__wh_loading + + - to_role: transformer_dbt + roles: + - z_account__create_database + + # RAW + - z_db__raw + - z_schemas__db__raw + + # Shared DBs + - z_db__covid19_epidemiological_data + - z_db__snowflake + + - z_wh__wh_orchestrating + - z_wh__wh_integration + - z_wh__wh_transforming + - z_wh__wh_transforming_dynamic_tables + + - z_stage__raw__dbt_artifacts__artifacts__write + + - z_tables_views__select diff --git a/secure/titan/resources/schemas.yml b/secure/titan/resources/schemas.yml new file mode 100644 index 000000000..8693d8841 --- /dev/null +++ b/secure/titan/resources/schemas.yml @@ -0,0 +1,47 @@ +# Schemas +vars: + - name: schemas + type: list + default: + # RAW DB + - name: RAW.AB_LOANS + - name: RAW.AIRBYTE_INTERNAL + - name: RAW.COUNTRY_GEO + - name: RAW.DBT_ARTIFACTS + owner: transformer_dbt + - name: RAW.GOOGLE_ANALYTICS_4 + - name: RAW.LOANS + - name: RAW.RAW + - name: RAW.SNAPSHOTS + owner: transformer_dbt + - name: RAW.USGS__EARTHQUAKE_DATA + - name: RAW.US_POPULATION + + # BALBOA DB + - name: BALBOA.DBT_TEST__AUDIT + + - name: BALBOA.L1_ACCOUNT_USAGE + - name: BALBOA.L1_COUNTRY_DATA + - name: BALBOA.L1_COUNTRY_GEO + - name: BALBOA.L1_COVID19_EPIDEMIOLOGICAL_DATA + - name: BALBOA.L1_GOOGLE_ANALYTICS_4 + - name: BALBOA.L1_LOANS + - name: BALBOA.L1_OBSERVE + - name: BALBOA.L1_USGS__EARTHQUAKE_DATA + - name: BALBOA.L1_US_POPULATION + + - name: BALBOA.L2_COUNTRY_DEMOGRAPHICS + - name: BALBOA.L2_COVID_OBSERVATIONS + - name: BALBOA.L2_SNOWFLAKE_USAGE + + - name: BALBOA.L3_COVID_ANALYTICS + - name: BALBOA.L3_EARTHQUAKE_ANALYTICS + - name: BALBOA.L3_LOAN_ANALYTICS + - name: BALBOA.MASKING_POLICIES + - name: BALBOA.SEEDS + + # BALBOA_APPS DB + - name: BALBOA_APPS.RESOURCES + + # GREAT_BAY DB + - name: GREAT_BAY.COVE_MARKETING diff --git a/secure/titan/resources/stages.yml b/secure/titan/resources/stages.yml new file mode 100644 index 000000000..1f2a8b9fb --- /dev/null +++ b/secure/titan/resources/stages.yml @@ -0,0 +1,22 @@ +stages: + - name: raw.dbt_artifacts.artifacts + type: internal + owner: transformer_dbt + directory: + enable: true + comment: Used to store dbt artifacts + +roles: + - name: z_stage__raw__dbt_artifacts__artifacts__read + - name: z_stage__raw__dbt_artifacts__artifacts__write + +grants: + - priv: "READ" + on: "stage raw.dbt_artifacts.artifacts" + to: z_stage__raw__dbt_artifacts__artifacts__read + + - priv: + - READ + - WRITE + on: "stage raw.dbt_artifacts.artifacts" + to: z_stage__raw__dbt_artifacts__artifacts__write diff --git a/secure/titan/resources/users.yml b/secure/titan/resources/users.yml new file mode 100644 index 000000000..789c5367b --- /dev/null +++ b/secure/titan/resources/users.yml @@ -0,0 +1,56 @@ +# Users should not be created on titan since keypairs can be lost on every run +# users: +# - name: svc_airbyte +# owner: SECURITYADMIN +# type: SERVICE +# - name: svc_airflow +# owner: SECURITYADMIN +# type: SERVICE +# - name: svc_balboa_ci +# owner: SECURITYADMIN +# type: SERVICE +# - name: svc_fivetran +# owner: SECURITYADMIN +# type: SERVICE + +# create or replace user svc_fivetran +# type = 'service' +# default_warehouse = 'wh_loading' +# rsa_public_key = '' + + +role_grants: + - to_user: gomezn + roles: + - analyst + - loader + - transformer_dbt + - accountadmin + - securityadmin + + - to_user: sebastian + roles: + - analyst + - loader + - transformer_dbt + - accountadmin + - securityadmin + + + + - to_user: svc_airbyte + roles: + - loader + + - to_user: svc_airflow + roles: + - loader + - transformer_dbt + + - to_user: svc_balboa_ci + roles: + - transformer_dbt + + - to_user: svc_fivetran + roles: + - loader diff --git a/secure/titan/resources/warehouses.yml b/secure/titan/resources/warehouses.yml new file mode 100644 index 000000000..999a83a94 --- /dev/null +++ b/secure/titan/resources/warehouses.yml @@ -0,0 +1,29 @@ +vars: + - name: warehouses + type: list + default: + - name: wh_admin + size: x-small + auto_suspend: 60 # seconds 0 means it never suspends + statement_timeout_in_seconds: 7200 # seconds, defaults to 3600 if not defined + - name: wh_catalog + size: x-small + auto_suspend: 60 + - name: wh_integration + size: x-small + auto_suspend: 60 + - name: wh_loading + size: x-small + auto_suspend: 60 + - name: wh_orchestrating + size: x-small + auto_suspend: 60 + - name: wh_transforming + size: x-small + auto_suspend: 60 + - name: wh_transforming_dynamic_tables + size: x-small + auto_suspend: 60 + - name: wh_transforming_sqlmesh + size: x-small + auto_suspend: 60 diff --git a/transform/dbt_project.yml b/transform/dbt_project.yml index 65300484a..b0705dce1 100644 --- a/transform/dbt_project.yml +++ b/transform/dbt_project.yml @@ -106,8 +106,8 @@ models: loan_analytics: +schema: L3_LOAN_ANALYTICS # cannot persist docs on dynamic tables - +persist_docs: - relation: false + # +persist_docs: + # relation: false +materialized: table # L4_shares: diff --git a/transform/macros/tooling/ci-cd/get_last_artifacts.sql b/transform/macros/tooling/ci-cd/get_last_artifacts.sql index 80331a228..da94aaf17 100644 --- a/transform/macros/tooling/ci-cd/get_last_artifacts.sql +++ b/transform/macros/tooling/ci-cd/get_last_artifacts.sql @@ -20,11 +20,41 @@ {% set artifacts_destination = "file://" + logs_dir %} - {% set get_query %} - get @{{ stage }}/manifest.json {{ artifacts_destination }}; - get @{{ stage }}/catalog.json {{ artifacts_destination }}; - {% endset %} + {# Check if files exist by looking at the LIST results #} + {% set manifest_found = false %} + {% set catalog_found = false %} + + {% for row in results.rows %} + {% if 'manifest.json' in row[0] %} + {% set manifest_found = true %} + {% endif %} + {% if 'catalog.json' in row[0] %} + {% set catalog_found = true %} + {% endif %} + {% endfor %} + + {# Only attempt to get files that exist #} + {% if manifest_found or catalog_found %} + {% set get_commands = [] %} + {% if manifest_found %} + {% set _ = get_commands.append("get @" ~ stage ~ "/manifest.json " ~ artifacts_destination ~ ";") %} + {% endif %} + {% if catalog_found %} + {% set _ = get_commands.append("get @" ~ stage ~ "/catalog.json " ~ artifacts_destination ~ ";") %} + {% endif %} + + {% set get_query = get_commands | join('\n ') %} + {% set results = run_query(get_query) %} - {% set results = run_query(get_query) %} + {{ print("✓ Successfully retrieved available artifact files") }} + {% if not manifest_found %} + {{ print("manifest.json not found in stage") }} + {% endif %} + {% if not catalog_found %} + {{ print("catalog.json not found in stage") }} + {% endif %} + {% else %} + {{ print("File not found - no artifact files (manifest.json or catalog.json) found in stage " ~ stage) }} + {% endif %} {% endmacro %} diff --git a/transform/macros/tooling/ci-cd/grant_access_to_pr_database.sql b/transform/macros/tooling/ci-cd/grant_access_to_pr_database.sql index 273753da6..40bbf7bd8 100644 --- a/transform/macros/tooling/ci-cd/grant_access_to_pr_database.sql +++ b/transform/macros/tooling/ci-cd/grant_access_to_pr_database.sql @@ -1,11 +1,11 @@ {# This macro grants access to a test database #} {# - To run: + To run: dbt run-operation grant_access_to_pr_database #} {%- macro grant_access_to_pr_database() -%} - {% set db_role_name = 'z_db_balboa_tst' %} + {% set db_role_name = 'z_db__balboa_tst' %} {% set db_name = target.database %} {% set apply_db_grants_sql %} @@ -15,7 +15,7 @@ {% do run_query(apply_db_grants_sql) %} {% set schemas_list %} - select schema_name + select schema_name from {{ db_name }}.information_schema.schemata where schema_name not in ('INFORMATION_SCHEMA','PUBLIC','DBT_TEST__AUDIT') {% endset %} @@ -24,9 +24,9 @@ {% for schema in schemas %} {% set apply_schema_grants_sql %} - grant usage on schema {{db_name}}.{{ schema[0] }} to z_schema_{{schema[0]}}; - grant select on all tables in schema {{db_name}}.{{ schema[0] }} to role z_tables_views_general; - grant select on all views in schema {{db_name}}.{{ schema[0] }} to role z_tables_views_general; + grant usage on schema {{db_name}}.{{ schema[0] }} to z_schema__{{schema[0]}}; + grant select on all tables in schema {{db_name}}.{{ schema[0] }} to role z_tables_views__select; + grant select on all views in schema {{db_name}}.{{ schema[0] }} to role z_tables_views__select; {% endset %} {% do run_query(apply_schema_grants_sql) %} diff --git a/transform/models/L1_inlets/country_geo/stg_country_polygons.sql b/transform/models/L1_inlets/country_geo/stg_country_polygons.sql index ae6a659b0..c05aab4f1 100644 --- a/transform/models/L1_inlets/country_geo/stg_country_polygons.sql +++ b/transform/models/L1_inlets/country_geo/stg_country_polygons.sql @@ -11,10 +11,8 @@ final as ( name as country_name, iso3166_1_alpha_2 as country_code_2, iso3166_1_alpha_3 as country_code_3, - type as feature_type, geometry_type, - try_to_geography(geometry) as geography, - features as raw_geojson + try_to_geography(geometry) as geography from raw_source ) @@ -23,11 +21,9 @@ select country_name, country_code_2, country_code_3, - feature_type, geometry_type, geography, case when geography is not null then st_area(geography) end as area_m2, case when geography is not null then st_perimeter(geography) end as perimeter_m, - case when geography is not null then st_centroid(geography) end as centroid, - raw_geojson + case when geography is not null then st_centroid(geography) end as centroid from final diff --git a/transform/models/L1_inlets/country_geo/stg_country_polygons.yml b/transform/models/L1_inlets/country_geo/stg_country_polygons.yml index 87edd254c..d8453a0f7 100644 --- a/transform/models/L1_inlets/country_geo/stg_country_polygons.yml +++ b/transform/models/L1_inlets/country_geo/stg_country_polygons.yml @@ -14,8 +14,6 @@ models: description: 'The two-letter ISO 3166-1 alpha-2 code of the country.' - name: country_code_3 description: 'The three-letter ISO 3166-1 alpha-3 code of the country.' - - name: feature_type - description: 'The type of geographical feature the polygon represents, such as a country or territory.' - name: geometry_type description: 'The geometric shape type used for the polygon, such as Polygon or Multipolygon.' - name: geography @@ -26,5 +24,3 @@ models: description: 'The perimeter of the polygon in meters, indicating the boundary length of the country’s representation.' - name: centroid description: 'The centroid point of the polygon, typically used to determine the geographic center or balancing point of the shape.' - - name: raw_geojson - description: 'The raw GeoJSON data used to create and describe the country’s polygon features, suitable for mapping applications.'