Feature/add titan

.github/workflows/10_feature_dbt_checks.yml

@@ -99,6 +99,7 @@ jobs:
         run: "../automate/dbt/get_artifacts.sh"
 
       - name: Clone incremental models that are directly or indirectly affected by the change
+        if: ${{ steps.prod_manifest.outputs.manifest_found == 'true' }}
         run: |
           dbt clone -s state:modified+,config.materialized:incremental,state:old --state logs
           dbt clone -s state:modified+,config.materialized:snapshot,state:old --state logs
@@ -110,9 +111,15 @@ jobs:
       - name: Run Observe Model
         run: "dbt build --fail-fast -s L1_inlets.observe"
 
-      # There is an issue with --empty and dynamic tables so need to exclude them
+      # There is an issue with --empty and dynamic tables so need to run them by themselves
       - name: Governance run of dynamic tables
-        run: "dbt build --fail-fast -s config.materialized:dynamic_table -s test_failures --defer --state logs"
+        if: ${{ steps.prod_manifest.outputs.manifest_found == 'false' }}
+        run: "dbt build --fail-fast -s config.materialized:dynamic_table stg_test_failures resource_type:seed"
+
+      # There is an issue with --empty and dynamic tables so need to run them by themselves
+      - name: Governance run of dynamic tables
+        if: ${{ steps.prod_manifest.outputs.manifest_found == 'true' }}
+        run: "dbt build --fail-fast -s config.materialized:dynamic_table stg_test_failures --defer --state logs"
 
       # There is an issue with --empty and dynamic tables so need to exclude them
       - name: Governance run of dbt with EMPTY models using slim mode
@@ -125,8 +132,13 @@ jobs:
         run: "dbt build --fail-fast --empty --exclude config.materialized:dynamic_table ${{ env.FULL_REFRESH_FLAG }}"
 
       - name: Generate Docs Combining Prod and branch catalog.json
+        if: ${{ steps.prod_manifest.outputs.catalog_found == 'true' }}
         run: "dbt-coves generate docs --merge-deferred --state logs"
 
+      - name: Generate dbt Docs
+        if: ${{ steps.prod_manifest.outputs.catalog_found == 'false' }}
+        run: "dbt docs generate"
+
       - name: Run governance checks
         run: "pre-commit run --from-ref origin/${{ github.event.pull_request.base.ref }} --to-ref HEAD"
 

.github/workflows/20_release_dbt_checks.yml

@@ -3,8 +3,8 @@ name: "20: 🏎️ dbt Release Branch Validations"
 on:  # yamllint disable-line rule:truthy
   pull_request:
     types: [opened, synchronize, labeled, unlabeled]
-    # No branch restrictions for validate-branch job to run on all PRs
-    # dbt job will be conditionally run based on target branch
+    branches:
+      - main
     paths:
       - transform/**/*
 
@@ -112,7 +112,7 @@ jobs:
       run: automate/git/branch_validator.py
 
   dbt:
-    if: github.base_ref == 'main' && github.event.action != 'unlabeled' && (github.event.action != 'labeled' || github.event.label.name == 'full-refresh')
+    if: github.event.action != 'unlabeled' && (github.event.action != 'labeled' || github.event.label.name == 'full-refresh')
     name: Pull Request dbt Tests
     runs-on: ubuntu-latest
     needs: [validate-branch]

automate/dbt/get_artifacts.sh

@@ -8,22 +8,57 @@ cd $DATACOVES__DBT_HOME
 mkdir -p logs
 
 dbt run-operation get_last_artifacts
-LINES_IN_MANIFEST="$(grep -c '^' logs/manifest.json)"
+
+# Check if manifest.json exists before trying to grep it
+if [ -f "logs/manifest.json" ]; then
+    LINES_IN_MANIFEST="$(grep -c '^' logs/manifest.json)"
+else
+    LINES_IN_MANIFEST=0
+fi
 
 if [ $LINES_IN_MANIFEST -eq 0 ]
 then
-    echo "Manifest not found in Snowflake stage, contact the Snowflake administrator to load a updated manifest to snowflake."
+    echo "Manifest not found in Snowflake stage, contact the Snowflake administrator to load a updated manifest.json to snowflake."
     # This is used by github actions
-    echo "manifest_found=false" >> $GITHUB_OUTPUT
+    if [ -n "$GITHUB_OUTPUT" ]; then
+        echo "manifest_found=false" >> $GITHUB_OUTPUT
+    fi
 
     # This is used by Jenkins
     # echo "false" > temp_MANIFEST_FOUND.txt
 else
     echo "Updated manifest from production"
 
     # This is used by github actions
-    echo "manifest_found=true" >> $GITHUB_OUTPUT
+    if [ -n "$GITHUB_OUTPUT" ]; then
+        echo "manifest_found=true" >> $GITHUB_OUTPUT
+    fi
 
     # This is used by Jenkins
     # echo "true" > temp_MANIFEST_FOUND.txt
 fi
+
+
+# Check if catalog.json exists before trying to grep it
+if [ -f "logs/catalog.json" ]; then
+    LINES_IN_CATALOG="$(grep -c '^' logs/catalog.json)"
+else
+    LINES_IN_CATALOG=0
+fi
+
+
+if [ $LINES_IN_CATALOG -eq 0 ]
+then
+    echo "Catalog not found in Snowflake stage, contact the Snowflake administrator to load a updated catalog.json to snowflake."
+    # This is used by github actions
+    if [ -n "$GITHUB_OUTPUT" ]; then
+        echo "catalog_found=false" >> $GITHUB_OUTPUT
+    fi
+else
+    echo "Updated catalog from production"
+
+    # This is used by github actions
+    if [ -n "$GITHUB_OUTPUT" ]; then
+        echo "catalog_found=true" >> $GITHUB_OUTPUT
+    fi
+fi

secure/titan/.env.sample

@@ -0,0 +1,5 @@
+SNOWFLAKE_ACCOUNT=
+SNOWFLAKE_USER=
+SNOWFLAKE_ROLE=securityadmin
+SNOWFLAKE_PRIVATE_KEY_PATH=/config/.ssl/dev-private.pem
+SNOWFLAKE_AUTHENTICATOR=SNOWFLAKE_JWT

secure/titan/README.md

@@ -0,0 +1,16 @@
+# Securing Snowflake Objects
+
+Using Titan allows us to create Snowflake objects and to also apply permissions to them
+
+## Running Plan
+
+First run `plan.sh` to see the changes that will be applied to Snowflake
+
+## Running Apple
+
+First run `apply.sh` will apply all changes to Snowflake
+
+### Titan Docs
+https://titan-core.gitbook.io/titan-core
+
+https://github.com/datacoves/titan/tree/rc/docs

secure/titan/apply.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+secure_path=/config/workspace/secure/titan
+cd $secure_path
+
+if [ -f .env ]; then
+    echo "File .env found."
+else
+    echo "File .env does not exist. Please create a .env file with the following variables:"
+    echo ""
+    echo "SNOWFLAKE_ACCOUNT="
+    echo "SNOWFLAKE_USER="
+    echo "SNOWFLAKE_ROLE="
+    echo "SNOWFLAKE_PRIVATE_KEY_PATH="
+    echo "SNOWFLAKE_AUTHENTICATOR=SNOWFLAKE_JWT"
+    echo ""
+    exit 1
+fi
+
+export $(cat .env | xargs)
+
+uvx --from titan-core@git+https://github.com/datacoves/titan.git@rc \
+    --refresh \
+    titan apply \
+    --config resources/ \
+    --sync_resources role,grant,role_grant

secure/titan/plan.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+secure_path=/config/workspace/secure/titan
+cd $secure_path
+
+if [ -f .env ]; then
+    echo "File .env found."
+else
+    echo "File .env does not exist. Please create a .env file with the following variables:"
+    echo ""
+    echo "SNOWFLAKE_ACCOUNT="
+    echo "SNOWFLAKE_USER="
+    echo "SNOWFLAKE_ROLE="
+    echo "SNOWFLAKE_PRIVATE_KEY_PATH="
+    echo "SNOWFLAKE_AUTHENTICATOR=SNOWFLAKE_JWT"
+    echo ""
+    exit 1
+fi
+
+export $(cat .env | xargs)
+
+uvx --from titan-core@git+https://github.com/datacoves/titan.git@rc \
+    --refresh \
+    titan plan \
+    --config resources/ \
+    --sync_resources role,grant,role_grant

secure/titan/resources/databases.yml

@@ -0,0 +1,37 @@
+vars:
+  - name: databases
+    type: list
+    default:
+      - name: raw
+        owner: loader
+        max_data_extension_time_in_days: 10
+      - name: balboa
+        owner: transformer_dbt
+        max_data_extension_time_in_days: 30
+      - name: balboa_apps
+        owner: transformer_dbt
+        max_data_extension_time_in_days: 10
+      - name: balboa_dev
+        owner: transformer_dbt
+        max_data_extension_time_in_days: 5
+      - name: great_bay
+        owner: transformer_dbt
+        max_data_extension_time_in_days: 10
+      - name: sushi_db
+        owner: transformer_dbt
+        max_data_extension_time_in_days: 10
+
+roles:
+  # Used to grant access to test db
+  - name: z_db__balboa_tst
+  - name: z_db__covid19_epidemiological_data
+  - name: z_db__snowflake
+
+grants:
+  - priv: "IMPORTED PRIVILEGES"
+    on: "database covid19_epidemiological_data"
+    to: z_db__covid19_epidemiological_data
+
+  - priv: "IMPORTED PRIVILEGES"
+    on: "database snowflake"
+    to: z_db__snowflake

secure/titan/resources/object_templates/database.yml

@@ -0,0 +1,18 @@
+# Databases
+databases:
+  - for_each: var.databases
+    name: "{{ each.value.name }}"
+    owner: "{{ each.value.owner }}"
+    max_data_extension_time_in_days: "{{ each.value.max_data_extension_time_in_days }}"
+
+# Database roles
+roles:
+  - for_each: var.databases
+    name: "z_db__{{ each.value.name }}"
+
+# Database grants
+grants:
+  - for_each: var.databases
+    priv: USAGE
+    on: "database {{ each.value.name }}"
+    to: "z_db__{{ each.value.name }}"

secure/titan/resources/object_templates/schema.yml

@@ -0,0 +1,18 @@
+# Schemas
+schemas:
+  - for_each: var.schemas
+    name: "{{ each.value.name.split('.')[1] }}"
+    database: "{{ each.value.name.split('.')[0] }}"
+    owner: "{{ each.value.get('owner', parent.owner) }}"
+
+# Schema roles
+roles:
+  - for_each: var.schemas
+    name: "z_schema__{{ each.value.name.split('.')[1] }}"
+
+# Schemas grants
+grants:
+  - for_each: var.schemas
+    priv: USAGE
+    on: "schema {{ each.value.name }}"
+    to: "z_schema__{{ each.value.name.split('.')[1] }}"

secure/titan/resources/object_templates/warehouses.yml

@@ -0,0 +1,24 @@
+# Warehouses
+warehouses:
+  - for_each: var.warehouses
+    name: "{{ each.value.name }}"
+    warehouse_size: "{{ each.value.size }}"
+    auto_suspend: "{{ each.value.auto_suspend }}"
+    auto_resume: true
+    initially_suspended: true
+    statement_timeout_in_seconds: "{{ each.value.statement_timeout_in_seconds | default(3600) }}"
+
+# Warehouse roles
+roles:
+  - for_each: var.warehouses
+    name: "z_wh__{{ each.value.name }}"
+
+
+# Warehouse grants
+grants:
+  - for_each: var.warehouses
+    priv:
+      - USAGE
+      - MONITOR
+    on: "warehouse {{ each.value.name }}"
+    to: "z_wh__{{ each.value.name }}"

secure/titan/resources/roles__base.yml

@@ -0,0 +1,30 @@
+roles:
+  - name: z_account__create_database
+  - name: z_db__tst_balboa  # Used to grant access to TST databases
+  - name: z_db__balboa_dev__create_schema
+  - name: z_schemas__db__raw
+  - name: z_tables_views__select
+
+grants:
+  - priv: "CREATE DATABASE"
+    on: "ACCOUNT"
+    to: z_account__create_database
+
+  - priv: "CREATE SCHEMA"
+    on: "database balboa_dev"
+    to: z_db__balboa_dev__create_schema
+
+  - priv: "USAGE"
+    on:
+      - "all schemas in database raw"
+      - "future schemas in database raw"
+    to: z_schemas__db__raw
+
+  - for_each: var.schemas
+    priv: "SELECT"
+    on:
+      - "all tables in schema {{ each.value.name }}"
+      - "all views in schema {{ each.value.name }}"
+      - "future tables in schema {{ each.value.name }}"
+      - "future views in schema {{ each.value.name }}"
+    to: z_tables_views__select