Updates needed for dbt 1.10

.github/workflows/10_feature_airflow_checks.yml

@@ -36,9 +36,10 @@ jobs:
   airflow:
     name: Pull Request Airflow Tests
     runs-on: ubuntu-latest
-    container: datacoves/ci-airflow-dbt-snowflake:4.0
     needs: [validate-branch]
 
+    container: datacoves/ci-airflow-dbt-snowflake:4.1
+
     env:
       AIRFLOW__CORE__DAGS_FOLDER: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}/orchestrate/dags
       AIRFLOW__CORE__DAGBAG_IMPORT_TIMEOUT: 300

.github/workflows/10_feature_dbt_checks.yml

@@ -46,7 +46,7 @@ jobs:
     # environment: PR_ENV
 
     # most people should use this one
-    container: datacoves/ci-basic-dbt-snowflake:4.0
+    container: datacoves/ci-basic-dbt-snowflake:4.1
 
     defaults:
       run:

.github/workflows/15_drop_feature_db.yml

@@ -23,7 +23,7 @@ jobs:
     # Alternatively, You can define multiple ENV for different workflows.
     # https://github.com/<org>/<repo>/settings/environments
     # environment: PR_ENV
-    container: datacoves/ci-basic-dbt-snowflake:4.0
+    container: datacoves/ci-basic-dbt-snowflake:4.1
 
     defaults:
       run:

.github/workflows/20_release_dbt_checks.yml

@@ -125,7 +125,7 @@ jobs:
     # environment: PR_ENV
 
     # Most people should use this docker image
-    container: datacoves/ci-basic-dbt-snowflake:4.0
+    container: datacoves/ci-basic-dbt-snowflake:4.1
 
     defaults:
       run:

.github/workflows/30_deploy_changes_to_production.yml

@@ -25,7 +25,7 @@ jobs:
     # Alternatively, You can define multiple ENV for different workflows.
     # https://github.com/<org>/<repo>/settings/environments
     # environment: PR_ENV
-    container: datacoves/ci-basic-dbt-snowflake:4.0
+    container: datacoves/ci-basic-dbt-snowflake:4.1
 
     defaults:
       run:

.pre-commit-config.yaml

@@ -25,8 +25,8 @@ repos:
         additional_dependencies:
           [
             "sqlfluff-templater-dbt==3.1.1",
-            "dbt-core==1.8.8",
-            "dbt-snowflake==1.8.4",
+            "dbt-core==1.10.15",
+            "dbt-snowflake==1.10.3",
           ]
         args: [--config, transform/.sqlfluff]
 

training_and_demos/dbt-api/dbt_api_files.py

@@ -18,6 +18,7 @@
 load_dotenv()
 base_url = os.getenv("DATACOVES__API_ENDPOINT")
 token = os.getenv("DATACOVES__API_TOKEN")
+account_id = os.getenv("DATACOVES__ACCOUNT_ID")
 project_slug = os.getenv("DATACOVES__PROJECT_SLUG")
 environment_slug = os.getenv("DATACOVES__ENVIRONMENT_SLUG")
 dbt_home = os.getenv("DATACOVES__DBT_HOME")
@@ -363,62 +364,57 @@ def delete_project_file(account_id: int, project_slug: str, filename: str):
 
     # health_check()
 
-    # get_account(1)
+    # get_account(account_id)
 
-    # get_projects(1)
-    # get_projects(1,"balboa-analytics-datacoves")
+    # get_projects(account_id)
+    # get_projects(account_id, project_slug)
 
-    # get_environments(1, "balboa-analytics-datacoves")
-    # get_environments(1, "balboa-analytics-datacoves", "zpg497")
+    # get_environments(account_id, project_slug)
+    # get_environments(account_id, project_slug, environment_slug)
 
     # Work with files
     cols = ["environment_slug",'filename', 'metadata', 'inserted_at']
 
-    # files = list_project_files(1, "balboa-analytics-datacoves")
-    # print_table(files, cols)
+    # project_files = list_project_files(account_id, project_slug)
+    # print_table(project_files, cols)
 
-    # files = list_environment_files(1, "balboa-analytics-datacoves", "zpg497")
-    # print_table(files, cols)
-    # print(files)
+    # environment_files = list_environment_files(account_id, project_slug, environment_slug)
+    # print_table(environment_files, cols)
 
     filenames = ["graph.gpickle", "graph_summary.json", "partial_parse.msgpack"]
 
     # UPLOAD FILES
     # for filename in filenames:
-    #     upload_env_file(1, "balboa-analytics-datacoves", "zpg497", filename)
+    #     upload_env_file(account_id, project_slug, environment_slug, filename)
 
-    # upload_env_file(1, "balboa-analytics-datacoves", "zpg497", "manifest.json", is_manifest=True )
+    # upload_env_file(account_id, project_slug, environment_slug, "manifest.json", is_manifest=True )
 
     # DELETE FILES
     # for filename in filenames:
-    #     delete_env_file(1, "balboa-analytics-datacoves", "zpg497", filename)
+    #     delete_env_file(account_id, project_slug, environment_slug, filename)
 
-    # delete_env_file(1, "balboa-analytics-datacoves", "zpg497", "manifest.json")
+    # delete_env_file(account_id, project_slug, environment_slug, "manifest.json")
 
     # SHOW FILE DETAILS
     # for filename in filenames:
-    #     show_env_file_details(1, "balboa-analytics-datacoves", "zpg497", filename)
+    #     show_env_file_details(account_id, project_slug, environment_slug, filename)
 
     # DOWNLOAD Files
     # for filename in filenames:
-    #     download_env_file(1, "balboa-analytics-datacoves", "zpg497", filename)
+    #     download_env_file(account_id, project_slug, environment_slug, filename)
 
-    download_env_manifest(1, "balboa-analytics-datacoves", "zpg497", trimmed = True)
-
-    # download_project_manifest(1, "balboa-analytics-datacoves", trimmed = True)
+    # download_env_manifest(account_id, project_slug, environment_slug, trimmed = True)
 
     # for filename in filenames:
-    #     promote_env_file(1, "balboa-analytics-datacoves", "zpg497", filename)
-    #     download_project_file(1, "balboa-analytics-datacoves", filename)
+    #     promote_env_file(account_id, project_slug, environment_slug, filename)
+    #     download_project_file(account_id, project_slug, filename)
 
     # DELETE FILES
     # for filename in filenames:
-    #     delete_project_file(1, "balboa-analytics-datacoves", filename)
-
+    #     delete_project_file(account_id, project_slug, filename)
 
-    # files = list_project_files(1, "balboa-analytics-datacoves")
-    # print_table(files, cols)
-    # promote_env_file(1, "balboa-analytics-datacoves", "zpg497", "manifest.json")
+    # promote_env_file(account_id, project_slug, environment_slug, "manifest.json" )
+    # download_project_manifest(account_id, project_slug, trimmed = True)
 
-    files = list_project_files(1, "balboa-analytics-datacoves")
+    files = list_project_files(account_id, project_slug)
     print_table(files, cols)

training_and_demos/dbt_fusion/readme.md

@@ -11,12 +11,6 @@ rm package-lock.yml
 dbt deps
 ```
 
-# Remove unsupported Features
-
-As of 7/5/2025, Dynamic tables in Snowflake are not supported
-
-
-
 # Fix configs
 
 Use [dbt tool](https://github.com/dbt-labs/dbt-autofix) to fix deprecations

transform/dbt_project.yml

@@ -25,90 +25,88 @@ analysis-paths: ["analysis"]
 test-paths: ["tests"]
 seed-paths: ["seeds"]
 macro-paths: ["macros"]
-
-target-path: "target"  # directory which will store compiled SQL files
 clean-targets:         # directories to be removed by `dbt clean`
-    - "target"
-    - "dbt_packages"
+  - "target"
+  - "dbt_packages"
 
 
 # Configuring models
 # Full documentation: https://docs.getdbt.com/docs/configuring-models
 # These settings can be overridden in the individual model files
 seeds:
-    +schema: SEEDS
-    +docs:
-        node_color: "#379965"
-    +quote_columns: false
-    +persist_docs:
-        relation: true
-        columns: true
-    +dbt-osmosis: "{model}.yml"
+  +schema: SEEDS
+  +docs:
+    node_color: "#379965"
+  +quote_columns: false
+  +persist_docs:
+    relation: true
+    columns: true
+  # +dbt-osmosis: "{model}.yml"
 
 snapshots:
-    +enabled: "{{ True if target.database == 'balboa' else False }}"
+  +enabled: "{{ True if target.database == 'balboa' else False }}"
 
-    balboa:
-        +target_database: raw
-        +target_schema: SNAPSHOTS
-    +docs:
-        node_color: "#0f703d"
+  balboa:
+    +target_database: raw
+    +target_schema: SNAPSHOTS
+  +docs:
+    node_color: "#0f703d"
 
 models:
     # elementary:
     #     +schema: "elementary"
-    +dbt-osmosis: "{model}.yml"
-
-    balboa:
-        +materialized: view
-        +schema: Z_SCHEMA_NOT_CONFIGURED
-
-        L1_inlets:
-            account_usage:
-                +schema: L1_ACCOUNT_USAGE
-            country_data:
-                +schema: L1_COUNTRY_DATA
-            country_geo:
-                +schema: L1_COUNTRY_GEO
-            google_analytics_4:
-                +schema: L1_GOOGLE_ANALYTICS_4
-            covid19_epidemiological_data:
-                +schema: L1_COVID19_EPIDEMIOLOGICAL_DATA
-            loans:
-                +schema: L1_LOANS
-            observe:
-                +schema: L1_OBSERVE
-            us_population:
-                +schema: L1_US_POPULATION
-            usgs__earthquake_data:
-                +schema: L1_USGS__EARTHQUAKE_DATA
-
-        L2_bays:
-            +group: marketing
-            +docs:
-                node_color: "#000899"
-            country_demographics:
-                +schema: L2_COUNTRY_DEMOGRAPHICS
-            covid_observations:
-                +schema: L2_COVID_OBSERVATIONS
-            snowflake_usage:
-                +schema: L2_SNOWFLAKE_USAGE
-
-        L3_coves:
-            +group: marketing
-            +access: public
-            +docs:
-                node_color: "#366ccf"
-            covid_analytics:
-                +schema: L3_COVID_ANALYTICS
-            earthquake_analytics:
-                +schema: L3_EARTHQUAKE_ANALYTICS
-            loan_analytics:
-                +schema: L3_LOAN_ANALYTICS
+  # +dbt-osmosis: "{model}.yml"
+
+  balboa:
+    +materialized: view
+    +schema: Z_SCHEMA_NOT_CONFIGURED
+
+    L1_inlets:
+      account_usage:
+        +schema: L1_ACCOUNT_USAGE
+      country_data:
+        +schema: L1_COUNTRY_DATA
+      country_geo:
+        +schema: L1_COUNTRY_GEO
+      google_analytics_4:
+        +schema: L1_GOOGLE_ANALYTICS_4
+      covid19_epidemiological_data:
+        +schema: L1_COVID19_EPIDEMIOLOGICAL_DATA
+      loans:
+        +schema: L1_LOANS
+      observe:
+        +schema: L1_OBSERVE
+      us_population:
+        +schema: L1_US_POPULATION
+      usgs__earthquake_data:
+        +schema: L1_USGS__EARTHQUAKE_DATA
+
+    L2_bays:
+      +group: marketing
+      +docs:
+        node_color: "#000899"
+      country_demographics:
+        +schema: L2_COUNTRY_DEMOGRAPHICS
+      covid_observations:
+        +schema: L2_COVID_OBSERVATIONS
+      snowflake_usage:
+        +schema: L2_SNOWFLAKE_USAGE
+
+    L3_coves:
+      +group: marketing
+      +access: public
+      +docs:
+        node_color: "#366ccf"
+      covid_analytics:
+        +schema: L3_COVID_ANALYTICS
+      earthquake_analytics:
+        +schema: L3_EARTHQUAKE_ANALYTICS
+      loan_analytics:
+        +schema: L3_LOAN_ANALYTICS
                 # cannot persist docs on dynamic tables
                 # +persist_docs:
                 #     relation: false
-            +materialized: table
+      +materialized: table
 
         # L4_shares:
         #   +docs:
@@ -125,50 +123,49 @@ models:
         #       columns: false
 
 
-        +persist_docs:
-            relation: true
-            columns: true
+    +persist_docs:
+      relation: true
+      columns: true
 
-    post-hook:
-        - "{{ dbt_snow_mask.apply_masking_policy('sources') if target.name == 'prd_pii' }}"
-        - "{{ dbt_snow_mask.apply_masking_policy('models') if target.name == 'prd_pii' }}"
+  +transient: "{{ 'false' if target.name == 'prd' else 'true' }}"
+  +post-hook:
+    - "{{ dbt_snow_mask.apply_masking_policy('sources') if target.name == 'prd_pii' }}"
+    - "{{ dbt_snow_mask.apply_masking_policy('models') if target.name == 'prd_pii' }}"
         # This macro creates a non_versioned view of a versioned dbt model
         # Useful if you want to have a versioned mart but dont want BI tool to break if version
         # is updated.
         # - "{{ create_latest_version_view() }}"
 
     # Snowflake Defaults
-    +transient: "{{ 'false' if target.name == 'prd' else 'true' }}"
-    copy_grants: true
-
-# This creates a view in snowflake that will track failures. Usefull for dynamic tables dbt isnt refreshing
-# Run dbt with:  --vars '{"persist_tests": "true", "tests_model": "test_failures"}'
+  +copy_grants: true
 on-run-end:
   - "{{ create_test_failure_view(results) }}"
 
 # In development, this will be created in your user schema
 data_tests:
-    +store_failures: true
-    +schema: DBT_TEST__AUDIT
-    +store_failures_as: view
+  +store_failures: true
+  +schema: DBT_TEST__AUDIT
+  +store_failures_as: view
 
 vars:
-    'dbt_date:time_zone': 'America/Los_Angeles'
+  'dbt_date:time_zone': 'America/Los_Angeles'
 
-    create_masking_policy_schema: "false"
-    use_common_masking_policy_db: "true"
-    common_masking_policy_db: "BALBOA"
-    common_masking_policy_schema: "MASKING_POLICIES"
-    use_force_applying_masking_policy: "true"
+  create_masking_policy_schema: "false"
+  use_common_masking_policy_db: "true"
+  common_masking_policy_db: "BALBOA"
+  common_masking_policy_schema: "MASKING_POLICIES"
+  use_force_applying_masking_policy: "true"
 
     # These are used in CI/CD to grant access to
-    db_usage_role_prefix: "z_db__"
-    schema_usage_role_prefix: "z_schema__"
-    pr_db_usage_role: "z_db__balboa_tst"
+  db_usage_role_prefix: "z_db__"
+  schema_usage_role_prefix: "z_schema__"
+  pr_db_usage_role: "z_db__balboa_tst"
 
 
 
 # Added for Elementary support
 # flags:
 #   require_explicit_package_overrides_for_builtin_materializations: False
 #   source_freshness_run_project_hooks: True
+flags:
+  require_generic_test_arguments_property: true