From 84f021d7fd6ee92ccea8b94a8665f257d2cc2833 Mon Sep 17 00:00:00 2001 From: skorulka Date: Wed, 25 Jun 2025 19:54:41 +0000 Subject: [PATCH 1/3] Sample test to create table not view based on git issue 178 --- dbt_adbs_test_project/dbt_test.env | 15 ++++++ .../models/property/property_data_product.sql | 15 ++++++ .../models/property/schema.yml | 49 +++++++++++++++++++ dbt_adbs_test_project/profiles.yml | 1 - .../seeds/mortgage_application_seed.csv | 4 ++ .../seeds/property_data_seed.csv | 5 ++ 6 files changed, 88 insertions(+), 1 deletion(-) create mode 100644 dbt_adbs_test_project/dbt_test.env create mode 100644 dbt_adbs_test_project/models/property/property_data_product.sql create mode 100644 dbt_adbs_test_project/models/property/schema.yml create mode 100644 dbt_adbs_test_project/seeds/mortgage_application_seed.csv create mode 100644 dbt_adbs_test_project/seeds/property_data_seed.csv diff --git a/dbt_adbs_test_project/dbt_test.env b/dbt_adbs_test_project/dbt_test.env new file mode 100644 index 0000000..156dfc0 --- /dev/null +++ b/dbt_adbs_test_project/dbt_test.env @@ -0,0 +1,15 @@ +export DBT_ORACLE_USER=dbt_test +export DBT_ORACLE_HOST=localhost +export DBT_ORACLE_PROTOCOL=tcps +export DBT_ORACLE_PORT=1522 +export DBT_ORACLE_SERVICE=myadw_medium.adb.oraclecloud.com +export DBT_ORACLE_PASSWORD=Auto_MY_DBT_1234 +export DBT_ORACLE_DATABASE=MYADW +export DBT_ORACLE_SCHEMA=dbt_test +export DBT_ORACLE_CUSTOM_SCHEMA=dbt_test +export DBT_MACRO_DEBUGGING=1 +export WALLET_LOCATION=/scratch/tls_wallet +export WALLET_PASSWORD=Auto_MY_ADB_Wallet +export TNS_ADMIN=/scratch/tls_wallet +export SSL_SERVER_CERT_DN="CN=3746b756d92b" +export DBT_ORACLE_OML_CLOUD_SERVICE_URL="" diff --git a/dbt_adbs_test_project/models/property/property_data_product.sql b/dbt_adbs_test_project/models/property/property_data_product.sql new file mode 100644 index 0000000..2dfbc56 --- /dev/null +++ b/dbt_adbs_test_project/models/property/property_data_product.sql @@ -0,0 +1,15 @@ +{{ config(materialized='table') }} + + SELECT + Cast (prop_data.property_id as int) property_id, + mort_data.mortgage_id, + prop_data.property_type, + prop_data.living_area, + --mort_data.create_date --zum zeigen von Enforce Model (contract sagt Date, aber ohne Spezification wird es Timestamp) (Null/Fehler bei Testdaten wird auch erkannt) + Cast( mort_data.create_date as date) create_date +FROM + {{ ref('mortgage_application_seed') }} mort_data +LEFT JOIN + {{ ref('property_data_seed') }} prop_data +ON + prop_data.property_id = mort_data.mortgage_id diff --git a/dbt_adbs_test_project/models/property/schema.yml b/dbt_adbs_test_project/models/property/schema.yml new file mode 100644 index 0000000..d3b02ec --- /dev/null +++ b/dbt_adbs_test_project/models/property/schema.yml @@ -0,0 +1,49 @@ +version: 2 +models: +- name: property_data_product + config: + meta: + data_contract: property-data-product + owner: property_onboarding + materialized: table + contract: + enforced: True + description: This table contains all active mortgages and their financed properties. + columns: + - name: property_id + data_tests: + - dbt_expectations.expect_column_values_to_be_between: + min_value: 0 + data_type: NUMBER + description: Unique identifier for the property + constraints: + - type: not_null + - type: unique + meta: + pii: false + - name: mortgage_id + data_tests: + - dbt_expectations.expect_column_values_to_be_between: + min_value: 0 + data_type: NUMBER + description: Unique identifier for the mortgage agreement + constraints: + - type: not_null + meta: + pii: false + - name: property_type + data_type: STRING + description: Type of the property like single-family home. + meta: + pii: false + - name: living_area + data_type: FLOAT + description: Living area of the property + meta: + pii: false + - name: create_date + data_type: DATE + description: Creation date of the mortgage agreement + meta: + pii: false + diff --git a/dbt_adbs_test_project/profiles.yml b/dbt_adbs_test_project/profiles.yml index c837a56..b77bfb9 100644 --- a/dbt_adbs_test_project/profiles.yml +++ b/dbt_adbs_test_project/profiles.yml @@ -11,7 +11,6 @@ dbt_test: service: "{{ env_var('DBT_ORACLE_SERVICE') }}" database: "{{ env_var('DBT_ORACLE_DATABASE') }}" schema: "{{ env_var('DBT_ORACLE_SCHEMA') }}" - oml_cloud_service_url: "{{ env_var('DBT_ORACLE_OML_CLOUD_SERVICE_URL')}}" session_info: action: "dbt run" client_identifier: "dbt-mac-abhisoms" diff --git a/dbt_adbs_test_project/seeds/mortgage_application_seed.csv b/dbt_adbs_test_project/seeds/mortgage_application_seed.csv new file mode 100644 index 0000000..4c7f1fd --- /dev/null +++ b/dbt_adbs_test_project/seeds/mortgage_application_seed.csv @@ -0,0 +1,4 @@ +mortgage_id,create_date +1,2025-06-03 +2,2025-01-20 +3,2024-02-03 diff --git a/dbt_adbs_test_project/seeds/property_data_seed.csv b/dbt_adbs_test_project/seeds/property_data_seed.csv new file mode 100644 index 0000000..320c91b --- /dev/null +++ b/dbt_adbs_test_project/seeds/property_data_seed.csv @@ -0,0 +1,5 @@ +property_id,property_type,living_area +1,commercial,200.34 +2,office,3000.0 +3,retail,2020.0 +4,apartment,690.4 From 7b54b51e5bbdc0ef1b11f19c12e23dae74f8362b Mon Sep 17 00:00:00 2001 From: skorulka Date: Thu, 3 Jul 2025 14:58:43 +0000 Subject: [PATCH 2/3] bug_179_Fix to avoid inserting redundant entries in snapshot table if is_deleted is True --- .../materializations/snapshot/snapshot.sql | 35 +++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/dbt/include/oracle/macros/materializations/snapshot/snapshot.sql b/dbt/include/oracle/macros/materializations/snapshot/snapshot.sql index 05c547b..71d1111 100644 --- a/dbt/include/oracle/macros/materializations/snapshot/snapshot.sql +++ b/dbt/include/oracle/macros/materializations/snapshot/snapshot.sql @@ -119,7 +119,9 @@ on {{ unique_key_join_on(strategy.unique_key, "snapshotted_data", "source_data") }} where {{ unique_key_is_null(strategy.unique_key, "snapshotted_data") }} or ({{ unique_key_is_not_null(strategy.unique_key, "snapshotted_data") }} and ({{ strategy.row_changed }}) - + {%- if strategy.hard_deletes == 'new_record' -%} + or ({{ unique_key_is_not_null(strategy.unique_key, "snapshotted_data") }} and snapshotted_data.{{ columns.dbt_is_deleted }} = 'True') + {%- endif %} ) ), @@ -140,6 +142,10 @@ where ( {{ strategy.row_changed }} ) + {%- if strategy.hard_deletes == 'new_record' -%} + or snapshotted_data.{{ columns.dbt_is_deleted }} = 'True' + {%- endif %} + ) {%- if strategy.hard_deletes == 'invalidate' or strategy.hard_deletes == 'new_record' -%} @@ -162,6 +168,20 @@ left join deletes_source_data source_data on {{ unique_key_join_on(strategy.unique_key, "snapshotted_data", "source_data") }} where {{ unique_key_is_null(strategy.unique_key, "source_data") }} + {%- if strategy.hard_deletes == 'new_record' %} + and not ( + --avoid updating the record's valid_to if the latest entry is marked as deleted + snapshotted_data.{{ columns.dbt_is_deleted }} = 'True' + and + {% if config.get('dbt_valid_to_current') %} + {% set source_unique_key = columns.dbt_valid_to | trim %} + {% set target_unique_key = config.get('dbt_valid_to_current') | trim %} + ( {{ equals(source_unique_key, target_unique_key) }} or {{ source_unique_key }} is null ) + {% else %} + {{ columns.dbt_valid_to }} is null + {% endif %} + ) + {%- endif %} ) {%- endif %} @@ -191,7 +211,18 @@ left join deletes_source_data source_data on {{ unique_key_join_on(strategy.unique_key, "snapshotted_data", "source_data") }} where {{ unique_key_is_null(strategy.unique_key, "source_data") }} - + and not ( + --avoid inserting a new record if the latest one is marked as deleted + snapshotted_data.{{ columns.dbt_is_deleted }} = 'True' + and + {% if config.get('dbt_valid_to_current') %} + {% set source_unique_key = columns.dbt_valid_to | trim %} + {% set target_unique_key = config.get('dbt_valid_to_current') | trim %} + ( {{ equals(source_unique_key, target_unique_key) }} or {{ source_unique_key }} is null ) + {% else %} + {{ columns.dbt_valid_to }} is null + {% endif %} + ) ) {%- endif %} From 6e7ab7baeaa4fedd22fe87f06829747a5e5b5c57 Mon Sep 17 00:00:00 2001 From: skorulka Date: Thu, 3 Jul 2025 15:00:14 +0000 Subject: [PATCH 3/3] Sample test to reproduce bug_179 --- dbt_adbs_test_project/models/dummy_model.sql | 8 ++++++++ dbt_adbs_test_project/models/filter_table.sql | 5 +++++ dbt_adbs_test_project/seeds/dummy_table.csv | 3 +++ dbt_adbs_test_project/snapshots/dummy_table.sql | 15 +++++++++++++++ 4 files changed, 31 insertions(+) create mode 100644 dbt_adbs_test_project/models/dummy_model.sql create mode 100644 dbt_adbs_test_project/models/filter_table.sql create mode 100644 dbt_adbs_test_project/seeds/dummy_table.csv create mode 100644 dbt_adbs_test_project/snapshots/dummy_table.sql diff --git a/dbt_adbs_test_project/models/dummy_model.sql b/dbt_adbs_test_project/models/dummy_model.sql new file mode 100644 index 0000000..d80bcbb --- /dev/null +++ b/dbt_adbs_test_project/models/dummy_model.sql @@ -0,0 +1,8 @@ +{{ config(materialized='table') }} + +SELECT + id, + label, + current_timestamp AS created_at +FROM {{ ref('dummy_table') }} + diff --git a/dbt_adbs_test_project/models/filter_table.sql b/dbt_adbs_test_project/models/filter_table.sql new file mode 100644 index 0000000..e902873 --- /dev/null +++ b/dbt_adbs_test_project/models/filter_table.sql @@ -0,0 +1,5 @@ +select id, + label, + created_at +from {{ ref ('dummy_model') }} +where id != 2 diff --git a/dbt_adbs_test_project/seeds/dummy_table.csv b/dbt_adbs_test_project/seeds/dummy_table.csv new file mode 100644 index 0000000..7b94182 --- /dev/null +++ b/dbt_adbs_test_project/seeds/dummy_table.csv @@ -0,0 +1,3 @@ +id,label +1,first +2,second diff --git a/dbt_adbs_test_project/snapshots/dummy_table.sql b/dbt_adbs_test_project/snapshots/dummy_table.sql new file mode 100644 index 0000000..a3a0a69 --- /dev/null +++ b/dbt_adbs_test_project/snapshots/dummy_table.sql @@ -0,0 +1,15 @@ + +{% snapshot new_snapshot_dummy_table %} + + {{ + config( + strategy = 'timestamp' + , unique_key = 'id' + , updated_at = 'created_at' + , hard_deletes = 'new_record' + ) + }} + + SELECT * FROM {{ ref ('dummy_model') }} + +{% endsnapshot %}