From 08b413642e44f5ef708d3fe05289a9de3b65761f Mon Sep 17 00:00:00 2001 From: gnguyen87 Date: Wed, 16 Oct 2024 17:18:58 -0500 Subject: [PATCH 01/11] Create a base stuAssessmentEdOrg model --- models/staging/edfi_3/base/_edfi_3__base.yml | 5 ++- ...ent_assessment_education_organizations.sql | 38 +++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 models/staging/edfi_3/base/base_ef3__student_assessment_education_organizations.sql diff --git a/models/staging/edfi_3/base/_edfi_3__base.yml b/models/staging/edfi_3/base/_edfi_3__base.yml index 67b1fc36..d60d415e 100644 --- a/models/staging/edfi_3/base/_edfi_3__base.yml +++ b/models/staging/edfi_3/base/_edfi_3__base.yml @@ -190,4 +190,7 @@ models: - name: base_ef3__surveys config: tags: ['survey'] - enabled: "{{ var('src:program:survey:enabled', True) }}" \ No newline at end of file + enabled: "{{ var('src:program:survey:enabled', True) }}" + - name: base_ef3__student_assessment_education_organizations + config: + tags: ['core'] \ No newline at end of file diff --git a/models/staging/edfi_3/base/base_ef3__student_assessment_education_organizations.sql b/models/staging/edfi_3/base/base_ef3__student_assessment_education_organizations.sql new file mode 100644 index 00000000..d8a852b8 --- /dev/null +++ b/models/staging/edfi_3/base/base_ef3__student_assessment_education_organizations.sql @@ -0,0 +1,38 @@ +with student_assessment_education_organizations as ( + select * from {{ source_edfi3('student_assessment_education_organizations') }} +), + + +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + file_row_number, + filename, + is_deleted, + v:id::string as record_guid, + v:educationOrganizationReference:educationOrganizationId::int as ed_org_identifier, + v:educationOrganizationReference:link:rel::string as ed_org_type, + {{ extract_descriptor('v:educationOrganizationAssociationTypeDescriptor::string') }} as ed_org_association_type, + v:schoolYearTypeReference:schoolYear::int as school_year, + v:studentAssessmentReference:assessmentIdentifier::string as assement_identifier, + v:studentAssessmentReference:namespace::string as assessment_name_space, + v:studentAssessmentReference:studentAssessmentIdentifier::string as student_assessment_identifier, + v:studentAssessmentReference:studentUniqueId::string as student_id, + -- references + v:educationOrganizationReference as education_organization_reference, + v:studentAssessmentReference as student_assessment_reference + -- edfi extensions + v:_ext as v_ext + from student_assessment_education_organizations +) + + +select * from renamed + + + + + From 34b48db4ca86f5936ec983c9dcfa17ba0467fdd0 Mon Sep 17 00:00:00 2001 From: gnguyen87 Date: Wed, 16 Oct 2024 17:19:16 -0500 Subject: [PATCH 02/11] Update gen_skey() macro to accommodate for new reference --- macros/gen_skey.sql | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/macros/gen_skey.sql b/macros/gen_skey.sql index 378d4a0a..b87fa224 100644 --- a/macros/gen_skey.sql +++ b/macros/gen_skey.sql @@ -210,6 +210,17 @@ 'annualize': False }, + 'k_student_assessment': { + 'reference_name': 'studentAssessmentReference'. + 'col_list': [ + 'assessmentIdentifier', + 'namespace', + 'studentAssessmentIdentifier', + 'studentUniqueId' + ], + 'annualize': True + + } %} {#- retrieve key def for then decompose parts -#} From d7cb9353105705c1b2e8bda646847e80f315ae1d Mon Sep 17 00:00:00 2001 From: gnguyen87 Date: Wed, 16 Oct 2024 17:19:30 -0500 Subject: [PATCH 03/11] Create stage stuAssessmentEdOrg model --- .../staging/edfi_3/stage/_edfi_3__stage.yml | 15 +++++++ ...ent_assessment_education_organizations.sql | 39 +++++++++++++++++++ 2 files changed, 54 insertions(+) create mode 100644 models/staging/edfi_3/stage/stg_ef3__student_assessment_education_organizations.sql diff --git a/models/staging/edfi_3/stage/_edfi_3__stage.yml b/models/staging/edfi_3/stage/_edfi_3__stage.yml index a7d73dd2..7f02489f 100644 --- a/models/staging/edfi_3/stage/_edfi_3__stage.yml +++ b/models/staging/edfi_3/stage/_edfi_3__stage.yml @@ -131,6 +131,13 @@ referential_integrity_tests: field: k_survey_response tags: ['ref_integrity'] + - k_student_assessment: &ref_k_student_assessment + - relationships: + to: ref('stg_ef3__student_assessment') + field: k_student_assessment + tags: ['ref_integrity'] + + models: - name: stg_ef3__assessments @@ -878,3 +885,11 @@ models: config: tags: ['survey'] enabled: "{{ var('src:domain:survey:enabled', True) }}" + + - name: stg_ef3__student_assessment_education_organizations + config: + tags: ['assessment'] + columns: + - name: k_student_assessment + tests: + *ref_k_student_assessment diff --git a/models/staging/edfi_3/stage/stg_ef3__student_assessment_education_organizations.sql b/models/staging/edfi_3/stage/stg_ef3__student_assessment_education_organizations.sql new file mode 100644 index 00000000..6a041bf1 --- /dev/null +++ b/models/staging/edfi_3/stage/stg_ef3__student_assessment_education_organizations.sql @@ -0,0 +1,39 @@ +with base as ( + select * from {{ ref('base_ef3__student_assessment_education_organizations') }} + where not is_deleted +), + + +keyed as ( + select + {{ + dbt_utils.generate_surrogate_key( + [ 'tenant_code', + 'api_year', + 'ed_org_identifier', + 'ed_org_association_type', + 'assessment_identifier', + 'student_assessment_identifier', + 'student_unique_id', + 'lower(namespace)' + ] + ) + }} as k_student_assessment_ed_org, + {{ gen_skey('k_ed_org') }}, + {{ gen_skey('k_student_assessment') }} + from base +), + + +deduped as ( + {{ + dbt_utils.deduplicate( + relation = 'keyed', + partition_by = 'k_student_assessment_ed_org', + order_by = 'pull_timestamp desc' + ) + }} +) + + +select * from deduped From a6dbd5332f099aff6542c97cb1ad7347380f2c3f Mon Sep 17 00:00:00 2001 From: gnguyen87 Date: Wed, 16 Oct 2024 17:28:15 -0500 Subject: [PATCH 04/11] Fix missing curly bracket --- macros/gen_skey.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/macros/gen_skey.sql b/macros/gen_skey.sql index b87fa224..12e413fb 100644 --- a/macros/gen_skey.sql +++ b/macros/gen_skey.sql @@ -219,9 +219,9 @@ 'studentUniqueId' ], 'annualize': True - - + } } + %} {#- retrieve key def for then decompose parts -#} {% set skey_def = skey_defs[k_name] %} From 520456ad348abe901c1f7ff530e912b3b88471bb Mon Sep 17 00:00:00 2001 From: rlittle08 Date: Fri, 15 Nov 2024 11:46:35 -0600 Subject: [PATCH 05/11] Update gen_skey.sql --- macros/gen_skey.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/macros/gen_skey.sql b/macros/gen_skey.sql index 12e413fb..fc8e69bb 100644 --- a/macros/gen_skey.sql +++ b/macros/gen_skey.sql @@ -211,7 +211,7 @@ }, 'k_student_assessment': { - 'reference_name': 'studentAssessmentReference'. + 'reference_name': 'studentAssessmentReference', 'col_list': [ 'assessmentIdentifier', 'namespace', From f59598d074936093748a7aef69de8435f8838c91 Mon Sep 17 00:00:00 2001 From: rlittle08 Date: Fri, 15 Nov 2024 14:19:13 -0600 Subject: [PATCH 06/11] Update and rename base_ef3__student_assessment_education_organizations.sql to base_ef3__student_assessment_education_organization_associations.sql --- ...dent_assessment_education_organization_associations.sql} | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) rename models/staging/edfi_3/base/{base_ef3__student_assessment_education_organizations.sql => base_ef3__student_assessment_education_organization_associations.sql} (91%) diff --git a/models/staging/edfi_3/base/base_ef3__student_assessment_education_organizations.sql b/models/staging/edfi_3/base/base_ef3__student_assessment_education_organization_associations.sql similarity index 91% rename from models/staging/edfi_3/base/base_ef3__student_assessment_education_organizations.sql rename to models/staging/edfi_3/base/base_ef3__student_assessment_education_organization_associations.sql index d8a852b8..b003960f 100644 --- a/models/staging/edfi_3/base/base_ef3__student_assessment_education_organizations.sql +++ b/models/staging/edfi_3/base/base_ef3__student_assessment_education_organization_associations.sql @@ -1,5 +1,5 @@ -with student_assessment_education_organizations as ( - select * from {{ source_edfi3('student_assessment_education_organizations') }} +with student_assessment_education_organization_associations as ( + select * from {{ source_edfi3('student_assessment_education_organization_associations') }} ), @@ -26,7 +26,7 @@ renamed as ( v:studentAssessmentReference as student_assessment_reference -- edfi extensions v:_ext as v_ext - from student_assessment_education_organizations + from student_assessment_education_organization_associations ) From 9cf94f0d5e82d9f3baec038ed635b5f90a2b17e8 Mon Sep 17 00:00:00 2001 From: rlittle08 Date: Fri, 15 Nov 2024 14:19:56 -0600 Subject: [PATCH 07/11] Update _edfi_3__stage.yml --- models/staging/edfi_3/stage/_edfi_3__stage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/staging/edfi_3/stage/_edfi_3__stage.yml b/models/staging/edfi_3/stage/_edfi_3__stage.yml index 7f02489f..bd4ba06d 100644 --- a/models/staging/edfi_3/stage/_edfi_3__stage.yml +++ b/models/staging/edfi_3/stage/_edfi_3__stage.yml @@ -886,7 +886,7 @@ models: tags: ['survey'] enabled: "{{ var('src:domain:survey:enabled', True) }}" - - name: stg_ef3__student_assessment_education_organizations + - name: stg_ef3__student_assessment_education_organization_associations config: tags: ['assessment'] columns: From aad06ff8ded51d2e570382ee5db6553005ce1f77 Mon Sep 17 00:00:00 2001 From: rlittle08 Date: Fri, 15 Nov 2024 14:20:47 -0600 Subject: [PATCH 08/11] Update and rename stg_ef3__student_assessment_education_organizations.sql to stg_ef3__student_assessment_education_organization_associations.sql --- ..._student_assessment_education_organization_associations.sql} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename models/staging/edfi_3/stage/{stg_ef3__student_assessment_education_organizations.sql => stg_ef3__student_assessment_education_organization_associations.sql} (96%) diff --git a/models/staging/edfi_3/stage/stg_ef3__student_assessment_education_organizations.sql b/models/staging/edfi_3/stage/stg_ef3__student_assessment_education_organization_associations.sql similarity index 96% rename from models/staging/edfi_3/stage/stg_ef3__student_assessment_education_organizations.sql rename to models/staging/edfi_3/stage/stg_ef3__student_assessment_education_organization_associations.sql index 6a041bf1..332206df 100644 --- a/models/staging/edfi_3/stage/stg_ef3__student_assessment_education_organizations.sql +++ b/models/staging/edfi_3/stage/stg_ef3__student_assessment_education_organization_associations.sql @@ -1,5 +1,5 @@ with base as ( - select * from {{ ref('base_ef3__student_assessment_education_organizations') }} + select * from {{ ref('base_ef3__student_assessment_education_organization_associations') }} where not is_deleted ), From c4d8333fd2b364f4a9eff7f09e3470a0a858bffc Mon Sep 17 00:00:00 2001 From: Jay Kaiser Date: Mon, 21 Oct 2024 14:17:49 -0500 Subject: [PATCH 09/11] Hotfix: Lower `grading_period_name` in surrogate key (#111) * Update stg_ef3__grading_periods.sql * Update CHANGELOG.md * Update dbt_project.yml --------- Co-authored-by: rlittle08 --- CHANGELOG.md | 4 ++++ dbt_project.yml | 2 +- models/staging/edfi_3/stage/stg_ef3__grading_periods.sql | 4 ++-- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f573dcc2..fe52d332 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,10 @@ ## Under the hood ## Fixes +# edu_edfi_source v0.4.2 +## Fixes +- Fix surrogate key creation for `stg_ef3__grading_periods` to properly hanlde lowering of alphanumeric column (grading_period_name) that is part of natural key + # edu_edfi_source v0.4.1 ## Fixes - Fix surrogate key creation for `stg_ef3__student_contact_associations` to properly hanlde lowering of alphanumeric columns that are part of natural keys diff --git a/dbt_project.yml b/dbt_project.yml index 87d41c37..2186eb0c 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,7 +1,7 @@ config-version: 2 name: 'edu_edfi_source' -version: '0.4.1' +version: '0.4.2' require-dbt-version: [">=1.0.0", "<2.0.0"] # This setting configures which "profile" dbt uses for this project. diff --git a/models/staging/edfi_3/stage/stg_ef3__grading_periods.sql b/models/staging/edfi_3/stage/stg_ef3__grading_periods.sql index d77bc0ea..e70ac3e5 100644 --- a/models/staging/edfi_3/stage/stg_ef3__grading_periods.sql +++ b/models/staging/edfi_3/stage/stg_ef3__grading_periods.sql @@ -19,7 +19,7 @@ keyed as ( {{ dbt_utils.generate_surrogate_key( ['tenant_code', 'lower(grading_period)', - 'grading_period_name', + 'lower(grading_period_name)', 'school_id', 'school_year'] ) }} @@ -40,4 +40,4 @@ deduped as ( }} ) select * from deduped -order by tenant_code, school_year desc, period_sequence \ No newline at end of file +order by tenant_code, school_year desc, period_sequence From c8b878850c6f9426de995104ac3c81cc09200892 Mon Sep 17 00:00:00 2001 From: gnguyen87 Date: Mon, 18 Nov 2024 15:30:38 -0600 Subject: [PATCH 10/11] Update base and stg_ef3__student_assessment_education_organization_associations after debugging --- macros/gen_skey.sql | 2 +- ...nt_assessment_education_organization_associations.sql | 9 ++++----- ...nt_assessment_education_organization_associations.sql | 7 ++++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/macros/gen_skey.sql b/macros/gen_skey.sql index fc8e69bb..11000b2f 100644 --- a/macros/gen_skey.sql +++ b/macros/gen_skey.sql @@ -211,7 +211,7 @@ }, 'k_student_assessment': { - 'reference_name': 'studentAssessmentReference', + 'reference_name': 'student_assessment_reference', 'col_list': [ 'assessmentIdentifier', 'namespace', diff --git a/models/staging/edfi_3/base/base_ef3__student_assessment_education_organization_associations.sql b/models/staging/edfi_3/base/base_ef3__student_assessment_education_organization_associations.sql index b003960f..12b006ff 100644 --- a/models/staging/edfi_3/base/base_ef3__student_assessment_education_organization_associations.sql +++ b/models/staging/edfi_3/base/base_ef3__student_assessment_education_organization_associations.sql @@ -1,5 +1,5 @@ with student_assessment_education_organization_associations as ( - select * from {{ source_edfi3('student_assessment_education_organization_associations') }} + {{ source_edfi3('student_assessment_education_organization_associations') }} ), @@ -8,7 +8,6 @@ renamed as ( tenant_code, api_year, pull_timestamp, - last_modified_timestamp, file_row_number, filename, is_deleted, @@ -17,13 +16,13 @@ renamed as ( v:educationOrganizationReference:link:rel::string as ed_org_type, {{ extract_descriptor('v:educationOrganizationAssociationTypeDescriptor::string') }} as ed_org_association_type, v:schoolYearTypeReference:schoolYear::int as school_year, - v:studentAssessmentReference:assessmentIdentifier::string as assement_identifier, - v:studentAssessmentReference:namespace::string as assessment_name_space, + v:studentAssessmentReference:assessmentIdentifier::string as assessment_identifier, + v:studentAssessmentReference:namespace::string as assessment_namespace, v:studentAssessmentReference:studentAssessmentIdentifier::string as student_assessment_identifier, v:studentAssessmentReference:studentUniqueId::string as student_id, -- references v:educationOrganizationReference as education_organization_reference, - v:studentAssessmentReference as student_assessment_reference + v:studentAssessmentReference as student_assessment_reference, -- edfi extensions v:_ext as v_ext from student_assessment_education_organization_associations diff --git a/models/staging/edfi_3/stage/stg_ef3__student_assessment_education_organization_associations.sql b/models/staging/edfi_3/stage/stg_ef3__student_assessment_education_organization_associations.sql index 332206df..aafc2b87 100644 --- a/models/staging/edfi_3/stage/stg_ef3__student_assessment_education_organization_associations.sql +++ b/models/staging/edfi_3/stage/stg_ef3__student_assessment_education_organization_associations.sql @@ -14,13 +14,14 @@ keyed as ( 'ed_org_association_type', 'assessment_identifier', 'student_assessment_identifier', - 'student_unique_id', - 'lower(namespace)' + 'student_id', + 'lower(assessment_namespace)' ] ) }} as k_student_assessment_ed_org, {{ gen_skey('k_ed_org') }}, - {{ gen_skey('k_student_assessment') }} + {{ gen_skey('k_student_assessment') }}, + base.* from base ), From 0b21cf562bf8ab78d10f669479efbcf2ed129c7a Mon Sep 17 00:00:00 2001 From: gnguyen87 Date: Mon, 18 Nov 2024 16:34:01 -0600 Subject: [PATCH 11/11] use ed_org macro instead so we can pull out both k_school k_lea --- ...__student_assessment_education_organization_associations.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/staging/edfi_3/stage/stg_ef3__student_assessment_education_organization_associations.sql b/models/staging/edfi_3/stage/stg_ef3__student_assessment_education_organization_associations.sql index aafc2b87..ad9ccb14 100644 --- a/models/staging/edfi_3/stage/stg_ef3__student_assessment_education_organization_associations.sql +++ b/models/staging/edfi_3/stage/stg_ef3__student_assessment_education_organization_associations.sql @@ -19,7 +19,7 @@ keyed as ( ] ) }} as k_student_assessment_ed_org, - {{ gen_skey('k_ed_org') }}, + {{ edorg_ref() }}, {{ gen_skey('k_student_assessment') }}, base.* from base