diff --git a/CHANGELOG.md b/CHANGELOG.md index f573dcc2..fe52d332 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,10 @@ ## Under the hood ## Fixes +# edu_edfi_source v0.4.2 +## Fixes +- Fix surrogate key creation for `stg_ef3__grading_periods` to properly hanlde lowering of alphanumeric column (grading_period_name) that is part of natural key + # edu_edfi_source v0.4.1 ## Fixes - Fix surrogate key creation for `stg_ef3__student_contact_associations` to properly hanlde lowering of alphanumeric columns that are part of natural keys diff --git a/dbt_project.yml b/dbt_project.yml index 87d41c37..2186eb0c 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,7 +1,7 @@ config-version: 2 name: 'edu_edfi_source' -version: '0.4.1' +version: '0.4.2' require-dbt-version: [">=1.0.0", "<2.0.0"] # This setting configures which "profile" dbt uses for this project. diff --git a/macros/gen_skey.sql b/macros/gen_skey.sql index 378d4a0a..11000b2f 100644 --- a/macros/gen_skey.sql +++ b/macros/gen_skey.sql @@ -210,7 +210,18 @@ 'annualize': False }, + 'k_student_assessment': { + 'reference_name': 'student_assessment_reference', + 'col_list': [ + 'assessmentIdentifier', + 'namespace', + 'studentAssessmentIdentifier', + 'studentUniqueId' + ], + 'annualize': True + } } + %} {#- retrieve key def for then decompose parts -#} {% set skey_def = skey_defs[k_name] %} diff --git a/models/staging/edfi_3/base/_edfi_3__base.yml b/models/staging/edfi_3/base/_edfi_3__base.yml index 67b1fc36..d60d415e 100644 --- a/models/staging/edfi_3/base/_edfi_3__base.yml +++ b/models/staging/edfi_3/base/_edfi_3__base.yml @@ -190,4 +190,7 @@ models: - name: base_ef3__surveys config: tags: ['survey'] - enabled: "{{ var('src:program:survey:enabled', True) }}" \ No newline at end of file + enabled: "{{ var('src:program:survey:enabled', True) }}" + - name: base_ef3__student_assessment_education_organizations + config: + tags: ['core'] \ No newline at end of file diff --git a/models/staging/edfi_3/base/base_ef3__student_assessment_education_organization_associations.sql b/models/staging/edfi_3/base/base_ef3__student_assessment_education_organization_associations.sql new file mode 100644 index 00000000..12b006ff --- /dev/null +++ b/models/staging/edfi_3/base/base_ef3__student_assessment_education_organization_associations.sql @@ -0,0 +1,37 @@ +with student_assessment_education_organization_associations as ( + {{ source_edfi3('student_assessment_education_organization_associations') }} +), + + +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + file_row_number, + filename, + is_deleted, + v:id::string as record_guid, + v:educationOrganizationReference:educationOrganizationId::int as ed_org_identifier, + v:educationOrganizationReference:link:rel::string as ed_org_type, + {{ extract_descriptor('v:educationOrganizationAssociationTypeDescriptor::string') }} as ed_org_association_type, + v:schoolYearTypeReference:schoolYear::int as school_year, + v:studentAssessmentReference:assessmentIdentifier::string as assessment_identifier, + v:studentAssessmentReference:namespace::string as assessment_namespace, + v:studentAssessmentReference:studentAssessmentIdentifier::string as student_assessment_identifier, + v:studentAssessmentReference:studentUniqueId::string as student_id, + -- references + v:educationOrganizationReference as education_organization_reference, + v:studentAssessmentReference as student_assessment_reference, + -- edfi extensions + v:_ext as v_ext + from student_assessment_education_organization_associations +) + + +select * from renamed + + + + + diff --git a/models/staging/edfi_3/stage/_edfi_3__stage.yml b/models/staging/edfi_3/stage/_edfi_3__stage.yml index a7d73dd2..bd4ba06d 100644 --- a/models/staging/edfi_3/stage/_edfi_3__stage.yml +++ b/models/staging/edfi_3/stage/_edfi_3__stage.yml @@ -131,6 +131,13 @@ referential_integrity_tests: field: k_survey_response tags: ['ref_integrity'] + - k_student_assessment: &ref_k_student_assessment + - relationships: + to: ref('stg_ef3__student_assessment') + field: k_student_assessment + tags: ['ref_integrity'] + + models: - name: stg_ef3__assessments @@ -878,3 +885,11 @@ models: config: tags: ['survey'] enabled: "{{ var('src:domain:survey:enabled', True) }}" + + - name: stg_ef3__student_assessment_education_organization_associations + config: + tags: ['assessment'] + columns: + - name: k_student_assessment + tests: + *ref_k_student_assessment diff --git a/models/staging/edfi_3/stage/stg_ef3__grading_periods.sql b/models/staging/edfi_3/stage/stg_ef3__grading_periods.sql index d77bc0ea..e70ac3e5 100644 --- a/models/staging/edfi_3/stage/stg_ef3__grading_periods.sql +++ b/models/staging/edfi_3/stage/stg_ef3__grading_periods.sql @@ -19,7 +19,7 @@ keyed as ( {{ dbt_utils.generate_surrogate_key( ['tenant_code', 'lower(grading_period)', - 'grading_period_name', + 'lower(grading_period_name)', 'school_id', 'school_year'] ) }} @@ -40,4 +40,4 @@ deduped as ( }} ) select * from deduped -order by tenant_code, school_year desc, period_sequence \ No newline at end of file +order by tenant_code, school_year desc, period_sequence diff --git a/models/staging/edfi_3/stage/stg_ef3__student_assessment_education_organization_associations.sql b/models/staging/edfi_3/stage/stg_ef3__student_assessment_education_organization_associations.sql new file mode 100644 index 00000000..ad9ccb14 --- /dev/null +++ b/models/staging/edfi_3/stage/stg_ef3__student_assessment_education_organization_associations.sql @@ -0,0 +1,40 @@ +with base as ( + select * from {{ ref('base_ef3__student_assessment_education_organization_associations') }} + where not is_deleted +), + + +keyed as ( + select + {{ + dbt_utils.generate_surrogate_key( + [ 'tenant_code', + 'api_year', + 'ed_org_identifier', + 'ed_org_association_type', + 'assessment_identifier', + 'student_assessment_identifier', + 'student_id', + 'lower(assessment_namespace)' + ] + ) + }} as k_student_assessment_ed_org, + {{ edorg_ref() }}, + {{ gen_skey('k_student_assessment') }}, + base.* + from base +), + + +deduped as ( + {{ + dbt_utils.deduplicate( + relation = 'keyed', + partition_by = 'k_student_assessment_ed_org', + order_by = 'pull_timestamp desc' + ) + }} +) + + +select * from deduped