feature/StudentAssessmentEducationOrg

CHANGELOG.md

@@ -3,6 +3,10 @@
 ## Under the hood
 ## Fixes
 
+# edu_edfi_source v0.4.2
+## Fixes
+- Fix surrogate key creation for `stg_ef3__grading_periods` to properly hanlde lowering of alphanumeric column (grading_period_name) that is part of natural key
+
 # edu_edfi_source v0.4.1
 ## Fixes
 - Fix surrogate key creation for `stg_ef3__student_contact_associations` to properly hanlde lowering of alphanumeric columns that are part of natural keys

dbt_project.yml

@@ -1,7 +1,7 @@
 config-version: 2
 
 name: 'edu_edfi_source'
-version: '0.4.1'
+version: '0.4.2'
 require-dbt-version: [">=1.0.0", "<2.0.0"]
 
 # This setting configures which "profile" dbt uses for this project.

macros/gen_skey.sql

@@ -210,7 +210,18 @@
             'annualize': False
         },
 
+         'k_student_assessment': {
+            'reference_name': 'student_assessment_reference',
+            'col_list': [
+                'assessmentIdentifier',
+                'namespace',
+                'studentAssessmentIdentifier',
+                'studentUniqueId'
+            ],
+            'annualize': True
+        }
     }
+
     %}
     {#- retrieve key def for then decompose parts -#}
     {% set skey_def = skey_defs[k_name] %}

models/staging/edfi_3/base/_edfi_3__base.yml

@@ -190,4 +190,7 @@ models:
   - name: base_ef3__surveys
     config:
       tags: ['survey']
-      enabled: "{{ var('src:program:survey:enabled', True) }}"
+      enabled: "{{ var('src:program:survey:enabled', True) }}"
+  - name: base_ef3__student_assessment_education_organizations
+    config:
+      tags: ['core']

models/staging/edfi_3/base/base_ef3__student_assessment_education_organization_associations.sql

@@ -0,0 +1,37 @@
+with student_assessment_education_organization_associations as (
+    {{ source_edfi3('student_assessment_education_organization_associations') }}
+),
+
+
+renamed as (
+    select
+        tenant_code,
+        api_year,
+        pull_timestamp,
+        file_row_number,
+        filename,
+        is_deleted,
+        v:id::string                                                                            as record_guid,
+        v:educationOrganizationReference:educationOrganizationId::int                           as ed_org_identifier,
+        v:educationOrganizationReference:link:rel::string                                       as ed_org_type,
+        {{ extract_descriptor('v:educationOrganizationAssociationTypeDescriptor::string') }}    as ed_org_association_type,
+        v:schoolYearTypeReference:schoolYear::int                                               as school_year,
+        v:studentAssessmentReference:assessmentIdentifier::string                               as assessment_identifier,
+        v:studentAssessmentReference:namespace::string                                          as assessment_namespace,
+        v:studentAssessmentReference:studentAssessmentIdentifier::string                        as student_assessment_identifier,
+        v:studentAssessmentReference:studentUniqueId::string                                    as student_id,
+        -- references
+        v:educationOrganizationReference                                                        as education_organization_reference,
+        v:studentAssessmentReference                                                            as student_assessment_reference,
+        -- edfi extensions
+        v:_ext                                                                                  as v_ext
+    from student_assessment_education_organization_associations
+)
+
+
+select  * from renamed
+
+
+
+
+

models/staging/edfi_3/stage/_edfi_3__stage.yml

@@ -131,6 +131,13 @@ referential_integrity_tests:
         field: k_survey_response
         tags: ['ref_integrity']
 
+  - k_student_assessment: &ref_k_student_assessment
+    - relationships:
+        to: ref('stg_ef3__student_assessment')
+        field: k_student_assessment
+        tags: ['ref_integrity']
+
+
 
 models:
   - name: stg_ef3__assessments
@@ -878,3 +885,11 @@ models:
     config: 
       tags: ['survey']
       enabled: "{{ var('src:domain:survey:enabled', True) }}"
+
+  - name: stg_ef3__student_assessment_education_organization_associations
+    config:
+      tags: ['assessment']
+    columns:
+      - name: k_student_assessment
+        tests:
+          *ref_k_student_assessment

models/staging/edfi_3/stage/stg_ef3__grading_periods.sql

@@ -19,7 +19,7 @@ keyed as (
             {{ dbt_utils.generate_surrogate_key(
             ['tenant_code', 
                 'lower(grading_period)',
-                'grading_period_name',
+                'lower(grading_period_name)',
                 'school_id', 
                 'school_year'] 
             ) }}
@@ -40,4 +40,4 @@ deduped as (
     }}
 )
 select * from deduped
-order by tenant_code, school_year desc, period_sequence
+order by tenant_code, school_year desc, period_sequence

models/staging/edfi_3/stage/stg_ef3__student_assessment_education_organization_associations.sql

@@ -0,0 +1,40 @@
+with base as (
+    select * from {{ ref('base_ef3__student_assessment_education_organization_associations') }}
+    where not is_deleted
+),
+
+
+keyed as (
+    select
+        {{
+            dbt_utils.generate_surrogate_key(
+                [   'tenant_code',
+                    'api_year',
+                    'ed_org_identifier',
+                    'ed_org_association_type',
+                    'assessment_identifier',
+                    'student_assessment_identifier',
+                    'student_id',
+                    'lower(assessment_namespace)'
+                ]
+            )
+        }} as k_student_assessment_ed_org,
+        {{ edorg_ref() }},
+        {{ gen_skey('k_student_assessment') }},
+        base.*
+    from base
+),
+
+
+deduped as (
+    {{
+        dbt_utils.deduplicate(
+            relation = 'keyed',
+            partition_by = 'k_student_assessment_ed_org',
+            order_by = 'pull_timestamp desc'
+        )
+    }}
+)
+
+
+select * from deduped