diff --git a/dbt/dbt_project.yml b/dbt/dbt_project.yml index 7462e7f2f..76f7c463d 100644 --- a/dbt/dbt_project.yml +++ b/dbt/dbt_project.yml @@ -49,6 +49,8 @@ models: +schema: census default: +schema: default + external: + +schema: external location: +schema: location model: diff --git a/dbt/models/default/schema/default.vw_pin_tax_roll.yml b/dbt/models/default/schema/default.vw_pin_tax_roll.yml index a66decc61..60ff2563e 100644 --- a/dbt/models/default/schema/default.vw_pin_tax_roll.yml +++ b/dbt/models/default/schema/default.vw_pin_tax_roll.yml @@ -32,7 +32,7 @@ models: - name: exe_wwii description: '{{ doc("shared_column_exe_wwii") }}' - name: final_eav - description: Final equalized AV + description: '{{ doc("shared_column_final_eav") }}' - name: mailed_taxable_av description: Mailed taxable AV - name: pin diff --git a/dbt/models/default/schema/default.vw_pin_universe.yml b/dbt/models/default/schema/default.vw_pin_universe.yml index cafc89563..40b35a64b 100644 --- a/dbt/models/default/schema/default.vw_pin_universe.yml +++ b/dbt/models/default/schema/default.vw_pin_universe.yml @@ -88,7 +88,7 @@ models: min_value: 100 max_value: 200 - name: combined_municipality_name - description: '{{ doc("column_combined_municipality_name") }}' + description: '{{ doc("shared_column_combined_municipality_name") }}' - name: cook_board_of_review_district_data_year description: '{{ doc("shared_column_data_year") }}' - name: cook_board_of_review_district_num diff --git a/dbt/models/external/docs.md b/dbt/models/external/docs.md new file mode 100644 index 000000000..7d4123cd2 --- /dev/null +++ b/dbt/models/external/docs.md @@ -0,0 +1,13 @@ +# vw_cmap + +{% docs view_vw_cmap %} +View to gather all necessary output for our yearly CMAP export. + +### Nuance + +- Assumes year built (`yrblt`) should be maxed within PIN. +- PINs with rows in `iasworld.dweldat` should be unique by pin, card, and year; +other PINs should be unique by pin and year. + +**Primary Key**: `year`, `pin`, `card` +{% enddocs %} diff --git a/dbt/models/external/external.vw_cmap.sql b/dbt/models/external/external.vw_cmap.sql new file mode 100644 index 000000000..10661bb69 --- /dev/null +++ b/dbt/models/external/external.vw_cmap.sql @@ -0,0 +1,224 @@ +/* For this query we keep every table unique to pin/year except +dweldat, since we want card-level characteristics for residential parcels. So +CTEs will aggregate oby and comdat to pin/year level. Other tables that don't +have CTEs are already unique by pin/year. We assume yrblt should be maxed +within PIN for this request. + +The basic idea here is that we're using asmt_all as the universe for parcels and +joining on residential characteristics from dweldat, condo characteristics from +oby, commercial characteristics from comdat, and land sf from our land view. We +also join on appeals and exemptions. */ + +-- This is our universe of pins. We're using BOR values since CMAP wants final +-- values and select arbitrary rows within PIN/year since there are unfixable +-- duplicates in asmt_all. +WITH asmt AS ( + SELECT + parid, + taxyr, + ARBITRARY(class) AS class, + ARBITRARY(valasm1) AS land, + ARBITRARY(valasm2) AS bldg, + ARBITRARY(valasm3) AS tot + FROM {{ source("iasworld", "asmt_all") }} + WHERE deactivat IS NULL + AND procname = 'BORVALUE' + AND valclass IS NULL + -- Class 999 are test pins + AND class NOT IN ('999') + + GROUP BY + parid, + taxyr +), + +-- Aggregate commercial characteristics to pin/year level +com AS ( + SELECT + parid, + taxyr, + COUNT(*) AS multi_imp_num, + -- Assuming gross building area should be summed within PIN + SUM(CAST(user20 AS DOUBLE)) AS gross_building_area, + -- Assuming net rentable area should be summed within PIN + SUM(CAST(user28 AS DOUBLE)) AS net_rentable_area, + MAX(yrblt) AS yrblt + FROM {{ source("iasworld", "comdat") }} + WHERE deactivat IS NULL + AND cur = 'Y' + GROUP BY + parid, + taxyr +), + +-- Aggregate condo characteristics to pin/year level +oby AS ( + SELECT + parid, + taxyr, + COUNT(*) AS multi_imp_num, + MAX(yrblt) AS yrblt + FROM {{ source("iasworld", "oby") }} + WHERE deactivat IS NULL + AND cur = 'Y' + GROUP BY + parid, + taxyr +), + +-- Card-level residential characteristics +dwel AS ( + SELECT + pin AS parid, + year AS taxyr, + card, + pin_num_cards AS multi_imp_num, + char_bldg_sf AS building_sf, + char_beds AS beds, + char_rooms AS rooms, + char_fbath AS full_bath, + char_hbath AS half_bath, + CASE WHEN char_frpl = 1 THEN 'Full' + WHEN char_frpl = 2 THEN 'Partial' + WHEN char_frpl = 3 THEN 'None' + END AS fireplace, + char_attic_type AS attic, + CASE WHEN char_bsmt = '1' THEN 'Full' + WHEN char_bsmt = '2' THEN 'Slab' + WHEN char_bsmt = '3' THEN 'Partial' + WHEN char_bsmt = '4' THEN 'Crawl' + END AS basement, + CASE WHEN char_ext_wall = '1' THEN 'Frame' + WHEN char_ext_wall = '2' THEN 'Masonry' + WHEN char_ext_wall = '3' THEN 'Frame + Masonry' + WHEN char_ext_wall = '4' THEN 'Stucco' + END AS wall_construction, + CASE WHEN char_roof_cnst = '1' THEN 'Shingle + Asphalt' + WHEN char_roof_cnst = '2' THEN 'Tar + Gravel' + WHEN char_roof_cnst = '3' THEN 'Slate' + WHEN char_roof_cnst = '4' THEN 'Shake' + WHEN char_roof_cnst = '5' THEN 'Tile' + WHEN char_roof_cnst = '6' THEN 'Other' + END AS roof_construction, + char_ncu AS num_apts, + char_yrblt AS yrblt + FROM {{ ref('default.vw_card_res_char') }} +), + +appeals AS ( + SELECT + pin, + year + FROM {{ ref('default.vw_pin_appeal') }} + GROUP BY pin, year +) + +SELECT + -- keys + asmt.parid AS pin, + dwel.card, + COALESCE(dwel.multi_imp_num, com.multi_imp_num, oby.multi_imp_num) + AS multi_imp_num, + asmt.taxyr AS year, + + -- address + vpa.prop_address_full, + vpa.prop_address_city_name, + vpa.prop_address_zipcode_1, + vpa.mail_address_name, + vpa.mail_address_full, + vpa.mail_address_city_name, + vpa.mail_address_state, + vpa.mail_address_zipcode_1, + + -- class + asmt.class AS class_code, + class_dict.class_desc AS class_description, + + -- town, muni, tax code + vpu.township_name AS township, + vpu.nbhd_code AS neighborhood, + -- Combined municipality name is an array. Empty arrays indicate + -- unincorporated parcels. Null values indicate missing data. + CASE + WHEN CARDINALITY(vpu.combined_municipality_name) = 0 + THEN 'UNINCORPORATED' + ELSE ARRAY_JOIN(vpu.combined_municipality_name, ', ') + END AS municipality, + vpu.tax_code, + + -- av + asmt.land AS land_av, + asmt.bldg AS building_av, + asmt.tot AS total_av, + vptr.final_eav AS equalized_total_av, + + -- exempt status + exempt.owner_name AS exempt_agency_name, + exempt.owner_num AS exempt_agency_num, + + -- exemptions + vptr.exe_disabled, + vptr.exe_freeze, + vptr.exe_homeowner, + vptr.exe_longtime_homeowner, + vptr.exe_senior, + vptr.exe_muni_built, + vptr.exe_vet_dis_lt50, + vptr.exe_vet_dis_50_69, + vptr.exe_vet_dis_ge70, + vptr.exe_vet_dis_100, + vptr.exe_vet_returning, + vptr.exe_wwii, + + -- characteristics + land.sf AS land_sf, + COALESCE(dwel.yrblt, com.yrblt, oby.yrblt) AS year_built, + CAST(asmt.taxyr AS INT) - COALESCE(dwel.yrblt, com.yrblt, oby.yrblt) AS age, + + com.gross_building_area, + com.net_rentable_area, + + dwel.building_sf, + dwel.beds, + dwel.rooms, + dwel.full_bath, + dwel.half_bath, + dwel.fireplace, + dwel.attic, + dwel.basement, + dwel.wall_construction, + dwel.roof_construction, + dwel.num_apts, + -- appeals + CASE WHEN appeals.pin IS NOT NULL THEN 'Yes' ELSE 'No' END AS appealed +FROM asmt +LEFT JOIN default.vw_pin_address AS vpa + ON asmt.parid = vpa.pin + AND asmt.taxyr = vpa.year +LEFT JOIN ccao.class_dict -- Join for class descriptions + ON asmt.class = class_dict.class_code +LEFT JOIN default.vw_pin_universe AS vpu + ON asmt.parid = vpu.pin + AND asmt.taxyr = vpu.year +LEFT JOIN default.vw_pin_exempt AS exempt + ON asmt.parid = exempt.pin + AND asmt.taxyr = exempt.year +LEFT JOIN default.vw_pin_tax_roll AS vptr + ON asmt.parid = vptr.pin + AND asmt.taxyr = vptr.year +LEFT JOIN default.vw_pin_land AS land + ON asmt.parid = land.pin + AND asmt.taxyr = land.year +LEFT JOIN oby + ON asmt.parid = oby.parid + AND asmt.taxyr = oby.taxyr +LEFT JOIN com + ON asmt.parid = com.parid + AND asmt.taxyr = com.taxyr +LEFT JOIN dwel + ON asmt.parid = dwel.parid + AND asmt.taxyr = dwel.taxyr +LEFT JOIN appeals + ON asmt.parid = appeals.pin + AND asmt.taxyr = appeals.year diff --git a/dbt/models/external/schema/external.vw_cmap.yml b/dbt/models/external/schema/external.vw_cmap.yml new file mode 100644 index 000000000..bb6afdc5e --- /dev/null +++ b/dbt/models/external/schema/external.vw_cmap.yml @@ -0,0 +1,139 @@ +models: + - name: external.vw_cmap + description: '{{ doc("view_vw_cmap") }}' + + columns: + - name: age + description: Age, calculated as year - year_built + - name: appealed + description: Parcel has an appeal in the given year + - name: attic + description: '{{ doc("shared_column_char_attic_type") }}' + - name: basement + description: '{{ doc("shared_column_char_bsmt") }}' + - name: beds + description: '{{ doc("shared_column_char_beds") }}' + - name: building_av + description: '{{ doc("shared_column_board_bldg") }}' + - name: building_sf + description: '{{ doc("shared_column_char_bldg_sf") }}' + - name: card + description: '{{ doc("shared_column_card") }}' + - name: class_code + description: '{{ doc("shared_column_class") }}' + - name: class_description + description: 'A short description of the class code' + - name: equalized_total_av + description: '{{ doc("shared_column_final_eav") }}' + - name: exe_disabled + description: '{{ doc("shared_column_exe_disabled") }}' + - name: exe_freeze + description: '{{ doc("shared_column_exe_freeze") }}' + - name: exe_homeowner + description: '{{ doc("shared_column_exe_homeowner") }}' + - name: exe_longtime_homeowner + description: '{{ doc("shared_column_exe_longtime_homeowner") }}' + - name: exempt_agency_name + description: '{{ doc("shared_column_owner_name") }}' + - name: exempt_agency_num + description: '{{ doc("shared_column_owner_num") }}' + - name: exe_muni_built + description: '{{ doc("shared_column_exe_muni_built") }}' + - name: exe_senior + description: '{{ doc("shared_column_exe_senior") }}' + - name: exe_vet_dis_100 + description: '{{ doc("shared_column_exe_vet_dis_100") }}' + - name: exe_vet_dis_50_69 + description: '{{ doc("shared_column_exe_vet_dis_50_69") }}' + - name: exe_vet_dis_ge70 + description: '{{ doc("shared_column_exe_vet_dis_ge70") }}' + - name: exe_vet_dis_lt50 + description: '{{ doc("shared_column_exe_vet_dis_lt50") }}' + - name: exe_vet_returning + description: '{{ doc("shared_column_exe_vet_returning") }}' + - name: exe_wwii + description: '{{ doc("shared_column_exe_wwii") }}' + - name: fireplace + description: '{{ doc("shared_column_char_frpl") }}' + - name: full_bath + description: '{{ doc("shared_column_char_fbath") }}' + - name: gross_building_area + description: 'Gross building area' + - name: half_bath + description: '{{ doc("shared_column_char_hbath") }}' + - name: land_av + description: '{{ doc("shared_column_board_land") }}' + - name: land_sf + description: '{{ doc("shared_column_char_land_sf") }}' + - name: mail_address_city_name + description: '{{ doc("shared_column_mail_address_city_name") }}' + - name: mail_address_full + description: '{{ doc("shared_column_mail_address_full") }}' + - name: mail_address_name + description: '{{ doc("shared_column_mail_address_name") }}' + - name: mail_address_state + description: '{{ doc("shared_column_mail_address_state") }}' + - name: mail_address_zipcode_1 + description: '{{ doc("shared_column_mail_address_zipcode_1") }}' + - name: multi_imp_num + description: '{{ doc("shared_column_pin_num_cards") }}' + - name: municipality + description: '{{ doc("shared_column_combined_municipality_name") }}' + - name: neighborhood + description: '{{ doc("shared_column_nbhd_code") }}' + - name: net_rentable_area + description: Net rentable area + - name: num_apts + description: '{{ doc("shared_column_char_ncu") }}' + - name: pin + description: '{{ doc("shared_column_pin") }}' + - name: prop_address_city_name + description: '{{ doc("shared_column_prop_address_city_name") }}' + - name: prop_address_full + description: '{{ doc("shared_column_prop_address_full") }}' + - name: prop_address_zipcode_1 + description: '{{ doc("shared_column_prop_address_zipcode_1") }}' + - name: roof_construction + description: '{{ doc("shared_column_char_roof_cnst") }}' + - name: rooms + description: '{{ doc("shared_column_char_rooms") }}' + - name: tax_code + description: '{{ doc("shared_column_tax_code") }}' + - name: total_av + description: '{{ doc("shared_column_board_tot") }}' + - name: township + description: '{{ doc("shared_column_township_name") }}' + - name: wall_construction + description: '{{ doc("shared_column_char_ext_wall") }}' + - name: year + description: '{{ doc("shared_column_year") }}' + - name: year_built + description: '{{ doc("shared_column_char_yrblt") }}' + + data_tests: + - row_count: + name: external_vw_cmap_row_count + equals: > + ( + SELECT COUNT(*) + FROM ( + SELECT DISTINCT parid, taxyr + FROM {{ source("iasworld", "asmt_all") }} + WHERE procname = 'BORVALUE' + AND valclass IS NULL + -- Class 999 are test pins + AND class NOT IN ('999') + ) AS asmt_all + LEFT JOIN {{ source("iasworld", "dweldat") }} AS dweldat + ON asmt_all.parid = dweldat.parid + AND asmt_all.taxyr = dweldat.taxyr + AND dweldat.deactivat IS NULL + AND dweldat.cur = 'Y' + ) + - unique_combination_of_columns: + name: external_vw_cmap_unique_by_pin_card_and_year + combination_of_columns: + - pin + - year + - card + diff --git a/dbt/models/location/columns.md b/dbt/models/location/columns.md index 9cf3f5b0a..8bfbd2a51 100644 --- a/dbt/models/location/columns.md +++ b/dbt/models/location/columns.md @@ -136,22 +136,6 @@ Chicago industrial corridor name 2-digit Chicago police district number {% enddocs %} -## combined_municipality_name - -{% docs column_combined_municipality_name %} -Combines `tax_municipality_name` and `cook_municipality_name`. -`tax_municipality_name` has both NULL values and empty array values. -If `tax_municipality_name` value is NULL, we replace it with `cook_municipality_name`. -If the `tax_municipality_name` is an empty array, we preserve -this as a representation for unincorporated and do -not change it. We then run these values through a crosswalk file to -make sure they align semantically with `tax_municipality_name`. - -Values which are NULL represent values which we do not have information for. -Values which are an empty array represent unincorporated parcels. - -{% enddocs %} - ## cook_board_of_review_district_num {% docs column_cook_board_of_review_district_num %} diff --git a/dbt/models/location/schema.yml b/dbt/models/location/schema.yml index 3e60552b4..fe780968e 100644 --- a/dbt/models/location/schema.yml +++ b/dbt/models/location/schema.yml @@ -347,7 +347,7 @@ models: - name: chicago_police_district_num description: '{{ doc("column_chicago_police_district_num") }}' - name: combined_municipality_name - description: '{{ doc("column_combined_municipality_name") }}' + description: '{{ doc("shared_column_combined_municipality_name") }}' - name: cook_board_of_review_district_data_year description: '{{ doc("shared_column_data_year") }}' - name: cook_board_of_review_district_num diff --git a/dbt/models/shared_columns.md b/dbt/models/shared_columns.md index 54031bcd0..4f0b82ddd 100644 --- a/dbt/models/shared_columns.md +++ b/dbt/models/shared_columns.md @@ -458,6 +458,12 @@ match the FMV. Calculated full market value. {% enddocs %} +## final_eav + +{% docs shared_column_final_eav %} +Final equalized assessed value (EAV). +{% enddocs %} + ## mailed_bldg {% docs shared_column_mailed_bldg %} @@ -1147,6 +1153,22 @@ percentage of fair cash value at which a property is assessed for taxing purposes. See `ccao.class_dict` for more information {% enddocs %} +## combined_municipality_name + +{% docs shared_column_combined_municipality_name %} +Combines `tax_municipality_name` and `cook_municipality_name`. +`tax_municipality_name` has both NULL values and empty array values. +If `tax_municipality_name` value is NULL, we replace it with `cook_municipality_name`. +If the `tax_municipality_name` is an empty array, we preserve +this as a representation for unincorporated and do +not change it. We then run these values through a crosswalk file to +make sure they align semantically with `tax_municipality_name`. + +Values which are NULL represent values which we do not have information for. +Values which are an empty array represent unincorporated parcels. + +{% enddocs %} + ## disabled_persons_exemption_amount {% docs shared_column_exe_disabled %}