From 1286765e4f97b6fe1f5e2eddd5656a2c1a0b8d16 Mon Sep 17 00:00:00 2001 From: Ross Perry Date: Thu, 12 Dec 2024 11:23:59 -0700 Subject: [PATCH 1/7] add or create labels during upload --- seed/data_importer/match.py | 14 +++++++++++ seed/data_importer/tasks.py | 6 ++++- seed/migrations/0235_state_incoming_labels.py | 23 +++++++++++++++++++ seed/models/properties.py | 1 + seed/models/tax_lots.py | 1 + seed/views/v3/import_files.py | 1 - 6 files changed, 44 insertions(+), 2 deletions(-) create mode 100644 seed/migrations/0235_state_incoming_labels.py diff --git a/seed/data_importer/match.py b/seed/data_importer/match.py index e51e2cc582..7d6e2ade8e 100644 --- a/seed/data_importer/match.py +++ b/seed/data_importer/match.py @@ -10,6 +10,7 @@ from celery import shared_task from celery.utils.log import get_task_logger +from django.apps import apps from django.contrib.postgres.aggregates.general import ArrayAgg from django.db import IntegrityError, transaction from django.db.models import Subquery @@ -31,6 +32,8 @@ PropertyAuditLog, PropertyState, PropertyView, + PropertyViewLabel, + StatusLabel, TaxLotAuditLog, TaxLotState, TaxLotView, @@ -793,6 +796,17 @@ def link_states(states, ViewClass, cycle, highest_ali, sub_progress_key, tuple_v state.raw_access_level_instance = ali view = state.promote(cycle=cycle) + # assign incoming labels to view + if view and state.incoming_labels: + incoming_label_names = state.incoming_labels.split(",") + for incoming_label_name in incoming_label_names: + incoming_label, _ = StatusLabel.objects.get_or_create(name=incoming_label_name, super_organization=cycle.organization) + if isinstance(view, PropertyView): + PropertyViewLabel.objects.get_or_create(statuslabel=incoming_label, propertyview=view) + elif isinstance(view, TaxLotView): + TaxLotViewLabel = apps.get_model("seed", "TaxLotView_labels") + TaxLotViewLabel.objects.get_or_create(statuslabel=incoming_label, taxlotview=view) + # link state link_count = _link_matches([*existing_views_matches, view], cycle.organization_id, view, ViewClass) if link_count == 0: diff --git a/seed/data_importer/tasks.py b/seed/data_importer/tasks.py index f69bbf7515..0c6e1cd8fb 100644 --- a/seed/data_importer/tasks.py +++ b/seed/data_importer/tasks.py @@ -408,7 +408,11 @@ def map_row_chunk(ids, file_pk, source_type, prog_key, **kwargs): if footprint_details.get("obj_field") and getattr(map_model_obj, footprint_details["obj_field"]) is None: _store_raw_footprint_and_create_rule(footprint_details, table, org, import_file, original_row, map_model_obj) - # There was an error with a field being too long [> 255 chars]. + # Store the incoming label names in state.incoming_labels. -ViewLabels will be created once a view is attatched + label_key = "Property Labels" if isinstance(map_model_obj, PropertyState) else "Tax Lot Labels" + if incoming_labels := map_model_obj.extra_data.pop(label_key, None): + map_model_obj.incoming_labels = incoming_labels + map_model_obj.save() # if importing BuildingSync create a BuildingFile for the property diff --git a/seed/migrations/0235_state_incoming_labels.py b/seed/migrations/0235_state_incoming_labels.py new file mode 100644 index 0000000000..8833092218 --- /dev/null +++ b/seed/migrations/0235_state_incoming_labels.py @@ -0,0 +1,23 @@ +# Generated by Django 3.2.25 on 2024-12-12 18:14 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('seed', '0234_transaction_goals'), + ] + + operations = [ + migrations.AddField( + model_name='propertystate', + name='incoming_labels', + field=models.TextField(blank=True, null=True), + ), + migrations.AddField( + model_name='taxlotstate', + name='incoming_labels', + field=models.TextField(blank=True, null=True), + ), + ] diff --git a/seed/models/properties.py b/seed/models/properties.py index ce482c6823..b36e170dfe 100644 --- a/seed/models/properties.py +++ b/seed/models/properties.py @@ -174,6 +174,7 @@ class PropertyState(models.Model): merge_state = models.IntegerField(choices=MERGE_STATE, default=MERGE_STATE_UNKNOWN, null=True) raw_access_level_instance = models.ForeignKey(AccessLevelInstance, null=True, on_delete=models.SET_NULL) raw_access_level_instance_error = models.TextField(null=True) + incoming_labels = models.TextField(null=True, blank=True) jurisdiction_property_id = models.TextField(null=True, blank=True, db_collation="natural_sort") diff --git a/seed/models/tax_lots.py b/seed/models/tax_lots.py index 9cf771345a..6d9a318b3c 100644 --- a/seed/models/tax_lots.py +++ b/seed/models/tax_lots.py @@ -72,6 +72,7 @@ class TaxLotState(models.Model): merge_state = models.IntegerField(choices=MERGE_STATE, default=MERGE_STATE_UNKNOWN, null=True) raw_access_level_instance = models.ForeignKey(AccessLevelInstance, null=True, on_delete=models.SET_NULL) raw_access_level_instance_error = models.TextField(null=True) + incoming_labels = models.TextField(null=True, blank=True) custom_id_1 = models.CharField(max_length=255, null=True, blank=True, db_collation="natural_sort") diff --git a/seed/views/v3/import_files.py b/seed/views/v3/import_files.py index b91633f67d..b106bbfafe 100644 --- a/seed/views/v3/import_files.py +++ b/seed/views/v3/import_files.py @@ -464,7 +464,6 @@ def map(self, request, pk=None): if not import_file.exists(): return {"status": "error", "message": f"ImportFile {pk} does not exist"} - # return remap_data(import_file_id) return JsonResponse(map_data(pk, remap, mark_as_done)) @swagger_auto_schema_org_query_param From dcfaba1f8e33098ca4fad0a61f44e461bb8970de Mon Sep 17 00:00:00 2001 From: Ross Perry Date: Thu, 12 Dec 2024 11:46:40 -0700 Subject: [PATCH 2/7] precommit --- seed/migrations/0235_state_incoming_labels.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/seed/migrations/0235_state_incoming_labels.py b/seed/migrations/0235_state_incoming_labels.py index 8833092218..bb261643e7 100644 --- a/seed/migrations/0235_state_incoming_labels.py +++ b/seed/migrations/0235_state_incoming_labels.py @@ -4,20 +4,19 @@ class Migration(migrations.Migration): - dependencies = [ - ('seed', '0234_transaction_goals'), + ("seed", "0234_transaction_goals"), ] operations = [ migrations.AddField( - model_name='propertystate', - name='incoming_labels', + model_name="propertystate", + name="incoming_labels", field=models.TextField(blank=True, null=True), ), migrations.AddField( - model_name='taxlotstate', - name='incoming_labels', + model_name="taxlotstate", + name="incoming_labels", field=models.TextField(blank=True, null=True), ), ] From 2858dbaa095e4f6fb8a28b3c947238c0c0f46af2 Mon Sep 17 00:00:00 2001 From: Ross Perry Date: Thu, 12 Dec 2024 13:05:17 -0700 Subject: [PATCH 3/7] exclude incoming_properties from columns --- seed/models/columns.py | 1 + seed/tests/test_columns.py | 1 + 2 files changed, 2 insertions(+) diff --git a/seed/models/columns.py b/seed/models/columns.py index c701d10205..cc5c400177 100644 --- a/seed/models/columns.py +++ b/seed/models/columns.py @@ -107,6 +107,7 @@ class Column(models.Model): "geocoding_confidence", "id", "import_file", + "incoming_labels", "long_lat", "merge_state", "raw_access_level_instance_error", diff --git a/seed/tests/test_columns.py b/seed/tests/test_columns.py index 4b47eef00d..1b3c844c21 100644 --- a/seed/tests/test_columns.py +++ b/seed/tests/test_columns.py @@ -1096,6 +1096,7 @@ def test_retrieve_db_field_name_from_db_tables(self): "generation_date", "gross_floor_area", "home_energy_score_id", + "incoming_labels", "indoor_water_use", "indoor_wui", "jurisdiction_property_id", From e0ae078fc59cf7e3b4a20e64fdbd557b16e5bc9a Mon Sep 17 00:00:00 2001 From: Ross Perry Date: Thu, 12 Dec 2024 15:04:11 -0700 Subject: [PATCH 4/7] fix tests fix tests --- seed/serializers/properties.py | 1 + seed/tests/test_columns.py | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/seed/serializers/properties.py b/seed/serializers/properties.py index df92676143..ad90ec4eef 100644 --- a/seed/serializers/properties.py +++ b/seed/serializers/properties.py @@ -241,6 +241,7 @@ class PropertyStatePromoteWritableSerializer(serializers.ModelSerializer): import_file_id = serializers.IntegerField(allow_null=True, read_only=True) organization_id = serializers.IntegerField() raw_access_level_instance_id = serializers.IntegerField() + incoming_labels = serializers.IntegerField(read_only=True) # read-only core fields id = serializers.IntegerField(read_only=True) diff --git a/seed/tests/test_columns.py b/seed/tests/test_columns.py index 1b3c844c21..4b47eef00d 100644 --- a/seed/tests/test_columns.py +++ b/seed/tests/test_columns.py @@ -1096,7 +1096,6 @@ def test_retrieve_db_field_name_from_db_tables(self): "generation_date", "gross_floor_area", "home_energy_score_id", - "incoming_labels", "indoor_water_use", "indoor_wui", "jurisdiction_property_id", From 6edc3a19d5cf87423aa8d2ad551751beec25fddc Mon Sep 17 00:00:00 2001 From: Ross Perry Date: Tue, 17 Dec 2024 12:00:51 -0700 Subject: [PATCH 5/7] migration order --- ...5_state_incoming_labels.py => 0242_state_incoming_labels.py} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename seed/migrations/{0235_state_incoming_labels.py => 0242_state_incoming_labels.py} (92%) diff --git a/seed/migrations/0235_state_incoming_labels.py b/seed/migrations/0242_state_incoming_labels.py similarity index 92% rename from seed/migrations/0235_state_incoming_labels.py rename to seed/migrations/0242_state_incoming_labels.py index bb261643e7..9e43b8492e 100644 --- a/seed/migrations/0235_state_incoming_labels.py +++ b/seed/migrations/0242_state_incoming_labels.py @@ -5,7 +5,7 @@ class Migration(migrations.Migration): dependencies = [ - ("seed", "0234_transaction_goals"), + ("seed", "0241_alter_meter_type"), ] operations = [ From 35cca11bbe9e0d6b848ccf66eff5dc64d143bc72 Mon Sep 17 00:00:00 2001 From: Ross Perry Date: Tue, 30 Sep 2025 08:43:09 -0600 Subject: [PATCH 6/7] migration order, default show in list, merge existing --- seed/data_importer/match.py | 4 +++- seed/lib/merging/merging.py | 9 +++++++++ ..._incoming_labels.py => 0248_state_incoming_labels.py} | 2 +- seed/models/columns.py | 8 ++++---- 4 files changed, 17 insertions(+), 6 deletions(-) rename seed/migrations/{0242_state_incoming_labels.py => 0248_state_incoming_labels.py} (91%) diff --git a/seed/data_importer/match.py b/seed/data_importer/match.py index 0f8688be2f..f117e6ebe6 100644 --- a/seed/data_importer/match.py +++ b/seed/data_importer/match.py @@ -800,7 +800,9 @@ def link_states(states, ViewClass, cycle, highest_ali, sub_progress_key, tuple_v if view and state.incoming_labels: incoming_label_names = state.incoming_labels.split(",") for incoming_label_name in incoming_label_names: - incoming_label, _ = StatusLabel.objects.get_or_create(name=incoming_label_name, super_organization=cycle.organization) + incoming_label, _ = StatusLabel.objects.get_or_create( + name=incoming_label_name, super_organization=cycle.organization, show_in_list=True + ) if isinstance(view, PropertyView): PropertyViewLabel.objects.get_or_create(statuslabel=incoming_label, propertyview=view) elif isinstance(view, TaxLotView): diff --git a/seed/lib/merging/merging.py b/seed/lib/merging/merging.py index edacd79e84..d68f2b876b 100644 --- a/seed/lib/merging/merging.py +++ b/seed/lib/merging/merging.py @@ -155,6 +155,13 @@ def _merge_extra_data(ed1, ed2, priorities, recognize_empty_columns, ignore_merg return extra_data +def _merge_incoming_labels(state1, state2, merged_state): + state1_labels = state1.incoming_labels.split(",") if state1.incoming_labels else [] + state2_labels = state2.incoming_labels.split(",") if state2.incoming_labels else [] + combined_labels = set(state1_labels + state2_labels) + merged_state.incoming_labels = ",".join(combined_labels) if combined_labels else None + + def merge_state(merged_state, state1, state2, priorities, ignore_merge_protection=False): """ Set attributes on our Canonical model, saving differences. @@ -217,6 +224,8 @@ def merge_state(merged_state, state1, state2, priorities, ignore_merge_protectio table_name=state2.__class__.__name__, recognize_empty=True, is_extra_data=True ).values_list("column_name", flat=True) + _merge_incoming_labels(state1, state2, merged_state) + merged_state.extra_data = _merge_extra_data( state1.extra_data, state2.extra_data, diff --git a/seed/migrations/0242_state_incoming_labels.py b/seed/migrations/0248_state_incoming_labels.py similarity index 91% rename from seed/migrations/0242_state_incoming_labels.py rename to seed/migrations/0248_state_incoming_labels.py index 9e43b8492e..a5a956cabc 100644 --- a/seed/migrations/0242_state_incoming_labels.py +++ b/seed/migrations/0248_state_incoming_labels.py @@ -5,7 +5,7 @@ class Migration(migrations.Migration): dependencies = [ - ("seed", "0241_alter_meter_type"), + ("seed", "0247_aggregatemetersystem"), ] operations = [ diff --git a/seed/models/columns.py b/seed/models/columns.py index d76229e243..11b1ada924 100644 --- a/seed/models/columns.py +++ b/seed/models/columns.py @@ -69,6 +69,7 @@ class Column(models.Model): "gross_floor_area_orig", "conditioned_floor_area_orig", "source_eui_weather_normalized_orig", + "incoming_labels", ] QUANTITY_UNIT_COLUMNS = [ @@ -107,7 +108,6 @@ class Column(models.Model): "geocoding_confidence", "id", "import_file", - "incoming_labels", "long_lat", "merge_state", "raw_access_level_instance_error", @@ -1426,9 +1426,9 @@ def retrieve_db_field_name_for_hash_comparison(inventory_type, organization_id): f.name for f in inventory_type._meta.fields if ( - (f.get_internal_type() != "ForeignKey") - and (f.name not in Column.COLUMN_EXCLUDE_FIELDS) - and (f.name not in excluded_columns) + f.get_internal_type() != "ForeignKey" + and (f.name not in Column.COLUMN_EXCLUDE_FIELDS or f.name == "incoming_labels") + and f.name not in excluded_columns ) ] From 63b686aab9ac5fc39df99ff5f49319cb60d0666c Mon Sep 17 00:00:00 2001 From: Ross Perry Date: Tue, 30 Sep 2025 11:26:15 -0600 Subject: [PATCH 7/7] update test --- seed/tests/test_columns.py | 1 + 1 file changed, 1 insertion(+) diff --git a/seed/tests/test_columns.py b/seed/tests/test_columns.py index b48a3108ad..0dced31bff 100644 --- a/seed/tests/test_columns.py +++ b/seed/tests/test_columns.py @@ -1096,6 +1096,7 @@ def test_retrieve_db_field_name_from_db_tables(self): "generation_date", "gross_floor_area", "home_energy_score_id", + "incoming_labels", "indoor_water_use", "indoor_wui", "jurisdiction_property_id",