diff --git a/apps/baseline/admin.py b/apps/baseline/admin.py index 70555a6d..8a2bc68a 100644 --- a/apps/baseline/admin.py +++ b/apps/baseline/admin.py @@ -28,6 +28,7 @@ FoodPurchase, Hazard, Hunting, + KeyParameter, LivelihoodActivity, LivelihoodStrategy, LivelihoodZone, @@ -204,7 +205,6 @@ def bss_uploaded_date_time(self, instance): return "" def get_fieldsets(self, request, obj=None): - fieldsets = super().get_fieldsets(request, obj=obj) if obj and obj.geography: # Check if 'geography' field has a value @@ -1013,7 +1013,6 @@ class EventAdmin(admin.ModelAdmin): class ExpandabilityFactorAdmin(admin.ModelAdmin): - fields = ( "livelihood_strategy", "wealth_group", @@ -1046,7 +1045,6 @@ class ExpandabilityFactorAdmin(admin.ModelAdmin): class CopingStrategyAdmin(admin.ModelAdmin): - fields = ( "community", "leaders", @@ -1075,6 +1073,20 @@ class CopingStrategyAdmin(admin.ModelAdmin): ) +class KeyParameterAdmin(admin.ModelAdmin): + list_display = ( + "livelihood_zone_baseline", + "strategy_type", + "key_parameter_type", + "name", + "description", + ) + search_fields = [ + *translation_fields("name"), + *translation_fields("description"), + ] + + admin.site.register(SourceOrganization, SourceOrganizationAdmin) admin.site.register(LivelihoodZone, LivelihoodZoneAdmin) admin.site.register(LivelihoodZoneBaseline, LivelihoodZoneBaselineAdmin) @@ -1098,3 +1110,5 @@ class CopingStrategyAdmin(admin.ModelAdmin): admin.site.register(LivelihoodActivity, LivelihoodActivityAdmin) admin.site.register(WealthGroupCharacteristicValue, WealthGroupCharacteristicValueAdmin) + +admin.site.register(KeyParameter, KeyParameterAdmin) diff --git a/apps/baseline/migrations/0017_keyparameter.py b/apps/baseline/migrations/0017_keyparameter.py new file mode 100644 index 00000000..7bbdbf53 --- /dev/null +++ b/apps/baseline/migrations/0017_keyparameter.py @@ -0,0 +1,127 @@ +import django.db.models.deletion +import django.utils.timezone +import model_utils.fields +from django.db import migrations, models + +import common.fields + + +class Migration(migrations.Migration): + dependencies = [ + ("baseline", "0016_alter_livelihoodstrategy_additional_identifier_and_more"), + ] + + operations = [ + migrations.CreateModel( + name="KeyParameter", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ( + "created", + model_utils.fields.AutoCreatedField( + default=django.utils.timezone.now, editable=False, verbose_name="created" + ), + ), + ( + "modified", + model_utils.fields.AutoLastModifiedField( + default=django.utils.timezone.now, editable=False, verbose_name="modified" + ), + ), + ( + "strategy_type", + models.CharField( + choices=[ + ("MilkProduction", "Milk Production"), + ("ButterProduction", "Butter Production"), + ("MeatProduction", "Meat Production"), + ("LivestockSale", "Livestock Sale"), + ("CropProduction", "Crop Production"), + ("FoodPurchase", "Food Purchase"), + ("PaymentInKind", "Payment in Kind"), + ("ReliefGiftOther", "Relief, Gift or Other Food"), + ("Hunting", "Hunting"), + ("Fishing", "Fishing"), + ("WildFoodGathering", "Wild Food Gathering"), + ("OtherCashIncome", "Other Cash Income"), + ("OtherPurchase", "Other Purchase"), + ], + db_index=True, + help_text="The type of livelihood strategy, such as crop production, or wild food gathering.", + max_length=30, + verbose_name="Strategy Type", + ), + ), + ( + "key_parameter_type", + models.CharField( + choices=[("price", "Price"), ("quantity", "Quantity")], + help_text="The type of key parameter, such as quantity or price.", + max_length=30, + verbose_name="Key Parameter Type", + ), + ), + ("name_en", common.fields.NameField(max_length=200, verbose_name="Name")), + ("name_fr", common.fields.NameField(blank=True, max_length=200, verbose_name="Name")), + ("name_es", common.fields.NameField(blank=True, max_length=200, verbose_name="Name")), + ("name_pt", common.fields.NameField(blank=True, max_length=200, verbose_name="Name")), + ("name_ar", common.fields.NameField(blank=True, max_length=200, verbose_name="Name")), + ( + "description_en", + common.fields.DescriptionField( + blank=True, + help_text="Any extra information or detail that is relevant to the object.", + max_length=2000, + verbose_name="Description", + ), + ), + ( + "description_fr", + common.fields.DescriptionField( + blank=True, + help_text="Any extra information or detail that is relevant to the object.", + max_length=2000, + verbose_name="Description", + ), + ), + ( + "description_es", + common.fields.DescriptionField( + blank=True, + help_text="Any extra information or detail that is relevant to the object.", + max_length=2000, + verbose_name="Description", + ), + ), + ( + "description_pt", + common.fields.DescriptionField( + blank=True, + help_text="Any extra information or detail that is relevant to the object.", + max_length=2000, + verbose_name="Description", + ), + ), + ( + "description_ar", + common.fields.DescriptionField( + blank=True, + help_text="Any extra information or detail that is relevant to the object.", + max_length=2000, + verbose_name="Description", + ), + ), + ( + "livelihood_zone_baseline", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="baseline.livelihoodzonebaseline", + verbose_name="Livelihood Zone Baseline", + ), + ), + ], + options={ + "abstract": False, + }, + ), + ] diff --git a/apps/baseline/models.py b/apps/baseline/models.py index 9dabfd3b..503ec4d2 100644 --- a/apps/baseline/models.py +++ b/apps/baseline/models.py @@ -2360,3 +2360,48 @@ class Strategy(models.TextChoices): class Meta: verbose_name = _("Coping Strategy") verbose_name_plural = _("Coping Strategies") + + +class KeyParameter(common_models.Model): + """ + These are 'key parameters' for a given BSS, as reported in the baseline Fact Sheets. + + These are defined as: + + > a source that contributes at least 10% of the kilocalories of one wealth group’s total food or income, + or at least 5% of two wealth groups' total food or income + + These are entered manually for three reasons: + 1. We expect to have to support manual overrides anyway. + 2. We lack sufficient data (eg, product kcals per unit, strategy sub-type production figures) [1] + 3. We haven't yet reverse-engineered the formulae in the LIAS for making kcals and cash comparable. + + [1] A price on a purchase LS is presumably what the Fact Sheets call a 'consumer price'. A price on a + livestock sale is presumably always a 'producer price'. Labor figures are presumably the ones from the + livelihood activity detail. And animal numbers and school stats are presumably wealth group characteristics. + """ + + class KeyParameterType(models.TextChoices): + PRICE = "price", _("Price") + QUANTITY = "quantity", _("Quantity") + + livelihood_zone_baseline = models.ForeignKey( + LivelihoodZoneBaseline, + on_delete=models.CASCADE, + verbose_name=_("Livelihood Zone Baseline"), + ) + strategy_type = models.CharField( + max_length=30, + choices=LivelihoodStrategyType.choices, + db_index=True, + verbose_name=_("Strategy Type"), + help_text=_("The type of livelihood strategy, such as crop production, or wild food gathering."), + ) + key_parameter_type = models.CharField( + max_length=30, + choices=KeyParameterType.choices, + verbose_name=_("Key Parameter Type"), + help_text=_("The type of key parameter, such as quantity or price."), + ) + name = TranslatedField(common_models.NameField(max_length=200)) + description = TranslatedField(common_models.DescriptionField()) diff --git a/apps/baseline/serializers.py b/apps/baseline/serializers.py index 83298a0e..3b2f84a3 100644 --- a/apps/baseline/serializers.py +++ b/apps/baseline/serializers.py @@ -19,6 +19,7 @@ FoodPurchase, Hazard, Hunting, + KeyParameter, LivelihoodActivity, LivelihoodProductCategory, LivelihoodStrategy, @@ -1466,3 +1467,34 @@ def get_strategy_label(self, obj): def get_wealth_group_label(self, obj): return str(obj.wealth_group) + + +class KeyParameterSerializer(serializers.ModelSerializer): + class Meta: + model = KeyParameter + fields = [ + "id", + "livelihood_zone_baseline", + "livelihood_zone_baseline_label", + "strategy_type", + "strategy_type_label", + "key_parameter_type", + "key_parameter_type_label", + "name", + "description", + ] + + strategy_type_label = serializers.SerializerMethodField() + + def get_strategy_type_label(self, obj): + return obj.get_strategy_type_display() + + key_parameter_type_label = serializers.SerializerMethodField() + + def get_key_parameter_type_label(self, obj): + return obj.get_key_parameter_type_display() + + livelihood_zone_baseline_label = serializers.SerializerMethodField() + + def get_livelihood_zone_baseline_label(self, obj): + return str(obj.livelihood_zone_baseline) diff --git a/apps/baseline/tests/factories.py b/apps/baseline/tests/factories.py index b9046a85..865812f9 100644 --- a/apps/baseline/tests/factories.py +++ b/apps/baseline/tests/factories.py @@ -21,6 +21,7 @@ FoodPurchase, Hazard, Hunting, + KeyParameter, LivelihoodActivity, LivelihoodProductCategory, LivelihoodStrategy, @@ -799,3 +800,44 @@ class Meta: ) strategy = factory.Iterator(["reduce", "increase"]) by_value = fuzzy.FuzzyInteger(0, 100) + + +class KeyParameterFactory(factory.django.DjangoModelFactory): + class Meta: + model = KeyParameter + django_get_or_create = [ + "livelihood_zone_baseline", + "strategy_type", + "key_parameter_type", + "name_en", + ] + + strategy_type = factory.Iterator( + [ + "MilkProduction", + "ButterProduction", + "MeatProduction", + "LivestockSale", + "CropProduction", + "FoodPurchase", + "PaymentInKind", + "ReliefGiftOther", + "Fishing", + "Hunting", + "WildFoodGathering", + "OtherCashIncome", + "OtherPurchase", + ] + ) + livelihood_zone_baseline = factory.SubFactory(LivelihoodZoneBaselineFactory) + key_parameter_type = factory.Iterator(["quantity", "price"]) + name_en = factory.Sequence(lambda n: f"Key parameter {n} en") + name_fr = factory.Sequence(lambda n: f"Key parameter {n} fr") + name_es = factory.Sequence(lambda n: f"Key parameter {n} es") + name_pt = factory.Sequence(lambda n: f"Key parameter {n} pt") + name_ar = factory.Sequence(lambda n: f"Key parameter {n} ar") + description_en = factory.LazyAttribute(lambda o: f"{o.name_en} description") + description_fr = factory.LazyAttribute(lambda o: f"{o.name_fr} description") + description_es = factory.LazyAttribute(lambda o: f"{o.name_es} description") + description_pt = factory.LazyAttribute(lambda o: f"{o.name_pt} description") + description_ar = factory.LazyAttribute(lambda o: f"{o.name_ar} description") diff --git a/apps/baseline/tests/test_factories.py b/apps/baseline/tests/test_factories.py index 0240e0b5..dfe86126 100644 --- a/apps/baseline/tests/test_factories.py +++ b/apps/baseline/tests/test_factories.py @@ -16,6 +16,7 @@ FoodPurchase, Hazard, Hunting, + KeyParameter, LivelihoodActivity, LivelihoodProductCategory, LivelihoodStrategy, @@ -55,6 +56,7 @@ FoodPurchaseFactory, HazardFactory, HuntingFactory, + KeyParameterFactory, LivelihoodActivityFactory, LivelihoodProductCategoryFactory, LivelihoodStrategyFactory, @@ -264,6 +266,11 @@ def test_copingstrategy_factory(self): CopingStrategyFactory() self.assertEqual(CopingStrategy.objects.count(), self.num_records) + def test_key_parameter_factory(self): + for _ in range(self.num_records): + KeyParameterFactory() + self.assertEqual(KeyParameter.objects.count(), self.num_records) + def test_all_factories(self): for _ in range(2): SourceOrganizationFactory() @@ -302,3 +309,4 @@ def test_all_factories(self): EventFactory() ExpandabilityFactorFactory() CopingStrategyFactory() + KeyParameterFactory() diff --git a/apps/baseline/tests/test_viewsets.py b/apps/baseline/tests/test_viewsets.py index c39ea0c1..36db8516 100644 --- a/apps/baseline/tests/test_viewsets.py +++ b/apps/baseline/tests/test_viewsets.py @@ -29,6 +29,7 @@ FoodPurchaseFactory, HazardFactory, HuntingFactory, + KeyParameterFactory, LivelihoodActivityFactory, LivelihoodProductCategoryFactory, LivelihoodStrategyFactory, @@ -5563,3 +5564,65 @@ def test_html(self): content = response.content df = pd.read_html(content)[0].fillna("") self.assertEqual(len(df), self.num_records + 1) + + +class KeyParameterViewSetTestCase(APITestCase): + @classmethod + def setUpTestData(cls): + cls.num_records = 5 + cls.data = [KeyParameterFactory() for _ in range(cls.num_records)] + cls.user = User.objects.create_superuser("test", "test@test.com", "password") + + def setUp(self): + self.url = reverse("keyparameter-list") + self.url_get = lambda n: reverse("keyparameter-detail", args=(self.data[n].pk,)) + + def test_get_record(self): + response = self.client.get(self.url_get(0)) + self.assertEqual(response.status_code, 200) + self.assertIsInstance(response.json(), dict) + # assertCountEqual checks elements match in any order + expected_fields = [ + "id", + "livelihood_zone_baseline", + "livelihood_zone_baseline_label", + "strategy_type", + "strategy_type_label", + "key_parameter_type", + "key_parameter_type_label", + "name", + "description", + ] + self.assertCountEqual( + response.json().keys(), + expected_fields, + f"KeyParameter: Fields expected: {expected_fields}. Fields found: {response.json().keys()}.", + ) + + def test_list_returns_filtered_data(self): + response = self.client.get( + self.url, + { + "name_en": self.data[0].name_en, + }, + ) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.json()), 1) + + def test_search(self): + response = self.client.get( + self.url, + { + "search": self.data[0].name_en, + }, + ) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.json()), 1) + response = self.client.get( + self.url, + { + "search": self.data[0].name_en + "xyz", + }, + ) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.json()), 0) diff --git a/apps/baseline/viewsets.py b/apps/baseline/viewsets.py index 31ef1c9d..5157d13e 100644 --- a/apps/baseline/viewsets.py +++ b/apps/baseline/viewsets.py @@ -21,6 +21,7 @@ FoodPurchase, Hazard, Hunting, + KeyParameter, LivelihoodActivity, LivelihoodProductCategory, LivelihoodStrategy, @@ -59,6 +60,7 @@ FoodPurchaseSerializer, HazardSerializer, HuntingSerializer, + KeyParameterSerializer, LivelihoodActivitySerializer, LivelihoodProductCategorySerializer, LivelihoodStrategySerializer, @@ -1570,3 +1572,31 @@ class CopingStrategyViewSet(BaseModelViewSet): "leaders", "strategy", ] + + +class KeyParameterFilterSet(filters.FilterSet): + class Meta: + model = KeyParameter + fields = ( + "livelihood_zone_baseline", + "strategy_type", + "key_parameter_type", + *translation_fields("description"), + *translation_fields("name"), + ) + + +class KeyParameterViewSet(BaseModelViewSet): + """ + API endpoint that allows key parameters to be viewed or edited. + """ + + queryset = KeyParameter.objects.select_related("livelihood_zone_baseline") + serializer_class = KeyParameterSerializer + filterset_class = KeyParameterFilterSet + search_fields = ( + "strategy_type", + "key_parameter_type", + *translation_fields("name"), + *translation_fields("description"), + ) diff --git a/hea/urls.py b/hea/urls.py index 1cc6733e..f509ed8b 100644 --- a/hea/urls.py +++ b/hea/urls.py @@ -23,6 +23,7 @@ FoodPurchaseViewSet, HazardViewSet, HuntingViewSet, + KeyParameterViewSet, LivelihoodActivityViewSet, LivelihoodProductCategoryViewSet, LivelihoodStrategyViewSet, @@ -113,6 +114,7 @@ router.register(r"event", EventViewSet) router.register(r"expandabilityfactor", ExpandabilityFactorViewSet) router.register(r"copingstrategy", CopingStrategyViewSet) +router.register(r"keyparameter", KeyParameterViewSet) urlpatterns = [ ########## LOCALE INDEPENDENT PATHS go here. ########## diff --git a/pipelines/jobs/metadata.py b/pipelines/jobs/metadata.py index 596a0069..7cb37dfd 100644 --- a/pipelines/jobs/metadata.py +++ b/pipelines/jobs/metadata.py @@ -16,6 +16,8 @@ from gdrivefs.core import GoogleDriveFile from upath import UPath +from common.fields import translation_fields + from ..configs import ReferenceDataConfig from ..utils import class_from_name @@ -28,11 +30,12 @@ from baseline.lookups import CommunityLookup # NOQA: E402 from baseline.models import ( # NOQA: E402 Community, + KeyParameter, LivelihoodZoneBaseline, LivelihoodZoneBaselineCorrection, ) from common.lookups import ClassifiedProductLookup, UserLookup # NOQA: E402 -from metadata.models import ActivityLabel # NOQA: E402 +from metadata.models import ActivityLabel, LivelihoodStrategyType # NOQA: E402 def load_metadata_for_model(context: OpExecutionContext, model: models.Model, df: pd.DataFrame): @@ -189,7 +192,7 @@ def load_all_corrections(context: OpExecutionContext): """ storage_options = {"token": "service_account", "access": "read_only", "root_file_id": "0AOJ0gJ8sjnO7Uk9PVA"} storage_options["creds"] = json.loads(os.environ["GOOGLE_APPLICATION_CREDENTIALS"]) - p = UPath("gdrive://Database Design/BSS Metadata", **storage_options) + p = UPath("gdrive://Database Design/BSS Metadata/Key Parameters", **storage_options) with p.fs.open(p.path, mode="rb", cache_type="bytes") as f: # Google Sheets have to be exported rather than read directly if isinstance(f, GoogleDriveFile) and (f.details["mimeType"] == "application/vnd.google-apps.spreadsheet"): @@ -229,7 +232,6 @@ def load_all_corrections(context: OpExecutionContext): raise ValueError(f"Found duplicate corrections:\n{duplicates_df.to_markdown()}") with transaction.atomic(): - df = df.set_index(["livelihood_zone_baseline_id", "worksheet_name", "cell_range"], drop=False) # Get the current set of corrections from the database, so we can see what has changed @@ -393,7 +395,6 @@ def load_all_community_aliases(context: OpExecutionContext): raise ValueError(f"Found duplicate aliases:\n{duplicates_df.to_markdown()}") with transaction.atomic(): - df = df.set_index(["livelihood_zone_baseline_id", "full_name"], drop=False) # Get the current set of aliases from the database, so we can see what has changed @@ -522,11 +523,81 @@ def load_all_fewsnet_geographies(context: OpExecutionContext): context.log.warning(f"Failed to find FEWS NET geometry for {livelihood_zone_baseline}") +@op +def load_all_key_parameters(context: OpExecutionContext): + """ + Load all Key Parameters from the BSS Metadata Google Sheet into Django. + + It looks for a file with the same name as the original BSS Excel file, in the shared Google Drive folder: + gdrive://Database Design/Key Parameters/{bss_filename} (without extension) + """ + storage_options = {"token": "service_account", "access": "read_only", "root_file_id": "0AOJ0gJ8sjnO7Uk9PVA"} + storage_options["creds"] = json.loads(os.environ["GOOGLE_APPLICATION_CREDENTIALS"]) + for lzb in LivelihoodZoneBaseline.objects.all(): + bss_filename = lzb.bss.name + bss_filename = bss_filename[bss_filename.rfind("/") + 1 : bss_filename.rfind(".")] + p = UPath(f"gdrive://Database Design/Key Parameters/{bss_filename}", **storage_options) + if not p.exists() or not p.is_file(): + context.log.info( + f"No key parameters file found for {lzb} at //Database Design/Key Parameters/{bss_filename}" + ) + continue + with p.fs.open(p.path, mode="rb", cache_type="bytes") as f: + # Google Sheets have to be exported rather than read directly + if isinstance(f, GoogleDriveFile) and (f.details["mimeType"] == "application/vnd.google-apps.spreadsheet"): + f = BytesIO(p.fs.export(p.path, "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")) + + df = pd.read_excel( + f, + sheet_name="Key Parameters", + engine="openpyxl", + usecols=[ + "strategy_type", + "key_parameter_type", + *translation_fields("name"), + *translation_fields("description"), + "created", + "modified", + ], + ) + + with transaction.atomic(): + # We don't have a definitive unique key, as there could be many key parameters for a single LZB, + # strategy type and parameter type, so we just delete and recreate them all. + KeyParameter.objects.filter(livelihood_zone_baseline=lzb).delete() + + instances = [ + KeyParameter(livelihood_zone_baseline_id=lzb.pk, **key_parameter) + for key_parameter in df.to_dict(orient="records") + if key_parameter.keys() + >= { + "strategy_type", + "key_parameter_type", + *translation_fields("name"), + *translation_fields("description"), + "created", + "modified", + } + and key_parameter["strategy_type"] in set(LivelihoodStrategyType.values) + and key_parameter["key_parameter_type"] in set(KeyParameter.KeyParameterType.values) + and any(key_parameter[lang_field] for lang_field in translation_fields("name")) + and any(key_parameter[lang_field] for lang_field in translation_fields("description")) + ] + # Save the corrections to the database using a bulk_create, updating any existing corrections + instances = LivelihoodZoneBaselineCorrection.objects.bulk_create(instances) + + # Log the insertions + context.log.info( + f"Loaded {len(instances)} key parameters for {lzb} from //Database Design/Key Parameters/{bss_filename}" + ) + + @job def update_metadata(): load_all_metadata() load_all_corrections() load_all_community_aliases() + load_all_key_parameters() @job