fix: support new images schema by converting to legacy schema (#41)

fix: support new images schema by converting to legacy schema

See openfoodfacts/openfoodfacts-server#11818 for more details about the schema changes.

Author: raphael0202

openfoodfacts_exports/exports/parquet/common.py

@@ -3,6 +3,7 @@
 
 import pyarrow as pa
 from huggingface_hub import HfApi
+from openfoodfacts.images import convert_to_legacy_schema
 from pydantic import BaseModel, Field, model_validator
 
 logger = logging.getLogger(__name__)
@@ -249,13 +250,15 @@ def parse_images(cls, data: dict) -> dict:
         key as the key and the image data as the value.
 
         To make the schema compatible with Parquet, we convert these fields
-        into a list of dictionaries with `key`, `imgid`, `rev`, `sizes`, `uploaded_t`,
-        and `uploader` keys. We copy the image key (ex: `3`, `nutrition_fr`,...)
-        from the original dictionary and add it as a field under the `key` key.
+        into a list of dictionaries with `key`, `imgid`, `rev`, `sizes`,
+        `uploaded_t`, and `uploader` keys. We copy the image key (ex: `3`,
+        `nutrition_fr`,...) from the original dictionary and add it as a field
+        under the `key` key.
         """
         images = data.pop("images", None)
         data["images"] = []
         if images:
+            images = convert_to_legacy_schema(images)
             for key, value in images.items():
                 data["images"].append({"key": key, **value})
         return data

pyproject.toml

@@ -10,7 +10,7 @@ dependencies = [
     "huggingface-hub>=0.26.2",
     "minio>=7.2.10",
     "more-itertools>=10.5.0",
-    "openfoodfacts>=2.2.0",
+    "openfoodfacts==2.5.2",
     "orjson>=3.10.11",
     "pyarrow>=18.0.0",
     "pytz>=2024.2",

tests/unit/exports/test_parquet.py

@@ -8,6 +8,7 @@
     BEAUTY_PRODUCT_SCHEMA,
     BeautyProduct,
 )
+from openfoodfacts_exports.exports.parquet.common import Product
 
 
 class TestConvertJSONLToParquet:
@@ -21,3 +22,114 @@ def test_convert_jsonl_to_parquet_data_missing(self):
                 schema=BEAUTY_PRODUCT_SCHEMA,
                 dtype_map=BEAUTY_DTYPE_MAP,
             )
+
+
+PARSED_IMAGES_WITH_LEGACY_SCHEMA = [
+    {
+        "key": "1",
+        "sizes": {
+            "100": {"h": 100, "w": 56},
+            "400": {"h": 400, "w": 225},
+            "full": {"h": 3555, "w": 2000},
+        },
+        "uploaded_t": "1490702616",
+        "uploader": "user1",
+    },
+    {
+        "key": "nutrition_fr",
+        "angle": None,
+        "geometry": "0x0-0-0",
+        "imgid": "1",
+        "normalize": "0",
+        "ocr": 1,
+        "orientation": "0",
+        "rev": "18",
+        "sizes": {
+            "100": {"h": 53, "w": 100},
+            "200": {"h": 107, "w": 200},
+            "400": {"h": 213, "w": 400},
+            "full": {"h": 1093, "w": 2050},
+        },
+        "white_magic": "0",
+        "x1": None,
+        "x2": None,
+        "y1": None,
+        "y2": None,
+    },
+]
+
+
+IMAGES_WITH_NEW_SCHEMA = {
+    "uploaded": {
+        "1": {
+            "sizes": {
+                "100": {
+                    "h": 100,
+                    "w": 56,
+                    "url": "https://images.openfoodfacts.org/images/products/326/385/950/6216/1.100.jpg",
+                },
+                "400": {
+                    "h": 400,
+                    "w": 225,
+                    "url": "https://images.openfoodfacts.org/images/products/326/385/950/6216/1.400.jpg",
+                },
+                "full": {
+                    "h": 3555,
+                    "w": 2000,
+                    "url": "https://images.openfoodfacts.org/images/products/326/385/950/6216/1.jpg",
+                },
+            },
+            "uploaded_t": "1490702616",
+            "uploader": "user1",
+        },
+    },
+    "selected": {
+        "nutrition": {
+            "fr": {
+                "imgid": "1",
+                "rev": "18",
+                "sizes": {
+                    "100": {
+                        "h": 53,
+                        "w": 100,
+                        "url": "https://images.openfoodfacts.org/images/products/326/385/950/6216/nutrition_fr.18.100.jpg",
+                    },
+                    "200": {
+                        "h": 107,
+                        "w": 200,
+                        "url": "https://images.openfoodfacts.org/images/products/326/385/950/6216/nutrition_fr.18.200.jpg",
+                    },
+                    "400": {
+                        "h": 213,
+                        "w": 400,
+                        "url": "https://images.openfoodfacts.org/images/products/326/385/950/6216/nutrition_fr.18.400.jpg",
+                    },
+                    "full": {
+                        "h": 1093,
+                        "w": 2050,
+                        "url": "https://images.openfoodfacts.org/images/products/326/385/950/6216/nutrition_fr.18.full.jpg",
+                    },
+                },
+                "generation": {
+                    "white_magic": "0",
+                    "x1": None,
+                    "x2": None,
+                    "y1": None,
+                    "y2": None,
+                    "normalize": "0",
+                    "ocr": 1,
+                    "orientation": "0",
+                    "angle": None,
+                    "geometry": "0x0-0-0",
+                },
+            },
+        }
+    },
+}
+
+
+class TestProduct:
+    def test_parse_images(self):
+        assert Product.parse_images({"images": IMAGES_WITH_NEW_SCHEMA}) == {
+            "images": PARSED_IMAGES_WITH_LEGACY_SCHEMA
+        }