Serialization util added

Author: ardila

nucleus/__init__.py

@@ -409,7 +409,9 @@ def preprocess_payload(batch):
                 (ITEMS_KEY, (None, json.dumps(batch), "application/json"))
             ]
             for item in batch:
-                image = open(item.get(IMAGE_URL_KEY), "rb")
+                image = open(  # pylint: disable=R1732
+                    item.get(IMAGE_URL_KEY), "rb"  # pylint: disable=R1732
+                )  # pylint: disable=R1732
                 img_name = os.path.basename(image.name)
                 img_type = (
                     f"image/{os.path.splitext(image.name)[1].strip('.')}"

nucleus/annotation.py

@@ -1,25 +1,27 @@
+import json
 from dataclasses import dataclass
 from enum import Enum
-from typing import Dict, Optional, Any, Union, List
+from typing import Any, Dict, List, Optional, Union
+
 from .constants import (
     ANNOTATION_ID_KEY,
+    ANNOTATIONS_KEY,
+    BOX_TYPE,
     DATASET_ITEM_ID_KEY,
-    REFERENCE_ID_KEY,
-    METADATA_KEY,
-    X_KEY,
-    Y_KEY,
-    WIDTH_KEY,
-    HEIGHT_KEY,
     GEOMETRY_KEY,
-    BOX_TYPE,
-    POLYGON_TYPE,
+    HEIGHT_KEY,
+    INDEX_KEY,
+    ITEM_ID_KEY,
     LABEL_KEY,
+    MASK_URL_KEY,
+    METADATA_KEY,
+    POLYGON_TYPE,
+    REFERENCE_ID_KEY,
     TYPE_KEY,
     VERTICES_KEY,
-    ITEM_ID_KEY,
-    MASK_URL_KEY,
-    INDEX_KEY,
-    ANNOTATIONS_KEY,
+    WIDTH_KEY,
+    X_KEY,
+    Y_KEY,
 )
 
 
@@ -42,6 +44,15 @@ def from_json(cls, payload: dict):
         else:
             return SegmentationAnnotation.from_json(payload)
 
+    def to_payload(self):
+        raise NotImplementedError(
+            "For serialization, use a specific subclass (i.e. SegmentationAnnotation), "
+            "not the base annotation class."
+        )
+
+    def to_json(self) -> str:
+        return json.dumps(self.to_payload())
+
 
 @dataclass
 class Segment:

nucleus/dataset_item.py

@@ -1,12 +1,14 @@
-from dataclasses import dataclass
+import json
 import os.path
+from dataclasses import dataclass
 from typing import Optional
+
 from .constants import (
+    DATASET_ITEM_ID_KEY,
     IMAGE_URL_KEY,
     METADATA_KEY,
-    REFERENCE_ID_KEY,
     ORIGINAL_IMAGE_URL_KEY,
-    DATASET_ITEM_ID_KEY,
+    REFERENCE_ID_KEY,
 )
 
 
@@ -55,3 +57,6 @@ def to_payload(self) -> dict:
         if self.item_id:
             payload[DATASET_ITEM_ID_KEY] = self.item_id
         return payload
+
+    def to_json(self) -> str:
+        return json.dumps(self.to_payload())

nucleus/utils.py

@@ -70,3 +70,25 @@ def format_dataset_item_response(response: dict) -> dict:
         ITEM_KEY: DatasetItem.from_json(item),
         ANNOTATIONS_KEY: annotation_response,
     }
+
+
+def serialize_and_write(
+    upload_unit: List[Union[DatasetItem, Annotation]], file_pointer
+):
+    for unit in upload_unit:
+        try:
+            file_pointer.write(unit.to_json())
+        except TypeError as e:
+            type_name = type(unit).__name__
+            message = (
+                f"The following {type_name} could not be serialized: {unit}\n"
+            )
+            message += (
+                "This is usally an issue with a custom python object being "
+                "present in the metadata. Please inspect this error and adjust the "
+                "metadata so it is json-serializable: only python primitives such as "
+                "strings, ints, floats, lists, and dicts. For example, you must "
+                "convert numpy arrays into list or lists of lists.\n"
+            )
+            message += f"The specific error was {e}"
+            raise ValueError(message) from e