Merge branch 'master' into claire/add_taxonomy_api_support

Author: Claire Pajot

nucleus/__init__.py

@@ -1,54 +1,7 @@
 """
 Nucleus Python Library.
 
-Data formats used:
-
-_____________________________________________________________________________________________________
-
-DatasetItem
-
-image_url       |   str     |   The URL containing the image for the given row of data.\n
-reference_id    |   str     |   An optional user-specified identifier to reference this given image.\n
-metadata        |   dict    |   All of column definitions for this item.
-                |           |   The keys should match the user-specified column names,
-                |           |   and the corresponding values will populate the cell under the column.\n
-_____________________________________________________________________________________________________
-
-
-Box2DGeometry:
-
-x               |   float   |   The distance, in pixels, between the left border of the bounding box
-                |           |   and the left border of the image.\n
-y               |   float   |   The distance, in pixels, between the top border of the bounding box
-                |           |   and the top border of the image.\n
-width	        |   float   |   The width in pixels of the annotation.\n
-height	        |   float   |   The height in pixels of the annotation.\n
-
-Box2DAnnotation:
-
-item_id         |   str     |   The internally-controlled item identifier to associate this annotation with.
-                |           |   The reference_id field should be empty if this field is populated.\n
-reference_id    |   str     |   The user-specified reference identifier to associate this annotation with.\n
-                |           |   The item_id field should be empty if this field is populated.
-label	        |   str     |	The label for this annotation (e.g. car, pedestrian, bicycle).\n
-type	        |   str     |   The type of this annotation. It should always be the box string literal.\n
-geometry        |   dict    |   Representation of the bounding box in the Box2DGeometry format.\n
-metadata        |   dict    |   An arbitrary metadata blob for the annotation.\n
-
-_____________________________________________________________________________________________________
-
-Box2DDetection:
-
-item_id         |   str     |   The internally-controlled item identifier to associate this annotation with.
-                |           |   The reference_id field should be empty if this field is populated.\n
-reference_id    |   str     |   The user-specified reference identifier to associate this annotation with.
-                |           |   The item_id field should be empty if this field is populated.\n
-label	        |   str     |	The label for this annotation (e.g. car, pedestrian, bicycle).\n
-type	        |   str     |   The type of this annotation. It should always be the box string literal.\n
-confidence      |   float   |   The optional confidence level of this annotation.
-                |           |   It should be between 0 and 1 (inclusive).\n
-geometry        |   dict    |   Representation of the bounding box in the Box2DGeometry format.\n
-metadata        |   dict    |   An arbitrary metadata blob for the annotation.\n
+For full documentation see: https://dashboard.scale.com/nucleus/docs/api?language=python
 """
 import asyncio
 import json
@@ -82,7 +35,6 @@
     ANNOTATIONS_PROCESSED_KEY,
     AUTOTAGS_KEY,
     DATASET_ID_KEY,
-    DATASET_ITEM_IDS_KEY,
     DEFAULT_NETWORK_TIMEOUT_SEC,
     EMBEDDING_DIMENSION_KEY,
     EMBEDDINGS_URL_KEY,
@@ -245,11 +197,8 @@ def get_dataset_items(self, dataset_id) -> List[DatasetItem]:
             for item in dataset_items:
                 image_url = item.get("original_image_url")
                 metadata = item.get("metadata", None)
-                item_id = item.get("id", None)
                 ref_id = item.get("ref_id", None)
-                dataset_item = DatasetItem(
-                    image_url, ref_id, item_id, metadata
-                )
+                dataset_item = DatasetItem(image_url, ref_id, metadata)
                 constructed_dataset_items.append(dataset_item)
         elif error:
             raise DatasetItemRetrievalError(message=error)
@@ -350,26 +299,19 @@ def delete_dataset(self, dataset_id: str) -> dict:
         return self.make_request({}, f"dataset/{dataset_id}", requests.delete)
 
     @sanitize_string_args
-    def delete_dataset_item(
-        self, dataset_id: str, item_id: str = None, reference_id: str = None
-    ) -> dict:
+    def delete_dataset_item(self, dataset_id: str, reference_id) -> dict:
         """
         Deletes a private dataset based on datasetId.
         Returns an empty payload where response status `200` indicates
         the dataset has been successfully deleted.
         :param payload: { "name": str }
         :return: { "dataset_id": str, "name": str }
         """
-        if item_id:
-            return self.make_request(
-                {}, f"dataset/{dataset_id}/{item_id}", requests.delete
-            )
-        else:  # Assume reference_id is provided
-            return self.make_request(
-                {},
-                f"dataset/{dataset_id}/refloc/{reference_id}",
-                requests.delete,
-            )
+        return self.make_request(
+            {},
+            f"dataset/{dataset_id}/refloc/{reference_id}",
+            requests.delete,
+        )
 
     def populate_dataset(
         self,
@@ -1016,17 +958,13 @@ def create_slice(self, dataset_id: str, payload: dict) -> Slice:
         as a means of identifying items in the dataset.
 
         "name" -- The human-readable name of the slice.
-
-        "dataset_item_ids" -- An optional list of dataset item ids for the items in the slice
-
         "reference_ids" -- An optional list of user-specified identifier for the items in the slice
 
         :param
         dataset_id: id of the dataset
         payload:
         {
             "name": str,
-            "dataset_item_ids": List[str],
             "reference_ids": List[str],
         }
         :return: new Slice object
@@ -1052,14 +990,12 @@ def slice_info(self, slice_id: str) -> dict:
 
         :param
         slice_id: id of the slice
-        id_type: the type of IDs you want in response (either "reference_id" or "dataset_item_id")
-        to identify the DatasetItems
 
         :return:
         {
             "name": str,
             "dataset_id": str,
-            "dataset_item_ids": List[str],
+            "reference_ids": List[str],
         }
         """
         response = self.make_request(
@@ -1111,35 +1047,25 @@ def delete_annotations(
     def append_to_slice(
         self,
         slice_id: str,
-        dataset_item_ids: List[str] = None,
-        reference_ids: List[str] = None,
+        reference_ids: List[str],
     ) -> dict:
         """
         Appends to a slice from items already present in a dataset.
         The caller must exclusively use either datasetItemIds or reference_ids
         as a means of identifying items in the dataset.
 
         :param
-        dataset_item_ids: List[str],
         reference_ids: List[str],
 
         :return:
         {
             "slice_id": str,
         }
         """
-        if dataset_item_ids and reference_ids:
-            raise Exception(
-                "You cannot specify both dataset_item_ids and reference_ids"
-            )
 
-        ids_to_append: Dict[str, Any] = {}
-        if dataset_item_ids:
-            ids_to_append[DATASET_ITEM_IDS_KEY] = dataset_item_ids
-        if reference_ids:
-            ids_to_append[REFERENCE_IDS_KEY] = reference_ids
-
-        response = self.make_request(ids_to_append, f"slice/{slice_id}/append")
+        response = self.make_request(
+            {REFERENCE_IDS_KEY: reference_ids}, f"slice/{slice_id}/append"
+        )
         return response
 
     def list_autotags(self, dataset_id: str) -> List[str]:
@@ -1192,7 +1118,7 @@ def create_custom_index(
 
         :param
         dataset_id: id of dataset that the custom index is being added to.
-        embeddings_urls: list of urls, each of which being a json mapping dataset_item_id -> embedding vector
+        embeddings_urls: list of urls, each of which being a json mapping reference_id -> embedding vector
         embedding_dim: the dimension of the embedding vectors, must be consistent for all embedding vectors in the index.
         """
         return self.make_request(

nucleus/annotation.py

@@ -9,12 +9,10 @@
     ANNOTATIONS_KEY,
     BOX_TYPE,
     CUBOID_TYPE,
-    DATASET_ITEM_ID_KEY,
     DIMENSIONS_KEY,
     GEOMETRY_KEY,
     HEIGHT_KEY,
     INDEX_KEY,
-    ITEM_ID_KEY,
     LABEL_KEY,
     MASK_TYPE,
     MASK_URL_KEY,
@@ -33,16 +31,7 @@
 
 
 class Annotation:
-    reference_id: Optional[str] = None
-    item_id: Optional[str] = None
-
-    def _check_ids(self):
-        if self.reference_id and self.item_id:
-            self.item_id = None  # Prefer reference id to item id.
-        if not (self.reference_id or self.item_id):
-            raise Exception(
-                "You must specify either a reference_id or an item_id for an annotation."
-            )
+    reference_id: str
 
     @classmethod
     def from_json(cls, payload: dict):
@@ -93,14 +82,12 @@ def to_payload(self) -> dict:
 class SegmentationAnnotation(Annotation):
     mask_url: str
     annotations: List[Segment]
+    reference_id: str
     annotation_id: Optional[str] = None
-    reference_id: Optional[str] = None
-    item_id: Optional[str] = None
 
     def __post_init__(self):
         if not self.mask_url:
             raise Exception("You must specify a mask_url.")
-        self._check_ids()
 
     @classmethod
     def from_json(cls, payload: dict):
@@ -112,8 +99,7 @@ def from_json(cls, payload: dict):
                 Segment.from_json(ann)
                 for ann in payload.get(ANNOTATIONS_KEY, [])
             ],
-            reference_id=payload.get(REFERENCE_ID_KEY, None),
-            item_id=payload.get(ITEM_ID_KEY, None),
+            reference_id=payload[REFERENCE_ID_KEY],
             annotation_id=payload.get(ANNOTATION_ID_KEY, None),
         )
 
@@ -124,10 +110,9 @@ def to_payload(self) -> dict:
             ANNOTATIONS_KEY: [ann.to_payload() for ann in self.annotations],
             ANNOTATION_ID_KEY: self.annotation_id,
         }
-        if self.reference_id:
-            payload[REFERENCE_ID_KEY] = self.reference_id
-        else:
-            payload[ITEM_ID_KEY] = self.item_id
+
+        payload[REFERENCE_ID_KEY] = self.reference_id
+
         return payload
 
 
@@ -144,14 +129,14 @@ class BoxAnnotation(Annotation):  # pylint: disable=R0902
     y: Union[float, int]
     width: Union[float, int]
     height: Union[float, int]
-    reference_id: Optional[str] = None
-    item_id: Optional[str] = None
+    reference_id: str
     annotation_id: Optional[str] = None
     metadata: Optional[Dict] = None
 
     def __post_init__(self):
-        self._check_ids()
         self.metadata = self.metadata if self.metadata else {}
+        if self.annotation_id is None:
+            self.annotation_id = f"{self.label}-{self.x}-{self.y}-{self.width}-{self.height}-{self.reference_id}"
 
     @classmethod
     def from_json(cls, payload: dict):
@@ -162,8 +147,7 @@ def from_json(cls, payload: dict):
             y=geometry.get(Y_KEY, 0),
             width=geometry.get(WIDTH_KEY, 0),
             height=geometry.get(HEIGHT_KEY, 0),
-            reference_id=payload.get(REFERENCE_ID_KEY, None),
-            item_id=payload.get(DATASET_ITEM_ID_KEY, None),
+            reference_id=payload[REFERENCE_ID_KEY],
             annotation_id=payload.get(ANNOTATION_ID_KEY, None),
             metadata=payload.get(METADATA_KEY, {}),
         )
@@ -215,13 +199,11 @@ def to_payload(self) -> dict:
 class PolygonAnnotation(Annotation):
     label: str
     vertices: List[Point]
-    reference_id: Optional[str] = None
-    item_id: Optional[str] = None
+    reference_id: str
     annotation_id: Optional[str] = None
     metadata: Optional[Dict] = None
 
     def __post_init__(self):
-        self._check_ids()
         self.metadata = self.metadata if self.metadata else {}
         if len(self.vertices) > 0:
             if not hasattr(self.vertices[0], X_KEY) or not hasattr(
@@ -245,8 +227,7 @@ def from_json(cls, payload: dict):
             vertices=[
                 Point.from_json(_) for _ in geometry.get(VERTICES_KEY, [])
             ],
-            reference_id=payload.get(REFERENCE_ID_KEY, None),
-            item_id=payload.get(DATASET_ITEM_ID_KEY, None),
+            reference_id=payload[REFERENCE_ID_KEY],
             annotation_id=payload.get(ANNOTATION_ID_KEY, None),
             metadata=payload.get(METADATA_KEY, {}),
         )
@@ -271,13 +252,11 @@ class CuboidAnnotation(Annotation):  # pylint: disable=R0902
     position: Point3D
     dimensions: Point3D
     yaw: float
-    reference_id: Optional[str] = None
-    item_id: Optional[str] = None
+    reference_id: str
     annotation_id: Optional[str] = None
     metadata: Optional[Dict] = None
 
     def __post_init__(self):
-        self._check_ids()
         self.metadata = self.metadata if self.metadata else {}
 
     @classmethod
@@ -288,8 +267,7 @@ def from_json(cls, payload: dict):
             position=Point3D.from_json(geometry.get(POSITION_KEY, {})),
             dimensions=Point3D.from_json(geometry.get(DIMENSIONS_KEY, {})),
             yaw=geometry.get(YAW_KEY, 0),
-            reference_id=payload.get(REFERENCE_ID_KEY, None),
-            item_id=payload.get(DATASET_ITEM_ID_KEY, None),
+            reference_id=payload[REFERENCE_ID_KEY],
             annotation_id=payload.get(ANNOTATION_ID_KEY, None),
             metadata=payload.get(METADATA_KEY, {}),
         )
@@ -304,8 +282,7 @@ def to_payload(self) -> dict:
                 YAW_KEY: self.yaw,
             },
         }
-        if self.reference_id:
-            payload[REFERENCE_ID_KEY] = self.reference_id
+        payload[REFERENCE_ID_KEY] = self.reference_id
         if self.annotation_id:
             payload[ANNOTATION_ID_KEY] = self.annotation_id
         if self.metadata:

nucleus/constants.py

@@ -20,8 +20,6 @@
 CX_KEY = "cx"
 CY_KEY = "cy"
 DATASET_ID_KEY = "dataset_id"
-DATASET_ITEM_IDS_KEY = "dataset_item_ids"
-DATASET_ITEM_ID_KEY = "dataset_item_id"
 DATASET_LENGTH_KEY = "length"
 DATASET_MODEL_RUNS_KEY = "model_run_ids"
 DATASET_NAME_KEY = "name"
@@ -48,7 +46,6 @@
 INDEX_KEY = "index"
 INDEX_CONTINUOUS_ENABLE_KEY = "enable"
 ITEMS_KEY = "items"
-ITEM_ID_KEY = "item_id"
 ITEM_KEY = "item"
 ITEMS_KEY = "items"
 ITEM_METADATA_SCHEMA_KEY = "item_metadata_schema"