Merge pull request #65 from scaleapi/da/validation_serialization

Add error check for duplicate reference ids

Author: ardila

nucleus/dataset.py

@@ -1,4 +1,5 @@
-from typing import Any, Dict, List, Optional
+from collections import Counter
+from typing import List, Dict, Any, Optional
 
 import requests
 
@@ -178,6 +179,19 @@ def append(
             'ignored_items': int,
         }
         """
+        ref_ids = []
+        for dataset_item in dataset_items:
+            if dataset_item.reference_id is not None:
+                ref_ids.append(dataset_item.reference_id)
+        if len(ref_ids) != len(set(ref_ids)):
+            duplicates = {
+                f"{key}": f"Count: {value}"
+                for key, value in Counter(ref_ids).items()
+            }
+            raise ValueError(
+                "Duplicate reference ids found among dataset_items: %s"
+                % duplicates
+            )
         return self._client.populate_dataset(
             self.id,
             dataset_items,

tests/test_annotation.py

@@ -57,7 +57,7 @@ def dataset(CLIENT):
     yield ds
 
     response = CLIENT.delete_dataset(ds.id)
-    assert response == {}
+    assert response == {"message": "Beginning dataset deletion..."}
 
 
 def test_box_gt_upload(dataset):

tests/test_dataset.py

@@ -58,7 +58,7 @@ def dataset(CLIENT):
     yield ds
 
     response = CLIENT.delete_dataset(ds.id)
-    assert response == {}
+    assert response == {"message": "Beginning dataset deletion..."}
 
 
 def test_dataset_create_and_delete(CLIENT):
@@ -73,7 +73,7 @@ def test_dataset_create_and_delete(CLIENT):
 
     # Deletion
     response = CLIENT.delete_dataset(ds.id)
-    assert response == {}
+    assert response == {"message": "Beginning dataset deletion..."}
 
 
 def test_dataset_append(dataset):
@@ -138,6 +138,22 @@ def test_dataset_list_autotags(CLIENT, dataset):
     assert autotag_response == []
 
 
+def test_raises_error_for_duplicate():
+    fake_dataset = Dataset("fake", NucleusClient("fake"))
+    with pytest.raises(ValueError) as error:
+        fake_dataset.append(
+            [
+                DatasetItem("fake", "duplicate"),
+                DatasetItem("fake", "duplicate"),
+            ]
+        )
+    assert (
+        str(error.value)
+        == "Duplicate reference ids found among dataset_items:"
+        " {'duplicate': 'Count: 2'}"
+    )
+
+
 def test_dataset_export_autotag_scores(CLIENT):
     # This test can only run for the test user who has an indexed dataset.
     # TODO: if/when we can create autotags via api, create one instead.

tests/test_indexing.py

@@ -34,7 +34,7 @@ def dataset(CLIENT):
     yield ds
 
     response = CLIENT.delete_dataset(ds.id)
-    assert response == {}
+    assert response == {"message": "Beginning dataset deletion..."}
 
 
 def test_index_integration(dataset):

tests/test_prediction.py

@@ -65,7 +65,7 @@ def model_run(CLIENT):
     yield run
 
     response = CLIENT.delete_dataset(ds.id)
-    assert response == {}
+    assert response == {"message": "Beginning dataset deletion..."}
     response = CLIENT.delete_model(model.id)
     assert response == {}
 

tests/test_slice.py

@@ -16,7 +16,7 @@ def dataset(CLIENT):
     yield ds
 
     response = CLIENT.delete_dataset(ds.id)
-    assert response == {}
+    assert response == {"message": "Beginning dataset deletion..."}
 
 
 def test_reprs():