Escape illegal characters on a per-field basis in order to support slashes in ref ids

Author: Diego Ardila

nucleus/__init__.py

@@ -54,6 +54,7 @@
 import json
 import logging
 import os
+import urllib.request
 from typing import Any, Dict, List, Optional, Union
 
 import aiohttp
@@ -62,12 +63,14 @@
 import tqdm
 import tqdm.notebook as tqdm_notebook
 
+from nucleus.url_utils import sanitize_string_args
+
 from .annotation import (
     BoxAnnotation,
+    Point,
     PolygonAnnotation,
     Segment,
     SegmentationAnnotation,
-    Point,
 )
 from .constants import (
     ANNOTATION_METADATA_SCHEMA_KEY,
@@ -298,6 +301,7 @@ def delete_dataset(self, dataset_id: str) -> dict:
         """
         return self.make_request({}, f"dataset/{dataset_id}", requests.delete)
 
+    @sanitize_string_args
     def delete_dataset_item(
         self, dataset_id: str, item_id: str = None, reference_id: str = None
     ) -> dict:
@@ -855,6 +859,7 @@ def model_run_info(self, model_run_id: str):
             {}, f"modelRun/{model_run_id}/info", requests.get
         )
 
+    @sanitize_string_args
     def dataitem_ref_id(self, dataset_id: str, reference_id: str):
         """
         :param dataset_id: internally controlled dataset id
@@ -865,6 +870,7 @@ def dataitem_ref_id(self, dataset_id: str, reference_id: str):
             {}, f"dataset/{dataset_id}/refloc/{reference_id}", requests.get
         )
 
+    @sanitize_string_args
     def predictions_ref_id(self, model_run_id: str, ref_id: str):
         """
         Returns Model Run info For Dataset Item by model_run_id and item reference_id.

nucleus/dataset.py

@@ -9,6 +9,8 @@
     serialize_and_write_to_presigned_url,
 )
 
+from nucleus.url_utils import sanitize_string_args
+
 from .annotation import Annotation, check_all_annotation_paths_remote
 from .constants import (
     DATASET_ITEM_IDS_KEY,
@@ -79,6 +81,7 @@ def size(self) -> int:
     def items(self) -> List[DatasetItem]:
         return self._client.get_dataset_items(self.id)
 
+    @sanitize_string_args
     def autotag_scores(self, autotag_name, for_scores_greater_than=0):
         """Export the autotag scores above a threshold, largest scores first.
 

nucleus/url_utils.py

@@ -0,0 +1,22 @@
+import urllib.request
+
+
+def sanitize_field(field):
+    return urllib.request.quote(field.encode("UTF-8"), safe="")
+
+
+def sanitize_string_args(function):
+    def sanitized_function(*args, **kwargs):
+        sanitized_args = []
+        sanitized_kwargs = {}
+        for arg in args:
+            if isinstance(arg, str):
+                arg = sanitize_field(arg)
+            sanitized_args.append(arg)
+        for key, value in kwargs.items():
+            if isinstance(value, str):
+                value = sanitize_field(value)
+            sanitized_kwargs[key] = value
+        return function(*sanitized_args, **sanitized_kwargs)
+
+    return sanitized_function

tests/test_dataset.py

@@ -37,7 +37,6 @@
     TEST_IMG_URLS,
     TEST_POLYGON_ANNOTATIONS,
     TEST_SEGMENTATION_ANNOTATIONS,
-    TEST_SLICE_NAME,
     reference_id_from_url,
 )
 
@@ -344,6 +343,20 @@ def test_annotate_async_with_error(dataset: Dataset):
     assert "Item with id fake_garbage doesn" in str(job.errors())
 
 
+def test_append_with_special_chars(dataset):
+    url = TEST_IMG_URLS[0]
+    ref_id = "test/reference/id"
+    ds_items = [
+        DatasetItem(
+            image_location=url,
+            reference_id=ref_id,
+            metadata={"test": "metadata"},
+        ),
+    ]
+    dataset.append(ds_items)
+    dataset.refloc(ref_id)
+
+
 def test_append_and_export(dataset):
     # Dataset upload
     url = TEST_IMG_URLS[0]

tests/test_slice.py

@@ -154,6 +154,7 @@ def sort_by_reference_id(items):
     )
 
 
+@pytest.mark.integration
 def test_slice_send_to_labeling(dataset):
     # Dataset upload
     ds_items = []