Remove upload_to_scale option (#407)

Author: jean-lucas

nucleus/constants.py

@@ -150,7 +150,6 @@
 TYPE_KEY = "type"
 UPDATED_ITEMS = "updated_items"
 UPDATE_KEY = "update"
-UPLOAD_TO_SCALE_KEY = "upload_to_scale"
 URL_KEY = "url"
 VERTICES_KEY = "vertices"
 VIDEO_LOCATION_KEY = "video_location"

nucleus/dataset_item.py

@@ -19,7 +19,6 @@
     POINTCLOUD_URL_KEY,
     REFERENCE_ID_KEY,
     TYPE_KEY,
-    UPLOAD_TO_SCALE_KEY,
     URL_KEY,
 )
 
@@ -112,13 +111,6 @@ class DatasetItem:  # pylint: disable=R0902
               Shorten this once we have a guide migrated for metadata, or maybe link
               from other places to here.
 
-        upload_to_scale (Optional[bool]): Set this to false in order to use
-          `privacy mode <https://nucleus.scale.com/docs/privacy-mode>`_.
-
-          Setting this to false means the actual data within the item will not be
-          uploaded to scale meaning that you can send in links that are only accessible
-          to certain users, and not to Scale. Skipping upload to Scale is currently only
-          implemented for images.
     """
 
     image_location: Optional[str] = None
@@ -127,24 +119,19 @@ class DatasetItem:  # pylint: disable=R0902
     )
     metadata: Optional[dict] = None
     pointcloud_location: Optional[str] = None
-    upload_to_scale: Optional[bool] = True
     embedding_info: Optional[DatasetItemEmbeddingInfo] = None
 
     def __post_init__(self):
         assert self.reference_id != "DUMMY_VALUE", "reference_id is required."
         assert bool(self.image_location) != bool(
             self.pointcloud_location
         ), "Must specify exactly one of the image_location or pointcloud_location parameters"
+
         if self.pointcloud_location and self.embedding_info:
             raise AssertionError(
                 "Cannot upload embedding vector if pointcloud_location is set"
             )
 
-        if (self.pointcloud_location) and not self.upload_to_scale:
-            raise NotImplementedError(
-                "Skipping upload to Scale is not currently implemented for pointclouds."
-            )
-
         self.local = (
             is_local_path(self.image_location) if self.image_location else None
         )
@@ -186,7 +173,6 @@ def from_json(cls, payload: dict):
             pointcloud_location=pointcloud_url,
             reference_id=payload.get(REFERENCE_ID_KEY, None),
             metadata=payload.get(METADATA_KEY, {}),
-            upload_to_scale=payload.get(UPLOAD_TO_SCALE_KEY, True),
         )
 
     def local_file_exists(self):
@@ -217,7 +203,6 @@ def to_payload(self, is_scene=False) -> dict:
                 self.image_location
             ), "Must specify image_location for DatasetItems not in a LidarScene or VideoScene"
             payload[IMAGE_URL_KEY] = self.image_location
-            payload[UPLOAD_TO_SCALE_KEY] = self.upload_to_scale
 
         return payload
 

nucleus/scene.py

@@ -14,7 +14,6 @@
     POINTCLOUD_LOCATION_KEY,
     REFERENCE_ID_KEY,
     TRACKS_KEY,
-    UPLOAD_TO_SCALE_KEY,
     VIDEO_LOCATION_KEY,
     VIDEO_URL_KEY,
 )
@@ -496,13 +495,6 @@ class VideoScene(ABC):
             Context Attachments may be provided to display the attachments side by side with the dataset
             item in the Detail View by specifying
             `{ "context_attachments": [ { "attachment": 'https://example.com/1' }, { "attachment": 'https://example.com/2' }, ... ] }`.
-        upload_to_scale (Optional[bool]): Set this to false in order to use
-            `privacy mode <https://nucleus.scale.com/docs/privacy-mode>`_. If using privacy mode
-            you must upload both a video_location and items to the VideoScene.
-
-            Setting this to false means the actual data within the video scene will not be
-            uploaded to scale meaning that you can send in links that are only accessible
-            to certain users, and not to Scale.
 
     Refer to our `guide to uploading video data
     <https://nucleus.scale.com/docs/uploading-video-data>`_ for more info!
@@ -513,7 +505,6 @@ class VideoScene(ABC):
     video_location: Optional[str] = None
     items: List[DatasetItem] = field(default_factory=list)
     metadata: Optional[dict] = field(default_factory=dict)
-    upload_to_scale: Optional[bool] = True
     attachment_type: Optional[str] = None
     tracks: List[Track] = field(default_factory=list)
 
@@ -540,34 +531,14 @@ def __eq__(self, other):
     @property
     def length(self) -> int:
         """Gets number of items in the scene for videos uploaded with an array of images."""
-        assert (
-            not self.upload_to_scale or not self.video_location
-        ), "Only videos with items have a length"
         return len(self.items)
 
     def validate(self):
         # TODO: make private
         assert (
             self.items or self.video_location
         ), "Please upload either a video_location or an array of dataset items representing frames"
-        if self.upload_to_scale is False:
-            assert (
-                self.frame_rate > 0
-            ), "When using privacy mode frame rate must be at least 1"
-            assert (
-                self.items and self.length > 0
-            ), "When using privacy mode scene must have a list of items of length at least 1"
-            for item in self.items:
-                assert isinstance(
-                    item, DatasetItem
-                ), "Each item in a scene must be a DatasetItem object"
-                assert (
-                    item.image_location is not None
-                ), "Each item in a video scene must have an image_location"
-                assert (
-                    item.upload_to_scale is not False
-                ), "Please specify whether to upload to scale in the VideoScene for videos"
-        elif self.items:
+        if self.items:
             assert (
                 self.frame_rate > 0
             ), "When uploading an array of items frame rate must be at least 1"
@@ -584,9 +555,6 @@ def validate(self):
                 assert (
                     item.image_location is not None
                 ), "Each item in a video scene must have an image_location"
-                assert (
-                    item.upload_to_scale is not False
-                ), "Please specify whether to upload to scale in the VideoScene for videos"
         else:
             assert (
                 not self.frame_rate
@@ -610,7 +578,7 @@ def add_item(
               exists. Default is False.
         """
         assert (
-            not self.upload_to_scale or not self.video_location
+            not self.video_location
         ), "Cannot add item to a video without items"
         if index is None:
             index = len(self.items)
@@ -631,7 +599,7 @@ def get_item(self, index: int) -> DatasetItem:
         Return:
             :class:`DatasetItem`: DatasetItem at the specified index."""
         assert (
-            not self.upload_to_scale or not self.video_location
+            not self.video_location
         ), "Cannot add item to a video without items"
         if index < 0 or index > len(self.items):
             raise ValueError(
@@ -646,7 +614,7 @@ def get_items(self) -> List[DatasetItem]:
             List[:class:`DatasetItem`]: List of DatasetItems, sorted by index ascending.
         """
         assert (
-            not self.upload_to_scale or not self.video_location
+            not self.video_location
         ), "Cannot add item to a video without items"
         return self.items
 
@@ -672,9 +640,6 @@ def info(self):
             payload[VIDEO_URL_KEY] = self.video_location
         if self.items:
             payload[LENGTH_KEY] = self.length
-        if self.upload_to_scale:
-            payload[UPLOAD_TO_SCALE_KEY] = self.upload_to_scale
-
         return payload
 
     @classmethod
@@ -699,7 +664,6 @@ def from_json(
             items=items,
             metadata=payload.get(METADATA_KEY, {}),
             video_location=payload.get(VIDEO_URL_KEY, None),
-            upload_to_scale=payload.get(UPLOAD_TO_SCALE_KEY, True),
             tracks=tracks,
         )
 
@@ -720,8 +684,6 @@ def to_payload(self) -> dict:
                 item.to_payload(is_scene=True) for item in self.items
             ]
             payload[FRAMES_KEY] = items_payload
-        if self.upload_to_scale is not None:
-            payload[UPLOAD_TO_SCALE_KEY] = self.upload_to_scale
         if self.tracks:
             payload[TRACKS_KEY] = [track.to_payload() for track in self.tracks]
         return payload

scripts/load_test.py

@@ -5,7 +5,6 @@
 from absl import app, flags
 
 import nucleus
-from nucleus import annotation
 from nucleus.async_job import JobError
 from nucleus.dataset import Dataset
 
@@ -89,7 +88,6 @@ def dataset_item_generator():
             image_location=IMAGE_URL,
             reference_id=str(i),
             metadata=generate_fake_metadata(i),
-            upload_to_scale=False,
         )
 
 

tests/helpers.py

@@ -121,27 +121,6 @@
             "video_url": TEST_VIDEO_URL,
             "metadata": {"timestamp": "1234", "weather": "rainy"},
         },
-        {
-            "reference_id": "scene_3",
-            "video_url": TEST_INACCESSIBLE_VIDEO_URL,
-            "frame_rate": 15,
-            "frames": [
-                {
-                    "image_url": TEST_INACCESSIBLE_IMG_URLS[0],
-                    "type": "image",
-                    "reference_id": "video_frame_2",
-                    "metadata": {"time": 123, "foo": "bar"},
-                },
-                {
-                    "image_url": TEST_INACCESSIBLE_IMG_URLS[1],
-                    "type": "image",
-                    "reference_id": "video_frame_3",
-                    "metadata": {"time": 124, "foo": "bar_2"},
-                },
-            ],
-            "metadata": {"timestamp": "1234", "weather": "rainy"},
-            "upload_to_scale": False,
-        },
     ],
     "update": False,
 }
@@ -251,21 +230,18 @@ def reference_id_from_url(url):
         reference_id_from_url(TEST_IMG_URLS[0]),
         None,
         None,
-        True,
     ),
     DatasetItem(
         TEST_IMG_URLS[1],
         reference_id_from_url(TEST_IMG_URLS[1]),
         None,
         None,
-        True,
     ),
     DatasetItem(
         TEST_IMG_URLS[2],
         reference_id_from_url(TEST_IMG_URLS[2]),
         None,
         None,
-        True,
     ),
 ]
 

tests/test_dataset.py

@@ -197,12 +197,9 @@ def check_is_expected_response(response):
     # Plain image upload
     ds_items_plain = []
     for i, url in enumerate(TEST_IMG_URLS):
-        # Upload just the first item in privacy mode
-        upload_to_scale = i == 0
         ds_items_plain.append(
             DatasetItem(
                 image_location=url,
-                upload_to_scale=upload_to_scale,
                 reference_id=url.split("/")[-1] + "_plain",
             )
         )
@@ -220,12 +217,9 @@ def test_scene_dataset_append(dataset_scene):
     # Plain image upload
     ds_items_plain = []
     for i, url in enumerate(TEST_IMG_URLS):
-        # Upload just the first item in privacy mode
-        upload_to_scale = i == 0
         ds_items_plain.append(
             DatasetItem(
                 image_location=url,
-                upload_to_scale=upload_to_scale,
                 reference_id=url.split("/")[-1] + "_plain",
             )
         )

tests/test_video_scene.py

@@ -12,7 +12,6 @@
     REFERENCE_ID_KEY,
     SCENES_KEY,
     TYPE_KEY,
-    UPLOAD_TO_SCALE_KEY,
     URL_KEY,
     VIDEO_URL_KEY,
 )
@@ -121,7 +120,6 @@ def test_video_scene_add_item():
                 METADATA_KEY: TEST_VIDEO_ITEMS[1].metadata or {},
             },
         ],
-        UPLOAD_TO_SCALE_KEY: True,
     }