Video MP4 Upload API (#275)

* initial updates

* rerun commit tests

* bug fix

* Fix check all paths remote

* Added export test

* Added scene property test

* remove frame rate for mp4

* clean up

* Docstring update

* fix ordering

* switch to optional

* added erroring tests

* fix VideoScene arg ordering

* lint

* lint again

* Changelog fix

* Isort lint

* Fix docs

* remove optional

Author: cmpajot

CHANGELOG.md

@@ -5,18 +5,24 @@ All notable changes to the [Nucleus Python Client](https://github.com/scaleapi/n
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [0.10.4](https://github.com/scaleapi/nucleus-python-client/releases/tag/v0.10.1)) - 2022-04-22
+## [0.10.4](https://github.com/scaleapi/nucleus-python-client/releases/tag/v0.10.4)) - 2022-05-02
+
+### Added
+
 - Additional check added for KeypointsAnnotation names validation
+- MP4 video upload
 
 ## [0.10.3](https://github.com/scaleapi/nucleus-python-client/releases/tag/v0.10.3) - 2022-04-22
 
 ### Fixed
+
 - Polygon and bounding box matching uses Shapely again providing faster evaluations
 - Evaluation function passing fixed for Polygon and Boundingbox configurations
 
 ## [0.10.1](https://github.com/scaleapi/nucleus-python-client/releases/tag/v0.10.1) - 2022-04-21
 
 ### Added
+
 - Added check for payload size
 
 ## [0.10.0](https://github.com/scaleapi/nucleus-python-client/releases/tag/v0.10.0)) - 2022-04-21

nucleus/constants.py

@@ -123,6 +123,8 @@
 UPLOAD_TO_SCALE_KEY = "upload_to_scale"
 URL_KEY = "url"
 VERTICES_KEY = "vertices"
+VIDEO_LOCATION_KEY = "video_location"
+VIDEO_URL_KEY = "video_url"
 VISIBLE_KEY = "visible"
 VIDEO_UPLOAD_TYPE_KEY = "video_upload_type"
 WIDTH_KEY = "width"

nucleus/scene.py

@@ -13,7 +13,9 @@
     NUM_SENSORS_KEY,
     POINTCLOUD_LOCATION_KEY,
     REFERENCE_ID_KEY,
+    VIDEO_LOCATION_KEY,
     VIDEO_UPLOAD_TYPE_KEY,
+    VIDEO_URL_KEY,
 )
 
 from .annotation import is_local_path
@@ -419,16 +421,17 @@ class _VideoUploadType(Enum):
 
 @dataclass
 class VideoScene(ABC):
-    """
-    Nucleus video datasets are comprised of VideoScenes, which are in turn
-    comprised of a sequence of :class:`DatasetItems <DatasetItem>` which are
-    equivalent to frames.
+    """Video or sequence of images over time.
+
+    Nucleus video datasets are comprised of VideoScenes. These can be
+    comprised of a single video, or a sequence of :class:`DatasetItems <DatasetItem>`
+    which are equivalent to frames.
 
     VideoScenes are uploaded to a :class:`Dataset` with any accompanying
     metadata. Each of :class:`DatasetItems <DatasetItem>` representing a frame
     also accepts metadata.
 
-    Note: Uploads with different items will error out (only on scenes that
+    Note: Updates with different items will error out (only on scenes that
     now differ). Existing video are expected to retain the same frames, and only
     metadata can be updated. If a video definition is changed (for example,
     additional frames added) the update operation will be ignored. If you would
@@ -437,30 +440,31 @@ class VideoScene(ABC):
 
     Parameters:
         reference_id (str): User-specified identifier to reference the scene.
-        frame_rate (int): Frame rate of the video.
         attachment_type (str): The type of attachments being uploaded as a string literal.
-            Currently, videos can only be uploaded as an array of frames, so the only
-            accepted attachment_type is "image".
-        items (Optional[List[:class:`DatasetItem`]]): List of items representing frames,
-            to be a part of the scene. A scene can be created before items have been added
-            to it, but must be non-empty when uploading to a :class:`Dataset`. A video scene
-            can contain a maximum of 3000 items.
+            If the video is uploaded as an array of frames, the attachment_type is "image".
+            If the video is uploaded as an MP4, the attachment_type is "video".
+        frame_rate (Optional[int]): Required if attachment_type is "image". Frame rate of the video.
+        video_location (Optional[str]): Required if attachment_type is "video". The remote URL
+            containing the video MP4. Remote formats supported include any URL (``http://``
+            or ``https://``) or URIs for AWS S3, Azure, or GCS (i.e. ``s3://``, ``gcs://``).
+        items (Optional[List[:class:`DatasetItem`]]): Required if attachment_type is "image".
+            List of items representing frames, to be a part of the scene. A scene can be created
+            before items have been added to it, but must be non-empty when uploading to
+            a :class:`Dataset`. A video scene can contain a maximum of 3000 items.
         metadata (Optional[Dict]): Optional metadata to include with the scene.
 
     Refer to our `guide to uploading video data
     <https://nucleus.scale.com/docs/uploading-video-data>`_ for more info!
     """
 
     reference_id: str
-    frame_rate: int
     attachment_type: _VideoUploadType
+    frame_rate: Optional[int] = None
+    video_location: Optional[str] = None
     items: List[DatasetItem] = field(default_factory=list)
     metadata: Optional[dict] = field(default_factory=dict)
 
     def __post_init__(self):
-        assert (
-            self.attachment_type != _VideoUploadType.IMAGE
-        ), "Videos can currently only be uploaded from frames"
         if self.metadata is None:
             self.metadata = {}
 
@@ -469,41 +473,67 @@ def __eq__(self, other):
             [
                 self.reference_id == other.reference_id,
                 self.items == other.items,
+                self.video_location == other.video_location,
                 self.metadata == other.metadata,
             ]
         )
 
     @property
     def length(self) -> int:
-        """Number of items in the scene."""
+        """Gets number of items in the scene for videos uploaded as an array of images."""
+        assert (
+            self.video_location is None
+        ), "Videos uploaded as an mp4 have no length"
         return len(self.items)
 
     def validate(self):
         # TODO: make private
-        assert self.frame_rate > 0, "Frame rate must be at least 1"
-        assert self.length > 0, "Must have at least 1 item in a scene"
-        for item in self.items:
-            assert isinstance(
-                item, DatasetItem
-            ), "Each item in a scene must be a DatasetItem object"
+        assert self.attachment_type in ("image", "video")
+        if self.attachment_type == "image":
+            assert (
+                self.frame_rate > 0
+            ), "When attachment_type='image' frame rate must be at least 1"
+            assert (
+                self.items and self.length > 0
+            ), "When attachment_type='image' scene must have a list of items of length at least 1"
             assert (
-                item.image_location is not None
-            ), "Each item in a video scene must have an image_location"
+                not self.video_location
+            ), "No video location is accepted when attachment_type='image'"
+            for item in self.items:
+                assert isinstance(
+                    item, DatasetItem
+                ), "Each item in a scene must be a DatasetItem object"
+                assert (
+                    item.image_location is not None
+                ), "Each item in a video scene must have an image_location"
+                assert (
+                    item.upload_to_scale is not False
+                ), "Skipping upload to Scale is not currently implemented for videos"
+        if self.attachment_type == "video":
             assert (
-                item.upload_to_scale is not False
-            ), "Skipping upload to Scale is not currently implemented for videos"
+                self.video_location
+            ), "When attachment_type='video' a video_location is required"
+            assert (
+                not self.frame_rate
+            ), "No frame rate is accepted when attachment_type='video'"
+            assert (
+                not self.items
+            ), "No list of items is accepted when attachment_type='video'"
 
     def add_item(
         self, item: DatasetItem, index: int = None, update: bool = False
     ) -> None:
-        """Adds DatasetItem to the specified index.
+        """Adds DatasetItem to the specified index for videos uploaded as an array of images.
 
         Parameters:
             item (:class:`DatasetItem`): Video item to add.
             index: Serial index at which to add the item.
             update: Whether to overwrite the item at the specified index, if it
               exists. Default is False.
         """
+        assert (
+            self.video_location is None
+        ), "Cannot add item to a video uploaded as an mp4"
         if index is None:
             index = len(self.items)
         assert (
@@ -515,44 +545,57 @@ def add_item(
             self.items.append(item)
 
     def get_item(self, index: int) -> DatasetItem:
-        """Fetches the DatasetItem at the specified index.
+        """Fetches the DatasetItem at the specified index for videos uploaded as an array of images.
 
         Parameters:
             index: Serial index for which to retrieve the DatasetItem.
 
         Return:
             :class:`DatasetItem`: DatasetItem at the specified index."""
+        assert (
+            self.video_location is None
+        ), "Cannot get item from a video uploaded as an mp4"
         if index < 0 or index > len(self.items):
             raise ValueError(
                 f"This scene does not have an item at index {index}"
             )
         return self.items[index]
 
     def get_items(self) -> List[DatasetItem]:
-        """Fetches a sorted list of DatasetItems of the scene.
+        """Fetches a sorted list of DatasetItems of the scene for videos uploaded as an array of images.
 
         Returns:
             List[:class:`DatasetItem`]: List of DatasetItems, sorted by index ascending.
         """
+        assert (
+            self.video_location is None
+        ), "Cannot get items from a video uploaded as an mp4"
         return self.items
 
     def info(self):
-        """Fetches information about the scene.
+        """Fetches information about the video scene.
 
         Returns:
             Payload containing::
 
                 {
                     "reference_id": str,
-                    "length": int,
-                    "num_sensors": int
+                    "length": Optional[int],
+                    "frame_rate": int,
+                    "video_url": Optional[str],
                 }
         """
-        return {
+        payload: Dict[str, Any] = {
             REFERENCE_ID_KEY: self.reference_id,
-            FRAME_RATE_KEY: self.frame_rate,
-            LENGTH_KEY: self.length,
         }
+        if self.frame_rate:
+            payload[FRAME_RATE_KEY] = self.frame_rate
+        if self.video_location:
+            payload[VIDEO_URL_KEY] = self.video_location
+        if self.items:
+            payload[LENGTH_KEY] = self.length
+
+        return payload
 
     @classmethod
     def from_json(cls, payload: dict):
@@ -561,24 +604,31 @@ def from_json(cls, payload: dict):
         items = [DatasetItem.from_json(item) for item in items_payload]
         return cls(
             reference_id=payload[REFERENCE_ID_KEY],
-            frame_rate=payload[FRAME_RATE_KEY],
+            frame_rate=payload.get(FRAME_RATE_KEY, None),
             attachment_type=payload[VIDEO_UPLOAD_TYPE_KEY],
             items=items,
             metadata=payload.get(METADATA_KEY, {}),
+            video_location=payload.get(VIDEO_URL_KEY, None),
         )
 
     def to_payload(self) -> dict:
         """Serializes scene object to schematized JSON dict."""
         self.validate()
-        items_payload = [item.to_payload(is_scene=True) for item in self.items]
         payload: Dict[str, Any] = {
             REFERENCE_ID_KEY: self.reference_id,
             VIDEO_UPLOAD_TYPE_KEY: self.attachment_type,
-            FRAME_RATE_KEY: self.frame_rate,
-            FRAMES_KEY: items_payload,
         }
+        if self.frame_rate:
+            payload[FRAME_RATE_KEY] = self.frame_rate
         if self.metadata:
             payload[METADATA_KEY] = self.metadata
+        if self.video_location:
+            payload[VIDEO_URL_KEY] = self.video_location
+        if self.items:
+            items_payload = [
+                item.to_payload(is_scene=True) for item in self.items
+            ]
+            payload[FRAMES_KEY] = items_payload
         return payload
 
     def to_json(self) -> str:
@@ -590,16 +640,24 @@ def check_all_scene_paths_remote(
     scenes: Union[List[LidarScene], List[VideoScene]]
 ):
     for scene in scenes:
-        for item in scene.get_items():
-            pointcloud_location = getattr(item, POINTCLOUD_LOCATION_KEY)
-            if pointcloud_location and is_local_path(pointcloud_location):
-                raise ValueError(
-                    f"All paths for DatasetItems in a Scene must be remote, but {item.pointcloud_location} is either "
-                    "local, or a remote URL type that is not supported."
-                )
-            image_location = getattr(item, IMAGE_LOCATION_KEY)
-            if image_location and is_local_path(image_location):
+        if isinstance(scene, VideoScene) and scene.video_location:
+            video_location = getattr(scene, VIDEO_LOCATION_KEY)
+            if video_location and is_local_path(video_location):
                 raise ValueError(
-                    f"All paths for DatasetItems in a Scene must be remote, but {item.image_location} is either "
+                    f"All paths for videos must be remote, but {scene.video_location} is either "
                     "local, or a remote URL type that is not supported."
                 )
+        else:
+            for item in scene.get_items():
+                pointcloud_location = getattr(item, POINTCLOUD_LOCATION_KEY)
+                if pointcloud_location and is_local_path(pointcloud_location):
+                    raise ValueError(
+                        f"All paths for DatasetItems in a Scene must be remote, but {item.pointcloud_location} is either "
+                        "local, or a remote URL type that is not supported."
+                    )
+                image_location = getattr(item, IMAGE_LOCATION_KEY)
+                if image_location and is_local_path(image_location):
+                    raise ValueError(
+                        f"All paths for DatasetItems in a Scene must be remote, but {item.image_location} is either "
+                        "local, or a remote URL type that is not supported."
+                    )