Nan fixes, back to python 3.6

Author: Diego Ardila

.circleci/config.yml

@@ -4,7 +4,7 @@ version: 2.1
 jobs:
   build_test:
     docker:
-      - image: python:3.7-slim-buster
+      - image: python:3.6-slim-buster
     resource_class: small
     steps:
       - checkout # checkout source code to working directory
@@ -18,7 +18,7 @@ jobs:
       - run:
           name: Black Formatting Check # Only validation, without re-formatting
           command: |
-            poetry run black --check -t py37 .
+            poetry run black --check -t py36 .
       - run:
           name: Flake8 Lint Check # Uses setup.cfg for configuration
           command: |

nucleus/__init__.py

@@ -402,7 +402,14 @@ def _process_append_requests_local(
     ):
         def get_files(batch):
             request_payload = [
-                (ITEMS_KEY, (None, json.dumps(batch), "application/json"))
+                (
+                    ITEMS_KEY,
+                    (
+                        None,
+                        json.dumps(batch, allow_nan=False),
+                        "application/json",
+                    ),
+                )
             ]
             for item in batch:
                 image = open(  # pylint: disable=R1732
@@ -426,7 +433,8 @@ def get_files(batch):
             files_per_request.append(get_files(batch))
             payload_items.append(batch)
 
-        responses = asyncio.run(
+        loop = asyncio.get_event_loop()
+        responses = loop.run_until_complete(
             self.make_many_files_requests_asynchronously(
                 files_per_request,
                 f"dataset/{dataset_id}/append",
@@ -476,7 +484,7 @@ async def _make_files_request(
         """
         Makes an async post request with files to a Nucleus endpoint.
 
-        :param files: A list of tuples (filename, file_pointer, file_type)
+        :param files: A list of tuples (name, (filename, file_pointer, file_type))
         :param route: route for the request
         :param session: Session to use for post.
         :return: response
@@ -502,12 +510,21 @@ async def _make_files_request(
             timeout=DEFAULT_NETWORK_TIMEOUT_SEC,
         ) as response:
             logger.info("API request has response code %s", response.status)
+
+            try:
+                data = await response.json()
+            except aiohttp.client_exceptions.ContentTypeError:
+                # In case of 404, the server returns text
+                data = await response.text()
+
             if not response.ok:
                 self.handle_bad_response(
-                    endpoint, session.post, aiohttp_response=response
+                    endpoint,
+                    session.post,
+                    aiohttp_response=(response.status, response.reason, data),
                 )
 
-            return await response.json()
+            return data
 
     def _process_append_requests(
         self,

nucleus/annotation.py

@@ -53,7 +53,7 @@ def to_payload(self):
         )
 
     def to_json(self) -> str:
-        return json.dumps(self.to_payload())
+        return json.dumps(self.to_payload(), allow_nan=False)
 
 
 @dataclass

nucleus/dataset_item.py

@@ -52,7 +52,7 @@ def to_payload(self) -> dict:
         return payload
 
     def to_json(self) -> str:
-        return json.dumps(self.to_payload())
+        return json.dumps(self.to_payload(), allow_nan=False)
 
 
 def is_local_path(path: str) -> bool:

nucleus/errors.py

@@ -38,10 +38,9 @@ def __init__(
                     )
 
         if aiohttp_response is not None:
-            message = f"Tried to {command.__name__} {endpoint}, but received {aiohttp_response.status}: {aiohttp_response.reason}."
-            if hasattr(requests_response, "text"):
-                text = requests_response.text()
-                if text:
-                    message += f"\nThe detailed error is:\n{text}"
+            status, reason, data = aiohttp_response
+            message = f"Tried to {command.__name__} {endpoint}, but received {status}: {reason}."
+            if data:
+                message += f"\nThe detailed error is:\n{data}"
 
         super().__init__(message)

pyproject.toml

@@ -32,8 +32,8 @@ documentation = "https://dashboard.scale.com/nucleus/docs/api"
 packages = [{include="nucleus"}]
 
 [tool.poetry.dependencies]
-python = "^3.7.0"
-requests = "^2.25.1"
+python = "^3.6.2"
+requests = "^2.23.0"
 tqdm = "^4.41.0"
 dataclasses = { version = "^0.7", python = "^3.6.1, <3.7" }
 aiohttp = "^3.7.4"

tests/test_dataset.py

@@ -1,39 +1,40 @@
+import math
+import os
+
+import pytest
+from nucleus import (
+    Dataset,
+    DatasetItem,
+    NucleusAPIError,
+    NucleusClient,
+    UploadResponse,
+)
 from nucleus.annotation import (
     BoxAnnotation,
     PolygonAnnotation,
     SegmentationAnnotation,
 )
+from nucleus.constants import (
+    DATASET_ID_KEY,
+    ERROR_ITEMS,
+    ERROR_PAYLOAD,
+    IGNORED_ITEMS,
+    NEW_ITEMS,
+    UPDATED_ITEMS,
+)
 from nucleus.job import AsyncJob, JobError
-import pytest
-import os
 
 from .helpers import (
+    LOCAL_FILENAME,
     TEST_BOX_ANNOTATIONS,
+    TEST_DATASET_NAME,
+    TEST_IMG_URLS,
     TEST_POLYGON_ANNOTATIONS,
     TEST_SEGMENTATION_ANNOTATIONS,
     TEST_SLICE_NAME,
-    TEST_DATASET_NAME,
-    TEST_IMG_URLS,
-    LOCAL_FILENAME,
     reference_id_from_url,
 )
 
-from nucleus import (
-    Dataset,
-    DatasetItem,
-    UploadResponse,
-    NucleusClient,
-    NucleusAPIError,
-)
-from nucleus.constants import (
-    NEW_ITEMS,
-    UPDATED_ITEMS,
-    IGNORED_ITEMS,
-    ERROR_ITEMS,
-    ERROR_PAYLOAD,
-    DATASET_ID_KEY,
-)
-
 TEST_AUTOTAG_DATASET = "ds_bz43jm2jwm70060b3890"
 
 
@@ -132,8 +133,20 @@ def check_is_expected_response(response):
 
 
 def test_dataset_append_local(CLIENT, dataset):
-    ds_items_local = [DatasetItem(image_location=LOCAL_FILENAME)]
+    ds_items_local_error = [
+        DatasetItem(image_location=LOCAL_FILENAME, metadata={"test": math.nan})
+    ]
+    with pytest.raises(ValueError) as e:
+        dataset.append(ds_items_local_error)
+        assert "Out of range float values are not JSON compliant" in str(
+            e.value
+        )
+    ds_items_local = [
+        DatasetItem(image_location=LOCAL_FILENAME, metadata={"test": 0})
+    ]
+
     response = dataset.append(ds_items_local)
+
     assert isinstance(response, UploadResponse)
     resp_json = response.json()
     assert resp_json[DATASET_ID_KEY] == dataset.id