Remove fsspec, s3fs and make the default loader and abstract interface (#345)

* Remove fsspec, s3fs and make the default loader and abstract interface

* Update lockfile --no-update

* Remove lock file

* Removing 3.10 from poetry install matrix again

* Bump version

Author: gatli

.circleci/config.yml

@@ -165,7 +165,7 @@ workflows:
       - test_poetry_installation:
           matrix:
             parameters:
-              python_version: ['3.6', '3.7', '3.8', '3.9'] #, '3.10'] Haven't gotten the 3.10 poetry install test to work
+              python_version: ['3.6', '3.7', '3.8', '3.9'] #, '3.10'] Need to work out a solution to support dev on 3.10
           context: Nucleus
       - test_client_installation:
           matrix:

CHANGELOG.md

@@ -5,6 +5,11 @@ All notable changes to the [Nucleus Python Client](https://github.com/scaleapi/n
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.14.15](https://github.com/scaleapi/nucleus-python-client/releases/tag/v0.14.15) - 2022-08-11
+
+### Removed
+- Removed s3fs, fsspec dependencies for simpler installation in various environments
+
 ## [0.14.14](https://github.com/scaleapi/nucleus-python-client/releases/tag/v0.14.14) - 2022-08-11
 
 ### Added

nucleus/metrics/segmentation_loader.py

@@ -1,27 +1,23 @@
+import abc
 from typing import Dict
 
 import numpy as np
-from PIL import Image
 
-try:
-    import fsspec
-except (ModuleNotFoundError, OSError):
-    from ..package_not_installed import PackageNotInstalled
 
-    fsspec = PackageNotInstalled
+class SegmentationMaskLoader(abc.ABC):
+    @abc.abstractmethod
+    def fetch(self, url: str) -> np.ndarray:
+        pass
 
 
-class SegmentationMaskLoader:
-    def __init__(self, fs: fsspec):
-        self.fs = fs
-
-    def fetch(self, url: str):
-        with self.fs.open(url) as fh:
-            img = Image.open(fh)
-        return np.asarray(img)
+class DummyLoader(SegmentationMaskLoader):
+    def fetch(self, url: str) -> np.ndarray:
+        raise NotImplementedError(
+            "This dummy loader has to be replaced with an actual implementation of an image loader"
+        )
 
 
-class InMemoryLoader:
+class InMemoryLoader(SegmentationMaskLoader):
     """We use this loader in the tests, this allows us to serve images from memory instead of fetching
     from a filesystem.
     """

nucleus/metrics/segmentation_metrics.py

@@ -9,7 +9,7 @@
 from nucleus.prediction import PredictionList, SegmentationPrediction
 
 from .base import Metric, ScalarResult
-from .segmentation_loader import SegmentationMaskLoader
+from .segmentation_loader import DummyLoader, SegmentationMaskLoader
 from .segmentation_utils import (
     FALSE_POSITIVES,
     convert_to_instance_seg_confusion,
@@ -18,14 +18,6 @@
     setup_iou_thresholds,
 )
 
-try:
-    from s3fs import S3FileSystem
-except (ModuleNotFoundError, OSError):
-    from ..package_not_installed import PackageNotInstalled
-
-    S3FileSystem = PackageNotInstalled
-
-
 # pylint: disable=useless-super-delegation
 
 
@@ -64,7 +56,7 @@ def __init__(
         """
         # TODO -> add custom filtering to Segmentation(Annotation|Prediction).annotations.(metadata|label)
         super().__init__(annotation_filters, prediction_filters)
-        self.loader = SegmentationMaskLoader(S3FileSystem(anon=False))
+        self.loader: SegmentationMaskLoader = DummyLoader()
         self.iou_threshold = iou_threshold
 
     def call_metric(

nucleus/metrics/segmentation_to_poly_metrics.py

@@ -20,28 +20,24 @@
 )
 from nucleus.prediction import PredictionList
 
-from .segmentation_loader import InMemoryLoader, SegmentationMaskLoader
-from .segmentation_metrics import (
-    SegmentationIOU,
-    SegmentationMAP,
-    SegmentationPrecision,
-    SegmentationRecall,
-)
-
-try:
-    from s3fs import S3FileSystem
-except (ModuleNotFoundError, OSError):
-    from ..package_not_installed import PackageNotInstalled
-
-    S3FileSystem = PackageNotInstalled
-
 from .base import Metric, ScalarResult
 from .polygon_metrics import (
     PolygonAveragePrecision,
     PolygonIOU,
     PolygonPrecision,
     PolygonRecall,
 )
+from .segmentation_loader import (
+    DummyLoader,
+    InMemoryLoader,
+    SegmentationMaskLoader,
+)
+from .segmentation_metrics import (
+    SegmentationIOU,
+    SegmentationMAP,
+    SegmentationPrecision,
+    SegmentationRecall,
+)
 
 
 class SegToPolyMode(str, Enum):
@@ -93,7 +89,7 @@ def __init__(
         self.enforce_label_match = enforce_label_match
         assert 0 <= confidence_threshold <= 1
         self.confidence_threshold = confidence_threshold
-        self.loader = SegmentationMaskLoader(S3FileSystem(anon=False))
+        self.loader: SegmentationMaskLoader = DummyLoader()
         self.mode = mode
 
     def call_metric(

nucleus/metrics/segmentation_utils.py

@@ -166,7 +166,7 @@ def non_max_suppress_confusion(confusion: np.ndarray, iou_threshold):
 
 
 def rasterize_polygons_to_segmentation_mask(
-    annotations: Sequence[BoxOrPolygonAnnotation], shape: Tuple[int, int]
+    annotations: Sequence[BoxOrPolygonAnnotation], shape: Tuple
 ) -> Tuple[np.ndarray, List[Segment]]:
     polys = [polygon_annotation_to_shape(a) for a in annotations]
     segments = [