Open up pydantic version to allow both v1 and v2 (#422)

* Open up pydantic version

* LOCK, STOCK, TWO SMOKING BARRELLS - (lock, changelog and bump)

* Fix mypy/pylint errors

* Ignore mypy error

Author: gatli

.pylintrc

@@ -18,7 +18,8 @@ disable=
     C0114,
     C0111,
     C0103,
-    R0904
+    R0904,
+    wrong-import-position
 
 [tool.pylint.REPORTS]
 reports=no

CHANGELOG.md

@@ -5,6 +5,11 @@ All notable changes to the [Nucleus Python Client](https://github.com/scaleapi/n
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.16.14](https://github.com/scaleapi/nucleus-python-client/releases/tag/v0.16.14) - 2024-01-03
+
+### Fixes
+- Open up Pydantic version requirements as was fixed in 0.16.11
+
 ## [0.16.13](https://github.com/scaleapi/nucleus-python-client/releases/tag/v0.16.13) - 2023-12-13
 
 ### Added

nucleus/__init__.py

@@ -45,13 +45,17 @@
 import datetime
 import os
 import warnings
-from typing import Any, Dict, List, Optional, Tuple, Union
-
-try:
-    # NOTE: we always use pydantic v1 but have to do these shenanigans to support both v1 and v2
-    import pydantic.v1 as pydantic
-except ImportError:
-    import pydantic
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
+
+if TYPE_CHECKING:
+    # Backwards compatibility is even uglier with mypy
+    from pydantic.v1 import parse_obj_as
+else:
+    try:
+        # NOTE: we always use pydantic v1 but have to do these shenanigans to support both v1 and v2
+        from pydantic.v1 import parse_obj_as
+    except ImportError:
+        from pydantic import parse_obj_as
 
 import requests
 import tqdm
@@ -216,7 +220,11 @@ def datasets(self) -> List[Dataset]:
             List of all datasets accessible to user
         """
         response = self.make_request({}, "dataset/details", requests.get)
-        dataset_details = pydantic.parse_obj_as(List[DatasetDetails], response)
+        dataset_details = (
+            parse_obj_as(  # pylint: disable=used-before-assignment
+                List[DatasetDetails], response
+            )
+        )
         return [
             Dataset(d.id, client=self, name=d.name) for d in dataset_details
         ]

nucleus/data_transfer_object/job_status.py

@@ -1,15 +1,19 @@
 # pylint: disable=E0213
 
 from datetime import datetime
-from typing import List, Optional, Union
+from typing import TYPE_CHECKING, List, Optional, Union
 
 from dateutil.parser import ParserError, parse
 
-try:
-    # NOTE: we always use pydantic v1 but have to do these shenanigans to support both v1 and v2
+if TYPE_CHECKING:
+    # Backwards compatibility is even uglier with mypy
     from pydantic.v1 import validator
-except ImportError:
-    from pydantic import validator
+else:
+    try:
+        # NOTE: we always use pydantic v1 but have to do these shenanigans to support both v1 and v2
+        from pydantic.v1 import validator
+    except ImportError:
+        from pydantic import validator
 
 from nucleus.constants import JOB_REQ_LIMIT
 from nucleus.job import CustomerJobTypes
@@ -24,7 +28,9 @@ class JobInfoRequestPayload(ImmutableModel):
     limit: Optional[int]
     show_completed: bool
 
-    @validator("from_date", "to_date")
+    @validator(  # pylint: disable=used-before-assignment
+        "from_date", "to_date"
+    )
     def ensure_date_format(cls, date):
         if date is None:
             return None

nucleus/pydantic_base.py

@@ -4,14 +4,20 @@
 As a library we want to support v1 and v2 such that we're not causing downstream problems for our users.
 This means we have to do some import shenanigans to support both v1 and v2.
 """
-try:
-    # NOTE: we always use pydantic v1 but have to do these shenanigans to support both v1 and v2
-    from pydantic.v1 import BaseModel  # pylint: disable=no-name-in-module
-except ImportError:
-    from pydantic import BaseModel
+from typing import TYPE_CHECKING
 
+if TYPE_CHECKING:
+    # Backwards compatibility is even uglier with mypy
+    from pydantic.v1 import BaseModel, Extra, ValidationError
+else:
+    try:
+        # NOTE: we always use pydantic v1 but have to do these shenanigans to support both v1 and v2
+        from pydantic.v1 import BaseModel  # pylint: disable=no-name-in-module
+    except ImportError:
+        from pydantic import BaseModel
 
-class ImmutableModel(BaseModel):
+
+class ImmutableModel(BaseModel):  # pylint: disable=used-before-assignment
     class Config:
         allow_mutation = False
 

nucleus/test_launch_integration.py

@@ -1,24 +1,28 @@
 import io
-from typing import Any, Callable, Dict, List, Optional, Type
+from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Type
 
 from PIL import Image, ImageDraw
 
-try:
-    # NOTE: we always use pydantic v1 but have to do these shenanigans to support both v1 and v2
+if TYPE_CHECKING:
+    # Backwards compatibility is even uglier with mypy
     from pydantic.v1 import BaseModel, Extra, ValidationError
-except ImportError:
-    from pydantic import BaseModel, Extra, ValidationError
+else:
+    try:
+        # NOTE: we always use pydantic v1 but have to do these shenanigans to support both v1 and v2
+        from pydantic.v1 import BaseModel, Extra, ValidationError
+    except ImportError:
+        from pydantic import BaseModel, Extra, ValidationError
 
 # From scaleapi/server/src/lib/select/api/types.ts
 # These classes specify how user models must pass output to Launch + Nucleus.
 
 
-class PointModel(BaseModel):
+class PointModel(BaseModel):  # pylint: disable=used-before-assignment
     x: float
     y: float
 
     class Config:
-        extra = Extra.forbid
+        extra = Extra.forbid  # pylint: disable=used-before-assignment
 
 
 class BoxGeometryModel(BaseModel):
@@ -106,7 +110,7 @@ def verify_output(
     for annotation in annotation_list:
         try:
             model.parse_obj(annotation)
-        except ValidationError as e:
+        except ValidationError as e:  # pylint: disable=used-before-assignment
             raise ValueError("Failed validation") from e
         if annotation["type"] != annotation_type:
             raise ValueError(

nucleus/utils.py

@@ -363,7 +363,7 @@ def upload_to_presigned_url(presigned_url: str, file_pointer: IO):
     # TODO optimize this further to deal with truly huge files and flaky internet connection.
     upload_response = requests.put(presigned_url, file_pointer)
     if not upload_response.ok:
-        raise HTTPError(
+        raise HTTPError(  # type: ignore
             f"Tried to put a file to url, but failed with status {upload_response.status_code}. The detailed error was: {upload_response.text}"
         )
 

nucleus/validate/data_transfer_objects/eval_function.py

@@ -1,10 +1,14 @@
-from typing import Any, Dict, List, Optional
+from typing import TYPE_CHECKING, Any, Dict, List, Optional
 
-try:
-    # NOTE: we always use pydantic v1 but have to do these shenanigans to support both v1 and v2
+if TYPE_CHECKING:
+    # Backwards compatibility is even uglier with mypy
     from pydantic.v1 import validator
-except ImportError:
-    from pydantic import validator
+else:
+    try:
+        # NOTE: we always use pydantic v1 but have to do these shenanigans to support both v1 and v2
+        from pydantic.v1 import validator
+    except ImportError:
+        from pydantic import validator
 
 from ...pydantic_base import ImmutableModel
 from ..constants import ThresholdComparison
@@ -63,7 +67,7 @@ class EvaluationCriterion(ImmutableModel):
     threshold: float
     eval_func_arguments: Dict[str, Any]
 
-    @validator("eval_function_id")
+    @validator("eval_function_id")  # pylint: disable=used-before-assignment
     def valid_eval_function_id(cls, v):  # pylint: disable=no-self-argument
         if not v.startswith("ef_"):
             raise ValueError(f"Expected field to start with 'ef_', got '{v}'")

nucleus/validate/data_transfer_objects/scenario_test.py

@@ -1,10 +1,14 @@
-from typing import List
+from typing import TYPE_CHECKING, List
 
-try:
-    # NOTE: we always use pydantic v1 but have to do these shenanigans to support both v1 and v2
+if TYPE_CHECKING:
+    # Backwards compatibility is even uglier with mypy
     from pydantic.v1 import validator
-except ImportError:
-    from pydantic import validator
+else:
+    try:
+        # NOTE: we always use pydantic v1 but have to do these shenanigans to support both v1 and v2
+        from pydantic.v1 import validator
+    except ImportError:
+        from pydantic import validator
 
 from nucleus.pydantic_base import ImmutableModel
 
@@ -19,7 +23,7 @@ class CreateScenarioTestRequest(ImmutableModel):
     slice_id: str
     evaluation_functions: List[EvalFunctionListEntry]
 
-    @validator("slice_id")
+    @validator("slice_id")  # pylint: disable=used-before-assignment
     def startswith_slice_indicator(cls, v):  # pylint: disable=no-self-argument
         if not v.startswith("slc_"):
             raise ValueError(f"Expected field to start with 'slc_', got '{v}'")

nucleus/validate/data_transfer_objects/scenario_test_evaluations.py

@@ -1,10 +1,14 @@
-from typing import Optional
+from typing import TYPE_CHECKING, Optional
 
-try:
-    # NOTE: we always use pydantic v1 but have to do these shenanigans to support both v1 and v2
+if TYPE_CHECKING:
+    # Backwards compatibility is even uglier with mypy
     from pydantic.v1 import root_validator, validator
-except ImportError:
-    from pydantic import root_validator, validator
+else:
+    try:
+        # NOTE: we always use pydantic v1 but have to do these shenanigans to support both v1 and v2
+        from pydantic.v1 import root_validator, validator
+    except ImportError:
+        from pydantic import root_validator, validator
 
 from nucleus.pydantic_base import ImmutableModel
 
@@ -16,7 +20,7 @@ class EvaluationResult(ImmutableModel):
     score: float = 0
     weight: float = 1
 
-    @root_validator()
+    @root_validator()  # pylint: disable=used-before-assignment
     def is_item_or_scene_provided(
         cls, values
     ):  # pylint: disable=no-self-argument
@@ -31,7 +35,7 @@ def is_item_or_scene_provided(
             )
         return values
 
-    @validator("weight")
+    @validator("weight")  # pylint: disable=used-before-assignment
     def is_normalized(cls, v):  # pylint: disable=no-self-argument
         if 0 <= v <= 1:
             return v