[Validate] Create custom evaluation functions (#305)

Author: jean-lucas

CHANGELOG.md

@@ -5,6 +5,13 @@ All notable changes to the [Nucleus Python Client](https://github.com/scaleapi/n
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.12.0](https://github.com/scaleapi/nucleus-python-client/releases/tag/v0.12.0) - 2022-05-27
+
+### Added
+
+- Allow users to create placeholder evaluation functions for Scenario Tests in Validate
+
+
 ## [0.11.2](https://github.com/scaleapi/nucleus-python-client/releases/tag/v0.11.2) - 2022-05-20
 
 ### Changed

nucleus/validate/client.py

@@ -3,8 +3,12 @@
 from nucleus.connection import Connection
 from nucleus.job import AsyncJob
 
-from .constants import SCENARIO_TEST_ID_KEY
-from .data_transfer_objects.eval_function import GetEvalFunctions
+from .constants import EVAL_FUNCTION_KEY, SCENARIO_TEST_ID_KEY
+from .data_transfer_objects.eval_function import (
+    CreateEvalFunction,
+    EvalFunctionEntry,
+    GetEvalFunctions,
+)
 from .data_transfer_objects.scenario_test import (
     CreateScenarioTestRequest,
     EvalFunctionListEntry,
@@ -175,3 +179,34 @@ def evaluate_model_on_scenario_tests(
             f"validate/{model_id}/evaluate",
         )
         return AsyncJob.from_json(response, self.connection)
+
+    def create_external_eval_function(
+        self,
+        name: str,
+    ) -> EvalFunctionEntry:
+        """Creates a new external evaluation function. This external function can be used to upload evaluation
+        results with functions defined and computed by the customer, without having to share the source code of the
+        respective function.
+
+        Args:
+            name: unique name of evaluation function
+
+        Raises:
+            - NucleusAPIError if the creation of the function fails on the server side
+            - ValidationError if the evaluation name is not well defined
+
+        Returns:
+            Created EvalFunctionConfig object.
+
+        """
+
+        response = self.connection.post(
+            CreateEvalFunction(
+                name=name,
+                is_external_function=True,
+                serialized_fn=None,
+                raw_source=None,
+            ).dict(),
+            "validate/eval_fn",
+        )
+        return EvalFunctionEntry.parse_obj(response[EVAL_FUNCTION_KEY])

nucleus/validate/constants.py

@@ -1,5 +1,6 @@
 from enum import Enum
 
+EVAL_FUNCTION_KEY = "eval_fn"
 EVALUATION_ID_KEY = "evaluation_id"
 EVAL_FUNCTION_ID_KEY = "eval_function_id"
 ID_KEY = "id"

nucleus/validate/data_transfer_objects/eval_function.py

@@ -72,6 +72,7 @@ class EvalFunctionEntry(ImmutableModel):
     id: str
     name: str
     is_public: bool
+    is_external_function: bool = False
     user_id: str
     serialized_fn: Optional[str] = None
     raw_source: Optional[str] = None
@@ -81,3 +82,24 @@ class GetEvalFunctions(ImmutableModel):
     """Expected format from GET validate/eval_fn"""
 
     eval_functions: List[EvalFunctionEntry]
+
+
+class CreateEvalFunction(ImmutableModel):
+    """Expected payload to POST validate/eval_fn"""
+
+    name: str
+    is_external_function: bool
+    serialized_fn: Optional[str] = None
+    raw_source: Optional[str] = None
+
+    @validator("name")
+    def name_is_valid(cls, v):  # pylint: disable=no-self-argument
+        if " " in v:
+            raise ValueError(
+                f"No spaces allowed in an evaluation function name, got '{v}'"
+            )
+        if len(v) == 0 or len(v) > 255:
+            raise ValueError(
+                "Name of evaluation function must be between 1-255 characters long"
+            )
+        return v

nucleus/validate/eval_functions/available_eval_functions.py

@@ -1131,10 +1131,19 @@ class CustomEvalFunction(EvalFunctionConfig):
     @classmethod
     def expected_name(cls) -> str:
         raise NotImplementedError(
-            "Custm evaluation functions are coming soon"
+            "Custom evaluation functions are coming soon"
         )  # Placeholder: See super().eval_func_entry for actual name
 
 
+class ExternalEvalFunction(EvalFunctionConfig):
+    def __call__(self, **kwargs):
+        raise NotImplementedError("Cannot call an external function")
+
+    @classmethod
+    def expected_name(cls) -> str:
+        return "external_function"
+
+
 class StandardEvalFunction(EvalFunctionConfig):
     """Class for standard Model CI eval functions that have not been added as attributes on
     AvailableEvalFunctions yet.
@@ -1186,6 +1195,7 @@ def expected_name(cls) -> str:
     CuboidPrecisionConfig,
     CategorizationF1Config,
     CustomEvalFunction,
+    ExternalEvalFunction,
     EvalFunctionNotAvailable,
     StandardEvalFunction,
     PolygonMAPConfig,
@@ -1229,7 +1239,12 @@ def __init__(self, available_functions: List[EvalFunctionEntry]):
         self._custom_to_function: Dict[str, CustomEvalFunction] = {
             f.name: CustomEvalFunction(f)
             for f in available_functions
-            if not f.is_public
+            if not f.is_public and not f.is_external_function
+        }
+        self._external_to_function: Dict[str, ExternalEvalFunction] = {
+            f.name: ExternalEvalFunction(f)
+            for f in available_functions
+            if f.is_external_function
         }
         self.bbox_iou: BoundingBoxIOUConfig = (
             self._assign_eval_function_if_defined(BoundingBoxIOUConfig)
@@ -1294,8 +1309,9 @@ def __repr__(self):
             str(name).lower() for name in self._public_func_entries.keys()
         ]
         return (
-            f"<AvailableEvaluationFunctions: public:{functions_lower} "
-            f"private: {list(self._custom_to_function.keys())}"
+            f"<AvailableEvaluationFunctions: public: {functions_lower} "
+            f"private: {list(self._custom_to_function.keys())} "
+            f"external: {list(self._external_to_function.keys())}   "
         )
 
     @property
@@ -1312,13 +1328,22 @@ def public_functions(self) -> Dict[str, EvalFunctionConfig]:
 
     @property
     def private_functions(self) -> Dict[str, CustomEvalFunction]:
-        """Custom functions uploaded to Model CI
+        """Private functions uploaded to Model CI
 
         Returns:
             Dict of function name to :class:`CustomEvalFunction`.
         """
         return self._custom_to_function
 
+    @property
+    def external_functions(self) -> Dict[str, ExternalEvalFunction]:
+        """External functions uploaded to Model CI
+
+        Returns:
+            Dict of function name to :class:`ExternalEvalFunction`.
+        """
+        return self._external_to_function
+
     def _assign_eval_function_if_defined(
         self,
         eval_function_constructor: Callable[[EvalFunctionEntry], EvalFunction],
@@ -1340,6 +1365,7 @@ def from_id(self, eval_function_id: str):
         for eval_func in itertools.chain(
             self._public_to_function.values(),
             self._custom_to_function.values(),
+            self._external_to_function.values(),
         ):
             if eval_func.id == eval_function_id:
                 return eval_func

nucleus/validate/scenario_test.py

@@ -93,7 +93,7 @@ def add_eval_function(
             ).dict(),
             "validate/scenario_test_eval_function",
         )
-        print(response)
+
         return ScenarioTestMetric(
             scenario_test_id=response[SCENARIO_TEST_ID_KEY],
             eval_function_id=response[EVAL_FUNCTION_ID_KEY],

pyproject.toml

@@ -21,7 +21,7 @@ exclude = '''
 
 [tool.poetry]
 name = "scale-nucleus"
-version = "0.11.2"
+version = "0.12.0"
 description = "The official Python client library for Nucleus, the Data Platform for AI"
 license =  "MIT"
 authors = ["Scale AI Nucleus Team <nucleusapi@scaleapi.com>"]