From 47044564f7575f2c0aa9dd1e8961a36ed781f17b Mon Sep 17 00:00:00 2001 From: Austin Raney Date: Mon, 14 Nov 2022 13:19:50 -0500 Subject: [PATCH 1/5] add ngen calibration request variant --- python/lib/communication/dmod/communication/maas_request.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/python/lib/communication/dmod/communication/maas_request.py b/python/lib/communication/dmod/communication/maas_request.py index 356ae671c..bb8fbc523 100644 --- a/python/lib/communication/dmod/communication/maas_request.py +++ b/python/lib/communication/dmod/communication/maas_request.py @@ -1364,3 +1364,7 @@ def get_request(model: str, config_data_id: str, session_secret: str = '', *args raise ValueError(err_msg.format(model, get_available_models())) return get_available_models()[model](config_data_id=config_data_id, session_secret=session_secret, *args, **kwargs) + + +class NGENCalibrationRequest(NGENRequest): + model_name: str = "ngen_cal" From 042b76acb2620f2f6502ebb17a13cefe4b8b1097 Mon Sep 17 00:00:00 2001 From: Austin Raney Date: Mon, 14 Nov 2022 14:24:37 -0500 Subject: [PATCH 2/5] get available models recursively --- .../dmod/communication/maas_request.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/python/lib/communication/dmod/communication/maas_request.py b/python/lib/communication/dmod/communication/maas_request.py index bb8fbc523..c248f7591 100644 --- a/python/lib/communication/dmod/communication/maas_request.py +++ b/python/lib/communication/dmod/communication/maas_request.py @@ -16,12 +16,18 @@ def get_available_models() -> dict: """ :return: The names of all models mapped to their class """ - available_models = dict() - for subclass in ModelExecRequest.__subclasses__(): # type: ModelExecRequest - available_models[subclass.model_name] = subclass + def recursively_get_all_model_subclasses(model_exec_request: "ModelExecRequest") -> dict: + available_models = dict() - return available_models + for subclass in model_exec_request.__subclasses__(): # type: ModelExecRequest + available_models[subclass.model_name] = subclass + # TODO: what to do if descendant subclass "overwrites" ancestor subclass? + available_models.update(recursively_get_all_model_subclasses(subclass)) + + return available_models + + return recursively_get_all_model_subclasses(ModelExecRequest) def get_available_outputs() -> set: From 35ab33e21e9be83b795b4e2576d90e4e4155f8e8 Mon Sep 17 00:00:00 2001 From: Austin Raney Date: Mon, 14 Nov 2022 14:33:16 -0500 Subject: [PATCH 3/5] adds NGENCalibrationRequest unit tests --- .../test/test_ngen_calibration_request.py | 85 +++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 python/lib/communication/dmod/test/test_ngen_calibration_request.py diff --git a/python/lib/communication/dmod/test/test_ngen_calibration_request.py b/python/lib/communication/dmod/test/test_ngen_calibration_request.py new file mode 100644 index 000000000..2dbd99b58 --- /dev/null +++ b/python/lib/communication/dmod/test/test_ngen_calibration_request.py @@ -0,0 +1,85 @@ +import unittest +from ..communication.maas_request import NGENCalibrationRequest +from dmod.core.meta_data import TimeRange + + +def create_time_range(begin, end, var=None) -> TimeRange: + serialized = { + "begin": begin, + "end": end, + "datetime_pattern": "%Y-%m-%d %H:%M:%S", + "subclass": TimeRange.__name__, + "variable": "Time" if var is None else var, + } + return TimeRange.factory_init_from_deserialized_json(serialized) + + +class TestNGENCalibrationRequest(unittest.TestCase): + @property + def time_range(self): + return create_time_range("2022-01-01 00:00:00", "2022-03-01 00:00:00") + + @property + def cat_ids_list(self): + return ["cat-1", "cat-2", "cat-3"] + + def test_model_name_eq_ngen_cal(self): + request = NGENCalibrationRequest( + session_secret="f21f27ac3d443c0948aab924bddefc64891c455a756ca77a4d86ec2f697cd13c", + cpu_count=1, + allocation_paradigm="ROUND_ROBIN", + time_range=self.time_range, + hydrofabric_uid="0123456789", + hydrofabric_data_id="9876543210", + config_data_id="02468", + bmi_cfg_data_id="02468", + catchments=self.cat_ids_list, + ) + self.assertEqual(request.model_name, NGENCalibrationRequest.model_name) + + def test_factory_init_from_deserialized_json(self): + msg = { + "model": { + "name": "ngen_cal", + "allocation_paradigm": "ROUND_ROBIN", + "cpu_count": 0, + "time_range": self.time_range.to_dict(), + "hydrofabric_data_id": "9876543210", + "hydrofabric_uid": "0123456789", + "config_data_id": "02468", + "bmi_config_data_id": "02468", + "catchments": self.cat_ids_list, + }, + "session-secret": "f21f27ac3d443c0948aab924bddefc64891c455a756ca77a4d86ec2f697cd13c", + } + request = NGENCalibrationRequest.factory_init_from_deserialized_json(msg) + self.maxDiff = None + self.assertDictEqual(msg, request.to_dict()) + + def test_factory_init_correct_response_subtype(self): + msg = { + "model": { + "name": "ngen_cal", + "allocation_paradigm": "ROUND_ROBIN", + "cpu_count": 0, + "time_range": self.time_range.to_dict(), + "hydrofabric_data_id": "9876543210", + "hydrofabric_uid": "0123456789", + "config_data_id": "02468", + "bmi_config_data_id": "02468", + "catchments": self.cat_ids_list, + }, + "session-secret": "f21f27ac3d443c0948aab924bddefc64891c455a756ca77a4d86ec2f697cd13c", + } + request = ( + NGENCalibrationRequest.factory_init_correct_subtype_from_deserialized_json( + msg + ) + ) + + self.assertEqual(request.model_name, NGENCalibrationRequest.model_name) + # use `type()` instead of `isinstance()` for specificity. + self.assertEqual(type(request), NGENCalibrationRequest) + +if __name__ == "__main__": + unittest.main() From 5b46a379221510d8b87dc728ed65aa13a78729f9 Mon Sep 17 00:00:00 2001 From: Austin Raney Date: Mon, 14 Nov 2022 14:41:25 -0500 Subject: [PATCH 4/5] expose NGENCalibrationRequest at module level --- python/lib/communication/dmod/communication/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/lib/communication/dmod/communication/__init__.py b/python/lib/communication/dmod/communication/__init__.py index 639493908..616ddd06f 100644 --- a/python/lib/communication/dmod/communication/__init__.py +++ b/python/lib/communication/dmod/communication/__init__.py @@ -3,7 +3,7 @@ PartitionerServiceClient, SchedulerClient from .maas_request import get_available_models, get_available_outputs, get_distribution_types, get_parameters, \ get_request, Distribution, ExternalRequest, ExternalRequestResponse, ModelExecRequest, ModelExecRequestResponse, \ - NWMRequest, NWMRequestResponse, Scalar, NGENRequest, NGENRequestResponse + NWMRequest, NWMRequestResponse, Scalar, NGENRequest, NGENRequestResponse, NGENCalibrationRequest from .message import AbstractInitRequest, MessageEventType, Message, Response, InvalidMessage, InvalidMessageResponse, \ InitRequestResponseReason from .metadata_message import MetadataPurpose, MetadataMessage, MetadataResponse From 3b2a73f8b37964350bb48a193c442e89ae692987 Mon Sep 17 00:00:00 2001 From: Austin Raney Date: Mon, 14 Nov 2022 14:43:35 -0500 Subject: [PATCH 5/5] add scheduler support for ngen_cal requests. NGENCalibrationRequest's, for now use container image 127.0.0.1:5000/ngen_cal:latest --- python/lib/scheduler/dmod/scheduler/scheduler.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python/lib/scheduler/dmod/scheduler/scheduler.py b/python/lib/scheduler/dmod/scheduler/scheduler.py index 2af9975ee..2f3b5dfff 100644 --- a/python/lib/scheduler/dmod/scheduler/scheduler.py +++ b/python/lib/scheduler/dmod/scheduler/scheduler.py @@ -2,7 +2,7 @@ import logging from requests.exceptions import ReadTimeout -from dmod.communication import MessageEventType, NGENRequest, NWMRequest +from dmod.communication import MessageEventType, NGENRequest, NGENCalibrationRequest, NWMRequest from dmod.core.exception import DmodRuntimeError from dmod.core.meta_data import DataCategory, DataFormat from os import getenv @@ -489,6 +489,9 @@ def determine_image_for_job(self, job: 'Job') -> str: str String name, including tag, of the appropriate Docker image for this job. """ + if isinstance(job.model_request, NGENCalibrationRequest): + return "127.0.0.1:5000/ngen_cal:latest" + if isinstance(job.model_request, NGENRequest): return "127.0.0.1:5000/ngen:latest" # For now, this is the only thing supported