From d9cfcf98b67e589fdf04ffdf199f6b836c69d667 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Mon, 3 Oct 2022 11:41:16 -0500 Subject: [PATCH 001/188] Fix fast-debug in dataservice image entrypoint. Fixing logic where pip installs the updated packages (to avoid the entire image rebuild), so make sure deps are ignored (as this was the slow part). --- docker/main/dataservice/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/main/dataservice/entrypoint.sh b/docker/main/dataservice/entrypoint.sh index e42ae462d..433ecc801 100644 --- a/docker/main/dataservice/entrypoint.sh +++ b/docker/main/dataservice/entrypoint.sh @@ -62,7 +62,7 @@ if [ -d ${UPDATED_PACKAGES_DIR:=/updated_packages} ]; then for srv in $(pip -qq freeze | grep dmod | awk -F= '{print $1}' | awk -F- '{print $2}'); do if [ $(ls ${UPDATED_PACKAGES_DIR} | grep dmod.${srv}- | wc -l) -eq 1 ]; then pip uninstall -y --no-input $(pip -qq freeze | grep dmod.${srv} | awk -F= '{print $1}') - pip install $(ls ${UPDATED_PACKAGES_DIR}/*.whl | grep dmod.${srv}-) + pip install --no-deps $(ls ${UPDATED_PACKAGES_DIR}/*.whl | grep dmod.${srv}-) fi done #pip install ${UPDATED_PACKAGES_DIR}/*.whl From 81062166471885543b62abb289095bbb4872483a Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Tue, 13 Sep 2022 16:49:07 -0500 Subject: [PATCH 002/188] Add package client name vars to example.env. --- example.env | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/example.env b/example.env index 637e9e5ce..bafbc8e63 100644 --- a/example.env +++ b/example.env @@ -108,6 +108,11 @@ TROUTE_BRANCH=ngen ## Python Packages Settings ## ######################################################################## +## The "name" of the built client Python distribution package, for purposes of installing (e.g., via pip) +PYTHON_PACKAGE_DIST_NAME_CLIENT=dmod-client +## The name of the actual Python communication package (i.e., for importing or specifying as a module on the command line) +PYTHON_PACKAGE_NAME_CLIENT=dmod.client + ## The "name" of the built communication Python distribution package, for purposes of installing (e.g., via pip) PYTHON_PACKAGE_DIST_NAME_COMMS=dmod-communication ## The name of the actual Python communication package (i.e., for importing or specifying as a module on the command line) From e7ad95d424506e95d1b898ccf4afe1b7c332f67a Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Tue, 13 Sep 2022 16:52:58 -0500 Subject: [PATCH 003/188] Add new QueryType for dataset management messages. Adding type GET_SERIALIZED_FORM to get the entire serialized state of a dataset. --- .../dmod/communication/dataset_management_message.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/lib/communication/dmod/communication/dataset_management_message.py b/python/lib/communication/dmod/communication/dataset_management_message.py index d148b6d0e..55f51e7b5 100644 --- a/python/lib/communication/dmod/communication/dataset_management_message.py +++ b/python/lib/communication/dmod/communication/dataset_management_message.py @@ -16,6 +16,7 @@ class QueryType(Enum): GET_VALUES = 6 GET_MIN_VALUE = 7 GET_MAX_VALUE = 8 + GET_SERIALIZED_FORM = 9 @classmethod def get_for_name(cls, name_str: str) -> 'QueryType': From 506e77bd4c5bc71239c8e8cb3d7dd10276a70cfb Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Mon, 3 Oct 2022 11:37:47 -0500 Subject: [PATCH 004/188] Add GET_SERIALIZED_FORM query dataservice support. --- python/services/dataservice/dmod/dataservice/service.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/python/services/dataservice/dmod/dataservice/service.py b/python/services/dataservice/dmod/dataservice/service.py index 04e1fa0af..60a27cec1 100644 --- a/python/services/dataservice/dmod/dataservice/service.py +++ b/python/services/dataservice/dmod/dataservice/service.py @@ -686,7 +686,13 @@ def _process_query(self, message: DatasetManagementMessage) -> DatasetManagement return DatasetManagementResponse(action=message.management_action, success=True, dataset_name=dataset_name, reason='Obtained {} Items List', data={DatasetManagementResponse._DATA_KEY_QUERY_RESULTS: list_of_files}) - # TODO: (later) add support for messages with other query types also + elif query_type == QueryType.GET_SERIALIZED_FORM: + dataset_name = message.dataset_name + serialized_form = self.get_known_datasets()[dataset_name].to_dict() + return DatasetManagementResponse(action=message.management_action, success=True, dataset_name=dataset_name, + reason='Obtained serialized {} dataset'.format(dataset_name), + data={DatasetManagementResponse._DATA_KEY_QUERY_RESULTS: serialized_form}) + # TODO: (later) add support for messages with other query types also else: reason = 'Unsupported {} Query Type - {}'.format(DatasetQuery.__class__.__name__, query_type.name) return DatasetManagementResponse(action=message.management_action, success=False, reason=reason) From 00cd95a85a1604180e235f1d61ebdaf63b55a7f8 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Mon, 3 Oct 2022 10:58:13 -0500 Subject: [PATCH 005/188] Refactor ExternalRequestClient async_make_request. Refactor to ensure it takes advantage of connection handling via its async context manager logic. --- python/lib/communication/dmod/communication/client.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/python/lib/communication/dmod/communication/client.py b/python/lib/communication/dmod/communication/client.py index 571f0f97d..5b5dbf68c 100644 --- a/python/lib/communication/dmod/communication/client.py +++ b/python/lib/communication/dmod/communication/client.py @@ -800,10 +800,8 @@ def _update_after_valid_response(self, response: EXTERN_REQ_R): # TODO: this can probably be taken out, as the superclass implementation should suffice async def async_make_request(self, request: EXTERN_REQ_M) -> EXTERN_REQ_R: - async with websockets.connect(self.endpoint_uri, ssl=self.client_ssl_context) as websocket: - await websocket.send(request.to_json()) - response = await websocket.recv() - return request.__class__.factory_init_correct_response_subtype(json_obj=json.loads(response)) + response = await self.async_send(request.to_json(), await_response=True) + return request.__class__.factory_init_correct_response_subtype(json_obj=json.loads(response)) @property def errors(self): From 314e36be14da47a4d480a7954696965c8e169a64 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Wed, 12 Oct 2022 13:43:43 -0500 Subject: [PATCH 006/188] Adding client methods for data/metadata retrieval. --- .../lib/client/dmod/client/request_clients.py | 126 +++++++++++++++++- 1 file changed, 123 insertions(+), 3 deletions(-) diff --git a/python/lib/client/dmod/client/request_clients.py b/python/lib/client/dmod/client/request_clients.py index 09251c01b..9a020119c 100644 --- a/python/lib/client/dmod/client/request_clients.py +++ b/python/lib/client/dmod/client/request_clients.py @@ -9,13 +9,13 @@ from dmod.communication.data_transmit_message import DataTransmitMessage, DataTransmitResponse from dmod.core.meta_data import DataCategory, DataDomain, TimeRange from pathlib import Path -from typing import List, Optional, Tuple, Type, Union +from typing import AnyStr, Dict, List, Optional, Tuple, Type, Union import json import websockets -#import logging -#logger = logging.getLogger("gui_log") +import logging +logger = logging.getLogger("client_log") class NgenRequestClient(ModelExecRequestClient[NGENRequest, NGENRequestResponse]): @@ -89,6 +89,25 @@ async def create_dataset(self, name: str, category: DataCategory, domain: DataDo async def delete_dataset(self, name: str, **kwargs) -> bool: pass + @abstractmethod + async def download_item_block(self, dataset_name: str, item_name: str, blk_start: int, blk_size: int) -> AnyStr: + """ + Download a block/chunk of a given size and start point from a specified dataset file. + + Parameters + ---------- + dataset_name + item_name + blk_start + blk_size + + Returns + ------- + AnyStr + The downloaded block/chunk. + """ + pass + @abstractmethod async def download_dataset(self, dataset_name: str, dest_dir: Path) -> bool: """ @@ -130,6 +149,14 @@ async def download_from_dataset(self, dataset_name: str, item_name: str, dest: P """ pass + @abstractmethod + async def get_dataset_content_details(self, name: str, **kwargs) -> bool: + pass + + @abstractmethod + async def get_item_size(self, dataset_name: str, item_name: str) -> int: + pass + @abstractmethod async def list_datasets(self, category: Optional[DataCategory] = None) -> List[str]: pass @@ -175,6 +202,50 @@ async def delete_dataset(self, name: str, **kwargs) -> bool: self.last_response = await self.async_make_request(request) return self.last_response is not None and self.last_response.success + async def get_dataset_content_details(self, name: str, **kwargs) -> dict: + # TODO: later add things like created and last updated perhaps + query = DatasetQuery(query_type=QueryType.GET_DATASET_ITEMS) + request = DatasetManagementMessage(action=ManagementAction.QUERY, query=query, dataset_name=name) + self.last_response: DatasetManagementResponse = await self.async_make_request(request) + if self.last_response.success: + return self.last_response.data + else: + return {} + + async def get_item_size(self, dataset_name: str, item_name: str) -> int: + query = DatasetQuery(query_type=QueryType.GET_ITEM_SIZE, item_name=item_name) + request = DatasetManagementMessage(action=ManagementAction.QUERY, query=query, dataset_name=dataset_name, + data_location=item_name) + self.last_response: DatasetManagementResponse = await self.async_make_request(request) + if self.last_response.success: + return self.last_response.data + else: + return -1 + + async def download_item_block(self, dataset_name: str, item_name: str, blk_start: int, blk_size: int) -> AnyStr: + """ + Download a block/chunk of a given size and start point from a specified dataset file. + + Parameters + ---------- + dataset_name + item_name + blk_start + blk_size + + Returns + ------- + AnyStr + The downloaded block/chunk. + """ + request = DatasetManagementMessage(action=ManagementAction.REQUEST_DATA, dataset_name=dataset_name, + data_location=item_name, blk_start=blk_start, blk_size=blk_size) + self.last_response: DatasetManagementResponse = await self.async_make_request(request) + if self.last_response.success: + return self.last_response.data + else: + return '' + async def download_dataset(self, dataset_name: str, dest_dir: Path) -> bool: """ Download an entire dataset to a local directory. @@ -460,6 +531,55 @@ async def delete_dataset(self, name: str, **kwargs) -> bool: self.last_response = await self.async_make_request(request) return self.last_response is not None and self.last_response.success + async def download_item_block(self, dataset_name: str, item_name: str, blk_start: int, blk_size: int) -> AnyStr: + """ + Download a block/chunk of a given size and start point from a specified dataset file. + + Parameters + ---------- + dataset_name + item_name + blk_start + blk_size + + Returns + ------- + AnyStr + The downloaded block/chunk. + """ + await self._async_acquire_session_info() + request = MaaSDatasetManagementMessage(action=ManagementAction.REQUEST_DATA, dataset_name=dataset_name, + session_secret=self.session_secret, data_location=item_name, + blk_start=blk_start, blk_size=blk_size) + self.last_response: DatasetManagementResponse = await self.async_make_request(request) + if self.last_response.success: + return self.last_response.data + else: + return '' + + async def get_item_size(self, dataset_name: str, item_name: str) -> int: + await self._async_acquire_session_info() + query = DatasetQuery(query_type=QueryType.GET_ITEM_SIZE, item_name=item_name) + request = MaaSDatasetManagementMessage(action=ManagementAction.QUERY, query=query, dataset_name=dataset_name, + session_secret=self._session_secret, data_location=item_name) + self.last_response: DatasetManagementResponse = await self.async_make_request(request) + if self.last_response.success: + return self.last_response.data + else: + return -1 + + async def get_dataset_content_details(self, name: str, **kwargs) -> List: + # TODO: later add things like created and last updated perhaps + await self._async_acquire_session_info() + query = DatasetQuery(query_type=QueryType.GET_DATASET_ITEMS) + request = MaaSDatasetManagementMessage(session_secret=self.session_secret, action=ManagementAction.QUERY, + query=query, dataset_name=name) + self.last_response: DatasetManagementResponse = await self.async_make_request(request) + if self.last_response.success: + return self.last_response.data[DatasetManagementResponse._DATA_KEY_QUERY_RESULTS] + else: + return [] + async def download_dataset(self, dataset_name: str, dest_dir: Path) -> bool: await self._async_acquire_session_info() try: From 30cbcca34a44f6b112c55efeb4c2e6733b9b8554 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Tue, 13 Sep 2022 16:50:05 -0500 Subject: [PATCH 007/188] Refactor dataset request handler to use try block. --- .../externalrequests/maas_request_handlers.py | 37 ++++++++++--------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/python/lib/externalrequests/dmod/externalrequests/maas_request_handlers.py b/python/lib/externalrequests/dmod/externalrequests/maas_request_handlers.py index dff0b45ac..1dec8137b 100644 --- a/python/lib/externalrequests/dmod/externalrequests/maas_request_handlers.py +++ b/python/lib/externalrequests/dmod/externalrequests/maas_request_handlers.py @@ -291,25 +291,28 @@ async def handle_request(self, request: MaaSDatasetManagementMessage, **kwargs) session, is_authorized, reason, msg = await self.get_authorized_session(request) if not is_authorized: return MaaSDatasetManagementResponse(success=False, reason=reason.name, message=msg) - # In this case, we actually can pass the request as-is straight through (i.e., after confirming authorization) - async with self.service_client as client: - # Have to handle these two slightly differently, since multiple message will be going over the websocket - if request.management_action == ManagementAction.REQUEST_DATA: - await client.connection.send(str(request)) - mgmt_response = await self._handle_data_download(client_websocket=kwargs['upstream_websocket'], - service_websocket=client.connection) - elif request.management_action == ManagementAction.ADD_DATA: - await client.connection.send(str(request)) - mgmt_response = await self._handle_data_upload(client_websocket=kwargs['upstream_websocket'], - service_websocket=client.connection) - else: - mgmt_response = await client.async_make_request(request) - logging.debug("************* {} received response:\n{}".format(self.__class__.__name__, str(mgmt_response))) - # Likewise, can just send back the response from the internal service client - return MaaSDatasetManagementResponse.factory_create(mgmt_response) + try: + # In this case, we actually can pass the request as-is straight through (i.e., after confirming authorization) + async with self.service_client as client: + # Have to handle these two slightly differently, since multiple message will be going over the websocket + if request.management_action == ManagementAction.REQUEST_DATA: + await client.connection.send(str(request)) + mgmt_response = await self._handle_data_download(client_websocket=kwargs['upstream_websocket'], + service_websocket=client.connection) + elif request.management_action == ManagementAction.ADD_DATA: + await client.connection.send(str(request)) + mgmt_response = await self._handle_data_upload(client_websocket=kwargs['upstream_websocket'], + service_websocket=client.connection) + else: + mgmt_response = await client.async_make_request(request) + logging.debug("************* {} received response:\n{}".format(self.__class__.__name__, str(mgmt_response))) + # Likewise, can just send back the response from the internal service client + return MaaSDatasetManagementResponse.factory_create(mgmt_response) + except Exception as e: + raise e @property def service_client(self) -> DataServiceClient: if self._service_client is None: - self._service_client = DataServiceClient(self.service_url, self.service_ssl_dir) + self._service_client = DataServiceClient(endpoint_uri=self.service_url, ssl_directory=self.service_ssl_dir) return self._service_client From b516c5c740b7a8b311dd2d20df90606f9194f188 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Tue, 13 Sep 2022 16:56:02 -0500 Subject: [PATCH 008/188] Add get_serialized_datasets to external client. Add new function to get serialized dataset details to DatasetExternalClient. --- .../lib/client/dmod/client/request_clients.py | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/python/lib/client/dmod/client/request_clients.py b/python/lib/client/dmod/client/request_clients.py index 9a020119c..2ea95d98b 100644 --- a/python/lib/client/dmod/client/request_clients.py +++ b/python/lib/client/dmod/client/request_clients.py @@ -625,6 +625,37 @@ async def download_from_dataset(self, dataset_name: str, item_name: str, dest: P if not has_data: return message_object + async def get_serialized_datasets(self, dataset_name: Optional[str] = None) -> Dict[str, dict]: + """ + Get dataset objects in serialized form, either for all datasets or for the one with the provided name. + + Parameters + ---------- + dataset_name : Optional[str] + The name of a specific dataset to get serialized details of, if only one should be obtained. + + Returns + ------- + Dict[str, dict] + A dictionary, keyed by dataset name, of serialized dataset objects. + """ + # TODO: may need to generalize this and add to super class + if dataset_name is None: + datasets = await self.list_datasets() + else: + datasets = [dataset_name] + serialized = dict() + action = ManagementAction.QUERY + query = DatasetQuery(query_type=QueryType.GET_SERIALIZED_FORM) + for d in datasets: + request = MaaSDatasetManagementMessage(action=action, query=query, dataset_name=d, + session_secret=self.session_secret) + self.last_response: DatasetManagementResponse = await self.async_make_request(request) + if self.last_response.success: + serialized[d] = self.last_response.data['dataset'] + # TODO: what to do if any are not successful + return serialized + async def list_datasets(self, category: Optional[DataCategory] = None) -> List[str]: await self._async_acquire_session_info() action = ManagementAction.LIST_ALL if category is None else ManagementAction.SEARCH From 8445296ddc8d4ae4227b921cd89580c72b5dfe81 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Mon, 3 Oct 2022 11:01:25 -0500 Subject: [PATCH 009/188] Improve DatasetExternalClient. Fix issue with not acquiring a session, and wrapping some things in try block to catch and log exceptions. --- .../lib/client/dmod/client/request_clients.py | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/python/lib/client/dmod/client/request_clients.py b/python/lib/client/dmod/client/request_clients.py index 2ea95d98b..bd89d40e7 100644 --- a/python/lib/client/dmod/client/request_clients.py +++ b/python/lib/client/dmod/client/request_clients.py @@ -643,18 +643,24 @@ async def get_serialized_datasets(self, dataset_name: Optional[str] = None) -> D if dataset_name is None: datasets = await self.list_datasets() else: + # TODO: improve how this is use so that it can be safely, efficiently put everywhere it **may** be needed + await self._async_acquire_session_info() datasets = [dataset_name] serialized = dict() action = ManagementAction.QUERY query = DatasetQuery(query_type=QueryType.GET_SERIALIZED_FORM) - for d in datasets: - request = MaaSDatasetManagementMessage(action=action, query=query, dataset_name=d, - session_secret=self.session_secret) - self.last_response: DatasetManagementResponse = await self.async_make_request(request) - if self.last_response.success: - serialized[d] = self.last_response.data['dataset'] - # TODO: what to do if any are not successful - return serialized + try: + for d in datasets: + request = MaaSDatasetManagementMessage(action=action, query=query, dataset_name=d, + session_secret=self.session_secret) + self.last_response: DatasetManagementResponse = await self.async_make_request(request) + if self.last_response.success: + serialized[d] = self.last_response.data[DatasetManagementResponse._DATA_KEY_QUERY_RESULTS] + # TODO: what to do if any are not successful + return serialized + except Exception as e: + logger.error(e) + raise e async def list_datasets(self, category: Optional[DataCategory] = None) -> List[str]: await self._async_acquire_session_info() From a2927b21cfc54d73197a679d56933bca3fb4aabb Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Tue, 13 Sep 2022 16:57:55 -0500 Subject: [PATCH 010/188] Bump dataservice dependencies. --- python/services/dataservice/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/services/dataservice/setup.py b/python/services/dataservice/setup.py index b839e55c9..0a9ceb885 100644 --- a/python/services/dataservice/setup.py +++ b/python/services/dataservice/setup.py @@ -17,7 +17,7 @@ author_email='', url='', license='', - install_requires=['dmod-core>=0.3.0', 'dmod-communication>=0.7.1', 'dmod-scheduler>=0.7.0', 'dmod-modeldata>=0.9.0', + install_requires=['dmod-core>=0.3.0', 'dmod-communication>=0.8.0', 'dmod-scheduler>=0.7.0', 'dmod-modeldata>=0.9.0', 'redis'], packages=find_namespace_packages(exclude=['dmod.test', 'deprecated', 'conf', 'schemas', 'ssl', 'src']) ) From f23c72186e07f85d452a71c13962ba1d34441c99 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Mon, 3 Oct 2022 11:36:20 -0500 Subject: [PATCH 011/188] Fix LIST_FILES query response bug in dataservice. Fixing bug in response to LIST_FILES query, where reason text was not being assembled entirely correctly. --- python/services/dataservice/dmod/dataservice/service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/services/dataservice/dmod/dataservice/service.py b/python/services/dataservice/dmod/dataservice/service.py index 60a27cec1..7926525d8 100644 --- a/python/services/dataservice/dmod/dataservice/service.py +++ b/python/services/dataservice/dmod/dataservice/service.py @@ -684,7 +684,7 @@ def _process_query(self, message: DatasetManagementMessage) -> DatasetManagement dataset_name = message.dataset_name list_of_files = self.get_known_datasets()[dataset_name].manager.list_files(dataset_name) return DatasetManagementResponse(action=message.management_action, success=True, dataset_name=dataset_name, - reason='Obtained {} Items List', + reason='Obtained {} Items List'.format(dataset_name), data={DatasetManagementResponse._DATA_KEY_QUERY_RESULTS: list_of_files}) elif query_type == QueryType.GET_SERIALIZED_FORM: dataset_name = message.dataset_name From 8e45fbd1e145b0e3e46045861860c2b662debee2 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Wed, 12 Oct 2022 13:30:53 -0500 Subject: [PATCH 012/188] Update dataset manager abstract interface. Adding optional offset and length params to abstract interface definition of get_data, and adding get_file_stat abstract method. --- python/lib/core/dmod/core/dataset.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/python/lib/core/dmod/core/dataset.py b/python/lib/core/dmod/core/dataset.py index b1e5ca2aa..bbb9701c5 100644 --- a/python/lib/core/dmod/core/dataset.py +++ b/python/lib/core/dmod/core/dataset.py @@ -819,7 +819,8 @@ def filter(self, base_dataset: Dataset, restrictions: List[Union[ContinuousRestr pass @abstractmethod - def get_data(self, dataset_name: str, item_name: str, **kwargs) -> Union[bytes, Any]: + def get_data(self, dataset_name: str, item_name: str, offset: Optional[int] = None, length: Optional[int] = None, + **kwargs) -> Union[bytes, Any]: """ Get data from this dataset. @@ -832,6 +833,10 @@ def get_data(self, dataset_name: str, item_name: str, **kwargs) -> Union[bytes, The dataset from which to get data. item_name : str The name of the object from which to get data. + offset : Optional[int] + Optional start byte position of object data. + length : Optional[int] + Optional number of bytes of object data from offset. kwargs Implementation-specific params for representing what data to get and how to get and deliver it. @@ -881,6 +886,26 @@ def link_user(self, user: DatasetUser, dataset: Dataset) -> bool: self._dataset_users[dataset.name].add(user.uuid) return True + @abstractmethod + def get_file_stat(self, dataset_name: str, file_name, **kwargs) -> Dict[str, Any]: + """ + Get the meta information about the given file. + + Parameters + ---------- + dataset_name : str + The name of the dataset containing the file of interest. + file_name : str + The name of the file of interest. + kwargs + + Returns + ------- + dict + Meta information about the given file, in dictionary form. + """ + pass + @abstractmethod def list_files(self, dataset_name: str, **kwargs) -> List[str]: """ From 60c6e8e7f0f4facc1ba386fab65888d0661ab722 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Wed, 12 Oct 2022 13:33:28 -0500 Subject: [PATCH 013/188] More QueryType elements and update DatasetQuery. Adding support for querying about dataset items. --- .../dataset_management_message.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/python/lib/communication/dmod/communication/dataset_management_message.py b/python/lib/communication/dmod/communication/dataset_management_message.py index 55f51e7b5..73a80b164 100644 --- a/python/lib/communication/dmod/communication/dataset_management_message.py +++ b/python/lib/communication/dmod/communication/dataset_management_message.py @@ -17,6 +17,10 @@ class QueryType(Enum): GET_MIN_VALUE = 7 GET_MAX_VALUE = 8 GET_SERIALIZED_FORM = 9 + GET_LAST_UPDATED = 10 + GET_SIZE = 11 + GET_ITEM_SIZE = 12 + GET_DATASET_ITEMS = 13 @classmethod def get_for_name(cls, name_str: str) -> 'QueryType': @@ -43,26 +47,32 @@ def get_for_name(cls, name_str: str) -> 'QueryType': class DatasetQuery(Serializable): _KEY_QUERY_TYPE = 'query_type' + _KEY_ITEM_NAME = 'item_name' @classmethod def factory_init_from_deserialized_json(cls, json_obj: dict) -> Optional['DatasetQuery']: try: - return cls(query_type=QueryType.get_for_name(json_obj[cls._KEY_QUERY_TYPE])) + return cls(query_type=QueryType.get_for_name(json_obj[cls._KEY_QUERY_TYPE]), + item_name=json_obj.get(cls._KEY_ITEM_NAME)) except Exception as e: return None def __hash__(self): - return hash(self.query_type) + return hash('{}{}'.format(self.query_type.name, self.item_name if self.item_name is not None else '')) def __eq__(self, other): - return isinstance(other, DatasetQuery) and self.query_type == other.query_type + return isinstance(other, DatasetQuery) and self.query_type == other.query_type \ + and self.item_name == other.item_name - def __init__(self, query_type: QueryType): + def __init__(self, query_type: QueryType, item_name: Optional[str] = None): self.query_type = query_type + self.item_name = item_name def to_dict(self) -> Dict[str, Union[str, Number, dict, list]]: serial = dict() serial[self._KEY_QUERY_TYPE] = self.query_type.name + if self.item_name is not None: + serial[self._KEY_ITEM_NAME] = self.item_name return serial From c893164eac6fc08129e86288ab27398f3edd91c7 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Wed, 12 Oct 2022 13:34:17 -0500 Subject: [PATCH 014/188] Update dataset message to support start and size. Updating to support indicating start and size of data for partial transfers. --- .../dataset_management_message.py | 32 +++++++++++++++++-- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/python/lib/communication/dmod/communication/dataset_management_message.py b/python/lib/communication/dmod/communication/dataset_management_message.py index 73a80b164..d21192906 100644 --- a/python/lib/communication/dmod/communication/dataset_management_message.py +++ b/python/lib/communication/dmod/communication/dataset_management_message.py @@ -192,6 +192,8 @@ class DatasetManagementMessage(AbstractInitRequest): _SERIAL_KEY_CATEGORY = 'category' _SERIAL_KEY_DATA_DOMAIN = 'data_domain' _SERIAL_KEY_DATA_LOCATION = 'data_location' + _SERIAL_KEY_DATA_BLK_START = 'data_blk_start' + _SERIAL_KEY_DATA_BLK_SIZE = 'data_blk_size' _SERIAL_KEY_DATASET_NAME = 'dataset_name' _SERIAL_KEY_IS_PENDING_DATA = 'pending_data' _SERIAL_KEY_QUERY = 'query' @@ -228,6 +230,8 @@ def factory_init_from_deserialized_json(cls, json_obj: dict) -> Optional['Datase category_str = json_obj.get(cls._SERIAL_KEY_CATEGORY) category = None if category_str is None else DataCategory.get_for_name(category_str) data_loc = json_obj.get(cls._SERIAL_KEY_DATA_LOCATION) + data_blk_start = json_obj.get(cls._SERIAL_KEY_DATA_BLK_START) + data_blk_size = json_obj.get(cls._SERIAL_KEY_DATA_BLK_SIZE) #page = json_obj[cls._SERIAL_KEY_PAGE] if cls._SERIAL_KEY_PAGE in json_obj else None if cls._SERIAL_KEY_QUERY in json_obj: query = DatasetQuery.factory_init_from_deserialized_json(json_obj[cls._SERIAL_KEY_QUERY]) @@ -240,7 +244,7 @@ def factory_init_from_deserialized_json(cls, json_obj: dict) -> Optional['Datase return deserialized_class(action=action, dataset_name=dataset_name, category=category, is_read_only_dataset=json_obj[cls._SERIAL_KEY_IS_READ_ONLY], domain=domain, - data_location=data_loc, + data_location=data_loc, blk_start=data_blk_start, blk_size=data_blk_size, is_pending_data=json_obj.get(cls._SERIAL_KEY_IS_PENDING_DATA), #page=page, query=query, **deserialized_class_kwargs) except Exception as e: @@ -272,8 +276,8 @@ def __hash__(self): def __init__(self, action: ManagementAction, dataset_name: Optional[str] = None, is_read_only_dataset: bool = False, category: Optional[DataCategory] = None, domain: Optional[DataDomain] = None, - data_location: Optional[str] = None, is_pending_data: bool = False, - query: Optional[DatasetQuery] = None, *args, **kwargs): + data_location: Optional[str] = None, blk_start: Optional[int] = None, blk_size: Optional[int] = None, + is_pending_data: bool = False, query: Optional[DatasetQuery] = None, *args, **kwargs): """ Initialize this instance. @@ -289,6 +293,10 @@ def __init__(self, action: ManagementAction, dataset_name: Optional[str] = None, The optional category of the involved dataset or datasets, when applicable; defaults to ``None``. data_location : Optional[str] Optional location/file/object/etc. for acted-upon data. + blk_start : Optional[int] + Optional starting point for when acting upon a block/chunk of data. + blk_size : Optional[int] + Optional block size for when acting upon a block/chunk of data. is_pending_data : bool Whether the sender has data pending transmission after this message (default: ``False``). query : Optional[DatasetQuery] @@ -313,9 +321,19 @@ def __init__(self, action: ManagementAction, dataset_name: Optional[str] = None, self._category = category self._domain = domain self._data_location = data_location + self._blk_start = blk_start + self._blk_size = blk_size self._query = query self._is_pending_data = is_pending_data + @property + def blk_size(self) -> Optional[int]: + return self._blk_size + + @property + def blk_start(self) -> Optional[int]: + return self._blk_start + @property def data_location(self) -> Optional[str]: """ @@ -417,6 +435,10 @@ def to_dict(self) -> Dict[str, Union[str, Number, dict, list]]: serial[self._SERIAL_KEY_CATEGORY] = self.data_category.name if self.data_location is not None: serial[self._SERIAL_KEY_DATA_LOCATION] = self.data_location + if self._blk_start is not None: + serial[self._SERIAL_KEY_DATA_BLK_START] = self._blk_start + if self._blk_size is not None: + serial[self._SERIAL_KEY_DATA_BLK_SIZE] = self._blk_size if self.data_domain is not None: serial[self._SERIAL_KEY_DATA_DOMAIN] = self.data_domain.to_dict() if self.query is not None: @@ -613,6 +635,10 @@ def __init__(self, session_secret: str, *args, **kwargs): is_read_only_dataset : bool category : Optional[DataCategory] data_location : Optional[str] + blk_start : Optional[int] + Optional starting point for when acting upon a block/chunk of data. + blk_size : Optional[int] + Optional block size for when acting upon a block/chunk of data. is_pending_data : bool query : Optional[DataQuery] """ From a9dbd5b97299bd192d7fa702a47b8ff64b65fd10 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Tue, 13 Sep 2022 16:58:11 -0500 Subject: [PATCH 015/188] Bump requestservice dependencies. --- python/services/requestservice/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/services/requestservice/setup.py b/python/services/requestservice/setup.py index fafbc703d..b62f49b99 100644 --- a/python/services/requestservice/setup.py +++ b/python/services/requestservice/setup.py @@ -17,7 +17,7 @@ author_email='', url='', license='', - install_requires=['websockets', 'dmod-core>=0.1.0', 'dmod-communication>=0.7.0', 'dmod-access>=0.2.0', + install_requires=['websockets', 'dmod-core>=0.2.0', 'dmod-communication>=0.8.0', 'dmod-access>=0.2.0', 'dmod-externalrequests>=0.3.0'], packages=find_namespace_packages(exclude=['dmod.test', 'schemas', 'ssl', 'src']) ) From 4198db5dd99e5a850a927fdb6439a96c38ce47b9 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Tue, 13 Sep 2022 16:57:33 -0500 Subject: [PATCH 016/188] Bump requestservice to 0.6.0. --- python/services/requestservice/dmod/requestservice/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/services/requestservice/dmod/requestservice/_version.py b/python/services/requestservice/dmod/requestservice/_version.py index f93e0653b..83e147c62 100644 --- a/python/services/requestservice/dmod/requestservice/_version.py +++ b/python/services/requestservice/dmod/requestservice/_version.py @@ -1 +1 @@ -__version__ = '0.5.2' \ No newline at end of file +__version__ = '0.6.0' \ No newline at end of file From 4444b62e790abaadac9237ae640f6459d162d88c Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Wed, 12 Oct 2022 13:41:13 -0500 Subject: [PATCH 017/188] Optimize object store dataset reloading. Optimizing the reloading of datasets on object store manager startup. --- .../dmod/modeldata/data/object_store_manager.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/python/lib/modeldata/dmod/modeldata/data/object_store_manager.py b/python/lib/modeldata/dmod/modeldata/data/object_store_manager.py index dffb5a8cf..910d98923 100644 --- a/python/lib/modeldata/dmod/modeldata/data/object_store_manager.py +++ b/python/lib/modeldata/dmod/modeldata/data/object_store_manager.py @@ -59,8 +59,13 @@ def __init__(self, obj_store_host_str: str, access_key: Optional[str] = None, se # For any buckets that have the standard serialized object (i.e., were for datasets previously), reload them for bucket_name in self.list_buckets(): serialized_item = self._gen_dataset_serial_obj_name(bucket_name) - if serialized_item in [o.object_name for o in self._client.list_objects(bucket_name)]: + try: self.reload(reload_from=bucket_name, serialized_item=serialized_item) + except minio.error.S3Error as e: + # Continue with looping through buckets and initializing if we get this particular exception and + # error code, but otherwise pass through the exception + if e.code != "NoSuchKey": + raise e except Exception as e: self._errors.append(e) # TODO: consider if we should not re-throw this (which would likely force us to ensure users checked this) @@ -552,12 +557,14 @@ def reload(self, reload_from: str, serialized_item: Optional[str] = None) -> Dat if serialized_item is None: serialized_item = self._gen_dataset_serial_obj_name(reload_from) + response_obj = None try: response_obj = self._client.get_object(bucket_name=reload_from, object_name=serialized_item) response_data = json.loads(response_obj.data.decode()) finally: - response_obj.close() - response_obj.release_conn() + if response_obj is not None: + response_obj.close() + response_obj.release_conn() # If we can safely infer it, make sure the "type" key is set in cases when it is missing if len(self.supported_dataset_types) == 1 and Dataset._KEY_TYPE not in response_data: From 7125fb887f598732c982d179c249f4aa3ccbd74e Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Wed, 12 Oct 2022 13:42:39 -0500 Subject: [PATCH 018/188] Update object store manager for interface changes. Accounting for changes to get_data parameters and implementing get_file_stat. --- .../modeldata/data/object_store_manager.py | 46 ++++++++++++++----- 1 file changed, 35 insertions(+), 11 deletions(-) diff --git a/python/lib/modeldata/dmod/modeldata/data/object_store_manager.py b/python/lib/modeldata/dmod/modeldata/data/object_store_manager.py index 910d98923..551cb2aa1 100644 --- a/python/lib/modeldata/dmod/modeldata/data/object_store_manager.py +++ b/python/lib/modeldata/dmod/modeldata/data/object_store_manager.py @@ -10,7 +10,7 @@ from minio.api import ObjectWriteResult from minio.deleteobjects import DeleteObject from pathlib import Path -from typing import Dict, List, Optional, Set, Tuple +from typing import Any, Dict, List, Optional, Set, Tuple from uuid import UUID @@ -423,7 +423,32 @@ def delete_data(self, dataset_name: str, **kwargs) -> bool: self._errors.extend(error_list) return False - def get_data(self, dataset_name: str, item_name: str, **kwargs) -> bytes: + def get_file_stat(self, dataset_name: str, file_name, **kwargs) -> Dict[str, Any]: + """ + Get the meta information about the given file. + + Parameters + ---------- + dataset_name : str + The name of the dataset containing the file of interest. + file_name : str + The name of the file of interest. + kwargs + + Returns + ------- + dict + Meta information about the given file, in dictionary form. + """ + obj_stat = self._client.stat_object(dataset_name, file_name) + as_dict = dict() + as_dict["name"] = obj_stat.object_name + as_dict["size"] = obj_stat.size + # TODO: get more of this if worth it + return as_dict + + def get_data(self, dataset_name: str, item_name: str, offset: Optional[int] = None, length: Optional[int] = None, + **kwargs) -> bytes: """ Get data from this dataset. @@ -437,15 +462,12 @@ def get_data(self, dataset_name: str, item_name: str, **kwargs) -> bytes: The name of the dataset (i.e., bucket) from which to get data. item_name : str The name of the object from which to get data. - kwargs - Implementation-specific params for representing what data to get and how to get and deliver it. - - Keyword Args - ------- - offset : int + offset : Optional[int] Optional start byte position of object data. - length : int + length : Optional[int] Optional number of bytes of object data from offset. + kwargs + Implementation-specific params for representing what data to get and how to get and deliver it. Returns ------- @@ -455,8 +477,10 @@ def get_data(self, dataset_name: str, item_name: str, **kwargs) -> bytes: if item_name not in self.list_files(dataset_name): raise RuntimeError('Cannot get data for non-existing {} file in {} dataset'.format(item_name, dataset_name)) optional_params = dict() - for key in [k for k in self.data_chunking_params if k in kwargs]: - optional_params[key] = kwargs[key] + if offset is not None: + optional_params['offset'] = offset + if length is not None: + optional_params['length'] = length response_object = self._client.get_object(bucket_name=dataset_name, object_name=item_name, **optional_params) return response_object.data From 9024a0c7f0130e8d27a16c10c014ed5332ef9b58 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Wed, 12 Oct 2022 13:35:57 -0500 Subject: [PATCH 019/188] Update modeldata dependency versions. --- python/lib/modeldata/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/lib/modeldata/setup.py b/python/lib/modeldata/setup.py index 7c389a0e7..0a3e64ac4 100644 --- a/python/lib/modeldata/setup.py +++ b/python/lib/modeldata/setup.py @@ -20,7 +20,7 @@ author_email='', url='', license='', - install_requires=['numpy>=1.20.1', 'pandas', 'geopandas', 'dmod-communication>=0.4.2', 'dmod-core>=0.3.0', 'minio', + install_requires=['numpy>=1.20.1', 'pandas', 'geopandas', 'dmod-communication>=0.9.1', 'dmod-core>=0.3.0', 'minio', 'aiohttp<=3.7.4', 'hypy@git+https://github.com/NOAA-OWP/hypy@master#egg=hypy&subdirectory=python'], packages=find_namespace_packages(exclude=['dmod.test', 'schemas', 'ssl', 'src']) ) From 6ecacedb193995975e7da88e8aac776c04d1b565 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Wed, 12 Oct 2022 14:32:19 -0500 Subject: [PATCH 020/188] Update client unit tests for interface changes. --- .../client/dmod/test/test_dataset_client.py | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/python/lib/client/dmod/test/test_dataset_client.py b/python/lib/client/dmod/test/test_dataset_client.py index b266658c7..37ff6e386 100644 --- a/python/lib/client/dmod/test/test_dataset_client.py +++ b/python/lib/client/dmod/test/test_dataset_client.py @@ -1,7 +1,7 @@ import unittest from ..client.request_clients import DataCategory, DatasetClient, DatasetManagementResponse, MaaSDatasetManagementResponse from pathlib import Path -from typing import List, Optional +from typing import List, Optional, AnyStr class SimpleMockDatasetClient(DatasetClient): @@ -28,6 +28,24 @@ async def download_from_dataset(self, dataset_name: str, item_name: str, dest: P """ Mock implementation, always returning ``False``. """ return False + async def download_item_block(self, dataset_name: str, item_name: str, blk_start: int, blk_size: int) -> AnyStr: + """ + Mock implementation, always returning empty string. + """ + return '' + + async def get_dataset_content_details(self, name: str, **kwargs) -> bool: + """ + Mock implementation, always returning ``False``. + """ + return False + + async def get_item_size(self, dataset_name: str, item_name: str) -> int: + """ + Mock implementation always returning ``1``. + """ + return 1 + async def list_datasets(self, category: Optional[DataCategory] = None) -> List[str]: """ Mock implementation, always returning an empty list. """ return [] From dbbcf5a572ff7e5d50aabc9e53ded007ce87f321 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Wed, 12 Oct 2022 13:43:54 -0500 Subject: [PATCH 021/188] Bump client package version to 0.2.0. --- python/lib/client/dmod/client/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/lib/client/dmod/client/_version.py b/python/lib/client/dmod/client/_version.py index df9144c54..7fd229a32 100644 --- a/python/lib/client/dmod/client/_version.py +++ b/python/lib/client/dmod/client/_version.py @@ -1 +1 @@ -__version__ = '0.1.1' +__version__ = '0.2.0' From f851e75fc1fbbd7cfb1262c8442cec45baa4e1cc Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Wed, 12 Oct 2022 13:38:21 -0500 Subject: [PATCH 022/188] Update dataservice response handling. Updating service to respond to GET_DATASET_ITEMS queries and requests for data with an offset start (i.e. partials). --- .../dataservice/dmod/dataservice/service.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/python/services/dataservice/dmod/dataservice/service.py b/python/services/dataservice/dmod/dataservice/service.py index 7926525d8..1d1c7c37d 100644 --- a/python/services/dataservice/dmod/dataservice/service.py +++ b/python/services/dataservice/dmod/dataservice/service.py @@ -692,6 +692,13 @@ def _process_query(self, message: DatasetManagementMessage) -> DatasetManagement return DatasetManagementResponse(action=message.management_action, success=True, dataset_name=dataset_name, reason='Obtained serialized {} dataset'.format(dataset_name), data={DatasetManagementResponse._DATA_KEY_QUERY_RESULTS: serialized_form}) + if query_type == QueryType.GET_DATASET_ITEMS: + dataset = self.get_known_datasets()[message.dataset_name] + mgr = dataset.manager + item_details: List[dict] = [mgr.get_file_stat(dataset.name, f) for f in mgr.list_files(dataset.name)] + return DatasetManagementResponse(action=message.management_action, success=True, dataset_name=dataset.name, + reason='Obtained file details for {} dataset'.format(dataset.name), + data={DatasetManagementResponse._DATA_KEY_QUERY_RESULTS: item_details}) # TODO: (later) add support for messages with other query types also else: reason = 'Unsupported {} Query Type - {}'.format(DatasetQuery.__class__.__name__, query_type.name) @@ -896,6 +903,14 @@ async def listener(self, websocket: WebSocketServerProtocol, path): partial_indx = 0 elif inbound_message.management_action == ManagementAction.CREATE: response = await self._async_process_dataset_create(message=inbound_message) + elif inbound_message.management_action == ManagementAction.REQUEST_DATA and inbound_message.blk_start is not None: + manager = self.get_known_datasets()[inbound_message.dataset_name].manager + raw_data = manager.get_data(dataset_name=inbound_message.dataset_name, + item_name=inbound_message.data_location, + offset=inbound_message.blk_start, length=inbound_message.blk_size) + response = DatasetManagementResponse(success=raw_data is not None, + action=inbound_message.management_action, + data=raw_data, reason="Data Block Retrieve Complete") elif inbound_message.management_action == ManagementAction.REQUEST_DATA: response = await self._async_process_data_request(message=inbound_message, websocket=websocket) elif inbound_message.management_action == ManagementAction.ADD_DATA: From f1ecc2cb2da53a544bee9d85b7b48f2553bd1832 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Wed, 12 Oct 2022 13:38:46 -0500 Subject: [PATCH 023/188] Update dataservice package dependency versions. --- python/services/dataservice/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/services/dataservice/setup.py b/python/services/dataservice/setup.py index 0a9ceb885..df3e2bc85 100644 --- a/python/services/dataservice/setup.py +++ b/python/services/dataservice/setup.py @@ -17,7 +17,7 @@ author_email='', url='', license='', - install_requires=['dmod-core>=0.3.0', 'dmod-communication>=0.8.0', 'dmod-scheduler>=0.7.0', 'dmod-modeldata>=0.9.0', + install_requires=['dmod-core>=0.3.0', 'dmod-communication>=0.9.1', 'dmod-scheduler>=0.7.0', 'dmod-modeldata>=0.9.0', 'redis'], packages=find_namespace_packages(exclude=['dmod.test', 'deprecated', 'conf', 'schemas', 'ssl', 'src']) ) From ab51dbf434277afea4de0d2928f12a45c9ab4574 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Tue, 13 Sep 2022 16:46:52 -0500 Subject: [PATCH 024/188] Add dataset view class and static page for GUI. --- python/gui/MaaS/cbv/DatasetManagementView.py | 132 +++++++ .../templates/maas/dataset_management.html | 359 ++++++++++++++++++ 2 files changed, 491 insertions(+) create mode 100644 python/gui/MaaS/cbv/DatasetManagementView.py create mode 100644 python/gui/MaaS/templates/maas/dataset_management.html diff --git a/python/gui/MaaS/cbv/DatasetManagementView.py b/python/gui/MaaS/cbv/DatasetManagementView.py new file mode 100644 index 000000000..bfaf7fd10 --- /dev/null +++ b/python/gui/MaaS/cbv/DatasetManagementView.py @@ -0,0 +1,132 @@ +""" +Defines a view that may be used to configure a MaaS request +""" +import asyncio +import os +from django.http import HttpRequest, HttpResponse +from django.views.generic.base import View +from django.shortcuts import render + +import dmod.communication as communication +from dmod.client.request_clients import DatasetExternalClient +from dmod.core.meta_data import DataCategory, DataFormat + +import logging +logger = logging.getLogger("gui_log") + +from pathlib import Path + +from .DMODProxy import DMODMixin, GUI_STATIC_SSL_DIR +from .utils import extract_log_data +from typing import Dict + + +class DatasetManagementView(View, DMODMixin): + + """ + A view used to configure a dataset management request or requests for transmitting dataset data. + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._dataset_client = None + + def _process_event_type(self, http_request: HttpRequest) -> communication.MessageEventType: + """ + Determine and return whether this request is for a ``DATASET_MANAGEMENT`` or ``DATA_TRANSMISSION`` event. + + Parameters + ---------- + http_request : HttpRequest + The raw HTTP request in question. + + Returns + ------- + communication.MessageEventType + Either ``communication.MessageEventType.DATASET_MANAGEMENT`` or + ``communication.MessageEventType.DATA_TRANSMISSION``. + """ + # TODO: + raise NotImplementedError("{}._process_event_type not implemented".format(self.__class__.__name__)) + + @property + def dataset_client(self) -> DatasetExternalClient: + if self._dataset_client is None: + self._dataset_client = DatasetExternalClient(endpoint_uri=self.maas_endpoint_uri, + ssl_directory=GUI_STATIC_SSL_DIR) + return self._dataset_client + + async def _get_datasets(self) -> Dict[str, dict]: + serial_datasets = await self.dataset_client.get_serialized_datasets() + return serial_datasets + + def get(self, http_request: HttpRequest, *args, **kwargs) -> HttpResponse: + """ + The handler for 'get' requests. + + This will render the 'maas/dataset_management.html' template after retrieving necessary information to initially + populate the forms it displays. + + Parameters + ---------- + http_request : HttpRequest + The request asking to render this page. + args + kwargs + + Returns + ------- + A rendered page. + """ + errors, warnings, info = extract_log_data(kwargs) + + # Gather map of serialized datasets, keyed by dataset name + serial_dataset_map = asyncio.get_event_loop().run_until_complete(self._get_datasets()) + + dataset_categories = [c.name.title() for c in DataCategory] + dataset_formats = [f.name for f in DataFormat] + + payload = { + 'datasets': serial_dataset_map, + 'dataset_categories': dataset_categories, + 'dataset_formats': dataset_formats, + 'errors': errors, + 'info': info, + 'warnings': warnings + } + + # TODO: create this file + return render(http_request, 'maas/dataset_management.html', payload) + + def post(self, http_request: HttpRequest, *args, **kwargs) -> HttpResponse: + """ + The handler for 'post' requests. + + This will attempt to submit the request and rerender the page like a 'get' request. + + Parameters + ---------- + http_request : HttpRequest + The request asking to render this page. + args + kwargs + + Returns + ------- + A rendered page. + """ + # TODO: implement this to figure out whether DATASET_MANAGEMENT or DATA_TRANSMISSION + event_type = self._process_event_type(http_request) + client, session_data, dmod_response = self.forward_request(http_request, event_type) + + # TODO: this probably isn't exactly correct, so review once closer to completion + if dmod_response is not None and 'dataset_id' in dmod_response.data: + session_data['new_dataset_id'] = dmod_response.data['dataset_id'] + + http_response = self.get(http_request=http_request, errors=client.errors, warnings=client.warnings, + info=client.info, *args, **kwargs) + + for k, v in session_data.items(): + http_response.set_cookie(k, v) + + return http_response diff --git a/python/gui/MaaS/templates/maas/dataset_management.html b/python/gui/MaaS/templates/maas/dataset_management.html new file mode 100644 index 000000000..9ac763131 --- /dev/null +++ b/python/gui/MaaS/templates/maas/dataset_management.html @@ -0,0 +1,359 @@ + + + + + OWP MaaS + {% load static %} + + + + + + + + +
+ + + {% if errors %} +
+
    + {% for error in errors %} +
  • {{ error }}
    • + {% endfor %} +
+
+ {% endif %} + + {% if warnings %} +
+
    + {% for warning in warnings %} +
  • {{ warning }}
    • + {% endfor %} +
+
+ {% endif %} + + {% if info %} +
+
    + {% for message in info %} +
  • {{ message }}
    • + {% endfor %} +
+
+ {% endif %} + + {# Cache jQuery scripts for UI scripting and styling #} +
+

Dataset Management

+
+ Manage + Create +
+
+

Create New Dataset:

+
+ {# Add the token to provide cross site request forgery protection #} + {% csrf_token %} +
+ + +
+ + + + +
+ + + + +
+ + +
+
+ + +
+
+
+
+ + From 60fbab5eaeac74635ad5b3c1e09959535f629e1a Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Tue, 13 Sep 2022 16:47:49 -0500 Subject: [PATCH 025/188] Refactor static dir usage in DMODProxy.py. --- python/gui/MaaS/cbv/DMODProxy.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python/gui/MaaS/cbv/DMODProxy.py b/python/gui/MaaS/cbv/DMODProxy.py index fc0fcb7a1..e9e308664 100644 --- a/python/gui/MaaS/cbv/DMODProxy.py +++ b/python/gui/MaaS/cbv/DMODProxy.py @@ -16,6 +16,8 @@ from pathlib import Path from typing import List, Optional, Tuple, Type +GUI_STATIC_SSL_DIR = Path('/usr/maas_portal/ssl') + class RequestFormProcessor(ABC): @@ -209,7 +211,7 @@ class PostFormRequestClient(ModelExecRequestClient): def _bootstrap_ssl_dir(cls, ssl_dir: Optional[Path] = None): if ssl_dir is None: ssl_dir = Path(__file__).resolve().parent.parent.parent.joinpath('ssl') - ssl_dir = Path('/usr/maas_portal/ssl') #Fixme + ssl_dir = GUI_STATIC_SSL_DIR #Fixme return ssl_dir def __init__(self, endpoint_uri: str, http_request: HttpRequest, ssl_dir: Optional[Path] = None): @@ -315,6 +317,7 @@ def forward_request(self, request: HttpRequest, event_type: MessageEventType) -> client = PostFormRequestClient(endpoint_uri=self.maas_endpoint_uri, http_request=request) if event_type == MessageEventType.MODEL_EXEC_REQUEST: form_processor_type = ModelExecRequestFormProcessor + # TODO: need a new type of form processor here (or 3 more, for management, uploading, and downloading) else: raise RuntimeError("{} got unsupported event type: {}".format(self.__class__.__name__, str(event_type))) From 89179adf410a04f350bab3c0f2f8e43ea6affe73 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Tue, 13 Sep 2022 16:59:42 -0500 Subject: [PATCH 026/188] Add Django GUI url def for dataset view. --- python/gui/MaaS/urls.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/python/gui/MaaS/urls.py b/python/gui/MaaS/urls.py index a6496fada..828dbe6c6 100644 --- a/python/gui/MaaS/urls.py +++ b/python/gui/MaaS/urls.py @@ -1,5 +1,6 @@ from django.urls import re_path from .cbv.EditView import EditView +from .cbv.DatasetManagementView import DatasetManagementView from .cbv.MapView import MapView, Fabrics, FabricNames, FabricTypes, ConnectedFeatures from .cbv.configuration import CreateConfiguration @@ -10,6 +11,9 @@ urlpatterns = [ re_path(r'^$', EditView.as_view()), + # TODO: add this later + #re_path(r'ngen$', NgenWorkflowView.as_view(), name="ngen-workflow"), + re_path(r'datasets', DatasetManagementView.as_view(), name="dataset-management"), re_path(r'map$', MapView.as_view(), name="map"), re_path(r'map/connections$', ConnectedFeatures.as_view(), name="connections"), re_path(r'fabric/names$', FabricNames.as_view(), name='fabric-names'), From 28e0168e31b423357c901d7995b2e6b6e0b98d14 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Tue, 13 Sep 2022 16:48:19 -0500 Subject: [PATCH 027/188] Add client package build arg for GUI Docker build. --- docker/nwm_gui/docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/nwm_gui/docker-compose.yml b/docker/nwm_gui/docker-compose.yml index 509441649..97c7a7b68 100644 --- a/docker/nwm_gui/docker-compose.yml +++ b/docker/nwm_gui/docker-compose.yml @@ -34,6 +34,7 @@ services: args: docker_internal_registry: ${DOCKER_INTERNAL_REGISTRY:?Missing DOCKER_INTERNAL_REGISTRY value (see 'Private Docker Registry ' section in example.env)} comms_package_name: ${PYTHON_PACKAGE_DIST_NAME_COMMS:?} + client_package_name: ${PYTHON_PACKAGE_DIST_NAME_CLIENT:?} networks: - request-listener-net # Call this when starting the container From 6df5754b8d643779b2fa161cdce1b0ccc9bee3d5 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Thu, 29 Sep 2022 10:38:26 -0500 Subject: [PATCH 028/188] Add second view and navigation. Adding initial second view display and manage existing datasets, along with navigation functionality to toggle between "manage" and "create" views. --- .../templates/maas/dataset_management.html | 97 ++++++++++++++++++- 1 file changed, 95 insertions(+), 2 deletions(-) diff --git a/python/gui/MaaS/templates/maas/dataset_management.html b/python/gui/MaaS/templates/maas/dataset_management.html index 9ac763131..c2fcd1827 100644 --- a/python/gui/MaaS/templates/maas/dataset_management.html +++ b/python/gui/MaaS/templates/maas/dataset_management.html @@ -16,6 +16,20 @@ $("#" + model + "_parameters").show(); } + /** + * Toggle what is displayed based on selection click in main nav bar for page. + * + * @param {string} selected_div_id - The id of the main-level div selected to be displayed. + */ + function toggleNav(selected_div_id) { + const all_nav_divs = ["dataset-manage", "dataset-create"]; + for (let i = 0; i <= all_nav_divs.length; i++) { + // TODO: consider a warning message for receiving an unexpected selection + let nav_div = document.getElementById(all_nav_divs[i]); + nav_div.style.display = (selected_div_id === all_nav_divs[i]) ? 'block' : 'none'; + } + } + /* {% for model, model_parameters in parameters.items %} {% for parameter in model_parameters %} @@ -152,6 +166,14 @@ font-size: 17px; } + #dataset-manage { + display: block; + } + + #dataset-create { + display: none; + } + @@ -196,8 +218,79 @@

Office of Water Prediction - Model as a Service

Dataset Management

- Manage - Create + Manage + Create +
+
+ + + + + + + + + + + {% for dataset in datasets %} + + + + + + + + + + {% endfor %} +
DatasetCategory
{{ dataset.name }}{{ dataset.data_category }}
+ + {% for dataset in datasets %} +
+ + +
+ × + + + + + + + + + + + + + + + + + + + + + + + + + + +
Name{{ dataset.name }}
Category{{ dataset.data_category }}
Type{{ dataset.type }}
Format{{ dataset.data_format }}
Created{{ dataset.create_on }}
Last Updated{{ dataset.last_updated }}
+
+
+ {% endfor %}

Create New Dataset:

From f68bbd2a49cbc3378b35252951bbfa8260127a0c Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Fri, 30 Sep 2022 10:11:10 -0500 Subject: [PATCH 029/188] Have DatasetManagementView.py send DS as list. Sending serialized datasets to HTML template as list/array rather than dict/map. --- python/gui/MaaS/cbv/DatasetManagementView.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/gui/MaaS/cbv/DatasetManagementView.py b/python/gui/MaaS/cbv/DatasetManagementView.py index bfaf7fd10..47f411888 100644 --- a/python/gui/MaaS/cbv/DatasetManagementView.py +++ b/python/gui/MaaS/cbv/DatasetManagementView.py @@ -82,12 +82,13 @@ def get(self, http_request: HttpRequest, *args, **kwargs) -> HttpResponse: # Gather map of serialized datasets, keyed by dataset name serial_dataset_map = asyncio.get_event_loop().run_until_complete(self._get_datasets()) + serial_dataset_list = [serial_dataset_map[d] for d in serial_dataset_map] dataset_categories = [c.name.title() for c in DataCategory] dataset_formats = [f.name for f in DataFormat] payload = { - 'datasets': serial_dataset_map, + 'datasets': serial_dataset_list, 'dataset_categories': dataset_categories, 'dataset_formats': dataset_formats, 'errors': errors, From 64957adee608d7097f298abeb74ee04f5e473f4a Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Fri, 30 Sep 2022 10:15:18 -0500 Subject: [PATCH 030/188] Update dataset management GUI with details view. Implementing layout and initial details-viewing behavior for dataset management GUI view. --- .../templates/maas/dataset_management.html | 154 +++++++++++++++++- 1 file changed, 150 insertions(+), 4 deletions(-) diff --git a/python/gui/MaaS/templates/maas/dataset_management.html b/python/gui/MaaS/templates/maas/dataset_management.html index c2fcd1827..f54c714f4 100644 --- a/python/gui/MaaS/templates/maas/dataset_management.html +++ b/python/gui/MaaS/templates/maas/dataset_management.html @@ -30,6 +30,125 @@ } } + function buildDatasetDetailsEmptyDiv() { + let details_div = document.createElement('div'); + details_div.id = 'details-div'; + details_div.class = 'details-modal'; + details_div.style.position = 'fixed'; + details_div.style.zIndex = '1'; + details_div.style.left = '25%'; + details_div.style.top = '5%'; + details_div.style.width = '50%'; + details_div.style.height = '50%'; + details_div.style.overflow = 'clip'; + details_div.style.backgroundColor = '#B7B5B5FF'; + details_div.style.border = '1px solid #888'; + details_div.style.padding = '5px'; + details_div.style.paddingTop = '0px'; + details_div.style.margin = '15% auto'; + + let close_x = document.createElement('span'); + close_x.style.padding = "0px"; + close_x.style.cursor = "pointer"; + close_x.style.fontWeight = "bold"; + close_x.style.fontSize = "20px"; + close_x.appendChild(document.createTextNode("\u00d7")); + close_x.onclick = function() { details_div.remove() }; + details_div.appendChild(close_x); + + let details_content_div = document.createElement('div'); + details_content_div.style.height = '90%'; + details_content_div.style.overflow = 'auto'; + + details_div.appendChild(details_content_div); + + let parent_div = document.getElementById('dataset-manage'); + parent_div.appendChild(details_div); + return details_content_div; + } + + function buildDetailsTableRow() { + let row = document.createElement('tr'); + // Skip the first arg, as this is the table itself + for (let i = 1; i < arguments.length; i++) { + let colCell = document.createElement('th'); + if (i == 1) { + colCell.style.textAlign = 'right'; + colCell.style.paddingRight = '8px'; + arguments[i] += ":"; + } + else { + colCell.style.textAlign = 'left'; + colCell.style.fontWeight = 'normal'; + } + let cellText = document.createTextNode(arguments[i]); + colCell.appendChild(cellText); + row.appendChild(colCell); + } + arguments[0].appendChild(row); + } + + function buildDatasetDetailsTable(dataset_name) { + {% for dataset in datasets %} + if (dataset_name == "{{ dataset.name }}") { + //let cellText = document.createTextNode("testing text {{ dataset.name }} (type: {{ dataset.type }})"); + //return cellText; + let table = document.createElement('table'); + table.class = 'datasets-details-table'; + buildDetailsTableRow(table, "Name", "{{ dataset.name }}") + buildDetailsTableRow(table, "Category", "{{ dataset.data_category }}") + buildDetailsTableRow(table, "Type", "{{ dataset.type }}") + buildDetailsTableRow(table, "Format", "{{ dataset.data_domain.data_format }}") + buildDetailsTableRow(table, "Read Only", "{{ dataset.is_read_only }}") + buildDetailsTableRow(table, "Created", "{{ dataset.create_on }}") + buildDetailsTableRow(table, "Last Updated", "{{ dataset.last_updated }}") + + // Put spacer row in to indicate the start of domain details + let domain_header_row = document.createElement('tr'); + let colCell = document.createElement('th'); + colCell.colSpan = 2; + colCell.style.textAlign = 'left'; + colCell.style.paddingTop = '20px'; + colCell.style.paddingLeft = '5%'; + colCell.style.fontStyle = 'italic'; + colCell.style.fontSize = '18px'; + colCell.appendChild(document.createTextNode("Data Domain")); + domain_header_row.appendChild(colCell); + table.appendChild(domain_header_row); + + // Then iteratively add domain details rows + {% for cont_rest in dataset.data_domain.continuous %} + buildDetailsTableRow(table, "{{ cont_rest.variable }}", "{{ cont_rest.begin }} to {{ cont_rest.end }}"); + {% endfor %} + {% for disc_rest in dataset.data_domain.discrete %} + buildDetailsTableRow(table, "{{ disc_rest.variable }}", "{{ disc_rest.values }}"); + {% endfor %} + + return table; + } + {% endfor %} + } + + function displayDatasetDetails(name) { + let details_div = buildDatasetDetailsEmptyDiv(); + + let detailsTable = buildDatasetDetailsTable(name); + details_div.appendChild(detailsTable); + details_div.style.display = 'block'; + } + + function clickDownloadDataset(dataset_name) { + return false; + } + + function clickUploadDataset(dataset_name) { + return false; + } + + function clickDeleteDataset(dataset_name) { + return false; + } + /* {% for model, model_parameters in parameters.items %} {% for parameter in model_parameters %} @@ -170,6 +289,25 @@ display: block; } + .datasets-table thead tr th { + padding-bottom: 5px; + } + + .mgr-tbl-dataset-header { + text-align: left; + } + + .mgr-tbl-category-header { + text-align: left; + } + + .mgr-tbl-content th { + font-weight: normal; + text-align: left; + padding-right: 15px; + padding-bottom: 2px; + } + #dataset-create { display: none; } @@ -225,8 +363,10 @@

Dataset Management

- - + + + + - + + + + + {% for dataset in datasets %} @@ -386,57 +380,9 @@

Dataset Management

- - - - {% endfor %}
DatasetCategoryDataset NameCategoryActions Details
- - {% for dataset in datasets %} -
- - -
- × - - - - - - - - - - - - - - - - - - - - - - - - - - -
Name{{ dataset.name }}
Category{{ dataset.data_category }}
Type{{ dataset.type }}
Format{{ dataset.data_format }}
Created{{ dataset.create_on }}
Last Updated{{ dataset.last_updated }}
-
-
- {% endfor %}

Create New Dataset:

From 7472e56cc899afcb7e60f6e8fbbcfbf45d3122a3 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Mon, 3 Oct 2022 11:42:30 -0500 Subject: [PATCH 032/188] Create AbstractDatasetView GUI Django view class. --- python/gui/MaaS/cbv/AbstractDatasetView.py | 29 ++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 python/gui/MaaS/cbv/AbstractDatasetView.py diff --git a/python/gui/MaaS/cbv/AbstractDatasetView.py b/python/gui/MaaS/cbv/AbstractDatasetView.py new file mode 100644 index 000000000..e1ae493c3 --- /dev/null +++ b/python/gui/MaaS/cbv/AbstractDatasetView.py @@ -0,0 +1,29 @@ +from abc import ABC +from django.views.generic.base import View +from dmod.client.request_clients import DatasetExternalClient +import logging +logger = logging.getLogger("gui_log") +from .DMODProxy import DMODMixin, GUI_STATIC_SSL_DIR +from typing import Dict + + +class AbstractDatasetView(View, DMODMixin, ABC): + + def __init__(self, *args, **kwargs): + super(AbstractDatasetView, self).__init__(*args, **kwargs) + self._dataset_client = None + + async def get_dataset(self, dataset_name: str) -> Dict[str, dict]: + serial_dataset = await self.dataset_client.get_serialized_datasets(dataset_name=dataset_name) + return serial_dataset + + async def get_datasets(self) -> Dict[str, dict]: + serial_datasets = await self.dataset_client.get_serialized_datasets() + return serial_datasets + + @property + def dataset_client(self) -> DatasetExternalClient: + if self._dataset_client is None: + self._dataset_client = DatasetExternalClient(endpoint_uri=self.maas_endpoint_uri, + ssl_directory=GUI_STATIC_SSL_DIR) + return self._dataset_client From 4715f65fbd332be2f2cc0ebeedc539ff3a66b8f8 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Mon, 3 Oct 2022 11:39:15 -0500 Subject: [PATCH 033/188] Add Django view for direct dataservice API calls. --- python/gui/MaaS/cbv/DatasetApiView.py | 29 +++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 python/gui/MaaS/cbv/DatasetApiView.py diff --git a/python/gui/MaaS/cbv/DatasetApiView.py b/python/gui/MaaS/cbv/DatasetApiView.py new file mode 100644 index 000000000..8697def2d --- /dev/null +++ b/python/gui/MaaS/cbv/DatasetApiView.py @@ -0,0 +1,29 @@ +import asyncio +from django.http import JsonResponse +from .AbstractDatasetView import AbstractDatasetView +import logging +logger = logging.getLogger("gui_log") + + +class DatasetApiView(AbstractDatasetView): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def _get_datasets_json(self) -> JsonResponse: + serial_dataset_map = asyncio.get_event_loop().run_until_complete(self.get_datasets()) + return JsonResponse({"datasets": serial_dataset_map}, status=200) + + def _get_dataset_json(self, dataset_name: str) -> JsonResponse: + serial_dataset = asyncio.get_event_loop().run_until_complete(self.get_dataset(dataset_name=dataset_name)) + return JsonResponse({"dataset": serial_dataset[dataset_name]}, status=200) + + def get(self, request, *args, **kwargs): + request_type = request.GET.get("request_type", None) + if request_type == 'datasets': + return self._get_datasets_json() + elif request_type == 'dataset': + return self._get_dataset_json(dataset_name=request.GET.get("name", None)) + + # TODO: finish + return JsonResponse({}, status=400) From 12d89b265a2579db89dd7a26e3cd200767c6d5a9 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Mon, 3 Oct 2022 11:39:31 -0500 Subject: [PATCH 034/188] Add GUI Django url for Dataset API AJAX calls. --- python/gui/MaaS/urls.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/gui/MaaS/urls.py b/python/gui/MaaS/urls.py index 828dbe6c6..9ca222ccb 100644 --- a/python/gui/MaaS/urls.py +++ b/python/gui/MaaS/urls.py @@ -1,6 +1,7 @@ from django.urls import re_path from .cbv.EditView import EditView from .cbv.DatasetManagementView import DatasetManagementView +from .cbv.DatasetApiView import DatasetApiView from .cbv.MapView import MapView, Fabrics, FabricNames, FabricTypes, ConnectedFeatures from .cbv.configuration import CreateConfiguration @@ -14,6 +15,7 @@ # TODO: add this later #re_path(r'ngen$', NgenWorkflowView.as_view(), name="ngen-workflow"), re_path(r'datasets', DatasetManagementView.as_view(), name="dataset-management"), + re_path(r'dataset-api', DatasetApiView.as_view(), name="dataset-api"), re_path(r'map$', MapView.as_view(), name="map"), re_path(r'map/connections$', ConnectedFeatures.as_view(), name="connections"), re_path(r'fabric/names$', FabricNames.as_view(), name='fabric-names'), From 88db9072b219ff535d2bbc122c94d1fb04ca1cac Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Mon, 3 Oct 2022 11:46:21 -0500 Subject: [PATCH 035/188] Add commented-out helper debug volume for GUI. Adding commented-out line for host bind-mount volume to GUI Docker service config to make debugging Django template and Javascript code faster by mounted code from the host into the running service. --- docker/nwm_gui/docker-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/nwm_gui/docker-compose.yml b/docker/nwm_gui/docker-compose.yml index 97c7a7b68..7eb8ef4fb 100644 --- a/docker/nwm_gui/docker-compose.yml +++ b/docker/nwm_gui/docker-compose.yml @@ -63,6 +63,8 @@ services: volumes: - ${DMOD_APP_STATIC:?}:/usr/maas_portal/static - ${DMOD_SSL_DIR}/requestservice:/usr/maas_portal/ssl + # Needed only for speeding debugging + #- ${DOCKER_GUI_HOST_SRC:?GUI sources path not configured in environment}/MaaS:/usr/maas_portal/MaaS #- ${DOCKER_GUI_HOST_VENV_DIR:-/tmp/blah}:${DOCKER_GUI_CONTAINER_VENV_DIR:-/tmp/blah} # Expose Django's port to the internal network so that the web server may access it expose: From 1cb89b08bc64ea8c3e494eb04759ee1b396c3310 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Mon, 3 Oct 2022 11:48:48 -0500 Subject: [PATCH 036/188] Refactor DatasetManagementView inheritance. Make DatasetManagementView inherit from AbstractDatasetView so certain required pieces could be centrally located and reused. --- python/gui/MaaS/cbv/DatasetManagementView.py | 25 +++----------------- 1 file changed, 3 insertions(+), 22 deletions(-) diff --git a/python/gui/MaaS/cbv/DatasetManagementView.py b/python/gui/MaaS/cbv/DatasetManagementView.py index 47f411888..b7db81112 100644 --- a/python/gui/MaaS/cbv/DatasetManagementView.py +++ b/python/gui/MaaS/cbv/DatasetManagementView.py @@ -2,26 +2,20 @@ Defines a view that may be used to configure a MaaS request """ import asyncio -import os from django.http import HttpRequest, HttpResponse -from django.views.generic.base import View from django.shortcuts import render import dmod.communication as communication -from dmod.client.request_clients import DatasetExternalClient from dmod.core.meta_data import DataCategory, DataFormat import logging logger = logging.getLogger("gui_log") -from pathlib import Path - -from .DMODProxy import DMODMixin, GUI_STATIC_SSL_DIR from .utils import extract_log_data -from typing import Dict +from .AbstractDatasetView import AbstractDatasetView -class DatasetManagementView(View, DMODMixin): +class DatasetManagementView(AbstractDatasetView): """ A view used to configure a dataset management request or requests for transmitting dataset data. @@ -29,7 +23,6 @@ class DatasetManagementView(View, DMODMixin): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self._dataset_client = None def _process_event_type(self, http_request: HttpRequest) -> communication.MessageEventType: """ @@ -49,17 +42,6 @@ def _process_event_type(self, http_request: HttpRequest) -> communication.Messag # TODO: raise NotImplementedError("{}._process_event_type not implemented".format(self.__class__.__name__)) - @property - def dataset_client(self) -> DatasetExternalClient: - if self._dataset_client is None: - self._dataset_client = DatasetExternalClient(endpoint_uri=self.maas_endpoint_uri, - ssl_directory=GUI_STATIC_SSL_DIR) - return self._dataset_client - - async def _get_datasets(self) -> Dict[str, dict]: - serial_datasets = await self.dataset_client.get_serialized_datasets() - return serial_datasets - def get(self, http_request: HttpRequest, *args, **kwargs) -> HttpResponse: """ The handler for 'get' requests. @@ -81,7 +63,7 @@ def get(self, http_request: HttpRequest, *args, **kwargs) -> HttpResponse: errors, warnings, info = extract_log_data(kwargs) # Gather map of serialized datasets, keyed by dataset name - serial_dataset_map = asyncio.get_event_loop().run_until_complete(self._get_datasets()) + serial_dataset_map = asyncio.get_event_loop().run_until_complete(self.get_datasets()) serial_dataset_list = [serial_dataset_map[d] for d in serial_dataset_map] dataset_categories = [c.name.title() for c in DataCategory] @@ -96,7 +78,6 @@ def get(self, http_request: HttpRequest, *args, **kwargs) -> HttpResponse: 'warnings': warnings } - # TODO: create this file return render(http_request, 'maas/dataset_management.html', payload) def post(self, http_request: HttpRequest, *args, **kwargs) -> HttpResponse: From 2848189f2c265283b37d19b8bd6ad9fa9283c1e1 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Mon, 3 Oct 2022 11:51:29 -0500 Subject: [PATCH 037/188] More progress on dataset management template. --- .../templates/maas/dataset_management.html | 287 +++++++++++++----- 1 file changed, 215 insertions(+), 72 deletions(-) diff --git a/python/gui/MaaS/templates/maas/dataset_management.html b/python/gui/MaaS/templates/maas/dataset_management.html index 1ca18311c..97c694b73 100644 --- a/python/gui/MaaS/templates/maas/dataset_management.html +++ b/python/gui/MaaS/templates/maas/dataset_management.html @@ -16,6 +16,143 @@ $("#" + model + "_parameters").show(); } + /** + * Build ``thead`` for the management overview table summarizing datasets. + */ + function buildDatasetsOverviewTableHeader() { + // Build the table header row + let thead = document.createElement('thead'); + let header = document.createElement('tr'); + thead.appendChild(header); + + let colCell = document.createElement('th'); + colCell.className = "mgr-tbl-dataset-header"; + colCell.appendChild(document.createTextNode('Dataset Name')); + header.appendChild(colCell); + + colCell = document.createElement('th'); + colCell.className = "mgr-tbl-category-header"; + colCell.appendChild(document.createTextNode('Category')); + header.appendChild(colCell); + + header.appendChild(document.createElement('th')); + + colCell = document.createElement('th'); + colCell.appendChild(document.createTextNode('Actions')); + header.appendChild(colCell); + + header.appendChild(document.createElement('th')); + header.appendChild(document.createElement('th')); + + return thead; + } + + function buildDatasetOverviewTableRow(serial_dataset) { + let row = document.createElement('tr'); + row.className = "mgr-tbl-content"; + + let colCell = document.createElement('th'); + colCell.appendChild(document.createTextNode(serial_dataset['name'])); + row.appendChild(colCell); + + colCell = document.createElement('th'); + colCell.appendChild(document.createTextNode(serial_dataset['data_category'])); + row.appendChild(colCell); + + let helper_create_links = function create_links_func (is_a, text, ds_name, on_click) { + let cell = document.createElement('th'); + let content; + if (is_a) { + content = document.createElement('a') + content.href = "javascript:void(0);"; + } + else { + content = document.createElement('button'); + } + content.onclick = function() { on_click(ds_name); }; + content.appendChild(document.createTextNode(text)); + cell.appendChild(content); + row.appendChild(cell); + } + + helper_create_links(true, "Details", serial_dataset['name'], displayDatasetDetails); + helper_create_links(true, "Download", serial_dataset['name'], clickDownloadDataset); + helper_create_links(true, "Upload Files", serial_dataset['name'], clickUploadDataset); + helper_create_links(true, "Delete", serial_dataset['name'], clickDeleteDataset); + + return row; + } + + function buildDatasetsOverviewTable() { + let table_id = 'dataset-manage-overview-table'; + let parent_div = document.getElementById('dataset-manage'); + + // First find and remove any existing table, in case we are refreshing + let table = document.getElementById(table_id); + if (table != null) { + table.remove(); + } + + table = document.createElement('table'); + parent_div.appendChild(table); + table.id = table_id; + table.className = 'datasets-table'; + + table.appendChild(buildDatasetsOverviewTableHeader()); + + $.ajax({ + url: "/dataset-api", + type: 'get', + data: {"request_type": "datasets"}, + success: function(response) { + for (const ds_name in response["datasets"]) { + table.appendChild(buildDatasetOverviewTableRow(response["datasets"][ds_name])); + } + }, + error: function(response) { + alert('Received an error'); + console.log(response); + } + }); + } + + function initDatasetCreateForm() { + let category_select = document.getElementById("create-dataset-category"); + category_select.selectedIndex = -1; + let format_select = document.getElementById("create-dataset-format"); + format_select.selectedIndex = -1; + } + + function initGuiViews() { + buildDatasetsOverviewTable(); + initDatasetCreateForm(); + } + + function changeCreateDatasetFormatSelection(selection) { + let dy_div = document.getElementById("dataset-create-form-dynamic-vars-div"); + for (let i = 0; i < dy_div.children.length; i++) { + dy_div.children[i].remove(); + } + let addUploadSelection = false; + if (selection === "NETCDF_FORCING_CANONICAL") { + addUploadSelection = true; + } + + if (addUploadSelection) { + let upload_select_label = document.createElement('label'); + upload_select_label.appendChild(document.createTextNode('Data Files:')); + dy_div.appendChild(upload_select_label); + upload_select_label.htmlFor = 'create-dataset-upload'; + let upload_select = document.createElement('input'); + upload_select.type = 'file'; + upload_select.name = 'create-dataset-upload'; + upload_select.id = 'create-dataset-upload'; + upload_select.style.float = 'right'; + upload_select.style.textAlign = 'right'; + dy_div.appendChild(upload_select); + } + } + /** * Toggle what is displayed based on selection click in main nav bar for page. * @@ -70,9 +207,12 @@ function buildDetailsTableRow() { let row = document.createElement('tr'); // Skip the first arg, as this is the table itself + let colCell; + let cellText; for (let i = 1; i < arguments.length; i++) { - let colCell = document.createElement('th'); + colCell = document.createElement('th'); if (i == 1) { + colCell.style.verticalAlign = 'top'; colCell.style.textAlign = 'right'; colCell.style.paddingRight = '8px'; arguments[i] += ":"; @@ -81,52 +221,69 @@ colCell.style.textAlign = 'left'; colCell.style.fontWeight = 'normal'; } - let cellText = document.createTextNode(arguments[i]); + cellText = document.createTextNode(arguments[i]); colCell.appendChild(cellText); row.appendChild(colCell); } arguments[0].appendChild(row); } + function ajaxBuildDatasetDetailsTable(dataset_json, table) { + buildDetailsTableRow(table, "Name", dataset_json["name"]); + buildDetailsTableRow(table, "Category", dataset_json["data_category"]) + buildDetailsTableRow(table, "Type", dataset_json["type"]); + buildDetailsTableRow(table, "Format", dataset_json["data_domain"]["data_format"]); + buildDetailsTableRow(table, "Read Only", dataset_json["is_read_only"]); + buildDetailsTableRow(table, "Created", dataset_json["create_on"]); + buildDetailsTableRow(table, "Last Updated", dataset_json["last_updated"]); + + // TODO: add something for dataset size + // TODO: add something for number of files. + // TODO: add something for indicating if data aligns properly with configured domain + + // Put spacer row in to indicate the start of domain details + let domain_header_row = document.createElement('tr'); + let colCell = document.createElement('th'); + colCell.colSpan = 2; + colCell.style.textAlign = 'left'; + colCell.style.paddingTop = '20px'; + colCell.style.paddingLeft = '5%'; + colCell.style.fontStyle = 'italic'; + colCell.style.fontSize = '18px'; + colCell.appendChild(document.createTextNode("Data Domain")); + domain_header_row.appendChild(colCell); + table.appendChild(domain_header_row); + + // Then iteratively add domain details rows + let c_restrict; + for (let c_i = 0; c_i < dataset_json["data_domain"]["continuous"].length; c_i++) { + c_restrict = dataset_json["data_domain"]["continuous"][c_i]; + buildDetailsTableRow(table, c_restrict["variable"], c_restrict["begin"] + " to " + c_restrict["end"]); + } + let d_restrict; + for (let d_i = 0; d_i < dataset_json["data_domain"]["discrete"].length; d_i++) { + d_restrict = dataset_json["data_domain"]["discrete"][d_i]; + buildDetailsTableRow(table, d_restrict["variable"], d_restrict["values"]); + } + } + function buildDatasetDetailsTable(dataset_name) { - {% for dataset in datasets %} - if (dataset_name == "{{ dataset.name }}") { - //let cellText = document.createTextNode("testing text {{ dataset.name }} (type: {{ dataset.type }})"); - //return cellText; - let table = document.createElement('table'); - table.class = 'datasets-details-table'; - buildDetailsTableRow(table, "Name", "{{ dataset.name }}") - buildDetailsTableRow(table, "Category", "{{ dataset.data_category }}") - buildDetailsTableRow(table, "Type", "{{ dataset.type }}") - buildDetailsTableRow(table, "Format", "{{ dataset.data_domain.data_format }}") - buildDetailsTableRow(table, "Read Only", "{{ dataset.is_read_only }}") - buildDetailsTableRow(table, "Created", "{{ dataset.create_on }}") - buildDetailsTableRow(table, "Last Updated", "{{ dataset.last_updated }}") - - // Put spacer row in to indicate the start of domain details - let domain_header_row = document.createElement('tr'); - let colCell = document.createElement('th'); - colCell.colSpan = 2; - colCell.style.textAlign = 'left'; - colCell.style.paddingTop = '20px'; - colCell.style.paddingLeft = '5%'; - colCell.style.fontStyle = 'italic'; - colCell.style.fontSize = '18px'; - colCell.appendChild(document.createTextNode("Data Domain")); - domain_header_row.appendChild(colCell); - table.appendChild(domain_header_row); - - // Then iteratively add domain details rows - {% for cont_rest in dataset.data_domain.continuous %} - buildDetailsTableRow(table, "{{ cont_rest.variable }}", "{{ cont_rest.begin }} to {{ cont_rest.end }}"); - {% endfor %} - {% for disc_rest in dataset.data_domain.discrete %} - buildDetailsTableRow(table, "{{ disc_rest.variable }}", "{{ disc_rest.values }}"); - {% endfor %} - - return table; + let table = document.createElement('table'); + table.class = 'datasets-details-table'; + + $.ajax({ + url: "/dataset-api", + type: 'get', + data: {"request_type": "dataset", "name": dataset_name}, + success: function(response) { + ajaxBuildDatasetDetailsTable(response["dataset"], table); + }, + error: function(response) { + alert('Received an error getting serialized ' + dataset_name + ' dataset'); + console.log(response); } - {% endfor %} + }); + return table; } function displayDatasetDetails(name) { @@ -260,9 +417,14 @@ background-color: palegreen; } - #standard-vars { + #dataset-create-form-universal-vars-div { line-height: 25px; - max-width: 300px; + max-width: 350px; + } + + #dataset-create-form-dynamic-vars-div { + line-height: 25px; + max-width: 350px; } .disabled { @@ -314,7 +476,7 @@ - +