From 020c12edb48156de07cbf55caba0c0af72af92a3 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Mon, 3 Oct 2022 11:41:16 -0500 Subject: [PATCH 01/46] Fix fast-debug in dataservice image entrypoint. Fixing logic where pip installs the updated packages (to avoid the entire image rebuild), so make sure deps are ignored (as this was the slow part). --- docker/main/dataservice/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/main/dataservice/entrypoint.sh b/docker/main/dataservice/entrypoint.sh index e42ae462d..433ecc801 100644 --- a/docker/main/dataservice/entrypoint.sh +++ b/docker/main/dataservice/entrypoint.sh @@ -62,7 +62,7 @@ if [ -d ${UPDATED_PACKAGES_DIR:=/updated_packages} ]; then for srv in $(pip -qq freeze | grep dmod | awk -F= '{print $1}' | awk -F- '{print $2}'); do if [ $(ls ${UPDATED_PACKAGES_DIR} | grep dmod.${srv}- | wc -l) -eq 1 ]; then pip uninstall -y --no-input $(pip -qq freeze | grep dmod.${srv} | awk -F= '{print $1}') - pip install $(ls ${UPDATED_PACKAGES_DIR}/*.whl | grep dmod.${srv}-) + pip install --no-deps $(ls ${UPDATED_PACKAGES_DIR}/*.whl | grep dmod.${srv}-) fi done #pip install ${UPDATED_PACKAGES_DIR}/*.whl From 9901d4fea9e51f497f74d12f1660cb4edd4becee Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Tue, 13 Sep 2022 16:49:07 -0500 Subject: [PATCH 02/46] Add package client name vars to example.env. --- example.env | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/example.env b/example.env index 637e9e5ce..bafbc8e63 100644 --- a/example.env +++ b/example.env @@ -108,6 +108,11 @@ TROUTE_BRANCH=ngen ## Python Packages Settings ## ######################################################################## +## The "name" of the built client Python distribution package, for purposes of installing (e.g., via pip) +PYTHON_PACKAGE_DIST_NAME_CLIENT=dmod-client +## The name of the actual Python communication package (i.e., for importing or specifying as a module on the command line) +PYTHON_PACKAGE_NAME_CLIENT=dmod.client + ## The "name" of the built communication Python distribution package, for purposes of installing (e.g., via pip) PYTHON_PACKAGE_DIST_NAME_COMMS=dmod-communication ## The name of the actual Python communication package (i.e., for importing or specifying as a module on the command line) From 371cd569c70df55293e115dc3a4f7448efd7e53d Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Tue, 13 Sep 2022 16:52:58 -0500 Subject: [PATCH 03/46] Add new QueryType for dataset management messages. Adding type GET_SERIALIZED_FORM to get the entire serialized state of a dataset. --- .../dmod/communication/dataset_management_message.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/lib/communication/dmod/communication/dataset_management_message.py b/python/lib/communication/dmod/communication/dataset_management_message.py index d148b6d0e..55f51e7b5 100644 --- a/python/lib/communication/dmod/communication/dataset_management_message.py +++ b/python/lib/communication/dmod/communication/dataset_management_message.py @@ -16,6 +16,7 @@ class QueryType(Enum): GET_VALUES = 6 GET_MIN_VALUE = 7 GET_MAX_VALUE = 8 + GET_SERIALIZED_FORM = 9 @classmethod def get_for_name(cls, name_str: str) -> 'QueryType': From df3d259ca33de7e848fe06d3e32266b893070d41 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Mon, 3 Oct 2022 11:37:47 -0500 Subject: [PATCH 04/46] Add GET_SERIALIZED_FORM query dataservice support. --- python/services/dataservice/dmod/dataservice/service.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/python/services/dataservice/dmod/dataservice/service.py b/python/services/dataservice/dmod/dataservice/service.py index 04e1fa0af..60a27cec1 100644 --- a/python/services/dataservice/dmod/dataservice/service.py +++ b/python/services/dataservice/dmod/dataservice/service.py @@ -686,7 +686,13 @@ def _process_query(self, message: DatasetManagementMessage) -> DatasetManagement return DatasetManagementResponse(action=message.management_action, success=True, dataset_name=dataset_name, reason='Obtained {} Items List', data={DatasetManagementResponse._DATA_KEY_QUERY_RESULTS: list_of_files}) - # TODO: (later) add support for messages with other query types also + elif query_type == QueryType.GET_SERIALIZED_FORM: + dataset_name = message.dataset_name + serialized_form = self.get_known_datasets()[dataset_name].to_dict() + return DatasetManagementResponse(action=message.management_action, success=True, dataset_name=dataset_name, + reason='Obtained serialized {} dataset'.format(dataset_name), + data={DatasetManagementResponse._DATA_KEY_QUERY_RESULTS: serialized_form}) + # TODO: (later) add support for messages with other query types also else: reason = 'Unsupported {} Query Type - {}'.format(DatasetQuery.__class__.__name__, query_type.name) return DatasetManagementResponse(action=message.management_action, success=False, reason=reason) From 277732276c295b2d117337cd828a3afec158afd2 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Mon, 3 Oct 2022 10:58:13 -0500 Subject: [PATCH 05/46] Refactor ExternalRequestClient async_make_request. Refactor to ensure it takes advantage of connection handling via its async context manager logic. --- python/lib/communication/dmod/communication/client.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/python/lib/communication/dmod/communication/client.py b/python/lib/communication/dmod/communication/client.py index 571f0f97d..5b5dbf68c 100644 --- a/python/lib/communication/dmod/communication/client.py +++ b/python/lib/communication/dmod/communication/client.py @@ -800,10 +800,8 @@ def _update_after_valid_response(self, response: EXTERN_REQ_R): # TODO: this can probably be taken out, as the superclass implementation should suffice async def async_make_request(self, request: EXTERN_REQ_M) -> EXTERN_REQ_R: - async with websockets.connect(self.endpoint_uri, ssl=self.client_ssl_context) as websocket: - await websocket.send(request.to_json()) - response = await websocket.recv() - return request.__class__.factory_init_correct_response_subtype(json_obj=json.loads(response)) + response = await self.async_send(request.to_json(), await_response=True) + return request.__class__.factory_init_correct_response_subtype(json_obj=json.loads(response)) @property def errors(self): From 13486ba25677e73731eb22ba6f28ab3e9cefc269 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Wed, 12 Oct 2022 13:43:43 -0500 Subject: [PATCH 06/46] Adding client methods for data/metadata retrieval. --- .../lib/client/dmod/client/request_clients.py | 126 +++++++++++++++++- 1 file changed, 123 insertions(+), 3 deletions(-) diff --git a/python/lib/client/dmod/client/request_clients.py b/python/lib/client/dmod/client/request_clients.py index 09251c01b..9a020119c 100644 --- a/python/lib/client/dmod/client/request_clients.py +++ b/python/lib/client/dmod/client/request_clients.py @@ -9,13 +9,13 @@ from dmod.communication.data_transmit_message import DataTransmitMessage, DataTransmitResponse from dmod.core.meta_data import DataCategory, DataDomain, TimeRange from pathlib import Path -from typing import List, Optional, Tuple, Type, Union +from typing import AnyStr, Dict, List, Optional, Tuple, Type, Union import json import websockets -#import logging -#logger = logging.getLogger("gui_log") +import logging +logger = logging.getLogger("client_log") class NgenRequestClient(ModelExecRequestClient[NGENRequest, NGENRequestResponse]): @@ -89,6 +89,25 @@ async def create_dataset(self, name: str, category: DataCategory, domain: DataDo async def delete_dataset(self, name: str, **kwargs) -> bool: pass + @abstractmethod + async def download_item_block(self, dataset_name: str, item_name: str, blk_start: int, blk_size: int) -> AnyStr: + """ + Download a block/chunk of a given size and start point from a specified dataset file. + + Parameters + ---------- + dataset_name + item_name + blk_start + blk_size + + Returns + ------- + AnyStr + The downloaded block/chunk. + """ + pass + @abstractmethod async def download_dataset(self, dataset_name: str, dest_dir: Path) -> bool: """ @@ -130,6 +149,14 @@ async def download_from_dataset(self, dataset_name: str, item_name: str, dest: P """ pass + @abstractmethod + async def get_dataset_content_details(self, name: str, **kwargs) -> bool: + pass + + @abstractmethod + async def get_item_size(self, dataset_name: str, item_name: str) -> int: + pass + @abstractmethod async def list_datasets(self, category: Optional[DataCategory] = None) -> List[str]: pass @@ -175,6 +202,50 @@ async def delete_dataset(self, name: str, **kwargs) -> bool: self.last_response = await self.async_make_request(request) return self.last_response is not None and self.last_response.success + async def get_dataset_content_details(self, name: str, **kwargs) -> dict: + # TODO: later add things like created and last updated perhaps + query = DatasetQuery(query_type=QueryType.GET_DATASET_ITEMS) + request = DatasetManagementMessage(action=ManagementAction.QUERY, query=query, dataset_name=name) + self.last_response: DatasetManagementResponse = await self.async_make_request(request) + if self.last_response.success: + return self.last_response.data + else: + return {} + + async def get_item_size(self, dataset_name: str, item_name: str) -> int: + query = DatasetQuery(query_type=QueryType.GET_ITEM_SIZE, item_name=item_name) + request = DatasetManagementMessage(action=ManagementAction.QUERY, query=query, dataset_name=dataset_name, + data_location=item_name) + self.last_response: DatasetManagementResponse = await self.async_make_request(request) + if self.last_response.success: + return self.last_response.data + else: + return -1 + + async def download_item_block(self, dataset_name: str, item_name: str, blk_start: int, blk_size: int) -> AnyStr: + """ + Download a block/chunk of a given size and start point from a specified dataset file. + + Parameters + ---------- + dataset_name + item_name + blk_start + blk_size + + Returns + ------- + AnyStr + The downloaded block/chunk. + """ + request = DatasetManagementMessage(action=ManagementAction.REQUEST_DATA, dataset_name=dataset_name, + data_location=item_name, blk_start=blk_start, blk_size=blk_size) + self.last_response: DatasetManagementResponse = await self.async_make_request(request) + if self.last_response.success: + return self.last_response.data + else: + return '' + async def download_dataset(self, dataset_name: str, dest_dir: Path) -> bool: """ Download an entire dataset to a local directory. @@ -460,6 +531,55 @@ async def delete_dataset(self, name: str, **kwargs) -> bool: self.last_response = await self.async_make_request(request) return self.last_response is not None and self.last_response.success + async def download_item_block(self, dataset_name: str, item_name: str, blk_start: int, blk_size: int) -> AnyStr: + """ + Download a block/chunk of a given size and start point from a specified dataset file. + + Parameters + ---------- + dataset_name + item_name + blk_start + blk_size + + Returns + ------- + AnyStr + The downloaded block/chunk. + """ + await self._async_acquire_session_info() + request = MaaSDatasetManagementMessage(action=ManagementAction.REQUEST_DATA, dataset_name=dataset_name, + session_secret=self.session_secret, data_location=item_name, + blk_start=blk_start, blk_size=blk_size) + self.last_response: DatasetManagementResponse = await self.async_make_request(request) + if self.last_response.success: + return self.last_response.data + else: + return '' + + async def get_item_size(self, dataset_name: str, item_name: str) -> int: + await self._async_acquire_session_info() + query = DatasetQuery(query_type=QueryType.GET_ITEM_SIZE, item_name=item_name) + request = MaaSDatasetManagementMessage(action=ManagementAction.QUERY, query=query, dataset_name=dataset_name, + session_secret=self._session_secret, data_location=item_name) + self.last_response: DatasetManagementResponse = await self.async_make_request(request) + if self.last_response.success: + return self.last_response.data + else: + return -1 + + async def get_dataset_content_details(self, name: str, **kwargs) -> List: + # TODO: later add things like created and last updated perhaps + await self._async_acquire_session_info() + query = DatasetQuery(query_type=QueryType.GET_DATASET_ITEMS) + request = MaaSDatasetManagementMessage(session_secret=self.session_secret, action=ManagementAction.QUERY, + query=query, dataset_name=name) + self.last_response: DatasetManagementResponse = await self.async_make_request(request) + if self.last_response.success: + return self.last_response.data[DatasetManagementResponse._DATA_KEY_QUERY_RESULTS] + else: + return [] + async def download_dataset(self, dataset_name: str, dest_dir: Path) -> bool: await self._async_acquire_session_info() try: From 50b0ba17df0bcb68407a4bd9c8b44b43a98f5d67 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Tue, 13 Sep 2022 16:50:05 -0500 Subject: [PATCH 07/46] Refactor dataset request handler to use try block. --- .../externalrequests/maas_request_handlers.py | 37 ++++++++++--------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/python/lib/externalrequests/dmod/externalrequests/maas_request_handlers.py b/python/lib/externalrequests/dmod/externalrequests/maas_request_handlers.py index dff0b45ac..1dec8137b 100644 --- a/python/lib/externalrequests/dmod/externalrequests/maas_request_handlers.py +++ b/python/lib/externalrequests/dmod/externalrequests/maas_request_handlers.py @@ -291,25 +291,28 @@ async def handle_request(self, request: MaaSDatasetManagementMessage, **kwargs) session, is_authorized, reason, msg = await self.get_authorized_session(request) if not is_authorized: return MaaSDatasetManagementResponse(success=False, reason=reason.name, message=msg) - # In this case, we actually can pass the request as-is straight through (i.e., after confirming authorization) - async with self.service_client as client: - # Have to handle these two slightly differently, since multiple message will be going over the websocket - if request.management_action == ManagementAction.REQUEST_DATA: - await client.connection.send(str(request)) - mgmt_response = await self._handle_data_download(client_websocket=kwargs['upstream_websocket'], - service_websocket=client.connection) - elif request.management_action == ManagementAction.ADD_DATA: - await client.connection.send(str(request)) - mgmt_response = await self._handle_data_upload(client_websocket=kwargs['upstream_websocket'], - service_websocket=client.connection) - else: - mgmt_response = await client.async_make_request(request) - logging.debug("************* {} received response:\n{}".format(self.__class__.__name__, str(mgmt_response))) - # Likewise, can just send back the response from the internal service client - return MaaSDatasetManagementResponse.factory_create(mgmt_response) + try: + # In this case, we actually can pass the request as-is straight through (i.e., after confirming authorization) + async with self.service_client as client: + # Have to handle these two slightly differently, since multiple message will be going over the websocket + if request.management_action == ManagementAction.REQUEST_DATA: + await client.connection.send(str(request)) + mgmt_response = await self._handle_data_download(client_websocket=kwargs['upstream_websocket'], + service_websocket=client.connection) + elif request.management_action == ManagementAction.ADD_DATA: + await client.connection.send(str(request)) + mgmt_response = await self._handle_data_upload(client_websocket=kwargs['upstream_websocket'], + service_websocket=client.connection) + else: + mgmt_response = await client.async_make_request(request) + logging.debug("************* {} received response:\n{}".format(self.__class__.__name__, str(mgmt_response))) + # Likewise, can just send back the response from the internal service client + return MaaSDatasetManagementResponse.factory_create(mgmt_response) + except Exception as e: + raise e @property def service_client(self) -> DataServiceClient: if self._service_client is None: - self._service_client = DataServiceClient(self.service_url, self.service_ssl_dir) + self._service_client = DataServiceClient(endpoint_uri=self.service_url, ssl_directory=self.service_ssl_dir) return self._service_client From 853a8c86edd55d5f4d248f6c67863685794f3112 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Tue, 13 Sep 2022 16:56:02 -0500 Subject: [PATCH 08/46] Add get_serialized_datasets to external client. Add new function to get serialized dataset details to DatasetExternalClient. --- .../lib/client/dmod/client/request_clients.py | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/python/lib/client/dmod/client/request_clients.py b/python/lib/client/dmod/client/request_clients.py index 9a020119c..2ea95d98b 100644 --- a/python/lib/client/dmod/client/request_clients.py +++ b/python/lib/client/dmod/client/request_clients.py @@ -625,6 +625,37 @@ async def download_from_dataset(self, dataset_name: str, item_name: str, dest: P if not has_data: return message_object + async def get_serialized_datasets(self, dataset_name: Optional[str] = None) -> Dict[str, dict]: + """ + Get dataset objects in serialized form, either for all datasets or for the one with the provided name. + + Parameters + ---------- + dataset_name : Optional[str] + The name of a specific dataset to get serialized details of, if only one should be obtained. + + Returns + ------- + Dict[str, dict] + A dictionary, keyed by dataset name, of serialized dataset objects. + """ + # TODO: may need to generalize this and add to super class + if dataset_name is None: + datasets = await self.list_datasets() + else: + datasets = [dataset_name] + serialized = dict() + action = ManagementAction.QUERY + query = DatasetQuery(query_type=QueryType.GET_SERIALIZED_FORM) + for d in datasets: + request = MaaSDatasetManagementMessage(action=action, query=query, dataset_name=d, + session_secret=self.session_secret) + self.last_response: DatasetManagementResponse = await self.async_make_request(request) + if self.last_response.success: + serialized[d] = self.last_response.data['dataset'] + # TODO: what to do if any are not successful + return serialized + async def list_datasets(self, category: Optional[DataCategory] = None) -> List[str]: await self._async_acquire_session_info() action = ManagementAction.LIST_ALL if category is None else ManagementAction.SEARCH From 7f4abae977bc9e1e75504d212d5fdc5c672e2b0d Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Mon, 3 Oct 2022 11:01:25 -0500 Subject: [PATCH 09/46] Improve DatasetExternalClient. Fix issue with not acquiring a session, and wrapping some things in try block to catch and log exceptions. --- .../lib/client/dmod/client/request_clients.py | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/python/lib/client/dmod/client/request_clients.py b/python/lib/client/dmod/client/request_clients.py index 2ea95d98b..bd89d40e7 100644 --- a/python/lib/client/dmod/client/request_clients.py +++ b/python/lib/client/dmod/client/request_clients.py @@ -643,18 +643,24 @@ async def get_serialized_datasets(self, dataset_name: Optional[str] = None) -> D if dataset_name is None: datasets = await self.list_datasets() else: + # TODO: improve how this is use so that it can be safely, efficiently put everywhere it **may** be needed + await self._async_acquire_session_info() datasets = [dataset_name] serialized = dict() action = ManagementAction.QUERY query = DatasetQuery(query_type=QueryType.GET_SERIALIZED_FORM) - for d in datasets: - request = MaaSDatasetManagementMessage(action=action, query=query, dataset_name=d, - session_secret=self.session_secret) - self.last_response: DatasetManagementResponse = await self.async_make_request(request) - if self.last_response.success: - serialized[d] = self.last_response.data['dataset'] - # TODO: what to do if any are not successful - return serialized + try: + for d in datasets: + request = MaaSDatasetManagementMessage(action=action, query=query, dataset_name=d, + session_secret=self.session_secret) + self.last_response: DatasetManagementResponse = await self.async_make_request(request) + if self.last_response.success: + serialized[d] = self.last_response.data[DatasetManagementResponse._DATA_KEY_QUERY_RESULTS] + # TODO: what to do if any are not successful + return serialized + except Exception as e: + logger.error(e) + raise e async def list_datasets(self, category: Optional[DataCategory] = None) -> List[str]: await self._async_acquire_session_info() From 8a05d7191b0592c9102af14d681d38573f5c92fd Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Tue, 13 Sep 2022 16:57:55 -0500 Subject: [PATCH 10/46] Bump dataservice dependencies. --- python/services/dataservice/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/services/dataservice/setup.py b/python/services/dataservice/setup.py index cd74d404d..7113dc975 100644 --- a/python/services/dataservice/setup.py +++ b/python/services/dataservice/setup.py @@ -14,7 +14,7 @@ author_email='', url='', license='', - install_requires=['dmod-core>=0.3.0', 'dmod-communication>=0.7.1', 'dmod-scheduler>=0.7.0', 'dmod-modeldata>=0.9.0', + install_requires=['dmod-core>=0.3.0', 'dmod-communication>=0.8.0', 'dmod-scheduler>=0.7.0', 'dmod-modeldata>=0.9.0', 'redis'], packages=find_namespace_packages(exclude=('tests', 'test', 'deprecated', 'conf', 'schemas', 'ssl', 'src')) ) From 274719bb3ce0ea3eb1ddcd222afc0d6c02908307 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Mon, 3 Oct 2022 11:36:20 -0500 Subject: [PATCH 11/46] Fix LIST_FILES query response bug in dataservice. Fixing bug in response to LIST_FILES query, where reason text was not being assembled entirely correctly. --- python/services/dataservice/dmod/dataservice/service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/services/dataservice/dmod/dataservice/service.py b/python/services/dataservice/dmod/dataservice/service.py index 60a27cec1..7926525d8 100644 --- a/python/services/dataservice/dmod/dataservice/service.py +++ b/python/services/dataservice/dmod/dataservice/service.py @@ -684,7 +684,7 @@ def _process_query(self, message: DatasetManagementMessage) -> DatasetManagement dataset_name = message.dataset_name list_of_files = self.get_known_datasets()[dataset_name].manager.list_files(dataset_name) return DatasetManagementResponse(action=message.management_action, success=True, dataset_name=dataset_name, - reason='Obtained {} Items List', + reason='Obtained {} Items List'.format(dataset_name), data={DatasetManagementResponse._DATA_KEY_QUERY_RESULTS: list_of_files}) elif query_type == QueryType.GET_SERIALIZED_FORM: dataset_name = message.dataset_name From 4f6da152e3812af16c86210e8b6ce24e3f2e4f70 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Wed, 12 Oct 2022 13:30:53 -0500 Subject: [PATCH 12/46] Update dataset manager abstract interface. Adding optional offset and length params to abstract interface definition of get_data, and adding get_file_stat abstract method. --- python/lib/core/dmod/core/dataset.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/python/lib/core/dmod/core/dataset.py b/python/lib/core/dmod/core/dataset.py index b1e5ca2aa..bbb9701c5 100644 --- a/python/lib/core/dmod/core/dataset.py +++ b/python/lib/core/dmod/core/dataset.py @@ -819,7 +819,8 @@ def filter(self, base_dataset: Dataset, restrictions: List[Union[ContinuousRestr pass @abstractmethod - def get_data(self, dataset_name: str, item_name: str, **kwargs) -> Union[bytes, Any]: + def get_data(self, dataset_name: str, item_name: str, offset: Optional[int] = None, length: Optional[int] = None, + **kwargs) -> Union[bytes, Any]: """ Get data from this dataset. @@ -832,6 +833,10 @@ def get_data(self, dataset_name: str, item_name: str, **kwargs) -> Union[bytes, The dataset from which to get data. item_name : str The name of the object from which to get data. + offset : Optional[int] + Optional start byte position of object data. + length : Optional[int] + Optional number of bytes of object data from offset. kwargs Implementation-specific params for representing what data to get and how to get and deliver it. @@ -881,6 +886,26 @@ def link_user(self, user: DatasetUser, dataset: Dataset) -> bool: self._dataset_users[dataset.name].add(user.uuid) return True + @abstractmethod + def get_file_stat(self, dataset_name: str, file_name, **kwargs) -> Dict[str, Any]: + """ + Get the meta information about the given file. + + Parameters + ---------- + dataset_name : str + The name of the dataset containing the file of interest. + file_name : str + The name of the file of interest. + kwargs + + Returns + ------- + dict + Meta information about the given file, in dictionary form. + """ + pass + @abstractmethod def list_files(self, dataset_name: str, **kwargs) -> List[str]: """ From a65ddfebba9ae74120aee335997261aabb896151 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Wed, 12 Oct 2022 13:33:28 -0500 Subject: [PATCH 13/46] More QueryType elements and update DatasetQuery. Adding support for querying about dataset items. --- .../dataset_management_message.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/python/lib/communication/dmod/communication/dataset_management_message.py b/python/lib/communication/dmod/communication/dataset_management_message.py index 55f51e7b5..73a80b164 100644 --- a/python/lib/communication/dmod/communication/dataset_management_message.py +++ b/python/lib/communication/dmod/communication/dataset_management_message.py @@ -17,6 +17,10 @@ class QueryType(Enum): GET_MIN_VALUE = 7 GET_MAX_VALUE = 8 GET_SERIALIZED_FORM = 9 + GET_LAST_UPDATED = 10 + GET_SIZE = 11 + GET_ITEM_SIZE = 12 + GET_DATASET_ITEMS = 13 @classmethod def get_for_name(cls, name_str: str) -> 'QueryType': @@ -43,26 +47,32 @@ def get_for_name(cls, name_str: str) -> 'QueryType': class DatasetQuery(Serializable): _KEY_QUERY_TYPE = 'query_type' + _KEY_ITEM_NAME = 'item_name' @classmethod def factory_init_from_deserialized_json(cls, json_obj: dict) -> Optional['DatasetQuery']: try: - return cls(query_type=QueryType.get_for_name(json_obj[cls._KEY_QUERY_TYPE])) + return cls(query_type=QueryType.get_for_name(json_obj[cls._KEY_QUERY_TYPE]), + item_name=json_obj.get(cls._KEY_ITEM_NAME)) except Exception as e: return None def __hash__(self): - return hash(self.query_type) + return hash('{}{}'.format(self.query_type.name, self.item_name if self.item_name is not None else '')) def __eq__(self, other): - return isinstance(other, DatasetQuery) and self.query_type == other.query_type + return isinstance(other, DatasetQuery) and self.query_type == other.query_type \ + and self.item_name == other.item_name - def __init__(self, query_type: QueryType): + def __init__(self, query_type: QueryType, item_name: Optional[str] = None): self.query_type = query_type + self.item_name = item_name def to_dict(self) -> Dict[str, Union[str, Number, dict, list]]: serial = dict() serial[self._KEY_QUERY_TYPE] = self.query_type.name + if self.item_name is not None: + serial[self._KEY_ITEM_NAME] = self.item_name return serial From c403de7f3444c48a69285b8cbf68491b977c56e8 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Wed, 12 Oct 2022 13:34:17 -0500 Subject: [PATCH 14/46] Update dataset message to support start and size. Updating to support indicating start and size of data for partial transfers. --- .../dataset_management_message.py | 32 +++++++++++++++++-- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/python/lib/communication/dmod/communication/dataset_management_message.py b/python/lib/communication/dmod/communication/dataset_management_message.py index 73a80b164..d21192906 100644 --- a/python/lib/communication/dmod/communication/dataset_management_message.py +++ b/python/lib/communication/dmod/communication/dataset_management_message.py @@ -192,6 +192,8 @@ class DatasetManagementMessage(AbstractInitRequest): _SERIAL_KEY_CATEGORY = 'category' _SERIAL_KEY_DATA_DOMAIN = 'data_domain' _SERIAL_KEY_DATA_LOCATION = 'data_location' + _SERIAL_KEY_DATA_BLK_START = 'data_blk_start' + _SERIAL_KEY_DATA_BLK_SIZE = 'data_blk_size' _SERIAL_KEY_DATASET_NAME = 'dataset_name' _SERIAL_KEY_IS_PENDING_DATA = 'pending_data' _SERIAL_KEY_QUERY = 'query' @@ -228,6 +230,8 @@ def factory_init_from_deserialized_json(cls, json_obj: dict) -> Optional['Datase category_str = json_obj.get(cls._SERIAL_KEY_CATEGORY) category = None if category_str is None else DataCategory.get_for_name(category_str) data_loc = json_obj.get(cls._SERIAL_KEY_DATA_LOCATION) + data_blk_start = json_obj.get(cls._SERIAL_KEY_DATA_BLK_START) + data_blk_size = json_obj.get(cls._SERIAL_KEY_DATA_BLK_SIZE) #page = json_obj[cls._SERIAL_KEY_PAGE] if cls._SERIAL_KEY_PAGE in json_obj else None if cls._SERIAL_KEY_QUERY in json_obj: query = DatasetQuery.factory_init_from_deserialized_json(json_obj[cls._SERIAL_KEY_QUERY]) @@ -240,7 +244,7 @@ def factory_init_from_deserialized_json(cls, json_obj: dict) -> Optional['Datase return deserialized_class(action=action, dataset_name=dataset_name, category=category, is_read_only_dataset=json_obj[cls._SERIAL_KEY_IS_READ_ONLY], domain=domain, - data_location=data_loc, + data_location=data_loc, blk_start=data_blk_start, blk_size=data_blk_size, is_pending_data=json_obj.get(cls._SERIAL_KEY_IS_PENDING_DATA), #page=page, query=query, **deserialized_class_kwargs) except Exception as e: @@ -272,8 +276,8 @@ def __hash__(self): def __init__(self, action: ManagementAction, dataset_name: Optional[str] = None, is_read_only_dataset: bool = False, category: Optional[DataCategory] = None, domain: Optional[DataDomain] = None, - data_location: Optional[str] = None, is_pending_data: bool = False, - query: Optional[DatasetQuery] = None, *args, **kwargs): + data_location: Optional[str] = None, blk_start: Optional[int] = None, blk_size: Optional[int] = None, + is_pending_data: bool = False, query: Optional[DatasetQuery] = None, *args, **kwargs): """ Initialize this instance. @@ -289,6 +293,10 @@ def __init__(self, action: ManagementAction, dataset_name: Optional[str] = None, The optional category of the involved dataset or datasets, when applicable; defaults to ``None``. data_location : Optional[str] Optional location/file/object/etc. for acted-upon data. + blk_start : Optional[int] + Optional starting point for when acting upon a block/chunk of data. + blk_size : Optional[int] + Optional block size for when acting upon a block/chunk of data. is_pending_data : bool Whether the sender has data pending transmission after this message (default: ``False``). query : Optional[DatasetQuery] @@ -313,9 +321,19 @@ def __init__(self, action: ManagementAction, dataset_name: Optional[str] = None, self._category = category self._domain = domain self._data_location = data_location + self._blk_start = blk_start + self._blk_size = blk_size self._query = query self._is_pending_data = is_pending_data + @property + def blk_size(self) -> Optional[int]: + return self._blk_size + + @property + def blk_start(self) -> Optional[int]: + return self._blk_start + @property def data_location(self) -> Optional[str]: """ @@ -417,6 +435,10 @@ def to_dict(self) -> Dict[str, Union[str, Number, dict, list]]: serial[self._SERIAL_KEY_CATEGORY] = self.data_category.name if self.data_location is not None: serial[self._SERIAL_KEY_DATA_LOCATION] = self.data_location + if self._blk_start is not None: + serial[self._SERIAL_KEY_DATA_BLK_START] = self._blk_start + if self._blk_size is not None: + serial[self._SERIAL_KEY_DATA_BLK_SIZE] = self._blk_size if self.data_domain is not None: serial[self._SERIAL_KEY_DATA_DOMAIN] = self.data_domain.to_dict() if self.query is not None: @@ -613,6 +635,10 @@ def __init__(self, session_secret: str, *args, **kwargs): is_read_only_dataset : bool category : Optional[DataCategory] data_location : Optional[str] + blk_start : Optional[int] + Optional starting point for when acting upon a block/chunk of data. + blk_size : Optional[int] + Optional block size for when acting upon a block/chunk of data. is_pending_data : bool query : Optional[DataQuery] """ From ca81028193f2f663c469146a981c7827a55e0b4d Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Tue, 13 Sep 2022 16:58:11 -0500 Subject: [PATCH 15/46] Bump requestservice dependencies. --- python/services/requestservice/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/services/requestservice/setup.py b/python/services/requestservice/setup.py index 06618889d..af7ed1e0a 100644 --- a/python/services/requestservice/setup.py +++ b/python/services/requestservice/setup.py @@ -14,7 +14,7 @@ author_email='', url='', license='', - install_requires=['websockets', 'dmod-core>=0.1.0', 'dmod-communication>=0.7.0', 'dmod-access>=0.2.0', + install_requires=['websockets', 'dmod-core>=0.2.0', 'dmod-communication>=0.8.0', 'dmod-access>=0.2.0', 'dmod-externalrequests>=0.3.0'], packages=find_namespace_packages(exclude=('tests', 'schemas', 'ssl', 'src')) ) From f823d81bfad0ed3eb666e2e80368f04c97e16ac2 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Tue, 13 Sep 2022 16:57:33 -0500 Subject: [PATCH 16/46] Bump requestservice to 0.6.0. --- python/services/requestservice/dmod/requestservice/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/services/requestservice/dmod/requestservice/_version.py b/python/services/requestservice/dmod/requestservice/_version.py index 08d79c0e9..83e147c62 100644 --- a/python/services/requestservice/dmod/requestservice/_version.py +++ b/python/services/requestservice/dmod/requestservice/_version.py @@ -1 +1 @@ -__version__ = '0.5.1' \ No newline at end of file +__version__ = '0.6.0' \ No newline at end of file From cb61094759863f9cc4845bbbed85321e085adb8e Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Wed, 12 Oct 2022 13:35:21 -0500 Subject: [PATCH 17/46] Bump communication package version to 0.10.1. --- python/lib/communication/dmod/communication/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/lib/communication/dmod/communication/_version.py b/python/lib/communication/dmod/communication/_version.py index 9d1bb721b..e754a834e 100644 --- a/python/lib/communication/dmod/communication/_version.py +++ b/python/lib/communication/dmod/communication/_version.py @@ -1 +1 @@ -__version__ = '0.10.0' +__version__ = '0.10.1' From 0199609af12af488bac19e1b4cf81175731221ac Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Wed, 12 Oct 2022 13:41:13 -0500 Subject: [PATCH 18/46] Optimize object store dataset reloading. Optimizing the reloading of datasets on object store manager startup. --- .../dmod/modeldata/data/object_store_manager.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/python/lib/modeldata/dmod/modeldata/data/object_store_manager.py b/python/lib/modeldata/dmod/modeldata/data/object_store_manager.py index dffb5a8cf..910d98923 100644 --- a/python/lib/modeldata/dmod/modeldata/data/object_store_manager.py +++ b/python/lib/modeldata/dmod/modeldata/data/object_store_manager.py @@ -59,8 +59,13 @@ def __init__(self, obj_store_host_str: str, access_key: Optional[str] = None, se # For any buckets that have the standard serialized object (i.e., were for datasets previously), reload them for bucket_name in self.list_buckets(): serialized_item = self._gen_dataset_serial_obj_name(bucket_name) - if serialized_item in [o.object_name for o in self._client.list_objects(bucket_name)]: + try: self.reload(reload_from=bucket_name, serialized_item=serialized_item) + except minio.error.S3Error as e: + # Continue with looping through buckets and initializing if we get this particular exception and + # error code, but otherwise pass through the exception + if e.code != "NoSuchKey": + raise e except Exception as e: self._errors.append(e) # TODO: consider if we should not re-throw this (which would likely force us to ensure users checked this) @@ -552,12 +557,14 @@ def reload(self, reload_from: str, serialized_item: Optional[str] = None) -> Dat if serialized_item is None: serialized_item = self._gen_dataset_serial_obj_name(reload_from) + response_obj = None try: response_obj = self._client.get_object(bucket_name=reload_from, object_name=serialized_item) response_data = json.loads(response_obj.data.decode()) finally: - response_obj.close() - response_obj.release_conn() + if response_obj is not None: + response_obj.close() + response_obj.release_conn() # If we can safely infer it, make sure the "type" key is set in cases when it is missing if len(self.supported_dataset_types) == 1 and Dataset._KEY_TYPE not in response_data: From 98ccf92ebda29e12bd2090838c905a95c6986747 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Wed, 12 Oct 2022 13:42:39 -0500 Subject: [PATCH 19/46] Update object store manager for interface changes. Accounting for changes to get_data parameters and implementing get_file_stat. --- .../modeldata/data/object_store_manager.py | 46 ++++++++++++++----- 1 file changed, 35 insertions(+), 11 deletions(-) diff --git a/python/lib/modeldata/dmod/modeldata/data/object_store_manager.py b/python/lib/modeldata/dmod/modeldata/data/object_store_manager.py index 910d98923..551cb2aa1 100644 --- a/python/lib/modeldata/dmod/modeldata/data/object_store_manager.py +++ b/python/lib/modeldata/dmod/modeldata/data/object_store_manager.py @@ -10,7 +10,7 @@ from minio.api import ObjectWriteResult from minio.deleteobjects import DeleteObject from pathlib import Path -from typing import Dict, List, Optional, Set, Tuple +from typing import Any, Dict, List, Optional, Set, Tuple from uuid import UUID @@ -423,7 +423,32 @@ def delete_data(self, dataset_name: str, **kwargs) -> bool: self._errors.extend(error_list) return False - def get_data(self, dataset_name: str, item_name: str, **kwargs) -> bytes: + def get_file_stat(self, dataset_name: str, file_name, **kwargs) -> Dict[str, Any]: + """ + Get the meta information about the given file. + + Parameters + ---------- + dataset_name : str + The name of the dataset containing the file of interest. + file_name : str + The name of the file of interest. + kwargs + + Returns + ------- + dict + Meta information about the given file, in dictionary form. + """ + obj_stat = self._client.stat_object(dataset_name, file_name) + as_dict = dict() + as_dict["name"] = obj_stat.object_name + as_dict["size"] = obj_stat.size + # TODO: get more of this if worth it + return as_dict + + def get_data(self, dataset_name: str, item_name: str, offset: Optional[int] = None, length: Optional[int] = None, + **kwargs) -> bytes: """ Get data from this dataset. @@ -437,15 +462,12 @@ def get_data(self, dataset_name: str, item_name: str, **kwargs) -> bytes: The name of the dataset (i.e., bucket) from which to get data. item_name : str The name of the object from which to get data. - kwargs - Implementation-specific params for representing what data to get and how to get and deliver it. - - Keyword Args - ------- - offset : int + offset : Optional[int] Optional start byte position of object data. - length : int + length : Optional[int] Optional number of bytes of object data from offset. + kwargs + Implementation-specific params for representing what data to get and how to get and deliver it. Returns ------- @@ -455,8 +477,10 @@ def get_data(self, dataset_name: str, item_name: str, **kwargs) -> bytes: if item_name not in self.list_files(dataset_name): raise RuntimeError('Cannot get data for non-existing {} file in {} dataset'.format(item_name, dataset_name)) optional_params = dict() - for key in [k for k in self.data_chunking_params if k in kwargs]: - optional_params[key] = kwargs[key] + if offset is not None: + optional_params['offset'] = offset + if length is not None: + optional_params['length'] = length response_object = self._client.get_object(bucket_name=dataset_name, object_name=item_name, **optional_params) return response_object.data From 6670dd1cd7edda6ddfbc687c7ddbd460ef2992ac Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Wed, 12 Oct 2022 13:35:57 -0500 Subject: [PATCH 20/46] Update modeldata dependency versions. --- python/lib/modeldata/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/lib/modeldata/setup.py b/python/lib/modeldata/setup.py index d00c51756..3f8f70565 100644 --- a/python/lib/modeldata/setup.py +++ b/python/lib/modeldata/setup.py @@ -14,7 +14,7 @@ author_email='', url='', license='', - install_requires=['numpy>=1.20.1', 'pandas', 'geopandas', 'dmod-communication>=0.4.2', 'dmod-core>=0.3.0', 'minio', + install_requires=['numpy>=1.20.1', 'pandas', 'geopandas', 'dmod-communication>=0.9.1', 'dmod-core>=0.3.0', 'minio', 'aiohttp<=3.7.4', 'hypy@git+https://github.com/NOAA-OWP/hypy@master#egg=hypy&subdirectory=python'], packages=find_namespace_packages(exclude=('tests', 'schemas', 'ssl', 'src')) ) From 0f0cd007914a017d4f980f4ee2b3469239bbd9cf Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Wed, 12 Oct 2022 14:32:19 -0500 Subject: [PATCH 21/46] Update client unit tests for interface changes. --- .../client/dmod/test/test_dataset_client.py | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/python/lib/client/dmod/test/test_dataset_client.py b/python/lib/client/dmod/test/test_dataset_client.py index b266658c7..37ff6e386 100644 --- a/python/lib/client/dmod/test/test_dataset_client.py +++ b/python/lib/client/dmod/test/test_dataset_client.py @@ -1,7 +1,7 @@ import unittest from ..client.request_clients import DataCategory, DatasetClient, DatasetManagementResponse, MaaSDatasetManagementResponse from pathlib import Path -from typing import List, Optional +from typing import List, Optional, AnyStr class SimpleMockDatasetClient(DatasetClient): @@ -28,6 +28,24 @@ async def download_from_dataset(self, dataset_name: str, item_name: str, dest: P """ Mock implementation, always returning ``False``. """ return False + async def download_item_block(self, dataset_name: str, item_name: str, blk_start: int, blk_size: int) -> AnyStr: + """ + Mock implementation, always returning empty string. + """ + return '' + + async def get_dataset_content_details(self, name: str, **kwargs) -> bool: + """ + Mock implementation, always returning ``False``. + """ + return False + + async def get_item_size(self, dataset_name: str, item_name: str) -> int: + """ + Mock implementation always returning ``1``. + """ + return 1 + async def list_datasets(self, category: Optional[DataCategory] = None) -> List[str]: """ Mock implementation, always returning an empty list. """ return [] From fc138048761b3d51a70b5559c98e0494ad6b2745 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Wed, 12 Oct 2022 13:43:54 -0500 Subject: [PATCH 22/46] Bump client package version to 0.2.0. --- python/lib/client/dmod/client/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/lib/client/dmod/client/_version.py b/python/lib/client/dmod/client/_version.py index b794fd409..7fd229a32 100644 --- a/python/lib/client/dmod/client/_version.py +++ b/python/lib/client/dmod/client/_version.py @@ -1 +1 @@ -__version__ = '0.1.0' +__version__ = '0.2.0' From 2afa9bde31b9dd0aef82c0a94cb3b189a6c720b5 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Wed, 12 Oct 2022 13:38:21 -0500 Subject: [PATCH 23/46] Update dataservice response handling. Updating service to respond to GET_DATASET_ITEMS queries and requests for data with an offset start (i.e. partials). --- .../dataservice/dmod/dataservice/service.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/python/services/dataservice/dmod/dataservice/service.py b/python/services/dataservice/dmod/dataservice/service.py index 7926525d8..1d1c7c37d 100644 --- a/python/services/dataservice/dmod/dataservice/service.py +++ b/python/services/dataservice/dmod/dataservice/service.py @@ -692,6 +692,13 @@ def _process_query(self, message: DatasetManagementMessage) -> DatasetManagement return DatasetManagementResponse(action=message.management_action, success=True, dataset_name=dataset_name, reason='Obtained serialized {} dataset'.format(dataset_name), data={DatasetManagementResponse._DATA_KEY_QUERY_RESULTS: serialized_form}) + if query_type == QueryType.GET_DATASET_ITEMS: + dataset = self.get_known_datasets()[message.dataset_name] + mgr = dataset.manager + item_details: List[dict] = [mgr.get_file_stat(dataset.name, f) for f in mgr.list_files(dataset.name)] + return DatasetManagementResponse(action=message.management_action, success=True, dataset_name=dataset.name, + reason='Obtained file details for {} dataset'.format(dataset.name), + data={DatasetManagementResponse._DATA_KEY_QUERY_RESULTS: item_details}) # TODO: (later) add support for messages with other query types also else: reason = 'Unsupported {} Query Type - {}'.format(DatasetQuery.__class__.__name__, query_type.name) @@ -896,6 +903,14 @@ async def listener(self, websocket: WebSocketServerProtocol, path): partial_indx = 0 elif inbound_message.management_action == ManagementAction.CREATE: response = await self._async_process_dataset_create(message=inbound_message) + elif inbound_message.management_action == ManagementAction.REQUEST_DATA and inbound_message.blk_start is not None: + manager = self.get_known_datasets()[inbound_message.dataset_name].manager + raw_data = manager.get_data(dataset_name=inbound_message.dataset_name, + item_name=inbound_message.data_location, + offset=inbound_message.blk_start, length=inbound_message.blk_size) + response = DatasetManagementResponse(success=raw_data is not None, + action=inbound_message.management_action, + data=raw_data, reason="Data Block Retrieve Complete") elif inbound_message.management_action == ManagementAction.REQUEST_DATA: response = await self._async_process_data_request(message=inbound_message, websocket=websocket) elif inbound_message.management_action == ManagementAction.ADD_DATA: From edd262252d5019909a1ffd00712dfd2fa5c76fc5 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Wed, 12 Oct 2022 13:38:46 -0500 Subject: [PATCH 24/46] Update dataservice package dependency versions. --- python/services/dataservice/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/services/dataservice/setup.py b/python/services/dataservice/setup.py index 7113dc975..55bb3221d 100644 --- a/python/services/dataservice/setup.py +++ b/python/services/dataservice/setup.py @@ -14,7 +14,7 @@ author_email='', url='', license='', - install_requires=['dmod-core>=0.3.0', 'dmod-communication>=0.8.0', 'dmod-scheduler>=0.7.0', 'dmod-modeldata>=0.9.0', + install_requires=['dmod-core>=0.3.0', 'dmod-communication>=0.9.1', 'dmod-scheduler>=0.7.0', 'dmod-modeldata>=0.9.0', 'redis'], packages=find_namespace_packages(exclude=('tests', 'test', 'deprecated', 'conf', 'schemas', 'ssl', 'src')) ) From 0d0ad3aaf2b4df368a9d053e1b0d5b9e71e568bc Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Tue, 13 Sep 2022 16:46:52 -0500 Subject: [PATCH 25/46] Add dataset view class and static page for GUI. --- python/gui/MaaS/cbv/DatasetManagementView.py | 132 +++++++ .../templates/maas/dataset_management.html | 359 ++++++++++++++++++ 2 files changed, 491 insertions(+) create mode 100644 python/gui/MaaS/cbv/DatasetManagementView.py create mode 100644 python/gui/MaaS/templates/maas/dataset_management.html diff --git a/python/gui/MaaS/cbv/DatasetManagementView.py b/python/gui/MaaS/cbv/DatasetManagementView.py new file mode 100644 index 000000000..bfaf7fd10 --- /dev/null +++ b/python/gui/MaaS/cbv/DatasetManagementView.py @@ -0,0 +1,132 @@ +""" +Defines a view that may be used to configure a MaaS request +""" +import asyncio +import os +from django.http import HttpRequest, HttpResponse +from django.views.generic.base import View +from django.shortcuts import render + +import dmod.communication as communication +from dmod.client.request_clients import DatasetExternalClient +from dmod.core.meta_data import DataCategory, DataFormat + +import logging +logger = logging.getLogger("gui_log") + +from pathlib import Path + +from .DMODProxy import DMODMixin, GUI_STATIC_SSL_DIR +from .utils import extract_log_data +from typing import Dict + + +class DatasetManagementView(View, DMODMixin): + + """ + A view used to configure a dataset management request or requests for transmitting dataset data. + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._dataset_client = None + + def _process_event_type(self, http_request: HttpRequest) -> communication.MessageEventType: + """ + Determine and return whether this request is for a ``DATASET_MANAGEMENT`` or ``DATA_TRANSMISSION`` event. + + Parameters + ---------- + http_request : HttpRequest + The raw HTTP request in question. + + Returns + ------- + communication.MessageEventType + Either ``communication.MessageEventType.DATASET_MANAGEMENT`` or + ``communication.MessageEventType.DATA_TRANSMISSION``. + """ + # TODO: + raise NotImplementedError("{}._process_event_type not implemented".format(self.__class__.__name__)) + + @property + def dataset_client(self) -> DatasetExternalClient: + if self._dataset_client is None: + self._dataset_client = DatasetExternalClient(endpoint_uri=self.maas_endpoint_uri, + ssl_directory=GUI_STATIC_SSL_DIR) + return self._dataset_client + + async def _get_datasets(self) -> Dict[str, dict]: + serial_datasets = await self.dataset_client.get_serialized_datasets() + return serial_datasets + + def get(self, http_request: HttpRequest, *args, **kwargs) -> HttpResponse: + """ + The handler for 'get' requests. + + This will render the 'maas/dataset_management.html' template after retrieving necessary information to initially + populate the forms it displays. + + Parameters + ---------- + http_request : HttpRequest + The request asking to render this page. + args + kwargs + + Returns + ------- + A rendered page. + """ + errors, warnings, info = extract_log_data(kwargs) + + # Gather map of serialized datasets, keyed by dataset name + serial_dataset_map = asyncio.get_event_loop().run_until_complete(self._get_datasets()) + + dataset_categories = [c.name.title() for c in DataCategory] + dataset_formats = [f.name for f in DataFormat] + + payload = { + 'datasets': serial_dataset_map, + 'dataset_categories': dataset_categories, + 'dataset_formats': dataset_formats, + 'errors': errors, + 'info': info, + 'warnings': warnings + } + + # TODO: create this file + return render(http_request, 'maas/dataset_management.html', payload) + + def post(self, http_request: HttpRequest, *args, **kwargs) -> HttpResponse: + """ + The handler for 'post' requests. + + This will attempt to submit the request and rerender the page like a 'get' request. + + Parameters + ---------- + http_request : HttpRequest + The request asking to render this page. + args + kwargs + + Returns + ------- + A rendered page. + """ + # TODO: implement this to figure out whether DATASET_MANAGEMENT or DATA_TRANSMISSION + event_type = self._process_event_type(http_request) + client, session_data, dmod_response = self.forward_request(http_request, event_type) + + # TODO: this probably isn't exactly correct, so review once closer to completion + if dmod_response is not None and 'dataset_id' in dmod_response.data: + session_data['new_dataset_id'] = dmod_response.data['dataset_id'] + + http_response = self.get(http_request=http_request, errors=client.errors, warnings=client.warnings, + info=client.info, *args, **kwargs) + + for k, v in session_data.items(): + http_response.set_cookie(k, v) + + return http_response diff --git a/python/gui/MaaS/templates/maas/dataset_management.html b/python/gui/MaaS/templates/maas/dataset_management.html new file mode 100644 index 000000000..9ac763131 --- /dev/null +++ b/python/gui/MaaS/templates/maas/dataset_management.html @@ -0,0 +1,359 @@ + + + + + OWP MaaS + {% load static %} + + + + + + + + +
+ + + {% if errors %} +
+
    + {% for error in errors %} +
  • {{ error }}
    • + {% endfor %} +
+
+ {% endif %} + + {% if warnings %} +
+
    + {% for warning in warnings %} +
  • {{ warning }}
    • + {% endfor %} +
+
+ {% endif %} + + {% if info %} +
+
    + {% for message in info %} +
  • {{ message }}
    • + {% endfor %} +
+
+ {% endif %} + + {# Cache jQuery scripts for UI scripting and styling #} +
+

Dataset Management

+
+ Manage + Create +
+
+

Create New Dataset:

+
+ {# Add the token to provide cross site request forgery protection #} + {% csrf_token %} +
+ + +
+ + + + +
+ + + + +
+ + +
+
+ + +
+
+
+
+ + From a6b0aa6afcbd8800806140ce97f4399c57c2fd86 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Tue, 13 Sep 2022 16:47:49 -0500 Subject: [PATCH 26/46] Refactor static dir usage in DMODProxy.py. --- python/gui/MaaS/cbv/DMODProxy.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python/gui/MaaS/cbv/DMODProxy.py b/python/gui/MaaS/cbv/DMODProxy.py index fc0fcb7a1..e9e308664 100644 --- a/python/gui/MaaS/cbv/DMODProxy.py +++ b/python/gui/MaaS/cbv/DMODProxy.py @@ -16,6 +16,8 @@ from pathlib import Path from typing import List, Optional, Tuple, Type +GUI_STATIC_SSL_DIR = Path('/usr/maas_portal/ssl') + class RequestFormProcessor(ABC): @@ -209,7 +211,7 @@ class PostFormRequestClient(ModelExecRequestClient): def _bootstrap_ssl_dir(cls, ssl_dir: Optional[Path] = None): if ssl_dir is None: ssl_dir = Path(__file__).resolve().parent.parent.parent.joinpath('ssl') - ssl_dir = Path('/usr/maas_portal/ssl') #Fixme + ssl_dir = GUI_STATIC_SSL_DIR #Fixme return ssl_dir def __init__(self, endpoint_uri: str, http_request: HttpRequest, ssl_dir: Optional[Path] = None): @@ -315,6 +317,7 @@ def forward_request(self, request: HttpRequest, event_type: MessageEventType) -> client = PostFormRequestClient(endpoint_uri=self.maas_endpoint_uri, http_request=request) if event_type == MessageEventType.MODEL_EXEC_REQUEST: form_processor_type = ModelExecRequestFormProcessor + # TODO: need a new type of form processor here (or 3 more, for management, uploading, and downloading) else: raise RuntimeError("{} got unsupported event type: {}".format(self.__class__.__name__, str(event_type))) From 354d3c5024ffedc7c7a162baf1940d1c3b1acd73 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Tue, 13 Sep 2022 16:59:42 -0500 Subject: [PATCH 27/46] Add Django GUI url def for dataset view. --- python/gui/MaaS/urls.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/python/gui/MaaS/urls.py b/python/gui/MaaS/urls.py index a6496fada..828dbe6c6 100644 --- a/python/gui/MaaS/urls.py +++ b/python/gui/MaaS/urls.py @@ -1,5 +1,6 @@ from django.urls import re_path from .cbv.EditView import EditView +from .cbv.DatasetManagementView import DatasetManagementView from .cbv.MapView import MapView, Fabrics, FabricNames, FabricTypes, ConnectedFeatures from .cbv.configuration import CreateConfiguration @@ -10,6 +11,9 @@ urlpatterns = [ re_path(r'^$', EditView.as_view()), + # TODO: add this later + #re_path(r'ngen$', NgenWorkflowView.as_view(), name="ngen-workflow"), + re_path(r'datasets', DatasetManagementView.as_view(), name="dataset-management"), re_path(r'map$', MapView.as_view(), name="map"), re_path(r'map/connections$', ConnectedFeatures.as_view(), name="connections"), re_path(r'fabric/names$', FabricNames.as_view(), name='fabric-names'), From 9916b4d67f7c5e23637c4b55334c5b17210fa42c Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Tue, 13 Sep 2022 16:48:19 -0500 Subject: [PATCH 28/46] Add client package build arg for GUI Docker build. --- docker/nwm_gui/docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/nwm_gui/docker-compose.yml b/docker/nwm_gui/docker-compose.yml index 509441649..97c7a7b68 100644 --- a/docker/nwm_gui/docker-compose.yml +++ b/docker/nwm_gui/docker-compose.yml @@ -34,6 +34,7 @@ services: args: docker_internal_registry: ${DOCKER_INTERNAL_REGISTRY:?Missing DOCKER_INTERNAL_REGISTRY value (see 'Private Docker Registry ' section in example.env)} comms_package_name: ${PYTHON_PACKAGE_DIST_NAME_COMMS:?} + client_package_name: ${PYTHON_PACKAGE_DIST_NAME_CLIENT:?} networks: - request-listener-net # Call this when starting the container From 14f675fc730b7f5d37e6db0dcfe93cbdca68d1f5 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Thu, 29 Sep 2022 10:38:26 -0500 Subject: [PATCH 29/46] Add second view and navigation. Adding initial second view display and manage existing datasets, along with navigation functionality to toggle between "manage" and "create" views. --- .../templates/maas/dataset_management.html | 97 ++++++++++++++++++- 1 file changed, 95 insertions(+), 2 deletions(-) diff --git a/python/gui/MaaS/templates/maas/dataset_management.html b/python/gui/MaaS/templates/maas/dataset_management.html index 9ac763131..c2fcd1827 100644 --- a/python/gui/MaaS/templates/maas/dataset_management.html +++ b/python/gui/MaaS/templates/maas/dataset_management.html @@ -16,6 +16,20 @@ $("#" + model + "_parameters").show(); } + /** + * Toggle what is displayed based on selection click in main nav bar for page. + * + * @param {string} selected_div_id - The id of the main-level div selected to be displayed. + */ + function toggleNav(selected_div_id) { + const all_nav_divs = ["dataset-manage", "dataset-create"]; + for (let i = 0; i <= all_nav_divs.length; i++) { + // TODO: consider a warning message for receiving an unexpected selection + let nav_div = document.getElementById(all_nav_divs[i]); + nav_div.style.display = (selected_div_id === all_nav_divs[i]) ? 'block' : 'none'; + } + } + /* {% for model, model_parameters in parameters.items %} {% for parameter in model_parameters %} @@ -152,6 +166,14 @@ font-size: 17px; } + #dataset-manage { + display: block; + } + + #dataset-create { + display: none; + } + @@ -196,8 +218,79 @@

Office of Water Prediction - Model as a Service

Dataset Management

- Manage - Create + Manage + Create +
+
+ + + + + + + + + + + {% for dataset in datasets %} + + + + + + + + + + {% endfor %} +
DatasetCategory
{{ dataset.name }}{{ dataset.data_category }}
+ + {% for dataset in datasets %} +
+ + +
+ × + + + + + + + + + + + + + + + + + + + + + + + + + + +
Name{{ dataset.name }}
Category{{ dataset.data_category }}
Type{{ dataset.type }}
Format{{ dataset.data_format }}
Created{{ dataset.create_on }}
Last Updated{{ dataset.last_updated }}
+
+
+ {% endfor %}

Create New Dataset:

From 57632b1706e18c3d958aec4144a9e671f677c478 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Fri, 30 Sep 2022 10:11:10 -0500 Subject: [PATCH 30/46] Have DatasetManagementView.py send DS as list. Sending serialized datasets to HTML template as list/array rather than dict/map. --- python/gui/MaaS/cbv/DatasetManagementView.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/gui/MaaS/cbv/DatasetManagementView.py b/python/gui/MaaS/cbv/DatasetManagementView.py index bfaf7fd10..47f411888 100644 --- a/python/gui/MaaS/cbv/DatasetManagementView.py +++ b/python/gui/MaaS/cbv/DatasetManagementView.py @@ -82,12 +82,13 @@ def get(self, http_request: HttpRequest, *args, **kwargs) -> HttpResponse: # Gather map of serialized datasets, keyed by dataset name serial_dataset_map = asyncio.get_event_loop().run_until_complete(self._get_datasets()) + serial_dataset_list = [serial_dataset_map[d] for d in serial_dataset_map] dataset_categories = [c.name.title() for c in DataCategory] dataset_formats = [f.name for f in DataFormat] payload = { - 'datasets': serial_dataset_map, + 'datasets': serial_dataset_list, 'dataset_categories': dataset_categories, 'dataset_formats': dataset_formats, 'errors': errors, From cf2256ab9670b35b30ed760b1c5b304cb409946b Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Fri, 30 Sep 2022 10:15:18 -0500 Subject: [PATCH 31/46] Update dataset management GUI with details view. Implementing layout and initial details-viewing behavior for dataset management GUI view. --- .../templates/maas/dataset_management.html | 154 +++++++++++++++++- 1 file changed, 150 insertions(+), 4 deletions(-) diff --git a/python/gui/MaaS/templates/maas/dataset_management.html b/python/gui/MaaS/templates/maas/dataset_management.html index c2fcd1827..f54c714f4 100644 --- a/python/gui/MaaS/templates/maas/dataset_management.html +++ b/python/gui/MaaS/templates/maas/dataset_management.html @@ -30,6 +30,125 @@ } } + function buildDatasetDetailsEmptyDiv() { + let details_div = document.createElement('div'); + details_div.id = 'details-div'; + details_div.class = 'details-modal'; + details_div.style.position = 'fixed'; + details_div.style.zIndex = '1'; + details_div.style.left = '25%'; + details_div.style.top = '5%'; + details_div.style.width = '50%'; + details_div.style.height = '50%'; + details_div.style.overflow = 'clip'; + details_div.style.backgroundColor = '#B7B5B5FF'; + details_div.style.border = '1px solid #888'; + details_div.style.padding = '5px'; + details_div.style.paddingTop = '0px'; + details_div.style.margin = '15% auto'; + + let close_x = document.createElement('span'); + close_x.style.padding = "0px"; + close_x.style.cursor = "pointer"; + close_x.style.fontWeight = "bold"; + close_x.style.fontSize = "20px"; + close_x.appendChild(document.createTextNode("\u00d7")); + close_x.onclick = function() { details_div.remove() }; + details_div.appendChild(close_x); + + let details_content_div = document.createElement('div'); + details_content_div.style.height = '90%'; + details_content_div.style.overflow = 'auto'; + + details_div.appendChild(details_content_div); + + let parent_div = document.getElementById('dataset-manage'); + parent_div.appendChild(details_div); + return details_content_div; + } + + function buildDetailsTableRow() { + let row = document.createElement('tr'); + // Skip the first arg, as this is the table itself + for (let i = 1; i < arguments.length; i++) { + let colCell = document.createElement('th'); + if (i == 1) { + colCell.style.textAlign = 'right'; + colCell.style.paddingRight = '8px'; + arguments[i] += ":"; + } + else { + colCell.style.textAlign = 'left'; + colCell.style.fontWeight = 'normal'; + } + let cellText = document.createTextNode(arguments[i]); + colCell.appendChild(cellText); + row.appendChild(colCell); + } + arguments[0].appendChild(row); + } + + function buildDatasetDetailsTable(dataset_name) { + {% for dataset in datasets %} + if (dataset_name == "{{ dataset.name }}") { + //let cellText = document.createTextNode("testing text {{ dataset.name }} (type: {{ dataset.type }})"); + //return cellText; + let table = document.createElement('table'); + table.class = 'datasets-details-table'; + buildDetailsTableRow(table, "Name", "{{ dataset.name }}") + buildDetailsTableRow(table, "Category", "{{ dataset.data_category }}") + buildDetailsTableRow(table, "Type", "{{ dataset.type }}") + buildDetailsTableRow(table, "Format", "{{ dataset.data_domain.data_format }}") + buildDetailsTableRow(table, "Read Only", "{{ dataset.is_read_only }}") + buildDetailsTableRow(table, "Created", "{{ dataset.create_on }}") + buildDetailsTableRow(table, "Last Updated", "{{ dataset.last_updated }}") + + // Put spacer row in to indicate the start of domain details + let domain_header_row = document.createElement('tr'); + let colCell = document.createElement('th'); + colCell.colSpan = 2; + colCell.style.textAlign = 'left'; + colCell.style.paddingTop = '20px'; + colCell.style.paddingLeft = '5%'; + colCell.style.fontStyle = 'italic'; + colCell.style.fontSize = '18px'; + colCell.appendChild(document.createTextNode("Data Domain")); + domain_header_row.appendChild(colCell); + table.appendChild(domain_header_row); + + // Then iteratively add domain details rows + {% for cont_rest in dataset.data_domain.continuous %} + buildDetailsTableRow(table, "{{ cont_rest.variable }}", "{{ cont_rest.begin }} to {{ cont_rest.end }}"); + {% endfor %} + {% for disc_rest in dataset.data_domain.discrete %} + buildDetailsTableRow(table, "{{ disc_rest.variable }}", "{{ disc_rest.values }}"); + {% endfor %} + + return table; + } + {% endfor %} + } + + function displayDatasetDetails(name) { + let details_div = buildDatasetDetailsEmptyDiv(); + + let detailsTable = buildDatasetDetailsTable(name); + details_div.appendChild(detailsTable); + details_div.style.display = 'block'; + } + + function clickDownloadDataset(dataset_name) { + return false; + } + + function clickUploadDataset(dataset_name) { + return false; + } + + function clickDeleteDataset(dataset_name) { + return false; + } + /* {% for model, model_parameters in parameters.items %} {% for parameter in model_parameters %} @@ -170,6 +289,25 @@ display: block; } + .datasets-table thead tr th { + padding-bottom: 5px; + } + + .mgr-tbl-dataset-header { + text-align: left; + } + + .mgr-tbl-category-header { + text-align: left; + } + + .mgr-tbl-content th { + font-weight: normal; + text-align: left; + padding-right: 15px; + padding-bottom: 2px; + } + #dataset-create { display: none; } @@ -225,8 +363,10 @@

Dataset Management

- - + + + + - + + + + + {% for dataset in datasets %} @@ -386,57 +380,9 @@

Dataset Management

- - - - {% endfor %}
DatasetCategoryDataset NameCategoryActions Details
- - {% for dataset in datasets %} -
- - -
- × - - - - - - - - - - - - - - - - - - - - - - - - - - -
Name{{ dataset.name }}
Category{{ dataset.data_category }}
Type{{ dataset.type }}
Format{{ dataset.data_format }}
Created{{ dataset.create_on }}
Last Updated{{ dataset.last_updated }}
-
-
- {% endfor %}

Create New Dataset:

From 09afa5888ce72178d69a5e9ac86ed7cd1ca7a745 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Mon, 3 Oct 2022 11:42:30 -0500 Subject: [PATCH 33/46] Create AbstractDatasetView GUI Django view class. --- python/gui/MaaS/cbv/AbstractDatasetView.py | 29 ++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 python/gui/MaaS/cbv/AbstractDatasetView.py diff --git a/python/gui/MaaS/cbv/AbstractDatasetView.py b/python/gui/MaaS/cbv/AbstractDatasetView.py new file mode 100644 index 000000000..e1ae493c3 --- /dev/null +++ b/python/gui/MaaS/cbv/AbstractDatasetView.py @@ -0,0 +1,29 @@ +from abc import ABC +from django.views.generic.base import View +from dmod.client.request_clients import DatasetExternalClient +import logging +logger = logging.getLogger("gui_log") +from .DMODProxy import DMODMixin, GUI_STATIC_SSL_DIR +from typing import Dict + + +class AbstractDatasetView(View, DMODMixin, ABC): + + def __init__(self, *args, **kwargs): + super(AbstractDatasetView, self).__init__(*args, **kwargs) + self._dataset_client = None + + async def get_dataset(self, dataset_name: str) -> Dict[str, dict]: + serial_dataset = await self.dataset_client.get_serialized_datasets(dataset_name=dataset_name) + return serial_dataset + + async def get_datasets(self) -> Dict[str, dict]: + serial_datasets = await self.dataset_client.get_serialized_datasets() + return serial_datasets + + @property + def dataset_client(self) -> DatasetExternalClient: + if self._dataset_client is None: + self._dataset_client = DatasetExternalClient(endpoint_uri=self.maas_endpoint_uri, + ssl_directory=GUI_STATIC_SSL_DIR) + return self._dataset_client From 5f0384dcb3cde68029f7b69c51bc82cf4f96a489 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Mon, 3 Oct 2022 11:39:15 -0500 Subject: [PATCH 34/46] Add Django view for direct dataservice API calls. --- python/gui/MaaS/cbv/DatasetApiView.py | 29 +++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 python/gui/MaaS/cbv/DatasetApiView.py diff --git a/python/gui/MaaS/cbv/DatasetApiView.py b/python/gui/MaaS/cbv/DatasetApiView.py new file mode 100644 index 000000000..8697def2d --- /dev/null +++ b/python/gui/MaaS/cbv/DatasetApiView.py @@ -0,0 +1,29 @@ +import asyncio +from django.http import JsonResponse +from .AbstractDatasetView import AbstractDatasetView +import logging +logger = logging.getLogger("gui_log") + + +class DatasetApiView(AbstractDatasetView): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def _get_datasets_json(self) -> JsonResponse: + serial_dataset_map = asyncio.get_event_loop().run_until_complete(self.get_datasets()) + return JsonResponse({"datasets": serial_dataset_map}, status=200) + + def _get_dataset_json(self, dataset_name: str) -> JsonResponse: + serial_dataset = asyncio.get_event_loop().run_until_complete(self.get_dataset(dataset_name=dataset_name)) + return JsonResponse({"dataset": serial_dataset[dataset_name]}, status=200) + + def get(self, request, *args, **kwargs): + request_type = request.GET.get("request_type", None) + if request_type == 'datasets': + return self._get_datasets_json() + elif request_type == 'dataset': + return self._get_dataset_json(dataset_name=request.GET.get("name", None)) + + # TODO: finish + return JsonResponse({}, status=400) From 45303a965b65f296bd7a05d7c0e5db6f059f8845 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Mon, 3 Oct 2022 11:39:31 -0500 Subject: [PATCH 35/46] Add GUI Django url for Dataset API AJAX calls. --- python/gui/MaaS/urls.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/gui/MaaS/urls.py b/python/gui/MaaS/urls.py index 828dbe6c6..9ca222ccb 100644 --- a/python/gui/MaaS/urls.py +++ b/python/gui/MaaS/urls.py @@ -1,6 +1,7 @@ from django.urls import re_path from .cbv.EditView import EditView from .cbv.DatasetManagementView import DatasetManagementView +from .cbv.DatasetApiView import DatasetApiView from .cbv.MapView import MapView, Fabrics, FabricNames, FabricTypes, ConnectedFeatures from .cbv.configuration import CreateConfiguration @@ -14,6 +15,7 @@ # TODO: add this later #re_path(r'ngen$', NgenWorkflowView.as_view(), name="ngen-workflow"), re_path(r'datasets', DatasetManagementView.as_view(), name="dataset-management"), + re_path(r'dataset-api', DatasetApiView.as_view(), name="dataset-api"), re_path(r'map$', MapView.as_view(), name="map"), re_path(r'map/connections$', ConnectedFeatures.as_view(), name="connections"), re_path(r'fabric/names$', FabricNames.as_view(), name='fabric-names'), From e98ef36cd3a1b42425bd7938c1d49656c17d6e89 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Mon, 3 Oct 2022 11:46:21 -0500 Subject: [PATCH 36/46] Add commented-out helper debug volume for GUI. Adding commented-out line for host bind-mount volume to GUI Docker service config to make debugging Django template and Javascript code faster by mounted code from the host into the running service. --- docker/nwm_gui/docker-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/nwm_gui/docker-compose.yml b/docker/nwm_gui/docker-compose.yml index 97c7a7b68..7eb8ef4fb 100644 --- a/docker/nwm_gui/docker-compose.yml +++ b/docker/nwm_gui/docker-compose.yml @@ -63,6 +63,8 @@ services: volumes: - ${DMOD_APP_STATIC:?}:/usr/maas_portal/static - ${DMOD_SSL_DIR}/requestservice:/usr/maas_portal/ssl + # Needed only for speeding debugging + #- ${DOCKER_GUI_HOST_SRC:?GUI sources path not configured in environment}/MaaS:/usr/maas_portal/MaaS #- ${DOCKER_GUI_HOST_VENV_DIR:-/tmp/blah}:${DOCKER_GUI_CONTAINER_VENV_DIR:-/tmp/blah} # Expose Django's port to the internal network so that the web server may access it expose: From 3a7e39f7b35a59dbcd223e207682b334d440bf1d Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Mon, 3 Oct 2022 11:48:48 -0500 Subject: [PATCH 37/46] Refactor DatasetManagementView inheritance. Make DatasetManagementView inherit from AbstractDatasetView so certain required pieces could be centrally located and reused. --- python/gui/MaaS/cbv/DatasetManagementView.py | 25 +++----------------- 1 file changed, 3 insertions(+), 22 deletions(-) diff --git a/python/gui/MaaS/cbv/DatasetManagementView.py b/python/gui/MaaS/cbv/DatasetManagementView.py index 47f411888..b7db81112 100644 --- a/python/gui/MaaS/cbv/DatasetManagementView.py +++ b/python/gui/MaaS/cbv/DatasetManagementView.py @@ -2,26 +2,20 @@ Defines a view that may be used to configure a MaaS request """ import asyncio -import os from django.http import HttpRequest, HttpResponse -from django.views.generic.base import View from django.shortcuts import render import dmod.communication as communication -from dmod.client.request_clients import DatasetExternalClient from dmod.core.meta_data import DataCategory, DataFormat import logging logger = logging.getLogger("gui_log") -from pathlib import Path - -from .DMODProxy import DMODMixin, GUI_STATIC_SSL_DIR from .utils import extract_log_data -from typing import Dict +from .AbstractDatasetView import AbstractDatasetView -class DatasetManagementView(View, DMODMixin): +class DatasetManagementView(AbstractDatasetView): """ A view used to configure a dataset management request or requests for transmitting dataset data. @@ -29,7 +23,6 @@ class DatasetManagementView(View, DMODMixin): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self._dataset_client = None def _process_event_type(self, http_request: HttpRequest) -> communication.MessageEventType: """ @@ -49,17 +42,6 @@ def _process_event_type(self, http_request: HttpRequest) -> communication.Messag # TODO: raise NotImplementedError("{}._process_event_type not implemented".format(self.__class__.__name__)) - @property - def dataset_client(self) -> DatasetExternalClient: - if self._dataset_client is None: - self._dataset_client = DatasetExternalClient(endpoint_uri=self.maas_endpoint_uri, - ssl_directory=GUI_STATIC_SSL_DIR) - return self._dataset_client - - async def _get_datasets(self) -> Dict[str, dict]: - serial_datasets = await self.dataset_client.get_serialized_datasets() - return serial_datasets - def get(self, http_request: HttpRequest, *args, **kwargs) -> HttpResponse: """ The handler for 'get' requests. @@ -81,7 +63,7 @@ def get(self, http_request: HttpRequest, *args, **kwargs) -> HttpResponse: errors, warnings, info = extract_log_data(kwargs) # Gather map of serialized datasets, keyed by dataset name - serial_dataset_map = asyncio.get_event_loop().run_until_complete(self._get_datasets()) + serial_dataset_map = asyncio.get_event_loop().run_until_complete(self.get_datasets()) serial_dataset_list = [serial_dataset_map[d] for d in serial_dataset_map] dataset_categories = [c.name.title() for c in DataCategory] @@ -96,7 +78,6 @@ def get(self, http_request: HttpRequest, *args, **kwargs) -> HttpResponse: 'warnings': warnings } - # TODO: create this file return render(http_request, 'maas/dataset_management.html', payload) def post(self, http_request: HttpRequest, *args, **kwargs) -> HttpResponse: From 2c44f7e83146ebc4f74257726eb321701e9945b2 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Mon, 3 Oct 2022 11:51:29 -0500 Subject: [PATCH 38/46] More progress on dataset management template. --- .../templates/maas/dataset_management.html | 287 +++++++++++++----- 1 file changed, 215 insertions(+), 72 deletions(-) diff --git a/python/gui/MaaS/templates/maas/dataset_management.html b/python/gui/MaaS/templates/maas/dataset_management.html index 1ca18311c..97c694b73 100644 --- a/python/gui/MaaS/templates/maas/dataset_management.html +++ b/python/gui/MaaS/templates/maas/dataset_management.html @@ -16,6 +16,143 @@ $("#" + model + "_parameters").show(); } + /** + * Build ``thead`` for the management overview table summarizing datasets. + */ + function buildDatasetsOverviewTableHeader() { + // Build the table header row + let thead = document.createElement('thead'); + let header = document.createElement('tr'); + thead.appendChild(header); + + let colCell = document.createElement('th'); + colCell.className = "mgr-tbl-dataset-header"; + colCell.appendChild(document.createTextNode('Dataset Name')); + header.appendChild(colCell); + + colCell = document.createElement('th'); + colCell.className = "mgr-tbl-category-header"; + colCell.appendChild(document.createTextNode('Category')); + header.appendChild(colCell); + + header.appendChild(document.createElement('th')); + + colCell = document.createElement('th'); + colCell.appendChild(document.createTextNode('Actions')); + header.appendChild(colCell); + + header.appendChild(document.createElement('th')); + header.appendChild(document.createElement('th')); + + return thead; + } + + function buildDatasetOverviewTableRow(serial_dataset) { + let row = document.createElement('tr'); + row.className = "mgr-tbl-content"; + + let colCell = document.createElement('th'); + colCell.appendChild(document.createTextNode(serial_dataset['name'])); + row.appendChild(colCell); + + colCell = document.createElement('th'); + colCell.appendChild(document.createTextNode(serial_dataset['data_category'])); + row.appendChild(colCell); + + let helper_create_links = function create_links_func (is_a, text, ds_name, on_click) { + let cell = document.createElement('th'); + let content; + if (is_a) { + content = document.createElement('a') + content.href = "javascript:void(0);"; + } + else { + content = document.createElement('button'); + } + content.onclick = function() { on_click(ds_name); }; + content.appendChild(document.createTextNode(text)); + cell.appendChild(content); + row.appendChild(cell); + } + + helper_create_links(true, "Details", serial_dataset['name'], displayDatasetDetails); + helper_create_links(true, "Download", serial_dataset['name'], clickDownloadDataset); + helper_create_links(true, "Upload Files", serial_dataset['name'], clickUploadDataset); + helper_create_links(true, "Delete", serial_dataset['name'], clickDeleteDataset); + + return row; + } + + function buildDatasetsOverviewTable() { + let table_id = 'dataset-manage-overview-table'; + let parent_div = document.getElementById('dataset-manage'); + + // First find and remove any existing table, in case we are refreshing + let table = document.getElementById(table_id); + if (table != null) { + table.remove(); + } + + table = document.createElement('table'); + parent_div.appendChild(table); + table.id = table_id; + table.className = 'datasets-table'; + + table.appendChild(buildDatasetsOverviewTableHeader()); + + $.ajax({ + url: "/dataset-api", + type: 'get', + data: {"request_type": "datasets"}, + success: function(response) { + for (const ds_name in response["datasets"]) { + table.appendChild(buildDatasetOverviewTableRow(response["datasets"][ds_name])); + } + }, + error: function(response) { + alert('Received an error'); + console.log(response); + } + }); + } + + function initDatasetCreateForm() { + let category_select = document.getElementById("create-dataset-category"); + category_select.selectedIndex = -1; + let format_select = document.getElementById("create-dataset-format"); + format_select.selectedIndex = -1; + } + + function initGuiViews() { + buildDatasetsOverviewTable(); + initDatasetCreateForm(); + } + + function changeCreateDatasetFormatSelection(selection) { + let dy_div = document.getElementById("dataset-create-form-dynamic-vars-div"); + for (let i = 0; i < dy_div.children.length; i++) { + dy_div.children[i].remove(); + } + let addUploadSelection = false; + if (selection === "NETCDF_FORCING_CANONICAL") { + addUploadSelection = true; + } + + if (addUploadSelection) { + let upload_select_label = document.createElement('label'); + upload_select_label.appendChild(document.createTextNode('Data Files:')); + dy_div.appendChild(upload_select_label); + upload_select_label.htmlFor = 'create-dataset-upload'; + let upload_select = document.createElement('input'); + upload_select.type = 'file'; + upload_select.name = 'create-dataset-upload'; + upload_select.id = 'create-dataset-upload'; + upload_select.style.float = 'right'; + upload_select.style.textAlign = 'right'; + dy_div.appendChild(upload_select); + } + } + /** * Toggle what is displayed based on selection click in main nav bar for page. * @@ -70,9 +207,12 @@ function buildDetailsTableRow() { let row = document.createElement('tr'); // Skip the first arg, as this is the table itself + let colCell; + let cellText; for (let i = 1; i < arguments.length; i++) { - let colCell = document.createElement('th'); + colCell = document.createElement('th'); if (i == 1) { + colCell.style.verticalAlign = 'top'; colCell.style.textAlign = 'right'; colCell.style.paddingRight = '8px'; arguments[i] += ":"; @@ -81,52 +221,69 @@ colCell.style.textAlign = 'left'; colCell.style.fontWeight = 'normal'; } - let cellText = document.createTextNode(arguments[i]); + cellText = document.createTextNode(arguments[i]); colCell.appendChild(cellText); row.appendChild(colCell); } arguments[0].appendChild(row); } + function ajaxBuildDatasetDetailsTable(dataset_json, table) { + buildDetailsTableRow(table, "Name", dataset_json["name"]); + buildDetailsTableRow(table, "Category", dataset_json["data_category"]) + buildDetailsTableRow(table, "Type", dataset_json["type"]); + buildDetailsTableRow(table, "Format", dataset_json["data_domain"]["data_format"]); + buildDetailsTableRow(table, "Read Only", dataset_json["is_read_only"]); + buildDetailsTableRow(table, "Created", dataset_json["create_on"]); + buildDetailsTableRow(table, "Last Updated", dataset_json["last_updated"]); + + // TODO: add something for dataset size + // TODO: add something for number of files. + // TODO: add something for indicating if data aligns properly with configured domain + + // Put spacer row in to indicate the start of domain details + let domain_header_row = document.createElement('tr'); + let colCell = document.createElement('th'); + colCell.colSpan = 2; + colCell.style.textAlign = 'left'; + colCell.style.paddingTop = '20px'; + colCell.style.paddingLeft = '5%'; + colCell.style.fontStyle = 'italic'; + colCell.style.fontSize = '18px'; + colCell.appendChild(document.createTextNode("Data Domain")); + domain_header_row.appendChild(colCell); + table.appendChild(domain_header_row); + + // Then iteratively add domain details rows + let c_restrict; + for (let c_i = 0; c_i < dataset_json["data_domain"]["continuous"].length; c_i++) { + c_restrict = dataset_json["data_domain"]["continuous"][c_i]; + buildDetailsTableRow(table, c_restrict["variable"], c_restrict["begin"] + " to " + c_restrict["end"]); + } + let d_restrict; + for (let d_i = 0; d_i < dataset_json["data_domain"]["discrete"].length; d_i++) { + d_restrict = dataset_json["data_domain"]["discrete"][d_i]; + buildDetailsTableRow(table, d_restrict["variable"], d_restrict["values"]); + } + } + function buildDatasetDetailsTable(dataset_name) { - {% for dataset in datasets %} - if (dataset_name == "{{ dataset.name }}") { - //let cellText = document.createTextNode("testing text {{ dataset.name }} (type: {{ dataset.type }})"); - //return cellText; - let table = document.createElement('table'); - table.class = 'datasets-details-table'; - buildDetailsTableRow(table, "Name", "{{ dataset.name }}") - buildDetailsTableRow(table, "Category", "{{ dataset.data_category }}") - buildDetailsTableRow(table, "Type", "{{ dataset.type }}") - buildDetailsTableRow(table, "Format", "{{ dataset.data_domain.data_format }}") - buildDetailsTableRow(table, "Read Only", "{{ dataset.is_read_only }}") - buildDetailsTableRow(table, "Created", "{{ dataset.create_on }}") - buildDetailsTableRow(table, "Last Updated", "{{ dataset.last_updated }}") - - // Put spacer row in to indicate the start of domain details - let domain_header_row = document.createElement('tr'); - let colCell = document.createElement('th'); - colCell.colSpan = 2; - colCell.style.textAlign = 'left'; - colCell.style.paddingTop = '20px'; - colCell.style.paddingLeft = '5%'; - colCell.style.fontStyle = 'italic'; - colCell.style.fontSize = '18px'; - colCell.appendChild(document.createTextNode("Data Domain")); - domain_header_row.appendChild(colCell); - table.appendChild(domain_header_row); - - // Then iteratively add domain details rows - {% for cont_rest in dataset.data_domain.continuous %} - buildDetailsTableRow(table, "{{ cont_rest.variable }}", "{{ cont_rest.begin }} to {{ cont_rest.end }}"); - {% endfor %} - {% for disc_rest in dataset.data_domain.discrete %} - buildDetailsTableRow(table, "{{ disc_rest.variable }}", "{{ disc_rest.values }}"); - {% endfor %} - - return table; + let table = document.createElement('table'); + table.class = 'datasets-details-table'; + + $.ajax({ + url: "/dataset-api", + type: 'get', + data: {"request_type": "dataset", "name": dataset_name}, + success: function(response) { + ajaxBuildDatasetDetailsTable(response["dataset"], table); + }, + error: function(response) { + alert('Received an error getting serialized ' + dataset_name + ' dataset'); + console.log(response); } - {% endfor %} + }); + return table; } function displayDatasetDetails(name) { @@ -260,9 +417,14 @@ background-color: palegreen; } - #standard-vars { + #dataset-create-form-universal-vars-div { line-height: 25px; - max-width: 300px; + max-width: 350px; + } + + #dataset-create-form-dynamic-vars-div { + line-height: 25px; + max-width: 350px; } .disabled { @@ -314,7 +476,7 @@ - +