diff --git a/docker/main/dataservice/entrypoint.sh b/docker/main/dataservice/entrypoint.sh index e42ae462d..433ecc801 100644 --- a/docker/main/dataservice/entrypoint.sh +++ b/docker/main/dataservice/entrypoint.sh @@ -62,7 +62,7 @@ if [ -d ${UPDATED_PACKAGES_DIR:=/updated_packages} ]; then for srv in $(pip -qq freeze | grep dmod | awk -F= '{print $1}' | awk -F- '{print $2}'); do if [ $(ls ${UPDATED_PACKAGES_DIR} | grep dmod.${srv}- | wc -l) -eq 1 ]; then pip uninstall -y --no-input $(pip -qq freeze | grep dmod.${srv} | awk -F= '{print $1}') - pip install $(ls ${UPDATED_PACKAGES_DIR}/*.whl | grep dmod.${srv}-) + pip install --no-deps $(ls ${UPDATED_PACKAGES_DIR}/*.whl | grep dmod.${srv}-) fi done #pip install ${UPDATED_PACKAGES_DIR}/*.whl diff --git a/docker/nwm_gui/app_server/entrypoint.sh b/docker/nwm_gui/app_server/entrypoint.sh index a2d8a531c..9eacc4bc8 100755 --- a/docker/nwm_gui/app_server/entrypoint.sh +++ b/docker/nwm_gui/app_server/entrypoint.sh @@ -34,6 +34,9 @@ echo "Starting dmod app" #Extract the DB secrets into correct ENV variables POSTGRES_SECRET_FILE="/run/secrets/${DOCKER_SECRET_POSTGRES_PASS:?}" export SQL_PASSWORD="$(cat ${POSTGRES_SECRET_FILE})" +export DMOD_SU_PASSWORD="$(cat ${POSTGRES_SECRET_FILE})" + +python manage.py migrate # Handle for debugging when appropriate if [ "$(echo "${PYCHARM_REMOTE_DEBUG_ACTIVE:-false}" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]')" == "true" ]; then diff --git a/docker/nwm_gui/docker-compose.yml b/docker/nwm_gui/docker-compose.yml index 509441649..04d6eac68 100644 --- a/docker/nwm_gui/docker-compose.yml +++ b/docker/nwm_gui/docker-compose.yml @@ -34,6 +34,7 @@ services: args: docker_internal_registry: ${DOCKER_INTERNAL_REGISTRY:?Missing DOCKER_INTERNAL_REGISTRY value (see 'Private Docker Registry ' section in example.env)} comms_package_name: ${PYTHON_PACKAGE_DIST_NAME_COMMS:?} + client_package_name: ${PYTHON_PACKAGE_DIST_NAME_CLIENT:?} networks: - request-listener-net # Call this when starting the container @@ -57,11 +58,15 @@ services: - SQL_USER=${DMOD_GUI_POSTGRES_USER:?} - SQL_HOST=db - SQL_PORT=5432 + - DMOD_SU_NAME=dmod_super_user + - DMOD_SU_EMAIL=none@noaa.gov - DATABASE=postgres - DOCKER_SECRET_POSTGRES_PASS=postgres_password volumes: - ${DMOD_APP_STATIC:?}:/usr/maas_portal/static - ${DMOD_SSL_DIR}/requestservice:/usr/maas_portal/ssl + # Needed only for speeding debugging + #- ${DOCKER_GUI_HOST_SRC:?GUI sources path not configured in environment}/MaaS:/usr/maas_portal/MaaS #- ${DOCKER_GUI_HOST_VENV_DIR:-/tmp/blah}:${DOCKER_GUI_CONTAINER_VENV_DIR:-/tmp/blah} # Expose Django's port to the internal network so that the web server may access it expose: diff --git a/example.env b/example.env index 637e9e5ce..bafbc8e63 100644 --- a/example.env +++ b/example.env @@ -108,6 +108,11 @@ TROUTE_BRANCH=ngen ## Python Packages Settings ## ######################################################################## +## The "name" of the built client Python distribution package, for purposes of installing (e.g., via pip) +PYTHON_PACKAGE_DIST_NAME_CLIENT=dmod-client +## The name of the actual Python communication package (i.e., for importing or specifying as a module on the command line) +PYTHON_PACKAGE_NAME_CLIENT=dmod.client + ## The "name" of the built communication Python distribution package, for purposes of installing (e.g., via pip) PYTHON_PACKAGE_DIST_NAME_COMMS=dmod-communication ## The name of the actual Python communication package (i.e., for importing or specifying as a module on the command line) diff --git a/python/gui/MaaS/cbv/AbstractDatasetView.py b/python/gui/MaaS/cbv/AbstractDatasetView.py new file mode 100644 index 000000000..e1ae493c3 --- /dev/null +++ b/python/gui/MaaS/cbv/AbstractDatasetView.py @@ -0,0 +1,29 @@ +from abc import ABC +from django.views.generic.base import View +from dmod.client.request_clients import DatasetExternalClient +import logging +logger = logging.getLogger("gui_log") +from .DMODProxy import DMODMixin, GUI_STATIC_SSL_DIR +from typing import Dict + + +class AbstractDatasetView(View, DMODMixin, ABC): + + def __init__(self, *args, **kwargs): + super(AbstractDatasetView, self).__init__(*args, **kwargs) + self._dataset_client = None + + async def get_dataset(self, dataset_name: str) -> Dict[str, dict]: + serial_dataset = await self.dataset_client.get_serialized_datasets(dataset_name=dataset_name) + return serial_dataset + + async def get_datasets(self) -> Dict[str, dict]: + serial_datasets = await self.dataset_client.get_serialized_datasets() + return serial_datasets + + @property + def dataset_client(self) -> DatasetExternalClient: + if self._dataset_client is None: + self._dataset_client = DatasetExternalClient(endpoint_uri=self.maas_endpoint_uri, + ssl_directory=GUI_STATIC_SSL_DIR) + return self._dataset_client diff --git a/python/gui/MaaS/cbv/DMODProxy.py b/python/gui/MaaS/cbv/DMODProxy.py index fc0fcb7a1..e9e308664 100644 --- a/python/gui/MaaS/cbv/DMODProxy.py +++ b/python/gui/MaaS/cbv/DMODProxy.py @@ -16,6 +16,8 @@ from pathlib import Path from typing import List, Optional, Tuple, Type +GUI_STATIC_SSL_DIR = Path('/usr/maas_portal/ssl') + class RequestFormProcessor(ABC): @@ -209,7 +211,7 @@ class PostFormRequestClient(ModelExecRequestClient): def _bootstrap_ssl_dir(cls, ssl_dir: Optional[Path] = None): if ssl_dir is None: ssl_dir = Path(__file__).resolve().parent.parent.parent.joinpath('ssl') - ssl_dir = Path('/usr/maas_portal/ssl') #Fixme + ssl_dir = GUI_STATIC_SSL_DIR #Fixme return ssl_dir def __init__(self, endpoint_uri: str, http_request: HttpRequest, ssl_dir: Optional[Path] = None): @@ -315,6 +317,7 @@ def forward_request(self, request: HttpRequest, event_type: MessageEventType) -> client = PostFormRequestClient(endpoint_uri=self.maas_endpoint_uri, http_request=request) if event_type == MessageEventType.MODEL_EXEC_REQUEST: form_processor_type = ModelExecRequestFormProcessor + # TODO: need a new type of form processor here (or 3 more, for management, uploading, and downloading) else: raise RuntimeError("{} got unsupported event type: {}".format(self.__class__.__name__, str(event_type))) diff --git a/python/gui/MaaS/cbv/DatasetApiView.py b/python/gui/MaaS/cbv/DatasetApiView.py new file mode 100644 index 000000000..6da6eff65 --- /dev/null +++ b/python/gui/MaaS/cbv/DatasetApiView.py @@ -0,0 +1,62 @@ +import asyncio +from django.http import JsonResponse +from wsgiref.util import FileWrapper +from django.http.response import StreamingHttpResponse +from .AbstractDatasetView import AbstractDatasetView +from .DatasetFileWebsocketFilelike import DatasetFileWebsocketFilelike +import logging +logger = logging.getLogger("gui_log") + + +class DatasetApiView(AbstractDatasetView): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def _get_dataset_content_details(self, dataset_name: str): + result = asyncio.get_event_loop().run_until_complete(self.dataset_client.get_dataset_content_details(name=dataset_name)) + logger.info(result) + return JsonResponse({"contents": result}, status=200) + + def _delete_dataset(self, dataset_name: str) -> JsonResponse: + result = asyncio.get_event_loop().run_until_complete(self.dataset_client.delete_dataset(name=dataset_name)) + return JsonResponse({"successful": result}, status=200) + + def _get_datasets_json(self) -> JsonResponse: + serial_dataset_map = asyncio.get_event_loop().run_until_complete(self.get_datasets()) + return JsonResponse({"datasets": serial_dataset_map}, status=200) + + def _get_dataset_json(self, dataset_name: str) -> JsonResponse: + serial_dataset = asyncio.get_event_loop().run_until_complete(self.get_dataset(dataset_name=dataset_name)) + return JsonResponse({"dataset": serial_dataset[dataset_name]}, status=200) + + def _get_download(self, request, *args, **kwargs): + dataset_name = request.GET.get("dataset_name", None) + item_name = request.GET.get("item_name", None) + chunk_size = 8192 + + custom_filelike = DatasetFileWebsocketFilelike(self.dataset_client, dataset_name, item_name) + + response = StreamingHttpResponse( + FileWrapper(custom_filelike, chunk_size), + content_type="application/octet-stream" + ) + response['Content-Length'] = asyncio.get_event_loop().run_until_complete(self.dataset_client.get_item_size(dataset_name, item_name)) + response['Content-Disposition'] = "attachment; filename=%s" % item_name + return response + + def get(self, request, *args, **kwargs): + request_type = request.GET.get("request_type", None) + if request_type == 'download_file': + return self._get_download(request) + elif request_type == 'datasets': + return self._get_datasets_json() + elif request_type == 'dataset': + return self._get_dataset_json(dataset_name=request.GET.get("name", None)) + elif request_type == 'contents': + return self._get_dataset_content_details(dataset_name=request.GET.get("name", None)) + if request_type == 'delete': + return self._delete_dataset(dataset_name=request.GET.get("name", None)) + + # TODO: finish + return JsonResponse({}, status=400) diff --git a/python/gui/MaaS/cbv/DatasetFileWebsocketFilelike.py b/python/gui/MaaS/cbv/DatasetFileWebsocketFilelike.py new file mode 100644 index 000000000..9e69409ad --- /dev/null +++ b/python/gui/MaaS/cbv/DatasetFileWebsocketFilelike.py @@ -0,0 +1,20 @@ +import asyncio +from typing import AnyStr +from dmod.client.request_clients import DatasetExternalClient + + +class DatasetFileWebsocketFilelike: + + def __init__(self, client: DatasetExternalClient, dataset_name: str, file_name: str): + self._client = client + self._dataset_name = dataset_name + self._file_name = file_name + self._read_index: int = 0 + + def read(self, blksize: int) -> AnyStr: + + result = asyncio.get_event_loop().run_until_complete( + self._client.download_item_block(dataset_name=self._dataset_name, item_name=self._file_name, + blk_start=self._read_index, blk_size=blksize)) + self._read_index += blksize + return result diff --git a/python/gui/MaaS/cbv/DatasetManagementView.py b/python/gui/MaaS/cbv/DatasetManagementView.py new file mode 100644 index 000000000..b7db81112 --- /dev/null +++ b/python/gui/MaaS/cbv/DatasetManagementView.py @@ -0,0 +1,114 @@ +""" +Defines a view that may be used to configure a MaaS request +""" +import asyncio +from django.http import HttpRequest, HttpResponse +from django.shortcuts import render + +import dmod.communication as communication +from dmod.core.meta_data import DataCategory, DataFormat + +import logging +logger = logging.getLogger("gui_log") + +from .utils import extract_log_data +from .AbstractDatasetView import AbstractDatasetView + + +class DatasetManagementView(AbstractDatasetView): + + """ + A view used to configure a dataset management request or requests for transmitting dataset data. + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def _process_event_type(self, http_request: HttpRequest) -> communication.MessageEventType: + """ + Determine and return whether this request is for a ``DATASET_MANAGEMENT`` or ``DATA_TRANSMISSION`` event. + + Parameters + ---------- + http_request : HttpRequest + The raw HTTP request in question. + + Returns + ------- + communication.MessageEventType + Either ``communication.MessageEventType.DATASET_MANAGEMENT`` or + ``communication.MessageEventType.DATA_TRANSMISSION``. + """ + # TODO: + raise NotImplementedError("{}._process_event_type not implemented".format(self.__class__.__name__)) + + def get(self, http_request: HttpRequest, *args, **kwargs) -> HttpResponse: + """ + The handler for 'get' requests. + + This will render the 'maas/dataset_management.html' template after retrieving necessary information to initially + populate the forms it displays. + + Parameters + ---------- + http_request : HttpRequest + The request asking to render this page. + args + kwargs + + Returns + ------- + A rendered page. + """ + errors, warnings, info = extract_log_data(kwargs) + + # Gather map of serialized datasets, keyed by dataset name + serial_dataset_map = asyncio.get_event_loop().run_until_complete(self.get_datasets()) + serial_dataset_list = [serial_dataset_map[d] for d in serial_dataset_map] + + dataset_categories = [c.name.title() for c in DataCategory] + dataset_formats = [f.name for f in DataFormat] + + payload = { + 'datasets': serial_dataset_list, + 'dataset_categories': dataset_categories, + 'dataset_formats': dataset_formats, + 'errors': errors, + 'info': info, + 'warnings': warnings + } + + return render(http_request, 'maas/dataset_management.html', payload) + + def post(self, http_request: HttpRequest, *args, **kwargs) -> HttpResponse: + """ + The handler for 'post' requests. + + This will attempt to submit the request and rerender the page like a 'get' request. + + Parameters + ---------- + http_request : HttpRequest + The request asking to render this page. + args + kwargs + + Returns + ------- + A rendered page. + """ + # TODO: implement this to figure out whether DATASET_MANAGEMENT or DATA_TRANSMISSION + event_type = self._process_event_type(http_request) + client, session_data, dmod_response = self.forward_request(http_request, event_type) + + # TODO: this probably isn't exactly correct, so review once closer to completion + if dmod_response is not None and 'dataset_id' in dmod_response.data: + session_data['new_dataset_id'] = dmod_response.data['dataset_id'] + + http_response = self.get(http_request=http_request, errors=client.errors, warnings=client.warnings, + info=client.info, *args, **kwargs) + + for k, v in session_data.items(): + http_response.set_cookie(k, v) + + return http_response diff --git a/python/gui/MaaS/static/maas/js/components/confirmDialog.js b/python/gui/MaaS/static/maas/js/components/confirmDialog.js new file mode 100644 index 000000000..a09710327 --- /dev/null +++ b/python/gui/MaaS/static/maas/js/components/confirmDialog.js @@ -0,0 +1,108 @@ +class ConfirmDialog { + constructor(parentDivName, id, styleClass, onConfirmFunc) { + this.parentDivName = parentDivName; + this.id = id; + this.styleClass = styleClass; + this.onConfirmFunc = onConfirmFunc; + + this.outer_div = null; + this.content_div = null; + } + + get parentDiv() { + return document.getElementById(this.parentDivName); + } +} + +class ConfirmDeleteDatasetDialog extends ConfirmDialog { + constructor(dataset_name, parentDivName, id, styleClass, onConfirmFunc) { + super(parentDivName, id, styleClass, onConfirmFunc); + this.dataset_name = dataset_name; + this.buttons_div = null; + } + + _style_outer_div() { + this.outer_div.style.position = 'fixed'; + this.outer_div.style.zIndex = '1'; + this.outer_div.style.left = '35%'; + this.outer_div.style.top = '5%'; + this.outer_div.style.width = '25%'; + this.outer_div.style.height = '25%'; + this.outer_div.style.overflow = 'clip'; + this.outer_div.style.backgroundColor = '#B7B5B5FF'; + this.outer_div.style.border = '1px solid #888'; + this.outer_div.style.padding = '15px'; + //this.outer_div.style.paddingTop = '0px'; + this.outer_div.style.margin = '15% auto'; + } + + _init_outer_div() { + if (this.outer_div == null) { + this.outer_div = document.createElement('div'); + this.outer_div.id = this.id; + this.outer_div.class = this.styleClass; + this._style_outer_div(); + this.parentDiv.appendChild(this.outer_div); + } + } + + _init_content() { + if (this.content_div == null) { + this.content_div = document.createElement('div'); + this.content_div.style.height = '70%'; + //this.content_div.style.overflow = 'fixed'; + this.content_div.style.padding = '10px'; + this.content_div.appendChild(document.createTextNode("This will permanently delete dataset: ")); + this.content_div.appendChild(document.createElement('br')); + this.content_div.appendChild(document.createElement('br')); + this.content_div.appendChild(document.createTextNode(this.dataset_name)); + this.content_div.appendChild(document.createElement('br')); + this.content_div.appendChild(document.createElement('br')); + this.content_div.appendChild(document.createTextNode("Proceed?")); + + if (this.outer_div == null) { + this._init_outer_div(); + } + this.outer_div.appendChild(this.content_div); + } + } + + _init_buttons() { + if (this.buttons_div == null) { + this.buttons_div = document.createElement('div'); + this.outer_div.appendChild(this.buttons_div); + this.buttons_div.style.padding = '10px'; + + let cancel_button = document.createElement('button'); + cancel_button.onclick = () => { + this.remove(); + }; + cancel_button.textContent = "Cancel"; + cancel_button.style.marginRight = '10px'; + this.buttons_div.appendChild(cancel_button); + + let confirm_button = document.createElement('button'); + confirm_button.onclick = this.onConfirmFunc; + confirm_button.textContent = "Confirm"; + this.buttons_div.appendChild(confirm_button); + } + } + + append() { + this._init_outer_div(); + this._init_content(); + this._init_buttons(); + } + + remove() { + this.buttons_div.remove(); + this.buttons_div = null; + + this.content_div.remove(); + this.content_div = null; + + this.outer_div.remove(); + this.outer_div = null; + } + +} \ No newline at end of file diff --git a/python/gui/MaaS/static/maas/js/components/createDatasetForm.js b/python/gui/MaaS/static/maas/js/components/createDatasetForm.js new file mode 100644 index 000000000..dcad86b36 --- /dev/null +++ b/python/gui/MaaS/static/maas/js/components/createDatasetForm.js @@ -0,0 +1,42 @@ +class CreateDatasetForm { + constructor(parentDivId) { + this.parentDivId = parentDivId; + this.formElementId = this.parentDivId + "-form"; + this.formContentDivId = this.formElementId + "-div-universal-inputs"; + this.dynamicVarsDivId = this.formElementId + "-div-dynamic-inputs"; + + } + + updateFormatChange(selection) { + let dy_div = document.getElementById(this.dynamicVarsDivId); + //let initial_length = dy_div.childNodes.length; + dy_div.childNodes.forEach(child => child.remove()); + + let addUploadSelection = false; + if (selection == "NETCDF_FORCING_CANONICAL") { + addUploadSelection = true; + } + + if (addUploadSelection) { + let upload_select_label = document.createElement('label'); + let selectId = this.parentDivId + '-inputs-upload'; + upload_select_label.appendChild(document.createTextNode('Data Files:')); + upload_select_label.htmlFor = selectId + dy_div.appendChild(upload_select_label); + + let upload_select = document.createElement('input'); + upload_select.type = 'file'; + upload_select.name = 'create-dataset-upload'; + upload_select.id = selectId + upload_select.style.float = 'right'; + upload_select.style.textAlign = 'right'; + dy_div.appendChild(upload_select); + } + } + + dynamicInputUpdate(formInput, selection) { + if (formInput.id == this.parentDivId + '-form-input-format') { + this.updateFormatChange(selection); + } + } +} \ No newline at end of file diff --git a/python/gui/MaaS/static/maas/js/components/datasetOverview.js b/python/gui/MaaS/static/maas/js/components/datasetOverview.js new file mode 100644 index 000000000..e6b404cba --- /dev/null +++ b/python/gui/MaaS/static/maas/js/components/datasetOverview.js @@ -0,0 +1,166 @@ +class DatasetOverviewTableRow { + constructor(parentTableId, serializedDataset, detailsOnClickFunc, filesOnClickFunc, downloadOnClickFunc, + uploadOnClickFunc, deleteOnClickFunc) { + this.parentTableId = parentTableId; + this.serializedDataset = serializedDataset; + + this.rowClassName = "mgr-tbl-content"; + + this.detailsOnClickFunc = detailsOnClickFunc; + this.filesOnClickFunc = filesOnClickFunc; + this.downloadOnClickFunc = downloadOnClickFunc; + this.uploadOnClickFunc = uploadOnClickFunc; + this.deleteOnClickFunc = deleteOnClickFunc; + + this.row = document.getElementById(this.rowId); + } + + get datasetName() { + return this.serializedDataset["name"]; + } + + get category() { + return this.serializedDataset["data_category"]; + } + + get rowId() { + return this.parentTableId + "-row-" + this.datasetName; + } + + get parentTable() { + return document.getElementById(this.parentTableId); + } + + _createLinks(is_anchor, text, onClickFunc) { + let cell = document.createElement('th'); + let content; + if (is_anchor) { + content = document.createElement('a'); + content.href = "javascript:void(0);"; + } + else { + content = document.createElement('button'); + } + + const ds_name = this.datasetName; + + let onclick; + switch (text) { + case 'Details': + onclick = this.detailsOnClickFunc; + break; + case 'Files': + onclick = this.filesOnClickFunc; + break; + case 'Download': + onclick = this.downloadOnClickFunc; + break; + case 'Upload Files': + onclick = this.uploadOnClickFunc; + break; + case 'Delete': + onclick = this.deleteOnClickFunc; + break; + } + + content.onclick = function() { onclick(ds_name); }; + content.appendChild(document.createTextNode(text)); + cell.appendChild(content); + this.row.appendChild(cell); + } + + build() { + if (this.row != null) { + this.row.remove(); + } + this.row = document.createElement('tr'); + this.row.id = this.rowId; + this.row.className = this.rowClassName; + + let colCell = document.createElement('th'); + colCell.appendChild(document.createTextNode(this.datasetName)); + this.row.appendChild(colCell); + + colCell = document.createElement('th'); + colCell.appendChild(document.createTextNode(this.category)); + this.row.appendChild(colCell); + + this._createLinks(true, "Details", this.datasetName, this.detailsOnClickFunc); + this._createLinks(true, "Files", this.datasetName, this.filesOnClickFunc); + this._createLinks(true, "Download", this.datasetName, this.downloadOnClickFunc); + this._createLinks(true, "Upload Files", this.datasetName, this.uploadOnClickFunc); + this._createLinks(true, "Delete", this.datasetName, this.deleteOnClickFunc); + } +} + +class DatasetOverviewTable { + constructor(parentDivId, tableClass, detailsOnClickFunc, filesOnClickFunc, downloadOnClickFunc, + uploadOnClickFunc, deleteOnClickFunc) { + this.parentDivId = parentDivId; + this.tableClass = tableClass; + this.tableId = this.parentDivId + "-overview-table"; + + this.detailsOnClickFunc = detailsOnClickFunc; + this.filesOnClickFunc = filesOnClickFunc; + this.downloadOnClickFunc = downloadOnClickFunc; + this.uploadOnClickFunc = uploadOnClickFunc; + this.deleteOnClickFunc = deleteOnClickFunc; + + this.table = document.getElementById(this.tableId); + } + + get parentDiv() { + return document.getElementById(this.parentDivId); + } + + get tableHeader() { + let thead = document.createElement('thead'); + let header = document.createElement('tr'); + thead.appendChild(header); + + let colCell = document.createElement('th'); + colCell.className = "mgr-tbl-dataset-header"; + colCell.appendChild(document.createTextNode('Dataset Name')); + header.appendChild(colCell); + + colCell = document.createElement('th'); + colCell.className = "mgr-tbl-category-header"; + colCell.appendChild(document.createTextNode('Category')); + header.appendChild(colCell); + + header.appendChild(document.createElement('th')); + + colCell = document.createElement('th'); + colCell.appendChild(document.createTextNode('Actions')); + header.appendChild(colCell); + + header.appendChild(document.createElement('th')); + header.appendChild(document.createElement('th')); + + return thead; + } + + buildAndAddRow(serializedDataset) { + let row = new DatasetOverviewTableRow(this.tableId, serializedDataset, this.detailsOnClickFunc, + this.filesOnClickFunc, this.downloadOnClickFunc, this.uploadOnClickFunc, this.deleteOnClickFunc); + row.build(); + this.table.appendChild(row.row); + } + + buildTable(contentResponse) { + if (this.table != null) { + this.table.remove(); + } + this.table = document.createElement('table'); + this.table.id = this.tableId; + this.table.className = this.tableClass; + + this.table.appendChild(this.tableHeader); + + for (const ds_name in contentResponse["datasets"]) { + this.buildAndAddRow(contentResponse["datasets"][ds_name]); + } + + this.parentDiv.appendChild(this.table); + } +} \ No newline at end of file diff --git a/python/gui/MaaS/templates/maas/dataset_management.html b/python/gui/MaaS/templates/maas/dataset_management.html new file mode 100644 index 000000000..8a847ba8c --- /dev/null +++ b/python/gui/MaaS/templates/maas/dataset_management.html @@ -0,0 +1,797 @@ + + + + + OWP MaaS + {% load static %} + + + + + + + + + + + +
+ + + {% if errors %} +
+ +
+ {% endif %} + + {% if warnings %} +
+ +
+ {% endif %} + + {% if info %} +
+ +
+ {% endif %} + + {# Cache jQuery scripts for UI scripting and styling #} +
+

Dataset Management

+
+ Manage + Create +
+
+
+
+

Create New Dataset:

+
+ {# Add the token to provide cross site request forgery protection #} + {% csrf_token %} +
+ + +
+ + + + + + +
+ + + + +
+ + +
+
+
+
+ + +
+
+
+
+ + diff --git a/python/gui/MaaS/urls.py b/python/gui/MaaS/urls.py index a6496fada..9ca222ccb 100644 --- a/python/gui/MaaS/urls.py +++ b/python/gui/MaaS/urls.py @@ -1,5 +1,7 @@ from django.urls import re_path from .cbv.EditView import EditView +from .cbv.DatasetManagementView import DatasetManagementView +from .cbv.DatasetApiView import DatasetApiView from .cbv.MapView import MapView, Fabrics, FabricNames, FabricTypes, ConnectedFeatures from .cbv.configuration import CreateConfiguration @@ -10,6 +12,10 @@ urlpatterns = [ re_path(r'^$', EditView.as_view()), + # TODO: add this later + #re_path(r'ngen$', NgenWorkflowView.as_view(), name="ngen-workflow"), + re_path(r'datasets', DatasetManagementView.as_view(), name="dataset-management"), + re_path(r'dataset-api', DatasetApiView.as_view(), name="dataset-api"), re_path(r'map$', MapView.as_view(), name="map"), re_path(r'map/connections$', ConnectedFeatures.as_view(), name="connections"), re_path(r'fabric/names$', FabricNames.as_view(), name='fabric-names'), diff --git a/python/gui/dependencies.txt b/python/gui/dependencies.txt index 5a8bba36e..8ed418e32 100644 --- a/python/gui/dependencies.txt +++ b/python/gui/dependencies.txt @@ -17,3 +17,4 @@ channels channels-redis djangorestframework psycopg2-binary # TODO: get source package in future. Note that psycopg2 cannot be used on Mac; psycopg2-binary must be used +numpy \ No newline at end of file diff --git a/python/lib/client/dmod/client/_version.py b/python/lib/client/dmod/client/_version.py index b794fd409..7fd229a32 100644 --- a/python/lib/client/dmod/client/_version.py +++ b/python/lib/client/dmod/client/_version.py @@ -1 +1 @@ -__version__ = '0.1.0' +__version__ = '0.2.0' diff --git a/python/lib/client/dmod/client/request_clients.py b/python/lib/client/dmod/client/request_clients.py index 09251c01b..bd89d40e7 100644 --- a/python/lib/client/dmod/client/request_clients.py +++ b/python/lib/client/dmod/client/request_clients.py @@ -9,13 +9,13 @@ from dmod.communication.data_transmit_message import DataTransmitMessage, DataTransmitResponse from dmod.core.meta_data import DataCategory, DataDomain, TimeRange from pathlib import Path -from typing import List, Optional, Tuple, Type, Union +from typing import AnyStr, Dict, List, Optional, Tuple, Type, Union import json import websockets -#import logging -#logger = logging.getLogger("gui_log") +import logging +logger = logging.getLogger("client_log") class NgenRequestClient(ModelExecRequestClient[NGENRequest, NGENRequestResponse]): @@ -89,6 +89,25 @@ async def create_dataset(self, name: str, category: DataCategory, domain: DataDo async def delete_dataset(self, name: str, **kwargs) -> bool: pass + @abstractmethod + async def download_item_block(self, dataset_name: str, item_name: str, blk_start: int, blk_size: int) -> AnyStr: + """ + Download a block/chunk of a given size and start point from a specified dataset file. + + Parameters + ---------- + dataset_name + item_name + blk_start + blk_size + + Returns + ------- + AnyStr + The downloaded block/chunk. + """ + pass + @abstractmethod async def download_dataset(self, dataset_name: str, dest_dir: Path) -> bool: """ @@ -130,6 +149,14 @@ async def download_from_dataset(self, dataset_name: str, item_name: str, dest: P """ pass + @abstractmethod + async def get_dataset_content_details(self, name: str, **kwargs) -> bool: + pass + + @abstractmethod + async def get_item_size(self, dataset_name: str, item_name: str) -> int: + pass + @abstractmethod async def list_datasets(self, category: Optional[DataCategory] = None) -> List[str]: pass @@ -175,6 +202,50 @@ async def delete_dataset(self, name: str, **kwargs) -> bool: self.last_response = await self.async_make_request(request) return self.last_response is not None and self.last_response.success + async def get_dataset_content_details(self, name: str, **kwargs) -> dict: + # TODO: later add things like created and last updated perhaps + query = DatasetQuery(query_type=QueryType.GET_DATASET_ITEMS) + request = DatasetManagementMessage(action=ManagementAction.QUERY, query=query, dataset_name=name) + self.last_response: DatasetManagementResponse = await self.async_make_request(request) + if self.last_response.success: + return self.last_response.data + else: + return {} + + async def get_item_size(self, dataset_name: str, item_name: str) -> int: + query = DatasetQuery(query_type=QueryType.GET_ITEM_SIZE, item_name=item_name) + request = DatasetManagementMessage(action=ManagementAction.QUERY, query=query, dataset_name=dataset_name, + data_location=item_name) + self.last_response: DatasetManagementResponse = await self.async_make_request(request) + if self.last_response.success: + return self.last_response.data + else: + return -1 + + async def download_item_block(self, dataset_name: str, item_name: str, blk_start: int, blk_size: int) -> AnyStr: + """ + Download a block/chunk of a given size and start point from a specified dataset file. + + Parameters + ---------- + dataset_name + item_name + blk_start + blk_size + + Returns + ------- + AnyStr + The downloaded block/chunk. + """ + request = DatasetManagementMessage(action=ManagementAction.REQUEST_DATA, dataset_name=dataset_name, + data_location=item_name, blk_start=blk_start, blk_size=blk_size) + self.last_response: DatasetManagementResponse = await self.async_make_request(request) + if self.last_response.success: + return self.last_response.data + else: + return '' + async def download_dataset(self, dataset_name: str, dest_dir: Path) -> bool: """ Download an entire dataset to a local directory. @@ -460,6 +531,55 @@ async def delete_dataset(self, name: str, **kwargs) -> bool: self.last_response = await self.async_make_request(request) return self.last_response is not None and self.last_response.success + async def download_item_block(self, dataset_name: str, item_name: str, blk_start: int, blk_size: int) -> AnyStr: + """ + Download a block/chunk of a given size and start point from a specified dataset file. + + Parameters + ---------- + dataset_name + item_name + blk_start + blk_size + + Returns + ------- + AnyStr + The downloaded block/chunk. + """ + await self._async_acquire_session_info() + request = MaaSDatasetManagementMessage(action=ManagementAction.REQUEST_DATA, dataset_name=dataset_name, + session_secret=self.session_secret, data_location=item_name, + blk_start=blk_start, blk_size=blk_size) + self.last_response: DatasetManagementResponse = await self.async_make_request(request) + if self.last_response.success: + return self.last_response.data + else: + return '' + + async def get_item_size(self, dataset_name: str, item_name: str) -> int: + await self._async_acquire_session_info() + query = DatasetQuery(query_type=QueryType.GET_ITEM_SIZE, item_name=item_name) + request = MaaSDatasetManagementMessage(action=ManagementAction.QUERY, query=query, dataset_name=dataset_name, + session_secret=self._session_secret, data_location=item_name) + self.last_response: DatasetManagementResponse = await self.async_make_request(request) + if self.last_response.success: + return self.last_response.data + else: + return -1 + + async def get_dataset_content_details(self, name: str, **kwargs) -> List: + # TODO: later add things like created and last updated perhaps + await self._async_acquire_session_info() + query = DatasetQuery(query_type=QueryType.GET_DATASET_ITEMS) + request = MaaSDatasetManagementMessage(session_secret=self.session_secret, action=ManagementAction.QUERY, + query=query, dataset_name=name) + self.last_response: DatasetManagementResponse = await self.async_make_request(request) + if self.last_response.success: + return self.last_response.data[DatasetManagementResponse._DATA_KEY_QUERY_RESULTS] + else: + return [] + async def download_dataset(self, dataset_name: str, dest_dir: Path) -> bool: await self._async_acquire_session_info() try: @@ -505,6 +625,43 @@ async def download_from_dataset(self, dataset_name: str, item_name: str, dest: P if not has_data: return message_object + async def get_serialized_datasets(self, dataset_name: Optional[str] = None) -> Dict[str, dict]: + """ + Get dataset objects in serialized form, either for all datasets or for the one with the provided name. + + Parameters + ---------- + dataset_name : Optional[str] + The name of a specific dataset to get serialized details of, if only one should be obtained. + + Returns + ------- + Dict[str, dict] + A dictionary, keyed by dataset name, of serialized dataset objects. + """ + # TODO: may need to generalize this and add to super class + if dataset_name is None: + datasets = await self.list_datasets() + else: + # TODO: improve how this is use so that it can be safely, efficiently put everywhere it **may** be needed + await self._async_acquire_session_info() + datasets = [dataset_name] + serialized = dict() + action = ManagementAction.QUERY + query = DatasetQuery(query_type=QueryType.GET_SERIALIZED_FORM) + try: + for d in datasets: + request = MaaSDatasetManagementMessage(action=action, query=query, dataset_name=d, + session_secret=self.session_secret) + self.last_response: DatasetManagementResponse = await self.async_make_request(request) + if self.last_response.success: + serialized[d] = self.last_response.data[DatasetManagementResponse._DATA_KEY_QUERY_RESULTS] + # TODO: what to do if any are not successful + return serialized + except Exception as e: + logger.error(e) + raise e + async def list_datasets(self, category: Optional[DataCategory] = None) -> List[str]: await self._async_acquire_session_info() action = ManagementAction.LIST_ALL if category is None else ManagementAction.SEARCH diff --git a/python/lib/client/dmod/test/test_dataset_client.py b/python/lib/client/dmod/test/test_dataset_client.py index b266658c7..37ff6e386 100644 --- a/python/lib/client/dmod/test/test_dataset_client.py +++ b/python/lib/client/dmod/test/test_dataset_client.py @@ -1,7 +1,7 @@ import unittest from ..client.request_clients import DataCategory, DatasetClient, DatasetManagementResponse, MaaSDatasetManagementResponse from pathlib import Path -from typing import List, Optional +from typing import List, Optional, AnyStr class SimpleMockDatasetClient(DatasetClient): @@ -28,6 +28,24 @@ async def download_from_dataset(self, dataset_name: str, item_name: str, dest: P """ Mock implementation, always returning ``False``. """ return False + async def download_item_block(self, dataset_name: str, item_name: str, blk_start: int, blk_size: int) -> AnyStr: + """ + Mock implementation, always returning empty string. + """ + return '' + + async def get_dataset_content_details(self, name: str, **kwargs) -> bool: + """ + Mock implementation, always returning ``False``. + """ + return False + + async def get_item_size(self, dataset_name: str, item_name: str) -> int: + """ + Mock implementation always returning ``1``. + """ + return 1 + async def list_datasets(self, category: Optional[DataCategory] = None) -> List[str]: """ Mock implementation, always returning an empty list. """ return [] diff --git a/python/lib/communication/dmod/communication/_version.py b/python/lib/communication/dmod/communication/_version.py index 9d1bb721b..e754a834e 100644 --- a/python/lib/communication/dmod/communication/_version.py +++ b/python/lib/communication/dmod/communication/_version.py @@ -1 +1 @@ -__version__ = '0.10.0' +__version__ = '0.10.1' diff --git a/python/lib/communication/dmod/communication/client.py b/python/lib/communication/dmod/communication/client.py index 571f0f97d..5b5dbf68c 100644 --- a/python/lib/communication/dmod/communication/client.py +++ b/python/lib/communication/dmod/communication/client.py @@ -800,10 +800,8 @@ def _update_after_valid_response(self, response: EXTERN_REQ_R): # TODO: this can probably be taken out, as the superclass implementation should suffice async def async_make_request(self, request: EXTERN_REQ_M) -> EXTERN_REQ_R: - async with websockets.connect(self.endpoint_uri, ssl=self.client_ssl_context) as websocket: - await websocket.send(request.to_json()) - response = await websocket.recv() - return request.__class__.factory_init_correct_response_subtype(json_obj=json.loads(response)) + response = await self.async_send(request.to_json(), await_response=True) + return request.__class__.factory_init_correct_response_subtype(json_obj=json.loads(response)) @property def errors(self): diff --git a/python/lib/communication/dmod/communication/dataset_management_message.py b/python/lib/communication/dmod/communication/dataset_management_message.py index d148b6d0e..d21192906 100644 --- a/python/lib/communication/dmod/communication/dataset_management_message.py +++ b/python/lib/communication/dmod/communication/dataset_management_message.py @@ -16,6 +16,11 @@ class QueryType(Enum): GET_VALUES = 6 GET_MIN_VALUE = 7 GET_MAX_VALUE = 8 + GET_SERIALIZED_FORM = 9 + GET_LAST_UPDATED = 10 + GET_SIZE = 11 + GET_ITEM_SIZE = 12 + GET_DATASET_ITEMS = 13 @classmethod def get_for_name(cls, name_str: str) -> 'QueryType': @@ -42,26 +47,32 @@ def get_for_name(cls, name_str: str) -> 'QueryType': class DatasetQuery(Serializable): _KEY_QUERY_TYPE = 'query_type' + _KEY_ITEM_NAME = 'item_name' @classmethod def factory_init_from_deserialized_json(cls, json_obj: dict) -> Optional['DatasetQuery']: try: - return cls(query_type=QueryType.get_for_name(json_obj[cls._KEY_QUERY_TYPE])) + return cls(query_type=QueryType.get_for_name(json_obj[cls._KEY_QUERY_TYPE]), + item_name=json_obj.get(cls._KEY_ITEM_NAME)) except Exception as e: return None def __hash__(self): - return hash(self.query_type) + return hash('{}{}'.format(self.query_type.name, self.item_name if self.item_name is not None else '')) def __eq__(self, other): - return isinstance(other, DatasetQuery) and self.query_type == other.query_type + return isinstance(other, DatasetQuery) and self.query_type == other.query_type \ + and self.item_name == other.item_name - def __init__(self, query_type: QueryType): + def __init__(self, query_type: QueryType, item_name: Optional[str] = None): self.query_type = query_type + self.item_name = item_name def to_dict(self) -> Dict[str, Union[str, Number, dict, list]]: serial = dict() serial[self._KEY_QUERY_TYPE] = self.query_type.name + if self.item_name is not None: + serial[self._KEY_ITEM_NAME] = self.item_name return serial @@ -181,6 +192,8 @@ class DatasetManagementMessage(AbstractInitRequest): _SERIAL_KEY_CATEGORY = 'category' _SERIAL_KEY_DATA_DOMAIN = 'data_domain' _SERIAL_KEY_DATA_LOCATION = 'data_location' + _SERIAL_KEY_DATA_BLK_START = 'data_blk_start' + _SERIAL_KEY_DATA_BLK_SIZE = 'data_blk_size' _SERIAL_KEY_DATASET_NAME = 'dataset_name' _SERIAL_KEY_IS_PENDING_DATA = 'pending_data' _SERIAL_KEY_QUERY = 'query' @@ -217,6 +230,8 @@ def factory_init_from_deserialized_json(cls, json_obj: dict) -> Optional['Datase category_str = json_obj.get(cls._SERIAL_KEY_CATEGORY) category = None if category_str is None else DataCategory.get_for_name(category_str) data_loc = json_obj.get(cls._SERIAL_KEY_DATA_LOCATION) + data_blk_start = json_obj.get(cls._SERIAL_KEY_DATA_BLK_START) + data_blk_size = json_obj.get(cls._SERIAL_KEY_DATA_BLK_SIZE) #page = json_obj[cls._SERIAL_KEY_PAGE] if cls._SERIAL_KEY_PAGE in json_obj else None if cls._SERIAL_KEY_QUERY in json_obj: query = DatasetQuery.factory_init_from_deserialized_json(json_obj[cls._SERIAL_KEY_QUERY]) @@ -229,7 +244,7 @@ def factory_init_from_deserialized_json(cls, json_obj: dict) -> Optional['Datase return deserialized_class(action=action, dataset_name=dataset_name, category=category, is_read_only_dataset=json_obj[cls._SERIAL_KEY_IS_READ_ONLY], domain=domain, - data_location=data_loc, + data_location=data_loc, blk_start=data_blk_start, blk_size=data_blk_size, is_pending_data=json_obj.get(cls._SERIAL_KEY_IS_PENDING_DATA), #page=page, query=query, **deserialized_class_kwargs) except Exception as e: @@ -261,8 +276,8 @@ def __hash__(self): def __init__(self, action: ManagementAction, dataset_name: Optional[str] = None, is_read_only_dataset: bool = False, category: Optional[DataCategory] = None, domain: Optional[DataDomain] = None, - data_location: Optional[str] = None, is_pending_data: bool = False, - query: Optional[DatasetQuery] = None, *args, **kwargs): + data_location: Optional[str] = None, blk_start: Optional[int] = None, blk_size: Optional[int] = None, + is_pending_data: bool = False, query: Optional[DatasetQuery] = None, *args, **kwargs): """ Initialize this instance. @@ -278,6 +293,10 @@ def __init__(self, action: ManagementAction, dataset_name: Optional[str] = None, The optional category of the involved dataset or datasets, when applicable; defaults to ``None``. data_location : Optional[str] Optional location/file/object/etc. for acted-upon data. + blk_start : Optional[int] + Optional starting point for when acting upon a block/chunk of data. + blk_size : Optional[int] + Optional block size for when acting upon a block/chunk of data. is_pending_data : bool Whether the sender has data pending transmission after this message (default: ``False``). query : Optional[DatasetQuery] @@ -302,9 +321,19 @@ def __init__(self, action: ManagementAction, dataset_name: Optional[str] = None, self._category = category self._domain = domain self._data_location = data_location + self._blk_start = blk_start + self._blk_size = blk_size self._query = query self._is_pending_data = is_pending_data + @property + def blk_size(self) -> Optional[int]: + return self._blk_size + + @property + def blk_start(self) -> Optional[int]: + return self._blk_start + @property def data_location(self) -> Optional[str]: """ @@ -406,6 +435,10 @@ def to_dict(self) -> Dict[str, Union[str, Number, dict, list]]: serial[self._SERIAL_KEY_CATEGORY] = self.data_category.name if self.data_location is not None: serial[self._SERIAL_KEY_DATA_LOCATION] = self.data_location + if self._blk_start is not None: + serial[self._SERIAL_KEY_DATA_BLK_START] = self._blk_start + if self._blk_size is not None: + serial[self._SERIAL_KEY_DATA_BLK_SIZE] = self._blk_size if self.data_domain is not None: serial[self._SERIAL_KEY_DATA_DOMAIN] = self.data_domain.to_dict() if self.query is not None: @@ -602,6 +635,10 @@ def __init__(self, session_secret: str, *args, **kwargs): is_read_only_dataset : bool category : Optional[DataCategory] data_location : Optional[str] + blk_start : Optional[int] + Optional starting point for when acting upon a block/chunk of data. + blk_size : Optional[int] + Optional block size for when acting upon a block/chunk of data. is_pending_data : bool query : Optional[DataQuery] """ diff --git a/python/lib/core/dmod/core/dataset.py b/python/lib/core/dmod/core/dataset.py index b1e5ca2aa..bbb9701c5 100644 --- a/python/lib/core/dmod/core/dataset.py +++ b/python/lib/core/dmod/core/dataset.py @@ -819,7 +819,8 @@ def filter(self, base_dataset: Dataset, restrictions: List[Union[ContinuousRestr pass @abstractmethod - def get_data(self, dataset_name: str, item_name: str, **kwargs) -> Union[bytes, Any]: + def get_data(self, dataset_name: str, item_name: str, offset: Optional[int] = None, length: Optional[int] = None, + **kwargs) -> Union[bytes, Any]: """ Get data from this dataset. @@ -832,6 +833,10 @@ def get_data(self, dataset_name: str, item_name: str, **kwargs) -> Union[bytes, The dataset from which to get data. item_name : str The name of the object from which to get data. + offset : Optional[int] + Optional start byte position of object data. + length : Optional[int] + Optional number of bytes of object data from offset. kwargs Implementation-specific params for representing what data to get and how to get and deliver it. @@ -881,6 +886,26 @@ def link_user(self, user: DatasetUser, dataset: Dataset) -> bool: self._dataset_users[dataset.name].add(user.uuid) return True + @abstractmethod + def get_file_stat(self, dataset_name: str, file_name, **kwargs) -> Dict[str, Any]: + """ + Get the meta information about the given file. + + Parameters + ---------- + dataset_name : str + The name of the dataset containing the file of interest. + file_name : str + The name of the file of interest. + kwargs + + Returns + ------- + dict + Meta information about the given file, in dictionary form. + """ + pass + @abstractmethod def list_files(self, dataset_name: str, **kwargs) -> List[str]: """ diff --git a/python/lib/externalrequests/dmod/externalrequests/maas_request_handlers.py b/python/lib/externalrequests/dmod/externalrequests/maas_request_handlers.py index dff0b45ac..1dec8137b 100644 --- a/python/lib/externalrequests/dmod/externalrequests/maas_request_handlers.py +++ b/python/lib/externalrequests/dmod/externalrequests/maas_request_handlers.py @@ -291,25 +291,28 @@ async def handle_request(self, request: MaaSDatasetManagementMessage, **kwargs) session, is_authorized, reason, msg = await self.get_authorized_session(request) if not is_authorized: return MaaSDatasetManagementResponse(success=False, reason=reason.name, message=msg) - # In this case, we actually can pass the request as-is straight through (i.e., after confirming authorization) - async with self.service_client as client: - # Have to handle these two slightly differently, since multiple message will be going over the websocket - if request.management_action == ManagementAction.REQUEST_DATA: - await client.connection.send(str(request)) - mgmt_response = await self._handle_data_download(client_websocket=kwargs['upstream_websocket'], - service_websocket=client.connection) - elif request.management_action == ManagementAction.ADD_DATA: - await client.connection.send(str(request)) - mgmt_response = await self._handle_data_upload(client_websocket=kwargs['upstream_websocket'], - service_websocket=client.connection) - else: - mgmt_response = await client.async_make_request(request) - logging.debug("************* {} received response:\n{}".format(self.__class__.__name__, str(mgmt_response))) - # Likewise, can just send back the response from the internal service client - return MaaSDatasetManagementResponse.factory_create(mgmt_response) + try: + # In this case, we actually can pass the request as-is straight through (i.e., after confirming authorization) + async with self.service_client as client: + # Have to handle these two slightly differently, since multiple message will be going over the websocket + if request.management_action == ManagementAction.REQUEST_DATA: + await client.connection.send(str(request)) + mgmt_response = await self._handle_data_download(client_websocket=kwargs['upstream_websocket'], + service_websocket=client.connection) + elif request.management_action == ManagementAction.ADD_DATA: + await client.connection.send(str(request)) + mgmt_response = await self._handle_data_upload(client_websocket=kwargs['upstream_websocket'], + service_websocket=client.connection) + else: + mgmt_response = await client.async_make_request(request) + logging.debug("************* {} received response:\n{}".format(self.__class__.__name__, str(mgmt_response))) + # Likewise, can just send back the response from the internal service client + return MaaSDatasetManagementResponse.factory_create(mgmt_response) + except Exception as e: + raise e @property def service_client(self) -> DataServiceClient: if self._service_client is None: - self._service_client = DataServiceClient(self.service_url, self.service_ssl_dir) + self._service_client = DataServiceClient(endpoint_uri=self.service_url, ssl_directory=self.service_ssl_dir) return self._service_client diff --git a/python/lib/modeldata/dmod/modeldata/data/object_store_manager.py b/python/lib/modeldata/dmod/modeldata/data/object_store_manager.py index dffb5a8cf..551cb2aa1 100644 --- a/python/lib/modeldata/dmod/modeldata/data/object_store_manager.py +++ b/python/lib/modeldata/dmod/modeldata/data/object_store_manager.py @@ -10,7 +10,7 @@ from minio.api import ObjectWriteResult from minio.deleteobjects import DeleteObject from pathlib import Path -from typing import Dict, List, Optional, Set, Tuple +from typing import Any, Dict, List, Optional, Set, Tuple from uuid import UUID @@ -59,8 +59,13 @@ def __init__(self, obj_store_host_str: str, access_key: Optional[str] = None, se # For any buckets that have the standard serialized object (i.e., were for datasets previously), reload them for bucket_name in self.list_buckets(): serialized_item = self._gen_dataset_serial_obj_name(bucket_name) - if serialized_item in [o.object_name for o in self._client.list_objects(bucket_name)]: + try: self.reload(reload_from=bucket_name, serialized_item=serialized_item) + except minio.error.S3Error as e: + # Continue with looping through buckets and initializing if we get this particular exception and + # error code, but otherwise pass through the exception + if e.code != "NoSuchKey": + raise e except Exception as e: self._errors.append(e) # TODO: consider if we should not re-throw this (which would likely force us to ensure users checked this) @@ -418,7 +423,32 @@ def delete_data(self, dataset_name: str, **kwargs) -> bool: self._errors.extend(error_list) return False - def get_data(self, dataset_name: str, item_name: str, **kwargs) -> bytes: + def get_file_stat(self, dataset_name: str, file_name, **kwargs) -> Dict[str, Any]: + """ + Get the meta information about the given file. + + Parameters + ---------- + dataset_name : str + The name of the dataset containing the file of interest. + file_name : str + The name of the file of interest. + kwargs + + Returns + ------- + dict + Meta information about the given file, in dictionary form. + """ + obj_stat = self._client.stat_object(dataset_name, file_name) + as_dict = dict() + as_dict["name"] = obj_stat.object_name + as_dict["size"] = obj_stat.size + # TODO: get more of this if worth it + return as_dict + + def get_data(self, dataset_name: str, item_name: str, offset: Optional[int] = None, length: Optional[int] = None, + **kwargs) -> bytes: """ Get data from this dataset. @@ -432,15 +462,12 @@ def get_data(self, dataset_name: str, item_name: str, **kwargs) -> bytes: The name of the dataset (i.e., bucket) from which to get data. item_name : str The name of the object from which to get data. - kwargs - Implementation-specific params for representing what data to get and how to get and deliver it. - - Keyword Args - ------- - offset : int + offset : Optional[int] Optional start byte position of object data. - length : int + length : Optional[int] Optional number of bytes of object data from offset. + kwargs + Implementation-specific params for representing what data to get and how to get and deliver it. Returns ------- @@ -450,8 +477,10 @@ def get_data(self, dataset_name: str, item_name: str, **kwargs) -> bytes: if item_name not in self.list_files(dataset_name): raise RuntimeError('Cannot get data for non-existing {} file in {} dataset'.format(item_name, dataset_name)) optional_params = dict() - for key in [k for k in self.data_chunking_params if k in kwargs]: - optional_params[key] = kwargs[key] + if offset is not None: + optional_params['offset'] = offset + if length is not None: + optional_params['length'] = length response_object = self._client.get_object(bucket_name=dataset_name, object_name=item_name, **optional_params) return response_object.data @@ -552,12 +581,14 @@ def reload(self, reload_from: str, serialized_item: Optional[str] = None) -> Dat if serialized_item is None: serialized_item = self._gen_dataset_serial_obj_name(reload_from) + response_obj = None try: response_obj = self._client.get_object(bucket_name=reload_from, object_name=serialized_item) response_data = json.loads(response_obj.data.decode()) finally: - response_obj.close() - response_obj.release_conn() + if response_obj is not None: + response_obj.close() + response_obj.release_conn() # If we can safely infer it, make sure the "type" key is set in cases when it is missing if len(self.supported_dataset_types) == 1 and Dataset._KEY_TYPE not in response_data: diff --git a/python/lib/modeldata/setup.py b/python/lib/modeldata/setup.py index d00c51756..3f8f70565 100644 --- a/python/lib/modeldata/setup.py +++ b/python/lib/modeldata/setup.py @@ -14,7 +14,7 @@ author_email='', url='', license='', - install_requires=['numpy>=1.20.1', 'pandas', 'geopandas', 'dmod-communication>=0.4.2', 'dmod-core>=0.3.0', 'minio', + install_requires=['numpy>=1.20.1', 'pandas', 'geopandas', 'dmod-communication>=0.9.1', 'dmod-core>=0.3.0', 'minio', 'aiohttp<=3.7.4', 'hypy@git+https://github.com/NOAA-OWP/hypy@master#egg=hypy&subdirectory=python'], packages=find_namespace_packages(exclude=('tests', 'schemas', 'ssl', 'src')) ) diff --git a/python/services/dataservice/dmod/dataservice/service.py b/python/services/dataservice/dmod/dataservice/service.py index 04e1fa0af..1d1c7c37d 100644 --- a/python/services/dataservice/dmod/dataservice/service.py +++ b/python/services/dataservice/dmod/dataservice/service.py @@ -684,9 +684,22 @@ def _process_query(self, message: DatasetManagementMessage) -> DatasetManagement dataset_name = message.dataset_name list_of_files = self.get_known_datasets()[dataset_name].manager.list_files(dataset_name) return DatasetManagementResponse(action=message.management_action, success=True, dataset_name=dataset_name, - reason='Obtained {} Items List', + reason='Obtained {} Items List'.format(dataset_name), data={DatasetManagementResponse._DATA_KEY_QUERY_RESULTS: list_of_files}) - # TODO: (later) add support for messages with other query types also + elif query_type == QueryType.GET_SERIALIZED_FORM: + dataset_name = message.dataset_name + serialized_form = self.get_known_datasets()[dataset_name].to_dict() + return DatasetManagementResponse(action=message.management_action, success=True, dataset_name=dataset_name, + reason='Obtained serialized {} dataset'.format(dataset_name), + data={DatasetManagementResponse._DATA_KEY_QUERY_RESULTS: serialized_form}) + if query_type == QueryType.GET_DATASET_ITEMS: + dataset = self.get_known_datasets()[message.dataset_name] + mgr = dataset.manager + item_details: List[dict] = [mgr.get_file_stat(dataset.name, f) for f in mgr.list_files(dataset.name)] + return DatasetManagementResponse(action=message.management_action, success=True, dataset_name=dataset.name, + reason='Obtained file details for {} dataset'.format(dataset.name), + data={DatasetManagementResponse._DATA_KEY_QUERY_RESULTS: item_details}) + # TODO: (later) add support for messages with other query types also else: reason = 'Unsupported {} Query Type - {}'.format(DatasetQuery.__class__.__name__, query_type.name) return DatasetManagementResponse(action=message.management_action, success=False, reason=reason) @@ -890,6 +903,14 @@ async def listener(self, websocket: WebSocketServerProtocol, path): partial_indx = 0 elif inbound_message.management_action == ManagementAction.CREATE: response = await self._async_process_dataset_create(message=inbound_message) + elif inbound_message.management_action == ManagementAction.REQUEST_DATA and inbound_message.blk_start is not None: + manager = self.get_known_datasets()[inbound_message.dataset_name].manager + raw_data = manager.get_data(dataset_name=inbound_message.dataset_name, + item_name=inbound_message.data_location, + offset=inbound_message.blk_start, length=inbound_message.blk_size) + response = DatasetManagementResponse(success=raw_data is not None, + action=inbound_message.management_action, + data=raw_data, reason="Data Block Retrieve Complete") elif inbound_message.management_action == ManagementAction.REQUEST_DATA: response = await self._async_process_data_request(message=inbound_message, websocket=websocket) elif inbound_message.management_action == ManagementAction.ADD_DATA: diff --git a/python/services/dataservice/setup.py b/python/services/dataservice/setup.py index cd74d404d..55bb3221d 100644 --- a/python/services/dataservice/setup.py +++ b/python/services/dataservice/setup.py @@ -14,7 +14,7 @@ author_email='', url='', license='', - install_requires=['dmod-core>=0.3.0', 'dmod-communication>=0.7.1', 'dmod-scheduler>=0.7.0', 'dmod-modeldata>=0.9.0', + install_requires=['dmod-core>=0.3.0', 'dmod-communication>=0.9.1', 'dmod-scheduler>=0.7.0', 'dmod-modeldata>=0.9.0', 'redis'], packages=find_namespace_packages(exclude=('tests', 'test', 'deprecated', 'conf', 'schemas', 'ssl', 'src')) ) diff --git a/python/services/requestservice/dmod/requestservice/_version.py b/python/services/requestservice/dmod/requestservice/_version.py index 08d79c0e9..83e147c62 100644 --- a/python/services/requestservice/dmod/requestservice/_version.py +++ b/python/services/requestservice/dmod/requestservice/_version.py @@ -1 +1 @@ -__version__ = '0.5.1' \ No newline at end of file +__version__ = '0.6.0' \ No newline at end of file diff --git a/python/services/requestservice/setup.py b/python/services/requestservice/setup.py index 06618889d..af7ed1e0a 100644 --- a/python/services/requestservice/setup.py +++ b/python/services/requestservice/setup.py @@ -14,7 +14,7 @@ author_email='', url='', license='', - install_requires=['websockets', 'dmod-core>=0.1.0', 'dmod-communication>=0.7.0', 'dmod-access>=0.2.0', + install_requires=['websockets', 'dmod-core>=0.2.0', 'dmod-communication>=0.8.0', 'dmod-access>=0.2.0', 'dmod-externalrequests>=0.3.0'], packages=find_namespace_packages(exclude=('tests', 'schemas', 'ssl', 'src')) )