Initial Dataset Management GUI

docker/main/dataservice/entrypoint.sh

@@ -62,7 +62,7 @@ if [ -d ${UPDATED_PACKAGES_DIR:=/updated_packages} ]; then
         for srv in $(pip -qq freeze | grep dmod | awk -F= '{print $1}' | awk -F- '{print $2}'); do
             if [ $(ls ${UPDATED_PACKAGES_DIR} | grep dmod.${srv}- | wc -l) -eq 1 ]; then
                 pip uninstall -y --no-input $(pip -qq freeze | grep dmod.${srv} | awk -F= '{print $1}')
-                pip install $(ls ${UPDATED_PACKAGES_DIR}/*.whl | grep dmod.${srv}-)
+                pip install --no-deps $(ls ${UPDATED_PACKAGES_DIR}/*.whl | grep dmod.${srv}-)
             fi
         done
         #pip install ${UPDATED_PACKAGES_DIR}/*.whl

docker/nwm_gui/app_server/entrypoint.sh

@@ -34,6 +34,9 @@ echo "Starting dmod app"
 #Extract the DB secrets into correct ENV variables
 POSTGRES_SECRET_FILE="/run/secrets/${DOCKER_SECRET_POSTGRES_PASS:?}"
 export SQL_PASSWORD="$(cat ${POSTGRES_SECRET_FILE})"
+export DMOD_SU_PASSWORD="$(cat ${POSTGRES_SECRET_FILE})"
+
+python manage.py migrate
 
 # Handle for debugging when appropriate
 if [ "$(echo "${PYCHARM_REMOTE_DEBUG_ACTIVE:-false}" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]')" == "true" ]; then

docker/nwm_gui/docker-compose.yml

@@ -34,6 +34,7 @@ services:
       args:
         docker_internal_registry: ${DOCKER_INTERNAL_REGISTRY:?Missing DOCKER_INTERNAL_REGISTRY value (see 'Private Docker Registry ' section in example.env)}
         comms_package_name: ${PYTHON_PACKAGE_DIST_NAME_COMMS:?}
+        client_package_name: ${PYTHON_PACKAGE_DIST_NAME_CLIENT:?}
     networks:
       - request-listener-net
     # Call this when starting the container
@@ -57,11 +58,15 @@ services:
       - SQL_USER=${DMOD_GUI_POSTGRES_USER:?}
       - SQL_HOST=db
       - SQL_PORT=5432
+      - DMOD_SU_NAME=dmod_super_user
+      - DMOD_SU_EMAIL=none@noaa.gov
       - DATABASE=postgres
       - DOCKER_SECRET_POSTGRES_PASS=postgres_password
     volumes:
       - ${DMOD_APP_STATIC:?}:/usr/maas_portal/static
       - ${DMOD_SSL_DIR}/requestservice:/usr/maas_portal/ssl
+      # Needed only for speeding debugging
+      #- ${DOCKER_GUI_HOST_SRC:?GUI sources path not configured in environment}/MaaS:/usr/maas_portal/MaaS
       #- ${DOCKER_GUI_HOST_VENV_DIR:-/tmp/blah}:${DOCKER_GUI_CONTAINER_VENV_DIR:-/tmp/blah}
     # Expose Django's port to the internal network so that the web server may access it
     expose:

example.env

@@ -108,6 +108,11 @@ TROUTE_BRANCH=ngen
 ## Python Packages Settings                                           ##
 ########################################################################
 
+## The "name" of the built client Python distribution package, for purposes of installing (e.g., via pip)
+PYTHON_PACKAGE_DIST_NAME_CLIENT=dmod-client
+## The name of the actual Python communication package (i.e., for importing or specifying as a module on the command line)
+PYTHON_PACKAGE_NAME_CLIENT=dmod.client
+
 ## The "name" of the built communication Python distribution package, for purposes of installing (e.g., via pip)
 PYTHON_PACKAGE_DIST_NAME_COMMS=dmod-communication
 ## The name of the actual Python communication package (i.e., for importing or specifying as a module on the command line)

python/gui/MaaS/cbv/AbstractDatasetView.py

@@ -0,0 +1,29 @@
+from abc import ABC
+from django.views.generic.base import View
+from dmod.client.request_clients import DatasetExternalClient
+import logging
+logger = logging.getLogger("gui_log")
+from .DMODProxy import DMODMixin, GUI_STATIC_SSL_DIR
+from typing import Dict
+
+
+class AbstractDatasetView(View, DMODMixin, ABC):
+
+    def __init__(self, *args, **kwargs):
+        super(AbstractDatasetView, self).__init__(*args, **kwargs)
+        self._dataset_client = None
+
+    async def get_dataset(self, dataset_name: str) -> Dict[str, dict]:
+        serial_dataset = await self.dataset_client.get_serialized_datasets(dataset_name=dataset_name)
+        return serial_dataset
+
+    async def get_datasets(self) -> Dict[str, dict]:
+        serial_datasets = await self.dataset_client.get_serialized_datasets()
+        return serial_datasets
+
+    @property
+    def dataset_client(self) -> DatasetExternalClient:
+        if self._dataset_client is None:
+            self._dataset_client = DatasetExternalClient(endpoint_uri=self.maas_endpoint_uri,
+                                                         ssl_directory=GUI_STATIC_SSL_DIR)
+        return self._dataset_client

python/gui/MaaS/cbv/DMODProxy.py

@@ -16,6 +16,8 @@
 from pathlib import Path
 from typing import List, Optional, Tuple, Type
 
+GUI_STATIC_SSL_DIR = Path('/usr/maas_portal/ssl')
+
 
 class RequestFormProcessor(ABC):
 
@@ -209,7 +211,7 @@ class PostFormRequestClient(ModelExecRequestClient):
     def _bootstrap_ssl_dir(cls, ssl_dir: Optional[Path] = None):
         if ssl_dir is None:
             ssl_dir = Path(__file__).resolve().parent.parent.parent.joinpath('ssl')
-            ssl_dir = Path('/usr/maas_portal/ssl') #Fixme
+            ssl_dir = GUI_STATIC_SSL_DIR #Fixme
         return ssl_dir
 
     def __init__(self, endpoint_uri: str, http_request: HttpRequest, ssl_dir: Optional[Path] = None):
@@ -315,6 +317,7 @@ def forward_request(self, request: HttpRequest, event_type: MessageEventType) ->
         client = PostFormRequestClient(endpoint_uri=self.maas_endpoint_uri, http_request=request)
         if event_type == MessageEventType.MODEL_EXEC_REQUEST:
             form_processor_type = ModelExecRequestFormProcessor
+        # TODO: need a new type of form processor here (or 3 more, for management, uploading, and downloading)
         else:
             raise RuntimeError("{} got unsupported event type: {}".format(self.__class__.__name__, str(event_type)))
 

python/gui/MaaS/cbv/DatasetApiView.py

@@ -0,0 +1,62 @@
+import asyncio
+from django.http import JsonResponse
+from wsgiref.util import FileWrapper
+from django.http.response import StreamingHttpResponse
+from .AbstractDatasetView import AbstractDatasetView
+from .DatasetFileWebsocketFilelike import DatasetFileWebsocketFilelike
+import logging
+logger = logging.getLogger("gui_log")
+
+
+class DatasetApiView(AbstractDatasetView):
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    def _get_dataset_content_details(self, dataset_name: str):
+        result = asyncio.get_event_loop().run_until_complete(self.dataset_client.get_dataset_content_details(name=dataset_name))
+        logger.info(result)
+        return JsonResponse({"contents": result}, status=200)
+
+    def _delete_dataset(self, dataset_name: str) -> JsonResponse:
+        result = asyncio.get_event_loop().run_until_complete(self.dataset_client.delete_dataset(name=dataset_name))
+        return JsonResponse({"successful": result}, status=200)
+
+    def _get_datasets_json(self) -> JsonResponse:
+        serial_dataset_map = asyncio.get_event_loop().run_until_complete(self.get_datasets())
+        return JsonResponse({"datasets": serial_dataset_map}, status=200)
+
+    def _get_dataset_json(self, dataset_name: str) -> JsonResponse:
+        serial_dataset = asyncio.get_event_loop().run_until_complete(self.get_dataset(dataset_name=dataset_name))
+        return JsonResponse({"dataset": serial_dataset[dataset_name]}, status=200)
+
+    def _get_download(self, request, *args, **kwargs):
+        dataset_name = request.GET.get("dataset_name", None)
+        item_name = request.GET.get("item_name", None)
+        chunk_size = 8192
+
+        custom_filelike = DatasetFileWebsocketFilelike(self.dataset_client, dataset_name, item_name)
+
+        response = StreamingHttpResponse(
+            FileWrapper(custom_filelike, chunk_size),
+            content_type="application/octet-stream"
+        )
+        response['Content-Length'] = asyncio.get_event_loop().run_until_complete(self.dataset_client.get_item_size(dataset_name, item_name))
+        response['Content-Disposition'] = "attachment; filename=%s" % item_name
+        return response
+
+    def get(self, request, *args, **kwargs):
+        request_type = request.GET.get("request_type", None)
+        if request_type == 'download_file':
+            return self._get_download(request)
+        elif request_type == 'datasets':
+            return self._get_datasets_json()
+        elif request_type == 'dataset':
+            return self._get_dataset_json(dataset_name=request.GET.get("name", None))
+        elif request_type == 'contents':
+            return self._get_dataset_content_details(dataset_name=request.GET.get("name", None))
+        if request_type == 'delete':
+            return self._delete_dataset(dataset_name=request.GET.get("name", None))
+
+        # TODO: finish
+        return JsonResponse({}, status=400)

python/gui/MaaS/cbv/DatasetFileWebsocketFilelike.py

@@ -0,0 +1,20 @@
+import asyncio
+from typing import AnyStr
+from dmod.client.request_clients import DatasetExternalClient
+
+
+class DatasetFileWebsocketFilelike:
+
+    def __init__(self, client: DatasetExternalClient, dataset_name: str, file_name: str):
+        self._client = client
+        self._dataset_name = dataset_name
+        self._file_name = file_name
+        self._read_index: int = 0
+
+    def read(self, blksize: int) -> AnyStr:
+
+        result = asyncio.get_event_loop().run_until_complete(
+            self._client.download_item_block(dataset_name=self._dataset_name, item_name=self._file_name,
+                                             blk_start=self._read_index, blk_size=blksize))
+        self._read_index += blksize
+        return result

python/gui/MaaS/cbv/DatasetManagementView.py

@@ -0,0 +1,114 @@
+"""
+Defines a view that may be used to configure a MaaS request
+"""
+import asyncio
+from django.http import HttpRequest, HttpResponse
+from django.shortcuts import render
+
+import dmod.communication as communication
+from dmod.core.meta_data import DataCategory, DataFormat
+
+import logging
+logger = logging.getLogger("gui_log")
+
+from .utils import extract_log_data
+from .AbstractDatasetView import AbstractDatasetView
+
+
+class DatasetManagementView(AbstractDatasetView):
+
+    """
+    A view used to configure a dataset management request or requests for transmitting dataset data.
+    """
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    def _process_event_type(self, http_request: HttpRequest) -> communication.MessageEventType:
+        """
+        Determine and return whether this request is for a ``DATASET_MANAGEMENT`` or ``DATA_TRANSMISSION`` event.
+
+        Parameters
+        ----------
+        http_request : HttpRequest
+            The raw HTTP request in question.
+
+        Returns
+        -------
+        communication.MessageEventType
+            Either ``communication.MessageEventType.DATASET_MANAGEMENT`` or
+            ``communication.MessageEventType.DATA_TRANSMISSION``.
+        """
+        # TODO:
+        raise NotImplementedError("{}._process_event_type not implemented".format(self.__class__.__name__))
+
+    def get(self, http_request: HttpRequest, *args, **kwargs) -> HttpResponse:
+        """
+        The handler for 'get' requests.
+
+        This will render the 'maas/dataset_management.html' template after retrieving necessary information to initially
+        populate the forms it displays.
+
+        Parameters
+        ----------
+        http_request : HttpRequest
+            The request asking to render this page.
+        args
+        kwargs
+
+        Returns
+        -------
+        A rendered page.
+        """
+        errors, warnings, info = extract_log_data(kwargs)
+
+        # Gather map of serialized datasets, keyed by dataset name
+        serial_dataset_map = asyncio.get_event_loop().run_until_complete(self.get_datasets())
+        serial_dataset_list = [serial_dataset_map[d] for d in serial_dataset_map]
+
+        dataset_categories = [c.name.title() for c in DataCategory]
+        dataset_formats = [f.name for f in DataFormat]
+
+        payload = {
+            'datasets': serial_dataset_list,
+            'dataset_categories': dataset_categories,
+            'dataset_formats': dataset_formats,
+            'errors': errors,
+            'info': info,
+            'warnings': warnings
+        }
+
+        return render(http_request, 'maas/dataset_management.html', payload)
+
+    def post(self, http_request: HttpRequest, *args, **kwargs) -> HttpResponse:
+        """
+        The handler for 'post' requests.
+
+        This will attempt to submit the request and rerender the page like a 'get' request.
+
+        Parameters
+        ----------
+        http_request : HttpRequest
+            The request asking to render this page.
+        args
+        kwargs
+
+        Returns
+        -------
+        A rendered page.
+        """
+        # TODO: implement this to figure out whether DATASET_MANAGEMENT or DATA_TRANSMISSION
+        event_type = self._process_event_type(http_request)
+        client, session_data, dmod_response = self.forward_request(http_request, event_type)
+
+        # TODO: this probably isn't exactly correct, so review once closer to completion
+        if dmod_response is not None and 'dataset_id' in dmod_response.data:
+            session_data['new_dataset_id'] = dmod_response.data['dataset_id']
+
+        http_response = self.get(http_request=http_request, errors=client.errors, warnings=client.warnings,
+                                 info=client.info, *args, **kwargs)
+
+        for k, v in session_data.items():
+            http_response.set_cookie(k, v)
+
+        return http_response