From e985af370e4bc893b2914b44f3a2fbf742e3897e Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Mon, 3 Oct 2022 11:41:16 -0500 Subject: [PATCH 1/9] Fix fast-debug in dataservice image entrypoint. Fixing logic where pip installs the updated packages (to avoid the entire image rebuild), so make sure deps are ignored (as this was the slow part). --- docker/main/dataservice/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/main/dataservice/entrypoint.sh b/docker/main/dataservice/entrypoint.sh index e42ae462d..433ecc801 100644 --- a/docker/main/dataservice/entrypoint.sh +++ b/docker/main/dataservice/entrypoint.sh @@ -62,7 +62,7 @@ if [ -d ${UPDATED_PACKAGES_DIR:=/updated_packages} ]; then for srv in $(pip -qq freeze | grep dmod | awk -F= '{print $1}' | awk -F- '{print $2}'); do if [ $(ls ${UPDATED_PACKAGES_DIR} | grep dmod.${srv}- | wc -l) -eq 1 ]; then pip uninstall -y --no-input $(pip -qq freeze | grep dmod.${srv} | awk -F= '{print $1}') - pip install $(ls ${UPDATED_PACKAGES_DIR}/*.whl | grep dmod.${srv}-) + pip install --no-deps $(ls ${UPDATED_PACKAGES_DIR}/*.whl | grep dmod.${srv}-) fi done #pip install ${UPDATED_PACKAGES_DIR}/*.whl From ecd9c58c845ed536a71c97de7ade53ea5ae4c654 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Wed, 15 Mar 2023 10:45:32 -0500 Subject: [PATCH 2/9] Add debug and fix entrypoint for subsetservice. Adding Docker environment variables to support remote PyCharm debugging, and removing explicit `entrypoint` setting in main stack deployment service config to ensure the included entrypoint.sh script is used. --- docker/main/docker-deploy.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker/main/docker-deploy.yml b/docker/main/docker-deploy.yml index 665668e58..67ee58170 100644 --- a/docker/main/docker-deploy.yml +++ b/docker/main/docker-deploy.yml @@ -128,9 +128,11 @@ services: - LISTEN_PORT=${DOCKER_SUBSET_CONTAINER_PORT:-5000} - SERVICE_PACKAGE_NAME=${PYTHON_PACKAGE_NAME_SUBSET_SERVICE:?} - FILES_DIRECTORY=/hydrofabric_data + - PYCHARM_REMOTE_DEBUG_ACTIVE=${PYCHARM_REMOTE_DEBUG_SUBSET_SERVICE_ACTIVE:-false} + - PYCHARM_REMOTE_DEBUG_SERVER_HOST=${PYCHARM_REMOTE_DEBUG_SERVER_HOST:-host.docker.internal} + - PYCHARM_REMOTE_DEBUG_SERVER_PORT=${PYCHARM_REMOTE_DEBUG_SERVER_PORT_SUBSET_SERVICE:-55874} # This typically needs to be commented out; intended for development use (see related 'volume' config below) #- UPDATED_PACKAGES_DIR=${UPDATED_PACKAGES_CONTAINER_DIR:?Updated packages directory not set, make sure this should be active} - entrypoint: ["python3", "-m", "${PYTHON_PACKAGE_NAME_SUBSET_SERVICE:?}"] command: ["--files-directory", "/hydrofabric_data", "--port", "${DOCKER_SUBSET_CONTAINER_PORT:-5000}", "--crosswalk-file", "${HYRDOFABRIC_CROSSWALK_FILE_BASENAME:-crosswalk.json}"] From eec98cddcdb1b5e89a8f5d260100b7f0ac2f56c1 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Fri, 20 Jan 2023 09:18:59 -0600 Subject: [PATCH 3/9] Add eval ENV vars to request-service def. --- docker/main/docker-deploy.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker/main/docker-deploy.yml b/docker/main/docker-deploy.yml index 67ee58170..cefdbaeb8 100644 --- a/docker/main/docker-deploy.yml +++ b/docker/main/docker-deploy.yml @@ -88,6 +88,9 @@ services: - DATA_SERVICE_ENDPOINT_HOST=${DOCKER_REQUESTS_DATASERVICE_ENDPOINT_HOST:-data-service} - DATA_SERVICE_ENDPOINT_PORT=${DOCKER_DATASERVICE_CONTAINER_PORT:-3015} - DATA_SERVICE_CLIENT_SSL_DIR=${DOCKER_REQUESTS_CONTAINER_DATASERVICE_CLIENT_SSL_DIR:-/ssl/dataservice} + - EVALUATION_SERVICE_ENDPOINT_HOST=${DOCKER_REQUESTS_EVALUATIONSERVICE_ENDPOINT_HOST:-evaluation-service} + - EVALUATION_SERVICE_ENDPOINT_PORT=${DOCKER_EVALUATIONSERVICE_CONTAINER_PORT:-3015} + - EVALUATION_SERVICE_CLIENT_SSL_DIR=${DOCKER_REQUESTS_CONTAINER_EVALUATIONSERVICE_CLIENT_SSL_DIR:-/ssl/evaluationservice} - PARTITIONER_SERVICE_ENDPOINT_HOST=${DOCKER_REQUESTS_PARTITIONERSERVICE_ENDPOINT_HOST:-partitioner-service} - PARTITIONER_SERVICE_ENDPOINT_PORT=${DOCKER_PARTITIONER_SERVICE_CONTAINER_PORT:-3014} - PARTITIONER_SERVICE_CLIENT_SSL_DIR=${DOCKER_REQUESTS_CONTAINER_PARTITIONERSERVICE_CLIENT_SSL_DIR:-/ssl/partitionerservice} From dac3cae4c99f1b0c2e70722f4e094a71d48f6bee Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Fri, 20 Jan 2023 09:19:42 -0600 Subject: [PATCH 4/9] Update requestservice entrypoint w/ eval settings. --- docker/main/requestservice/entrypoint.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker/main/requestservice/entrypoint.sh b/docker/main/requestservice/entrypoint.sh index da38e9ec7..2dcae593a 100755 --- a/docker/main/requestservice/entrypoint.sh +++ b/docker/main/requestservice/entrypoint.sh @@ -54,6 +54,9 @@ python -m ${SERVICE_PACKAGE_NAME:?} \ --data-service-host ${DATA_SERVICE_ENDPOINT_HOST:?} \ --data-service-port ${DATA_SERVICE_ENDPOINT_PORT:?} \ --data-service-ssl-dir ${DATA_SERVICE_CLIENT_SSL_DIR:?} \ + --evaluation-service-host ${EVALUATION_SERVICE_ENDPOINT_HOST:?} \ + --evaluation-service-port ${EVALUATION_SERVICE_ENDPOINT_PORT:?} \ + --evaluation-service-ssl-dir ${EVALUATION_SERVICE_CLIENT_SSL_DIR:?} \ --partitioner-service-host ${PARTITIONER_SERVICE_ENDPOINT_HOST:?} \ --partitioner-service-port ${PARTITIONER_SERVICE_ENDPOINT_PORT:?} \ --partitioner-service-ssl-dir ${PARTITIONER_SERVICE_CLIENT_SSL_DIR:?} From 6a1fbc5e2a02595c0e355a487f867dcac244cf71 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Fri, 4 Nov 2022 15:55:28 -0500 Subject: [PATCH 5/9] Fix strange issue with DockerFile for vol helper. Fixing issue where somehow (despite this working before) the scripts directory was not recursively being copied (i.e., just the top-level). --- docker/main/s3fs-volume-helper/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/main/s3fs-volume-helper/Dockerfile b/docker/main/s3fs-volume-helper/Dockerfile index b8a261aeb..39100f924 100644 --- a/docker/main/s3fs-volume-helper/Dockerfile +++ b/docker/main/s3fs-volume-helper/Dockerfile @@ -1,8 +1,8 @@ FROM alpine:3.15 -RUN apk update && apk upgrade && apk add docker bash && mkdir -p /dmod/scripts +RUN apk update && apk upgrade && apk add docker bash && mkdir -p /dmod -COPY ./scripts/* /dmod/scripts/. +COPY ./scripts /dmod/scripts ENV OUT_OF_GIT_REPO='true' From ce32ddbffa2a7c1f1f3b9eae165a321c317383f3 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Mon, 31 Oct 2022 14:28:57 -0500 Subject: [PATCH 6/9] Add debug support to subsetservice entrypoint. --- docker/main/subsetservice/entrypoint.sh | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/docker/main/subsetservice/entrypoint.sh b/docker/main/subsetservice/entrypoint.sh index cfc3728de..65eae6335 100755 --- a/docker/main/subsetservice/entrypoint.sh +++ b/docker/main/subsetservice/entrypoint.sh @@ -12,6 +12,23 @@ if [ -n "${VENV_DIR:-}" ]; then pip install --update -r /code/requirements.txt fi +# Install for debugging when appropriate +if [ "$(echo "${PYCHARM_REMOTE_DEBUG_ACTIVE:-false}" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]')" = "true" ]; then + _DEBUG_ARG="--pycharm-remote-debug" +fi + +# Handle some things in any cases when there is debugging +if [ -n "${_DEBUG_ARG:-}" ]; then + # Append these as well if appropriate, though defaults are coded (and they are somewhat agnostic to the debug setup) + if [ -n "${PYCHARM_REMOTE_DEBUG_SERVER_HOST:-}" ]; then + _DEBUG_ARG="${_DEBUG_ARG:-} --remote-debug-host ${PYCHARM_REMOTE_DEBUG_SERVER_HOST}" + fi + + if [ -n "${PYCHARM_REMOTE_DEBUG_SERVER_PORT:-}" ]; then + _DEBUG_ARG="${_DEBUG_ARG:-} --remote-debug-port ${PYCHARM_REMOTE_DEBUG_SERVER_PORT}" + fi +fi + # If we find this directory, and if there are wheels in it, then install those if [ -d ${UPDATED_PACKAGES_DIR:=/updated_packages} ]; then if [ $(ls ${UPDATED_PACKAGES_DIR}/*.whl | wc -l) -gt 0 ]; then @@ -44,4 +61,4 @@ fi #set +e #export PYTHONASYNCIODEBUG=1 -python3 -m ${SERVICE_PACKAGE_NAME:?} ${args} \ No newline at end of file +python3 -m ${SERVICE_PACKAGE_NAME:?} ${_DEBUG_ARG:-} ${args} \ No newline at end of file From 382c14ae5a287ca64539edc4add53c464bc7ef39 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Thu, 5 Jan 2023 07:11:00 -0600 Subject: [PATCH 7/9] Add OS dependency geos-dev to py-deps image. Adding the now-required dependency to list of installed apk packages in foundation stage of py-deps image. --- docker/py-sources/py-deps.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/py-sources/py-deps.Dockerfile b/docker/py-sources/py-deps.Dockerfile index 9a0ff75fe..da6d7e35b 100644 --- a/docker/py-sources/py-deps.Dockerfile +++ b/docker/py-sources/py-deps.Dockerfile @@ -1,4 +1,4 @@ -ARG REQUIRE="gcc g++ musl-dev gdal-dev libffi-dev openssl-dev rust cargo git proj proj-dev proj-util openblas openblas-dev lapack lapack-dev" +ARG REQUIRE="gcc g++ musl-dev gdal-dev libffi-dev openssl-dev rust cargo git proj proj-dev proj-util openblas openblas-dev lapack lapack-dev geos-dev" ################################################################################################################ ################################################################################################################ ##### Create foundational level build stage with initial structure From fa881dad94aefe279b64d88af90a27d5b0b7f8fd Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Tue, 13 Sep 2022 16:49:07 -0500 Subject: [PATCH 8/9] Add package client name vars to example.env. --- example.env | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/example.env b/example.env index 637e9e5ce..bafbc8e63 100644 --- a/example.env +++ b/example.env @@ -108,6 +108,11 @@ TROUTE_BRANCH=ngen ## Python Packages Settings ## ######################################################################## +## The "name" of the built client Python distribution package, for purposes of installing (e.g., via pip) +PYTHON_PACKAGE_DIST_NAME_CLIENT=dmod-client +## The name of the actual Python communication package (i.e., for importing or specifying as a module on the command line) +PYTHON_PACKAGE_NAME_CLIENT=dmod.client + ## The "name" of the built communication Python distribution package, for purposes of installing (e.g., via pip) PYTHON_PACKAGE_DIST_NAME_COMMS=dmod-communication ## The name of the actual Python communication package (i.e., for importing or specifying as a module on the command line) From 9cce3e21189b5e369f78fa2d1889ceb51d367628 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Fri, 4 Nov 2022 15:54:35 -0500 Subject: [PATCH 9/9] Fix bug in obj store vol script for sentinel. --- scripts/init_object_store_dataset_volumes.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/init_object_store_dataset_volumes.sh b/scripts/init_object_store_dataset_volumes.sh index 156a53646..284bdea50 100755 --- a/scripts/init_object_store_dataset_volumes.sh +++ b/scripts/init_object_store_dataset_volumes.sh @@ -167,7 +167,7 @@ while [ ${#} -gt 0 ]; do DO_REMOVE_ALL='true' ;; --sentinel|-S) - SENTINEL="/tmp/${2:?}" + SENTINEL="${2:?}" shift ;; --service-mode)