From f71849f949ca4aa0a0978fcc685ceab927776d42 Mon Sep 17 00:00:00 2001 From: veenhouse Date: Mon, 22 Aug 2022 17:37:32 +0000 Subject: [PATCH 001/132] updating for aws --- .env.dev | 9 -- .env.prod | 11 -- .env.prod.db | 3 - .env.stage | 11 -- .env.stage.db | 3 - .gitignore | 1 + .gitmodules | 0 .vscode/settings.json | 1 + Dockerfile.dev => Dockerfile | 18 +-- Dockerfile.prod | 105 ------------------ Dockerfile.stage | 104 ----------------- .../chp_api/{settings/base.py => settings.py} | 33 ++++-- chp_api/chp_api/settings/__init__.py | 0 chp_api/chp_api/settings/dev.py | 31 ------ chp_api/chp_api/settings/production.py | 31 ------ chp_api/chp_api/settings/staging.py | 33 ------ deploy/chp-api/Jenkinsfile | 2 +- deployment-script-jenkins | 20 ---- deployment-script-prod | 31 ------ deployment-script-stage | 31 ------ docker-compose-dev.yml | 23 ++++ docker-compose.dev.yml | 10 -- docker-compose.local.yml | 0 docker-compose.stage.yml | 17 --- docker-compose.prod.yml => docker-compose.yml | 6 +- entrypoint-dev.sh | 17 --- entrypoint.prod.sh => entrypoint.sh | 0 entrypoint.stage.sh | 25 ----- gunicorn.config-stage.py | 13 --- gunicorn.config-prod.py => gunicorn.config.py | 3 +- nginx/Dockerfile | 2 +- nginx/nginx.conf | 2 +- requirements-dev.txt | 12 +- 33 files changed, 71 insertions(+), 537 deletions(-) delete mode 100644 .env.dev delete mode 100644 .env.prod delete mode 100644 .env.prod.db delete mode 100644 .env.stage delete mode 100644 .env.stage.db delete mode 100644 .gitmodules rename Dockerfile.dev => Dockerfile (79%) delete mode 100644 Dockerfile.prod delete mode 100644 Dockerfile.stage rename chp_api/chp_api/{settings/base.py => settings.py} (83%) delete mode 100644 chp_api/chp_api/settings/__init__.py delete mode 100644 chp_api/chp_api/settings/dev.py delete mode 100644 chp_api/chp_api/settings/production.py delete mode 100644 chp_api/chp_api/settings/staging.py delete mode 100644 deployment-script-jenkins delete mode 100644 deployment-script-prod delete mode 100644 deployment-script-stage create mode 100644 docker-compose-dev.yml delete mode 100644 docker-compose.dev.yml delete mode 100644 docker-compose.local.yml delete mode 100644 docker-compose.stage.yml rename docker-compose.prod.yml => docker-compose.yml (51%) delete mode 100644 entrypoint-dev.sh rename entrypoint.prod.sh => entrypoint.sh (100%) delete mode 100644 entrypoint.stage.sh delete mode 100644 gunicorn.config-stage.py rename gunicorn.config-prod.py => gunicorn.config.py (93%) diff --git a/.env.dev b/.env.dev deleted file mode 100644 index 57ac5c5..0000000 --- a/.env.dev +++ /dev/null @@ -1,9 +0,0 @@ -DEBUG=1 -SECRET_KEY=foo -DJANGO_ALLOWED_HOSTS=localhost 127.0.0.1 [::1] 129.170.70.76 0.0.0.0 -SQL_ENGINE=django.db.backends.postgresql -SQL_DATABASE=chp_api_dev -SQL_USER=chp_api_user -SQL_PASSWORD=chp_api_user -SQL_HOST=db -SQL_PORT=5432 \ No newline at end of file diff --git a/.env.prod b/.env.prod deleted file mode 100644 index 7eb93ee..0000000 --- a/.env.prod +++ /dev/null @@ -1,11 +0,0 @@ -DEBUG=0 -SECRET_KEY=BkbsAreTheBest2020 -DJANGO_ALLOWED_HOSTS=chp.thayer.dartmouth.edu -SQL_ENGINE=django.db.backends.postgresql -SQL_DATABASE=chp_api_prod -SQL_USER=chp_api_user -SQL_PASSWORD=chp_api_user -SQL_HOST=db -SQL_PORT=5432 -DATABASE=postgres -DJANGO_SETTINGS_MODULE=chp_api.settings.production \ No newline at end of file diff --git a/.env.prod.db b/.env.prod.db deleted file mode 100644 index a05584e..0000000 --- a/.env.prod.db +++ /dev/null @@ -1,3 +0,0 @@ -POSTGRES_USER=chp_api_user -POSTGRES_PASSWORD=chp_api_user -POSTGRES_DB=chp_api_prod diff --git a/.env.stage b/.env.stage deleted file mode 100644 index 284f3ad..0000000 --- a/.env.stage +++ /dev/null @@ -1,11 +0,0 @@ -DEBUG=0 -SECRET_KEY=BkbsAreTheBest2020 -DJANGO_ALLOWED_HOSTS=chp-dev.thayer.dartmouth.edu breast.chp-dev.thayer.dartmouth.edu brain.chp-dev.thayer.dartmouth.edu lung.chp-dev.thayer.dartmouth.edu 127.0.0.1 -SQL_ENGINE=django.db.backends.postgresql -SQL_DATABASE=chp_api_prod -SQL_USER=chp_api_user -SQL_PASSWORD=chp_api_user -SQL_HOST=db -SQL_PORT=5432 -DATABASE=postgres -DJANGO_SETTINGS_MODULE=chp_api.settings.staging diff --git a/.env.stage.db b/.env.stage.db deleted file mode 100644 index a05584e..0000000 --- a/.env.stage.db +++ /dev/null @@ -1,3 +0,0 @@ -POSTGRES_USER=chp_api_user -POSTGRES_PASSWORD=chp_api_user -POSTGRES_DB=chp_api_prod diff --git a/.gitignore b/.gitignore index 60e8e5c..7ec3611 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ chp_db_fixture.json.gz #deployment-script deployment-script +chp.sql #SSH Keys id_rsa* diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index e69de29..0000000 diff --git a/.vscode/settings.json b/.vscode/settings.json index fe654df..c49a2b3 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,4 +1,5 @@ { "python.analysis.typeCheckingMode": "off", "workbench.editor.enablePreview": false, + "python.defaultInterpreterPath": "${workspaceFolder}/.venv/bin/python" } \ No newline at end of file diff --git a/Dockerfile.dev b/Dockerfile similarity index 79% rename from Dockerfile.dev rename to Dockerfile index c4df0b6..d045990 100644 --- a/Dockerfile.dev +++ b/Dockerfile @@ -12,12 +12,12 @@ WORKDIR /usr/src/chp_api RUN apt-get update \ && apt-get install -y git python3-pip python3-dev dos2unix -RUN git clone --single-branch --branch production https://github.com/di2ag/trapi_model.git -RUN git clone --single-branch --branch production https://github.com/di2ag/reasoner-validator.git -RUN git clone --single-branch --branch production https://github.com/di2ag/chp_utils.git -RUN git clone --single-branch --branch production https://github.com/di2ag/chp_look_up.git -RUN git clone --single-branch --branch production https://github.com/di2ag/chp_learn.git -RUN git clone --single-branch --branch production https://github.com/di2ag/gene-specificity.git +RUN git clone --single-branch --branch master https://github.com/di2ag/trapi_model.git +RUN git clone --single-branch --branch master https://github.com/di2ag/reasoner-validator.git +RUN git clone --single-branch --branch master https://github.com/di2ag/chp_utils.git +RUN git clone --single-branch --branch master https://github.com/di2ag/chp_look_up.git +RUN git clone --single-branch --branch master https://github.com/di2ag/chp_learn.git +RUN git clone --single-branch --branch master https://github.com/di2ag/gene-specificity.git # lint RUN pip3 install --upgrade pip @@ -86,13 +86,13 @@ RUN python3 -m pip install --upgrade pip RUN pip3 install --no-cache /wheels/* # copy entry point -COPY ./entrypoint.prod.sh $APP_HOME +COPY ./entrypoint.sh $APP_HOME # copy project COPY ./chp_api $APP_HOME/chp_api COPY ./manage.py $APP_HOME COPY ./dispatcher $APP_HOME/dispatcher -COPY ./gunicorn.config-prod.py $APP_HOME +COPY ./gunicorn.config.py $APP_HOME # chown all the files to the app user RUN chown -R chp_api:chp_api $APP_HOME @@ -101,4 +101,4 @@ RUN chown -R chp_api:chp_api $APP_HOME USER chp_api # run entrypoint.sh -ENTRYPOINT ["/home/chp_api/web/entrypoint.prod.sh"] \ No newline at end of file +ENTRYPOINT ["/home/chp_api/web/entrypoint.sh"] \ No newline at end of file diff --git a/Dockerfile.prod b/Dockerfile.prod deleted file mode 100644 index e33236c..0000000 --- a/Dockerfile.prod +++ /dev/null @@ -1,105 +0,0 @@ -########### -# BUILDER # -########### - -# first stage of build to pull repos -FROM ubuntu:20.04 as intermediate - -# set work directory -WORKDIR /usr/src/chp_api - -# install git -RUN apt-get update \ - && apt-get install -y git python3-pip python3-dev dos2unix - -RUN git clone --single-branch --branch production https://github.com/di2ag/trapi_model.git -RUN git clone --single-branch --branch production https://github.com/di2ag/reasoner-validator.git -RUN git clone --single-branch --branch production https://github.com/di2ag/chp_utils.git -RUN git clone --single-branch --branch production https://github.com/di2ag/chp_look_up.git -RUN git clone --single-branch --branch production https://github.com/di2ag/chp_learn.git -RUN git clone --single-branch --branch production https://github.com/di2ag/gene-specificity.git - -# lint -RUN pip3 install --upgrade pip -RUN pip3 install flake8 wheel -COPY . . - -# install dependencies -COPY ./requirements.txt . -RUN pip3 wheel --no-cache-dir --no-deps --wheel-dir /usr/src/chp_api/wheels -r requirements.txt - -# gather trapi model wheel -RUN cd trapi_model && python3 setup.py bdist_wheel && cd dist && cp trapi_model-*-py3-none-any.whl /usr/src/chp_api/wheels - -# gather reasoner-validator wheel -RUN cd reasoner-validator && python3 setup.py bdist_wheel && cd dist && cp reasoner_validator-*-py3-none-any.whl /usr/src/chp_api/wheels - -# gather chp-utils wheel -RUN cd chp_utils && python3 setup.py bdist_wheel && cd dist && cp chp_utils-*-py3-none-any.whl /usr/src/chp_api/wheels - -#gather chp_look_up wheel -RUN cd chp_look_up && python3 setup.py bdist_wheel && cd dist && cp chp_look_up-*-py3-none-any.whl /usr/src/chp_api/wheels - -#gather chp_learn wheel -RUN cd chp_learn && python3 setup.py bdist_wheel && cd dist && cp chp_learn-*-py3-none-any.whl /usr/src/chp_api/wheels - -#gather gene specificity wheel -RUN cd gene-specificity && python3 setup.py bdist_wheel && cd dist && cp gene_specificity-*-py3-none-any.whl /usr/src/chp_api/wheels - -######### -# FINAL # -######### - -#pull official base image -FROM ubuntu:20.04 - -# add app user -RUN groupadd chp_api && useradd -ms /bin/bash -g chp_api chp_api - -# create the appropriate directories -ENV HOME=/home/chp_api -ENV APP_HOME=/home/chp_api/web -RUN mkdir $APP_HOME -RUN mkdir $APP_HOME/staticfiles -WORKDIR $APP_HOME - -# set environment variables -ENV PYTHONDONTWRITEBYTECODE 1 -ENV PYTHONUNBUFFERED 1 -ENV TZ=America/New_York - -# set ARGs -ARG DEBIAN_FRONTEND=noninterative - -# install dependencies -RUN apt-get update \ - && apt-get install -y python3-pip graphviz openmpi-bin libopenmpi-dev build-essential libssl-dev libffi-dev python3-dev -RUN apt-get install -y libgraphviz-dev python3-pygraphviz -RUN apt-get install -y libpq-dev -RUN apt-get install -y netcat - -# copy repo to new image -COPY --from=intermediate /usr/src/chp_api/wheels /wheels -COPY --from=intermediate /usr/src/chp_api/requirements.txt . -RUN pip3 install --upgrade pip -RUN python3 -m pip install --upgrade pip -RUN pip3 install --no-cache /wheels/* - -# copy entry point -COPY ./entrypoint.prod.sh $APP_HOME - -# copy project -COPY ./chp_api $APP_HOME/chp_api -COPY ./manage.py $APP_HOME -COPY ./dispatcher $APP_HOME/dispatcher -COPY ./gunicorn.config-prod.py $APP_HOME -COPY ./chp_db_fixture.json.gz $APP_HOME - -# chown all the files to the app user -RUN chown -R chp_api:chp_api $APP_HOME - -# change to the app user -USER chp_api - -# run entrypoint.sh -ENTRYPOINT ["/home/chp_api/web/entrypoint.prod.sh"] \ No newline at end of file diff --git a/Dockerfile.stage b/Dockerfile.stage deleted file mode 100644 index 8d3c4e1..0000000 --- a/Dockerfile.stage +++ /dev/null @@ -1,104 +0,0 @@ -########### -# BUILDER # -########### - -# first stage of build to pull repos -FROM ubuntu:20.04 as intermediate - -# set work directory -WORKDIR /usr/src/chp_api - -# install git -RUN apt-get update \ - && apt-get install -y git python3-pip python3-dev dos2unix - -RUN git clone --single-branch --branch staging https://github.com/di2ag/trapi_model.git -RUN git clone --single-branch --branch staging https://github.com/di2ag/reasoner-validator.git -RUN git clone --single-branch --branch staging https://github.com/di2ag/chp_utils.git -RUN git clone --single-branch --branch staging https://github.com/di2ag/chp_look_up.git -RUN git clone --single-branch --branch staging https://github.com/di2ag/chp_learn.git -RUN git clone --single-branch --branch staging https://github.com/di2ag/gene-specificity.git - -# lint -RUN pip3 install --upgrade pip -RUN pip3 install flake8 wheel -COPY . . - -# install dependencies -COPY ./requirements.txt . -RUN pip3 wheel --no-cache-dir --no-deps --wheel-dir /usr/src/chp_api/wheels -r requirements.txt - -# gather trapi model wheel -RUN cd trapi_model && python3 setup.py bdist_wheel && cd dist && cp trapi_model-*-py3-none-any.whl /usr/src/chp_api/wheels - -# gather reasoner-validator wheel -RUN cd reasoner-validator && python3 setup.py bdist_wheel && cd dist && cp reasoner_validator-*-py3-none-any.whl /usr/src/chp_api/wheels - -# gather chp_utils wheel -RUN cd chp_utils && python3 setup.py bdist_wheel && cd dist && cp chp_utils-*-py3-none-any.whl /usr/src/chp_api/wheels - -#gather chp_look_up wheel -RUN cd chp_look_up && python3 setup.py bdist_wheel && cd dist && cp chp_look_up-*-py3-none-any.whl /usr/src/chp_api/wheels - -#gather chp_learn wheel -RUN cd chp_learn && python3 setup.py bdist_wheel && cd dist && cp chp_learn-*-py3-none-any.whl /usr/src/chp_api/wheels - -#grather gene-specificity wheel -RUN cd gene-specificity && python3 setup.py bdist_wheel && cd dist && cp gene_specificity-*-py3-none-any.whl /usr/src/chp_api/wheels - -######### -# FINAL # -######### - -#pull official base image -FROM ubuntu:20.04 - -# add app user -RUN groupadd chp_api && useradd -ms /bin/bash -g chp_api chp_api - -# create the appropriate directories -ENV HOME=/home/chp_api -ENV APP_HOME=/home/chp_api/web -RUN mkdir $APP_HOME -RUN mkdir $APP_HOME/staticfiles -WORKDIR $APP_HOME - -# set environment variables -ENV PYTHONDONTWRITEBYTECODE 1 -ENV PYTHONUNBUFFERED 1 -ENV TZ=America/New_York - -# set ARGs -ARG DEBIAN_FRONTEND=noninterative - -# install dependencies -RUN apt-get update \ - && apt-get install -y python3-pip graphviz openmpi-bin libopenmpi-dev build-essential libssl-dev libffi-dev python3-dev -RUN apt-get install -y libgraphviz-dev python3-pygraphviz -RUN apt-get install -y libpq-dev -RUN apt-get install -y netcat - -# copy repo to new image -COPY --from=intermediate /usr/src/chp_api/wheels /wheels -COPY --from=intermediate /usr/src/chp_api/requirements.txt . -RUN pip3 install --upgrade pip -RUN pip3 install --no-cache /wheels/* - -# copy entry point -COPY ./entrypoint.prod.sh $APP_HOME - -# copy project -COPY ./chp_api $APP_HOME/chp_api -COPY ./manage.py $APP_HOME -COPY ./dispatcher $APP_HOME/dispatcher -COPY ./gunicorn.config-stage.py $APP_HOME -COPY ./chp_db_fixture.json.gz $APP_HOME - -# chown all the files to the app user -RUN chown -R chp_api:chp_api $APP_HOME - -# change to the app user -USER chp_api - -# run entrypoint.sh -ENTRYPOINT ["/home/chp_api/web/entrypoint.stage.sh"] \ No newline at end of file diff --git a/chp_api/chp_api/settings/base.py b/chp_api/chp_api/settings.py similarity index 83% rename from chp_api/chp_api/settings/base.py rename to chp_api/chp_api/settings.py index 9500cfd..b2234c6 100644 --- a/chp_api/chp_api/settings/base.py +++ b/chp_api/chp_api/settings.py @@ -9,22 +9,27 @@ For the full list of settings and their values, see https://docs.djangoproject.com/en/3.0/ref/settings/ """ - import os from importlib import import_module +import environ as environ # type: ignore + +# Initialise environment variables +env = environ.Env() +environ.Env.read_env() + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = int(env("DEBUG", default=0)) # Build paths inside the project like this: os.path.join(BASE_DIR, ...) BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) DATA_UPLOAD_MAX_MEMORY_SIZE = None - REST_FRAMEWORK = { 'DEFAULT_PARSER_CLASSES': [ 'rest_framework.parsers.JSONParser', ] } - # Application definition INSTALLED_BASE_APPS = [ 'django.contrib.admin', @@ -56,8 +61,6 @@ INSTALLED_APPS = INSTALLED_BASE_APPS + INSTALLED_CHP_APPS MIDDLEWARE = [ - #'django_hosts.middleware.HostsRequestMiddleware', - #'django_hosts.middleware.HostsResponseMiddleware', 'django.middleware.security.SecurityMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', 'django.middleware.common.CommonMiddleware', @@ -102,7 +105,6 @@ WSGI_APPLICATION = 'chp_api.wsgi.application' - # Password validation # https://docs.djangoproject.com/en/3.0/ref/settings/#auth-password-validators @@ -121,7 +123,6 @@ }, ] - # Internationalization # https://docs.djangoproject.com/en/3.0/topics/i18n/ LANGUAGE_CODE = 'en-us' @@ -134,7 +135,6 @@ USE_TZ = True - # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/3.0/howto/static-files/ @@ -143,3 +143,20 @@ # Hosts Configuration #ROOT_HOSTCONF = 'chp_api.hosts' + +# Database +# https://docs.djangoproject.com/en/3.0/ref/settings/#databases +DATABASES = { + 'default': { + "ENGINE": "django.db.backends.postgresql_psycopg2", + "NAME": env("SQL_DATABASE"), + "USER": env("SQL_USER"), + "PASSWORD": env("SQL_PASSWORD"), + "HOST": env("SQL_HOST"), + "PORT": env("SQL_PORT"), + } +} + +ALLOWED_HOSTS = env("DJANGO_ALLOWED_HOSTS").split(" ") +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = env("SECRET_KEY") \ No newline at end of file diff --git a/chp_api/chp_api/settings/__init__.py b/chp_api/chp_api/settings/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/chp_api/chp_api/settings/dev.py b/chp_api/chp_api/settings/dev.py deleted file mode 100644 index 9e33ec2..0000000 --- a/chp_api/chp_api/settings/dev.py +++ /dev/null @@ -1,31 +0,0 @@ -""" -Local Development Django settings for chp_api project. -""" - -from .base import * -import environ - -# Initialise environment variables -env = environ.Env() -environ.Env.read_env() - -# SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = env("SECRET_KEY") - -# SECURITY WARNING: don't run with debug turned on in production! -DEBUG = True - -ALLOWED_HOSTS = ['localhost', '127.0.0.1'] - -# Database -# https://docs.djangoproject.com/en/3.0/ref/settings/#databases -DATABASES = { - 'default': { - "ENGINE": "django.db.backends.postgresql_psycopg2", - "NAME": env("DATABASE_NAME"), - "USER": env("DATABASE_USER"), - "PASSWORD": env("SQL_PASSWORD"), - "HOST": env("SQL_HOST"), - "PORT": env("SQL_PORT"), - } -} \ No newline at end of file diff --git a/chp_api/chp_api/settings/production.py b/chp_api/chp_api/settings/production.py deleted file mode 100644 index b3b62c0..0000000 --- a/chp_api/chp_api/settings/production.py +++ /dev/null @@ -1,31 +0,0 @@ -""" -Production Django settings for chp_api project. -""" - -from .base import * -import environ - -# Initialise environment variables -env = environ.Env() -environ.Env.read_env() - -# SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = env("SECRET_KEY") - -# SECURITY WARNING: don't run with debug turned on in production! -DEBUG = int(os.environ.get("DEBUG", default=0)) - -ALLOWED_HOSTS = os.environ.get("DJANGO_ALLOWED_HOSTS").split(" ") - -# Database -# https://docs.djangoproject.com/en/3.0/ref/settings/#databases -DATABASES = { - 'default': { - "ENGINE": "django.db.backends.postgresql_psycopg2", - "NAME": env("DATABASE_NAME"), - "USER": env("DATABASE_USER"), - "PASSWORD": env("SQL_PASSWORD"), - "HOST": env("SQL_HOST"), - "PORT": env("SQL_PORT"), - } -} \ No newline at end of file diff --git a/chp_api/chp_api/settings/staging.py b/chp_api/chp_api/settings/staging.py deleted file mode 100644 index 91e4485..0000000 --- a/chp_api/chp_api/settings/staging.py +++ /dev/null @@ -1,33 +0,0 @@ -""" -Staging Django settings for chp_api project. -""" - -from .base import * -import environ - -# Initialise environment variables -env = environ.Env() -environ.Env.read_env() - -# SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = env("SECRET_KEY") - -# SECURITY WARNING: don't run with debug turned on in production! -DEBUG = int(os.environ.get("DEBUG", default=0)) - -ALLOWED_HOSTS = os.environ.get("DJANGO_ALLOWED_HOSTS").split(" ") - -DATA_UPLOAD_MAX_MEMORY_SIZE = None - -# Database -# https://docs.djangoproject.com/en/3.0/ref/settings/#databases -DATABASES = { - 'default': { - "ENGINE": "django.db.backends.postgresql_psycopg2", - "NAME": env("DATABASE_NAME"), - "USER": env("DATABASE_USER"), - "PASSWORD": env("SQL_PASSWORD"), - "HOST": env("SQL_HOST"), - "PORT": env("SQL_PORT"), - } -} \ No newline at end of file diff --git a/deploy/chp-api/Jenkinsfile b/deploy/chp-api/Jenkinsfile index ac12cbd..155d079 100644 --- a/deploy/chp-api/Jenkinsfile +++ b/deploy/chp-api/Jenkinsfile @@ -43,7 +43,7 @@ pipeline { when { expression { return env.BUILD == 'true' }} steps { script { - docker.build(env.DOCKER_REPO_NAME, "--no-cache -f ./chp_api/Dockerfile.prod ./chp_api") + docker.build(env.DOCKER_REPO_NAME, "--no-cache -f ./chp_api/Dockerfile ./chp_api") docker.withRegistry('https://853771734544.dkr.ecr.us-east-1.amazonaws.com', 'ecr:us-east-1:aws-ifx-deploy') { docker.image(env.DOCKER_REPO_NAME).push("${BUILD_VERSION}") } diff --git a/deployment-script-jenkins b/deployment-script-jenkins deleted file mode 100644 index 3e26712..0000000 --- a/deployment-script-jenkins +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -intuition_user=$(whoami) - -echo "Taking down the NCATS Server." -docker-compose -f docker-compose.jenkins.yml down -v - -echo "Building back from scratch." -docker system prune - -docker-compose -f docker-compose.jenkins.yml build - -echo "Bringing up server." -docker-compose -f docker-compose.jenkins.yml up -d -docker-compose -f docker-compose.jenkins.yml exec web python3 manage.py makemigrations -docker-compose -f docker-compose.jenkins.yml exec web python3 manage.py migrate --noinput -docker-compose -f docker-compose.jenkins.yml exec web python3 manage.py collectstatic --no-input -echo "Server should now be up." - -echo "Check logs with: docker-compose -f docker-compose.prod.yml logs -f" diff --git a/deployment-script-prod b/deployment-script-prod deleted file mode 100644 index 4d01ffa..0000000 --- a/deployment-script-prod +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -INTUITION_USER=$1 - -GET_FIXTURE=${2:-0} - -echo "Taking down the NCATS Server." -docker-compose -f docker-compose.prod.yml down -v - -if [ $GET_FIXTURE -eq 1 ] -then - echo "Copying over database fixtures from intuition. If on AWS will need to SFTP to their servers." - scp $INTUITION_USER@intuition.thayer.dartmouth.edu:/home/public/data/ncats/chp_db/databases/chp_db_fixture.json.gz chp_api/chp_db_fixture.json.gz -fi - -echo "Building back from scratch." -docker system prune -docker-compose -f docker-compose.prod.yml build --no-cache - -echo "Bringing up server." -docker-compose -f docker-compose.prod.yml up -d -docker-compose -f docker-compose.prod.yml exec web python3 manage.py makemigrations --settings chp_api.settings.production -docker-compose -f docker-compose.prod.yml exec web python3 manage.py migrate --noinput --settings chp_api.settings.production -docker-compose -f docker-compose.prod.yml exec web python3 manage.py collectstatic --no-input --settings chp_api.settings.production - -echo "Loading in CHP DB fixture." -docker-compose -f docker-compose.prod.yml exec web python3 manage.py loaddata chp_db_fixture.json.gz -v3 --settings chp_api.settings.production - -echo "Server should now be up." - -echo "Check logs with: docker-compose -f docker-compose.prod.yml logs -f" diff --git a/deployment-script-stage b/deployment-script-stage deleted file mode 100644 index 0dbef9c..0000000 --- a/deployment-script-stage +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -INTUITION_USER=$1 - -GET_FIXTURE=${2:-0} - -echo "Taking down the NCATS Server." -docker-compose -f docker-compose.stage.yml down -v - -if [ $GET_FIXTURE -eq 1 ] -then - echo "Copying over database fixtures from intuition. If on AWS will need to SFTP to their servers." - scp $INTUITION_USER@intuition.thayer.dartmouth.edu:/home/public/data/ncats/chp_db/databases/chp_db_fixture.json.gz chp_api/chp_db_fixture.json.gz -fi - -echo "Building back from scratch." -docker system prune -docker-compose -f docker-compose.stage.yml build --no-cache - -echo "Bringing up server." -docker-compose -f docker-compose.stage.yml up -d -docker-compose -f docker-compose.stage.yml exec web python3 manage.py makemigrations --settings chp_api.settings.staging -docker-compose -f docker-compose.stage.yml exec web python3 manage.py migrate --noinput --settings chp_api.settings.staging -docker-compose -f docker-compose.stage.yml exec web python3 manage.py collectstatic --no-input --clear --settings chp_api.settings.staging - -echo "Loading in CHP DB fixture." -docker-compose -f docker-compose.stage.yml exec web python3 manage.py loaddata chp_db_fixture.json.gz -v3 --settings chp_api.settings.staging - -echo "Server should now be up." - -echo "Check logs with: docker-compose -f docker-compose.stage.yml logs -f" diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml new file mode 100644 index 0000000..d92a9a5 --- /dev/null +++ b/docker-compose-dev.yml @@ -0,0 +1,23 @@ +version: '3.7' + +services: + web: + build: + context: . + dockerfile: Dockerfile + command: gunicorn -c gunicorn.config.py --env DJANGO_SETTINGS_MODULE=chp_api.settings chp_api.wsgi:application --bind 0.0.0.0:8000 + volumes: + - static_volume:/home/chp_api/web/staticfiles + expose: + - 8000 + db: + image: postgres:12.0-alpine + volumes: + - postgres_data:/var/lib/postgresql/data/ + environment: + - POSTGRES_USER=chp_api_user + - POSTGRES_PASSWORD=chp_api_user + - POSTGRES_DB=chp_api_prod +volumes: + postgres_data: + static_volume: \ No newline at end of file diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml deleted file mode 100644 index 6894d6e..0000000 --- a/docker-compose.dev.yml +++ /dev/null @@ -1,10 +0,0 @@ -version: '3.7' - -services: - db: - image: postgres:12.0-alpine - volumes: - - postgres_data:/var/lib/postgresql/data/ -volumes: - postgres_data: - static_volume: \ No newline at end of file diff --git a/docker-compose.local.yml b/docker-compose.local.yml deleted file mode 100644 index e69de29..0000000 diff --git a/docker-compose.stage.yml b/docker-compose.stage.yml deleted file mode 100644 index cbd32b7..0000000 --- a/docker-compose.stage.yml +++ /dev/null @@ -1,17 +0,0 @@ -version: '3.7' - -services: - web: - build: - context: . - dockerfile: Dockerfile.stage - command: gunicorn -c gunicorn.config-stage.py --env DJANGO_SETTINGS_MODULE=chp_api.settings.staging chp_api.wsgi:application --bind 0.0.0.0:8000 --access-logfile gunicorn-access.log --error-logfile gunicorn-error.log --log-level debug - volumes: - - static_volume:/home/chp_api/web/staticfiles - expose: - - 8000 - env_file: - - ./.env.stage -volumes: - postgres_data: - static_volume: \ No newline at end of file diff --git a/docker-compose.prod.yml b/docker-compose.yml similarity index 51% rename from docker-compose.prod.yml rename to docker-compose.yml index 70f7c79..c8c8800 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.yml @@ -4,14 +4,12 @@ services: web: build: context: . - dockerfile: Dockerfile.prod - command: gunicorn -c gunicorn.config-prod.py --env DJANGO_SETTINGS_MODULE=chp_api.settings.production chp_api.wsgi:application --bind 0.0.0.0:8000 + dockerfile: Dockerfile + command: gunicorn -c gunicorn.config.py --env DJANGO_SETTINGS_MODULE=chp_api.settings chp_api.wsgi:application --bind 0.0.0.0:8000 volumes: - static_volume:/home/chp_api/web/staticfiles expose: - 8000 - env_file: - - ./.env.prod volumes: postgres_data: static_volume: \ No newline at end of file diff --git a/entrypoint-dev.sh b/entrypoint-dev.sh deleted file mode 100644 index 2e626f9..0000000 --- a/entrypoint-dev.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/sh - -if [ "$DATABASE" = "postgres" ] -then - echo "Waiting for postgres..." - - while ! nc -z $SQL_HOST $SQL_PORT; do - sleep 0.1 - done - - echo "PostgreSQL started" -fi - -python3 manage.py flush --no-input -python3 manage.py migrate - -exec "$@" \ No newline at end of file diff --git a/entrypoint.prod.sh b/entrypoint.sh similarity index 100% rename from entrypoint.prod.sh rename to entrypoint.sh diff --git a/entrypoint.stage.sh b/entrypoint.stage.sh deleted file mode 100644 index 40e5a12..0000000 --- a/entrypoint.stage.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/sh - -# Wait for Database image to start -if [ "$DATABASE" = "postgres" ] -then - echo "Waiting for postgres..." - - while ! nc -z $SQL_HOST $SQL_PORT; do - sleep 0.1 - done - - echo "PostgreSQL started" -fi - -# Run django migrations and collect static -echo "Collect static files" -python3 manage.py collectstatic --noinput - -echo "Make database migrations" -python3 manage.py makemigrations - -echo "Apply database migrations" -python3 manage.py migrate - -exec "$@" \ No newline at end of file diff --git a/gunicorn.config-stage.py b/gunicorn.config-stage.py deleted file mode 100644 index 2459b11..0000000 --- a/gunicorn.config-stage.py +++ /dev/null @@ -1,13 +0,0 @@ -### Gunicorn Configuration File ### - -timeout = 0 -graceful_timeout = 0 -limit_request_field_size = 0 -limit_request_line = 0 -limit_request_fields = 0 -proxy_allow_ips = '*' -workers=1 -errorlog='gunicorn-error.log' -accesslog='gunicorn-access.log' -loglevel='debug' - diff --git a/gunicorn.config-prod.py b/gunicorn.config.py similarity index 93% rename from gunicorn.config-prod.py rename to gunicorn.config.py index 32f99a5..6834b98 100644 --- a/gunicorn.config-prod.py +++ b/gunicorn.config.py @@ -9,5 +9,4 @@ workers=10 errorlog='gunicorn-error.log' accesslog='gunicorn-access.log' -loglevel='debug' - +loglevel='debug' \ No newline at end of file diff --git a/nginx/Dockerfile b/nginx/Dockerfile index c4570fd..179179a 100644 --- a/nginx/Dockerfile +++ b/nginx/Dockerfile @@ -1,4 +1,4 @@ FROM nginx:1.19.0-alpine RUN rm /etc/nginx/conf.d/default.conf -COPY nginx.conf /etc/nginx/conf.d/default.conf +COPY nginx.conf /etc/nginx/conf.d/default.conf \ No newline at end of file diff --git a/nginx/nginx.conf b/nginx/nginx.conf index b6613f4..3df1b0f 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -20,4 +20,4 @@ server { location /staticfiles/ { alias /home/chp_api/web/staticfiles/; } -} +} \ No newline at end of file diff --git a/requirements-dev.txt b/requirements-dev.txt index bf1faea..2ced061 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,6 +1,6 @@ -chp_learn @ git+https://github.com/di2ag/chp_learn.git@production -chp_utils @ git+https://github.com/di2ag/chp_utils.git@production -trapi_model @ git+https://github.com/di2ag/trapi_model.git@production -chp_look_up @ git+https://github.com/di2ag/chp_look_up.git@production -gene-specificity @ git+https://github.com/di2ag/gene-specificity.git@production -reasoner-validator @ git+https://github.com/di2ag/reasoner-validator.git@production \ No newline at end of file +chp_learn @ git+https://github.com/di2ag/chp_learn.git@master +chp_utils @ git+https://github.com/di2ag/chp_utils.git@master +trapi_model @ git+https://github.com/di2ag/trapi_model.git@master +chp_look_up @ git+https://github.com/di2ag/chp_look_up.git@master +gene-specificity @ git+https://github.com/di2ag/gene-specificity.git@master +reasoner-validator @ git+https://github.com/di2ag/reasoner-validator.git@master \ No newline at end of file From f54a2f39a84d1a3cfb6fc0c18e746e68e41c900d Mon Sep 17 00:00:00 2001 From: veenhouse Date: Mon, 22 Aug 2022 17:46:02 +0000 Subject: [PATCH 002/132] changing jenkins file to point to new docker file location --- deploy/chp-api/Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/chp-api/Jenkinsfile b/deploy/chp-api/Jenkinsfile index 155d079..0c3134c 100644 --- a/deploy/chp-api/Jenkinsfile +++ b/deploy/chp-api/Jenkinsfile @@ -43,7 +43,7 @@ pipeline { when { expression { return env.BUILD == 'true' }} steps { script { - docker.build(env.DOCKER_REPO_NAME, "--no-cache -f ./chp_api/Dockerfile ./chp_api") + docker.build(env.DOCKER_REPO_NAME, "--no-cache -f ./Dockerfile ./chp_api") docker.withRegistry('https://853771734544.dkr.ecr.us-east-1.amazonaws.com', 'ecr:us-east-1:aws-ifx-deploy') { docker.image(env.DOCKER_REPO_NAME).push("${BUILD_VERSION}") } From 61718228a1839fb5d525927e20c2ca13375f8ed8 Mon Sep 17 00:00:00 2001 From: veenhouse Date: Mon, 22 Aug 2022 18:22:02 +0000 Subject: [PATCH 003/132] fixing docker settings in jenkins --- deploy/chp-api/Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/chp-api/Jenkinsfile b/deploy/chp-api/Jenkinsfile index 0c3134c..04279cf 100644 --- a/deploy/chp-api/Jenkinsfile +++ b/deploy/chp-api/Jenkinsfile @@ -41,9 +41,9 @@ pipeline { } stage('Build Docker') { when { expression { return env.BUILD == 'true' }} - steps { + steps {= script { - docker.build(env.DOCKER_REPO_NAME, "--no-cache -f ./Dockerfile ./chp_api") + docker.build(env.DOCKER_REPO_NAME, "--no-cache -f Dockerfile .") docker.withRegistry('https://853771734544.dkr.ecr.us-east-1.amazonaws.com', 'ecr:us-east-1:aws-ifx-deploy') { docker.image(env.DOCKER_REPO_NAME).push("${BUILD_VERSION}") } From f76018ae38fb3656a0ceafa4c970ec4e9b1e807d Mon Sep 17 00:00:00 2001 From: veenhouse Date: Mon, 22 Aug 2022 18:43:02 +0000 Subject: [PATCH 004/132] fixing jenkins --- deploy/chp-api/Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/chp-api/Jenkinsfile b/deploy/chp-api/Jenkinsfile index 04279cf..02d5e28 100644 --- a/deploy/chp-api/Jenkinsfile +++ b/deploy/chp-api/Jenkinsfile @@ -45,7 +45,7 @@ pipeline { script { docker.build(env.DOCKER_REPO_NAME, "--no-cache -f Dockerfile .") docker.withRegistry('https://853771734544.dkr.ecr.us-east-1.amazonaws.com', 'ecr:us-east-1:aws-ifx-deploy') { - docker.image(env.DOCKER_REPO_NAME).push("${BUILD_VERSION}") + docker.image(env.DOCKER_REPO_NAME).push("${BUILD_VERSION}") } sh ''' cp deploy/chp-api/configs/nginx.conf deploy/chp-api/nginx/ From 7b17f0d4609d60c441690e952239ca2f4eb6d665 Mon Sep 17 00:00:00 2001 From: veenhouse Date: Mon, 22 Aug 2022 18:50:36 +0000 Subject: [PATCH 005/132] jenkins update --- deploy/chp-api/Jenkinsfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/deploy/chp-api/Jenkinsfile b/deploy/chp-api/Jenkinsfile index 02d5e28..7de9253 100644 --- a/deploy/chp-api/Jenkinsfile +++ b/deploy/chp-api/Jenkinsfile @@ -41,11 +41,11 @@ pipeline { } stage('Build Docker') { when { expression { return env.BUILD == 'true' }} - steps {= + steps { script { - docker.build(env.DOCKER_REPO_NAME, "--no-cache -f Dockerfile .") + docker.build(env.DOCKER_REPO_NAME, "--no-cache -f ./Dockerfile .") docker.withRegistry('https://853771734544.dkr.ecr.us-east-1.amazonaws.com', 'ecr:us-east-1:aws-ifx-deploy') { - docker.image(env.DOCKER_REPO_NAME).push("${BUILD_VERSION}") + docker.image(env.DOCKER_REPO_NAME).push("${BUILD_VERSION}") } sh ''' cp deploy/chp-api/configs/nginx.conf deploy/chp-api/nginx/ From 4c9a5113a45bb0ba3346f40b67a4d8e924a690fb Mon Sep 17 00:00:00 2001 From: veenhouse Date: Mon, 22 Aug 2022 19:11:53 +0000 Subject: [PATCH 006/132] updating docker file to point to new dirctory structure --- Dockerfile | 8 ++++---- chp_api/manage.py | 2 -- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index d045990..8c60a1d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -89,10 +89,10 @@ RUN pip3 install --no-cache /wheels/* COPY ./entrypoint.sh $APP_HOME # copy project -COPY ./chp_api $APP_HOME/chp_api -COPY ./manage.py $APP_HOME -COPY ./dispatcher $APP_HOME/dispatcher -COPY ./gunicorn.config.py $APP_HOME +COPY ./chp_api/chp_api $APP_HOME/chp_api +COPY ./chp_api/manage.py $APP_HOME +COPY ./chp_api/dispatcher $APP_HOME/dispatcher +COPY ./chp_api/gunicorn.config.py $APP_HOME # chown all the files to the app user RUN chown -R chp_api:chp_api $APP_HOME diff --git a/chp_api/manage.py b/chp_api/manage.py index 81b455b..45e6ffe 100644 --- a/chp_api/manage.py +++ b/chp_api/manage.py @@ -3,7 +3,6 @@ import os import sys - def main(): os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'chp_api.settings') try: @@ -16,6 +15,5 @@ def main(): ) from exc execute_from_command_line(sys.argv) - if __name__ == '__main__': main() \ No newline at end of file From 2c189f4da2dd3528566bc62d7cbc326c25568e06 Mon Sep 17 00:00:00 2001 From: veenhouse Date: Mon, 22 Aug 2022 19:27:22 +0000 Subject: [PATCH 007/132] fixing location of gunicorn script --- Dockerfile | 2 +- docker-compose-dev.yml | 23 ----------------------- docker-compose.yml | 15 --------------- 3 files changed, 1 insertion(+), 39 deletions(-) delete mode 100644 docker-compose-dev.yml delete mode 100644 docker-compose.yml diff --git a/Dockerfile b/Dockerfile index 8c60a1d..3bade97 100644 --- a/Dockerfile +++ b/Dockerfile @@ -92,7 +92,7 @@ COPY ./entrypoint.sh $APP_HOME COPY ./chp_api/chp_api $APP_HOME/chp_api COPY ./chp_api/manage.py $APP_HOME COPY ./chp_api/dispatcher $APP_HOME/dispatcher -COPY ./chp_api/gunicorn.config.py $APP_HOME +COPY ./gunicorn.config.py $APP_HOME # chown all the files to the app user RUN chown -R chp_api:chp_api $APP_HOME diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml deleted file mode 100644 index d92a9a5..0000000 --- a/docker-compose-dev.yml +++ /dev/null @@ -1,23 +0,0 @@ -version: '3.7' - -services: - web: - build: - context: . - dockerfile: Dockerfile - command: gunicorn -c gunicorn.config.py --env DJANGO_SETTINGS_MODULE=chp_api.settings chp_api.wsgi:application --bind 0.0.0.0:8000 - volumes: - - static_volume:/home/chp_api/web/staticfiles - expose: - - 8000 - db: - image: postgres:12.0-alpine - volumes: - - postgres_data:/var/lib/postgresql/data/ - environment: - - POSTGRES_USER=chp_api_user - - POSTGRES_PASSWORD=chp_api_user - - POSTGRES_DB=chp_api_prod -volumes: - postgres_data: - static_volume: \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index c8c8800..0000000 --- a/docker-compose.yml +++ /dev/null @@ -1,15 +0,0 @@ -version: '3.7' - -services: - web: - build: - context: . - dockerfile: Dockerfile - command: gunicorn -c gunicorn.config.py --env DJANGO_SETTINGS_MODULE=chp_api.settings chp_api.wsgi:application --bind 0.0.0.0:8000 - volumes: - - static_volume:/home/chp_api/web/staticfiles - expose: - - 8000 -volumes: - postgres_data: - static_volume: \ No newline at end of file From 228ce6f9d5ce256e03956703b9234a9e1a4c8d34 Mon Sep 17 00:00:00 2001 From: veenhouse Date: Mon, 22 Aug 2022 19:58:16 +0000 Subject: [PATCH 008/132] removing old file references from deployment configuration --- deploy/chp-api/templates/deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/chp-api/templates/deployment.yaml b/deploy/chp-api/templates/deployment.yaml index a9ed2d1..ae61aa6 100644 --- a/deploy/chp-api/templates/deployment.yaml +++ b/deploy/chp-api/templates/deployment.yaml @@ -26,7 +26,7 @@ spec: image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" imagePullPolicy: {{ .Values.image.pullPolicy }} command: ["/bin/sh"] - args: ["-c", "/bin/bash /home/chp_api/web/entrypoint.prod.sh && gunicorn -c gunicorn.config-prod.py --log-file=- --env DJANGO_SETTINGS_MODULE=chp_api.settings.production chp_api.wsgi:application --bind 0.0.0.0:8000"] + args: ["-c", "/bin/bash /home/chp_api/web/entrypoint.sh && gunicorn -c gunicorn.config.py --log-file=- --env DJANGO_SETTINGS_MODULE=chp_api.settings chp_api.wsgi:application --bind 0.0.0.0:8000"] ports: - name: http-app containerPort: 8000 From 0df6c9868989f9e497dbea4b59dc0fd3d746ec4e Mon Sep 17 00:00:00 2001 From: veenhouse Date: Mon, 22 Aug 2022 20:50:24 +0000 Subject: [PATCH 009/132] cleaning up outdated file reference --- deploy/chp-api/templates/deployment.yaml | 2 +- deploy/chp-api/values.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/chp-api/templates/deployment.yaml b/deploy/chp-api/templates/deployment.yaml index ae61aa6..1cee24b 100644 --- a/deploy/chp-api/templates/deployment.yaml +++ b/deploy/chp-api/templates/deployment.yaml @@ -103,4 +103,4 @@ spec: accessModes: [ "ReadWriteOnce" ] resources: requests: - storage: 1Gi + storage: 1Gi \ No newline at end of file diff --git a/deploy/chp-api/values.yaml b/deploy/chp-api/values.yaml index bc9b665..4635dd8 100644 --- a/deploy/chp-api/values.yaml +++ b/deploy/chp-api/values.yaml @@ -19,7 +19,7 @@ app: debug: "0" secret_key: "" djangoAllowedHosts: "" - djangoSettingsModule: "chp_api.settings.production" + djangoSettingsModule: "chp_api.settings" # database connection information db: From bdb4647c56337bf8a4f97ce6c858212bef6f6d7b Mon Sep 17 00:00:00 2001 From: veenhouse Date: Thu, 1 Sep 2022 17:27:33 -0500 Subject: [PATCH 010/132] adding new local dev support --- .devcontainer/Dockerfile | 12 ++++----- .devcontainer/devcontainer.json | 28 ++++++++----------- .devcontainer/docker-compose.yml | 46 ++++++++++++++++++++++++++++++++ .gitignore | 10 ++++--- Dockerfile.new | 38 ++++++++++++++++++++++++++ docker-compose.yml | 25 +++++++++++++++++ requirements-dev.txt | 10 +++---- 7 files changed, 138 insertions(+), 31 deletions(-) create mode 100644 .devcontainer/docker-compose.yml create mode 100644 Dockerfile.new create mode 100644 docker-compose.yml diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index ad21f07..f0b34cd 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,14 +1,14 @@ -# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.245.0/containers/python-3/.devcontainer/base.Dockerfile - # [Choice] Python version (use -bullseye variants on local arm64/Apple Silicon): 3, 3.10, 3.9, 3.8, 3.7, 3.6, 3-bullseye, 3.10-bullseye, 3.9-bullseye, 3.8-bullseye, 3.7-bullseye, 3.6-bullseye, 3-buster, 3.10-buster, 3.9-buster, 3.8-buster, 3.7-buster, 3.6-buster -ARG VARIANT="3.10-bullseye" +ARG VARIANT=3-bullseye FROM mcr.microsoft.com/vscode/devcontainers/python:0-${VARIANT} +ENV PYTHONUNBUFFERED 1 + # [Choice] Node.js version: none, lts/*, 16, 14, 12, 10 ARG NODE_VERSION="none" RUN if [ "${NODE_VERSION}" != "none" ]; then su vscode -c "umask 0002 && . /usr/local/share/nvm/nvm.sh && nvm install ${NODE_VERSION} 2>&1"; fi -# [Optional] If your pip requirements rarely change, uncomment this section to add them to the image. +# [Optional] If your requirements rarely change, uncomment this section to add them to the image. # COPY requirements.txt /tmp/pip-tmp/ # RUN pip3 --disable-pip-version-check --no-cache-dir install -r /tmp/pip-tmp/requirements.txt \ # && rm -rf /tmp/pip-tmp @@ -17,5 +17,5 @@ RUN if [ "${NODE_VERSION}" != "none" ]; then su vscode -c "umask 0002 && . /usr/ # RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ # && apt-get -y install --no-install-recommends -# [Optional] Uncomment this line to install global node packages. -# RUN su vscode -c "source /usr/local/share/nvm/nvm.sh && npm install -g " 2>&1 \ No newline at end of file + + diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 34929e3..be1db70 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,19 +1,11 @@ // For format details, see https://aka.ms/devcontainer.json. For config options, see the README at: -// https://github.com/microsoft/vscode-dev-containers/tree/v0.245.0/containers/python-3 +// https://github.com/microsoft/vscode-dev-containers/tree/v0.245.0/containers/python-3-postgres +// Update the VARIANT arg in docker-compose.yml to pick a Python version { - "name": "Python 3", - "build": { - "dockerfile": "Dockerfile", - "context": "..", - "args": { - // Update 'VARIANT' to pick a Python version: 3, 3.10, 3.9, 3.8, 3.7, 3.6 - // Append -bullseye or -buster to pin to an OS version. - // Use -bullseye variants on local on arm64/Apple Silicon. - "VARIANT": "3.8", - // Options - "NODE_VERSION": "lts/*" - } - }, + "name": "Python 3 & PostgreSQL", + "dockerComposeFile": "docker-compose.yml", + "service": "app", + "workspaceFolder": "/workspace", // Configure tool-specific properties. "customizations": { @@ -32,7 +24,8 @@ "python.linting.mypyPath": "/usr/local/py-utils/bin/mypy", "python.linting.pycodestylePath": "/usr/local/py-utils/bin/pycodestyle", "python.linting.pydocstylePath": "/usr/local/py-utils/bin/pydocstyle", - "python.linting.pylintPath": "/usr/local/py-utils/bin/pylint" + "python.linting.pylintPath": "/usr/local/py-utils/bin/pylint", + "python.testing.pytestPath": "/usr/local/py-utils/bin/pytest" }, // Add the IDs of extensions you want installed when the container is created. @@ -44,10 +37,11 @@ }, // Use 'forwardPorts' to make a list of ports inside the container available locally. - // "forwardPorts": [], + // This can be used to network with other containers or the host. + // "forwardPorts": [5000, 5432], // Use 'postCreateCommand' to run commands after the container is created. - // "postCreateCommand": "pip3 install --user -r requirements.txt", + // "postCreateCommand": "pip install --user -r requirements.txt", // Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root. "remoteUser": "vscode", diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml new file mode 100644 index 0000000..7485e64 --- /dev/null +++ b/.devcontainer/docker-compose.yml @@ -0,0 +1,46 @@ +version: '3.8' + +services: + app: + container_name: dev-env + build: + context: .. + dockerfile: .devcontainer/Dockerfile + args: + # Update 'VARIANT' to pick a version of Python: 3, 3.10, 3.9, 3.8, 3.7, 3.6 + # Append -bullseye or -buster to pin to an OS version. + # Use -bullseye variants on local arm64/Apple Silicon. + VARIANT: "3.8" + # Optional Node.js version to install + NODE_VERSION: "lts/*" + + volumes: + - ..:/workspace:cached + + # Overrides default command so things don't shut down after the process ends. + command: sleep infinity + + # Runs app on the same network as the database container, allows "forwardPorts" in devcontainer.json function. + network_mode: service:db + # Uncomment the next line to use a non-root user for all processes. + # user: vscode + + env_file: + - ../.dev.env + + # Use "forwardPorts" in **devcontainer.json** to forward an app port locally. + # (Adding the "ports" property to this file will not forward from a Codespace.) + + db: + image: postgres:latest + container_name: dev-db + restart: unless-stopped + volumes: + - postgresql:/var/lib/postgresql/data + env_file: + - ../.dev-db.env + # Add "forwardPorts": ["5432"] to **devcontainer.json** to forward PostgreSQL locally. + # (Adding the "ports" property to this file will not forward from a Codespace. + +volumes: + postgresql: diff --git a/.gitignore b/.gitignore index 7ec3611..77654d9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,10 @@ -chp_db_fixture.json.gz -#deployment-script -deployment-script +# env files for development +.dev-db.env +.dev.env + +# data files for development chp.sql +chp_db_fixture.json.gz #SSH Keys id_rsa* @@ -135,3 +138,4 @@ dmypy.json # Pyre type checker .pyre/ +/Dockerfile.dev-db \ No newline at end of file diff --git a/Dockerfile.new b/Dockerfile.new new file mode 100644 index 0000000..8812297 --- /dev/null +++ b/Dockerfile.new @@ -0,0 +1,38 @@ +################ +# venv builder # +################ +FROM python:3.8.3 as venv_builder + +COPY requirements.txt . +RUN python3 -m venv /opt/venv +RUN /opt/venv/bin/pip install -r requirements.txt + +########### +# CHP API # +########### +FROM python:3.8.3-slim as chp-api + +# set environment variables +ENV PYTHONDONTWRITEBYTECODE 1 +ENV PYTHONUNBUFFERED 1 +ENV TZ=America/New_York +ENV SERVER_DIR=/chp_api/ +ENV VIRTUAL_ENV_PATH=/opt/venv + +# copy venv from venv builder image +COPY --from=venv_builder ${VIRTUAL_ENV_PATH} ${VIRTUAL_ENV_PATH} + +# copy project +COPY ./chp_api $SERVER_DIR + +# copy entry point +COPY ./entrypoint.sh ${SERVER_DIR} + +# enter app directory +WORKDIR $SERVER_DIR + +# Enable venv +ENV PATH="/opt/venv/bin:$PATH" +# run server +# ENTRYPOINT ["./entrypoint.sh"] +CMD ["python", "manage.py", "runserver"] \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..227b45e --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,25 @@ +version: '3.8' + +services: + api: + build: + context: . + dockerfile: Dockerfile + container_name: chp-api + command: python manage.py runserver 0.0.0.0:8000 + volumes: + - static_volume:/chp_api/staticfiles + expose: + - 80 + env_file: + - ./.dev.env + db: + image: postgres:latest + restart: unless-stopped + volumes: + - ../chp.sql:/var/lib/postgresql/data + env_file: + - .dev-db.env + +volumes: + static_volume: \ No newline at end of file diff --git a/requirements-dev.txt b/requirements-dev.txt index 2ced061..39c2337 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,6 +1,6 @@ -chp_learn @ git+https://github.com/di2ag/chp_learn.git@master -chp_utils @ git+https://github.com/di2ag/chp_utils.git@master -trapi_model @ git+https://github.com/di2ag/trapi_model.git@master -chp_look_up @ git+https://github.com/di2ag/chp_look_up.git@master -gene-specificity @ git+https://github.com/di2ag/gene-specificity.git@master +chp_learn @ git+https://github.com/di2ag/chp_learn.git@master +chp_utils @ git+https://github.com/di2ag/chp_utils.git@master +trapi_model @ git+https://github.com/di2ag/trapi_model.git@master +chp_look_up @ git+https://github.com/di2ag/chp_look_up.git@master +gene-specificity @ git+https://github.com/di2ag/gene-specificity.git@master reasoner-validator @ git+https://github.com/di2ag/reasoner-validator.git@master \ No newline at end of file From 33cc876bbfb6f13fa2a2c4b5fd4a24e3b4d0ca51 Mon Sep 17 00:00:00 2001 From: veenhouse Date: Thu, 1 Sep 2022 17:47:39 -0500 Subject: [PATCH 011/132] removing api command in compose file --- docker-compose.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 227b45e..1c6b1e0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,7 +6,6 @@ services: context: . dockerfile: Dockerfile container_name: chp-api - command: python manage.py runserver 0.0.0.0:8000 volumes: - static_volume:/chp_api/staticfiles expose: From a219ddf47ac1d39568b40e72488365b258b0375f Mon Sep 17 00:00:00 2001 From: veenhouse Date: Thu, 1 Sep 2022 18:36:49 -0500 Subject: [PATCH 012/132] removing api command in compose file --- Dockerfile.new | 2 +- docker-compose.yml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile.new b/Dockerfile.new index 8812297..4a3a293 100644 --- a/Dockerfile.new +++ b/Dockerfile.new @@ -6,7 +6,7 @@ FROM python:3.8.3 as venv_builder COPY requirements.txt . RUN python3 -m venv /opt/venv RUN /opt/venv/bin/pip install -r requirements.txt - +RUN /opt/venv/bin/pip install -r requirements-dev.txt ########### # CHP API # ########### diff --git a/docker-compose.yml b/docker-compose.yml index 1c6b1e0..227b45e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,6 +6,7 @@ services: context: . dockerfile: Dockerfile container_name: chp-api + command: python manage.py runserver 0.0.0.0:8000 volumes: - static_volume:/chp_api/staticfiles expose: From 1c1e15e48f15e785555122d4fd034ce3e2f70374 Mon Sep 17 00:00:00 2001 From: GregHydeDartmouth <60626258+GregHydeDartmouth@users.noreply.github.com> Date: Wed, 7 Sep 2022 12:49:26 -0400 Subject: [PATCH 013/132] Update README.md removed deprecated chp_client link --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index a0a2ed5..f1fa59e 100644 --- a/README.md +++ b/README.md @@ -36,8 +36,6 @@ Our roadmap outlining or KP’s milestones and the progression of those mileston Our NCATS Wiki Page: https://github.com/NCATSTranslator/Translator-All/wiki/Connections-Hypothesis-Provider -Our CHP Client repository: https://github.com/di2ag/chp_client - A repository for our reasoning code: https://github.com/di2ag/chp From dbed7b76260bd7d3f14b99a83387b2a1ba260bd5 Mon Sep 17 00:00:00 2001 From: GregHydeDartmouth <60626258+GregHydeDartmouth@users.noreply.github.com> Date: Tue, 25 Oct 2022 15:58:31 -0400 Subject: [PATCH 014/132] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f1fa59e..e5727fa 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ We encourage anyone looking for tooling/instructions, to interface with our API, Our API is in active developement and is currently following [Translator Reasoner API standards 1.2.0](https://github.com/NCATSTranslator/ReasonerAPI) -Our API is currently live at: [chp.thayer.dartmouth.edu](http://chp.thayer.dartmouth.edu/) +Our API is currently live at: [https://chp-api.transltr.io](https://chp-api.transltr.io) ## Open Endpoints * [query](query.md) : `POST /query/` From 9cb0d6682618bd7d597ad47914e5dc7f2f288b0e Mon Sep 17 00:00:00 2001 From: Ojesanmi Date: Fri, 6 Jan 2023 14:26:58 -0500 Subject: [PATCH 015/132] fix: refactor chp ci pipeline --- deploy/chp-api/Jenkinsfile | 35 ++++++++++++++++++++--------------- deploy/chp-api/deploy.sh | 8 ++++++++ 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/deploy/chp-api/Jenkinsfile b/deploy/chp-api/Jenkinsfile index 7de9253..3fba214 100644 --- a/deploy/chp-api/Jenkinsfile +++ b/deploy/chp-api/Jenkinsfile @@ -43,33 +43,38 @@ pipeline { when { expression { return env.BUILD == 'true' }} steps { script { - docker.build(env.DOCKER_REPO_NAME, "--no-cache -f ./Dockerfile .") - docker.withRegistry('https://853771734544.dkr.ecr.us-east-1.amazonaws.com', 'ecr:us-east-1:aws-ifx-deploy') { - docker.image(env.DOCKER_REPO_NAME).push("${BUILD_VERSION}") - } + docker.build(env.DOCKER_REPO_NAME, "--no-cache -f ./chp_api/Dockerfile ./") + sh ''' + docker login -u AWS -p $(aws ecr get-login-password --region us-east-1) 853771734544.dkr.ecr.us-east-1.amazonaws.com + ''' + docker.image(env.DOCKER_REPO_NAME).push("${BUILD_VERSION}") sh ''' cp deploy/chp-api/configs/nginx.conf deploy/chp-api/nginx/ ''' - docker.build(env.DOCKER_REPO_NAME, "--no-cache ./deploy/chp-api/nginx") - docker.withRegistry('https://853771734544.dkr.ecr.us-east-1.amazonaws.com', 'ecr:us-east-1:aws-ifx-deploy') { - docker.image(env.DOCKER_REPO_NAME).push("${BUILD_VERSION}-nginx") - } + docker.build(env.DOCKER_REPO_NAME, "--no-cache -f ./deploy/chp-api/nginx") + sh ''' + docker login -u AWS -p $(aws ecr get-login-password --region us-east-1) 853771734544.dkr.ecr.us-east-1.amazonaws.com + ''' + docker.image(env.DOCKER_REPO_NAME).push("${BUILD_VERSION}-nginx") } } } stage('Deploy to AWS EKS') { + agent { label 'translator && ci && deploy'} steps { - configFileProvider([ - configFile(fileId: 'values-ci.yaml', targetLocation: 'deploy/chp-api/values.ncats.yaml') - ]){ - withAWS(credentials:'aws-ifx-deploy') - { + checkout scm + configFileProvider([ + configFile(fileId: 'values-ci.yaml', targetLocation: 'deploy/chp-api/values.ncats.yaml'), + configFile(fileId: 'chp-secrets.sh', targetLocation: 'deploy/chp-api/chp-secrets.sh') + ]){ + script { sh ''' aws --region ${AWS_REGION} eks update-kubeconfig --name ${KUBERNETES_CLUSTER_NAME} - cd deploy/chp-api && /bin/bash deploy.sh + cd deploy/chp-api && /bin/bash chp-secrets.sh && /bin/bash deploy.sh ''' - } + } } + cleanWs() } } } diff --git a/deploy/chp-api/deploy.sh b/deploy/chp-api/deploy.sh index eb2a20d..0406b77 100644 --- a/deploy/chp-api/deploy.sh +++ b/deploy/chp-api/deploy.sh @@ -22,6 +22,14 @@ do rm values.yaml.bak done +sed -i.bak \ + -e "s/SECRET_KEY_VALUE/$SECRET_KEY/g" \ + -e "s/ENGINE_VALUE/$ENGINE/g;s/DBNAME_VALUE/$DBNAME/g" \ + -e "s/USERNAME_VALUE/$USERNAME/g;s/PASSWORD_VALUE/$PASSWORD/g" \ + -e "s/HOST_VALUE/$HOST/g;s/PORT_VALUE/$PORT/g" \ + configs/settings.py +rm configs/settings.py.bak + kubectl apply -f namespace.yaml # deploy helm chart From 079eaf2e6a68171cdf1a3667b5ea6ae883038d11 Mon Sep 17 00:00:00 2001 From: Ojesanmi Date: Fri, 6 Jan 2023 14:38:40 -0500 Subject: [PATCH 016/132] fix: modified image tag --- deploy/chp-api/Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/chp-api/Jenkinsfile b/deploy/chp-api/Jenkinsfile index 3fba214..76ff601 100644 --- a/deploy/chp-api/Jenkinsfile +++ b/deploy/chp-api/Jenkinsfile @@ -16,7 +16,7 @@ pipeline { pollSCM('H/5 * * * *') } environment { - DOCKER_REPO_NAME = "translator-ea-chp-api" + DOCKER_REPO_NAME = "853771734544.dkr.ecr.us-east-1.amazonaws.com/translator-ea-chp-api" } stages { stage('Build Version'){ From e2bf3085479849af2f14cb74377421d6db15b535 Mon Sep 17 00:00:00 2001 From: Ojesanmi Date: Fri, 6 Jan 2023 14:40:06 -0500 Subject: [PATCH 017/132] fix: dockerfile path --- deploy/chp-api/Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/chp-api/Jenkinsfile b/deploy/chp-api/Jenkinsfile index 76ff601..470c09d 100644 --- a/deploy/chp-api/Jenkinsfile +++ b/deploy/chp-api/Jenkinsfile @@ -43,7 +43,7 @@ pipeline { when { expression { return env.BUILD == 'true' }} steps { script { - docker.build(env.DOCKER_REPO_NAME, "--no-cache -f ./chp_api/Dockerfile ./") + docker.build(env.DOCKER_REPO_NAME, "--no-cache -f ./Dockerfile ./") sh ''' docker login -u AWS -p $(aws ecr get-login-password --region us-east-1) 853771734544.dkr.ecr.us-east-1.amazonaws.com ''' From 9b33e4ec15d77d04e8c7cbb85b689a73111b50cc Mon Sep 17 00:00:00 2001 From: Ojesanmi Date: Fri, 6 Jan 2023 14:48:34 -0500 Subject: [PATCH 018/132] fix: dockerfile path --- deploy/chp-api/Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/chp-api/Jenkinsfile b/deploy/chp-api/Jenkinsfile index 470c09d..476101d 100644 --- a/deploy/chp-api/Jenkinsfile +++ b/deploy/chp-api/Jenkinsfile @@ -51,7 +51,7 @@ pipeline { sh ''' cp deploy/chp-api/configs/nginx.conf deploy/chp-api/nginx/ ''' - docker.build(env.DOCKER_REPO_NAME, "--no-cache -f ./deploy/chp-api/nginx") + docker.build(env.DOCKER_REPO_NAME, "--no-cache -f ./deploy/chp-api/nginx .") sh ''' docker login -u AWS -p $(aws ecr get-login-password --region us-east-1) 853771734544.dkr.ecr.us-east-1.amazonaws.com ''' From 99fcc8c88db387ec63e811ffbd579937c87d11de Mon Sep 17 00:00:00 2001 From: Ojesanmi Date: Fri, 6 Jan 2023 15:06:53 -0500 Subject: [PATCH 019/132] fix: dockerfile path --- deploy/chp-api/Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/chp-api/Jenkinsfile b/deploy/chp-api/Jenkinsfile index 476101d..90b79b4 100644 --- a/deploy/chp-api/Jenkinsfile +++ b/deploy/chp-api/Jenkinsfile @@ -51,7 +51,7 @@ pipeline { sh ''' cp deploy/chp-api/configs/nginx.conf deploy/chp-api/nginx/ ''' - docker.build(env.DOCKER_REPO_NAME, "--no-cache -f ./deploy/chp-api/nginx .") + docker.build(env.DOCKER_REPO_NAME, "--no-cache -f ./deploy/chp-api/nginx ./") sh ''' docker login -u AWS -p $(aws ecr get-login-password --region us-east-1) 853771734544.dkr.ecr.us-east-1.amazonaws.com ''' From 1bd6befe6461c56223b80b22a2c5caec05e10089 Mon Sep 17 00:00:00 2001 From: Ojesanmi Date: Fri, 6 Jan 2023 15:23:41 -0500 Subject: [PATCH 020/132] fix: build script --- deploy/chp-api/Jenkinsfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deploy/chp-api/Jenkinsfile b/deploy/chp-api/Jenkinsfile index 90b79b4..4406bf5 100644 --- a/deploy/chp-api/Jenkinsfile +++ b/deploy/chp-api/Jenkinsfile @@ -50,8 +50,9 @@ pipeline { docker.image(env.DOCKER_REPO_NAME).push("${BUILD_VERSION}") sh ''' cp deploy/chp-api/configs/nginx.conf deploy/chp-api/nginx/ + cd deploy.chp-api/nginx ''' - docker.build(env.DOCKER_REPO_NAME, "--no-cache -f ./deploy/chp-api/nginx ./") + docker.build(env.DOCKER_REPO_NAME, "--no-cache -f ./Dockerfile .") sh ''' docker login -u AWS -p $(aws ecr get-login-password --region us-east-1) 853771734544.dkr.ecr.us-east-1.amazonaws.com ''' From f286f38b720f7795ced29432862fca10c3c0d2d2 Mon Sep 17 00:00:00 2001 From: Ojesanmi Date: Fri, 6 Jan 2023 15:29:43 -0500 Subject: [PATCH 021/132] fix: build script --- deploy/chp-api/Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/chp-api/Jenkinsfile b/deploy/chp-api/Jenkinsfile index 4406bf5..40f3022 100644 --- a/deploy/chp-api/Jenkinsfile +++ b/deploy/chp-api/Jenkinsfile @@ -50,7 +50,7 @@ pipeline { docker.image(env.DOCKER_REPO_NAME).push("${BUILD_VERSION}") sh ''' cp deploy/chp-api/configs/nginx.conf deploy/chp-api/nginx/ - cd deploy.chp-api/nginx + cd deploy/chp-api/nginx ''' docker.build(env.DOCKER_REPO_NAME, "--no-cache -f ./Dockerfile .") sh ''' From ebb8859dd251057371a91b3fb9f307bd54285cfd Mon Sep 17 00:00:00 2001 From: Ojesanmi Date: Fri, 6 Jan 2023 15:54:28 -0500 Subject: [PATCH 022/132] fix: modified build script --- deploy/chp-api/Jenkinsfile | 2 +- deploy/chp-api/deploy.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/chp-api/Jenkinsfile b/deploy/chp-api/Jenkinsfile index 40f3022..c6d75de 100644 --- a/deploy/chp-api/Jenkinsfile +++ b/deploy/chp-api/Jenkinsfile @@ -65,7 +65,7 @@ pipeline { steps { checkout scm configFileProvider([ - configFile(fileId: 'values-ci.yaml', targetLocation: 'deploy/chp-api/values.ncats.yaml'), + configFile(fileId: 'values-ci.yaml', targetLocation: 'deploy/chp-api/values-ncats.yaml'), configFile(fileId: 'chp-secrets.sh', targetLocation: 'deploy/chp-api/chp-secrets.sh') ]){ script { diff --git a/deploy/chp-api/deploy.sh b/deploy/chp-api/deploy.sh index 0406b77..2ea4f84 100644 --- a/deploy/chp-api/deploy.sh +++ b/deploy/chp-api/deploy.sh @@ -33,4 +33,4 @@ rm configs/settings.py.bak kubectl apply -f namespace.yaml # deploy helm chart -helm -n ${namespace} upgrade --install ${projectName} -f values.ncats.yaml ./ \ No newline at end of file +helm -n ${namespace} upgrade --install ${projectName} -f values-ncats.yaml ./ \ No newline at end of file From ef79cbaf835629a87dda9c45bae0134a70e8eb9d Mon Sep 17 00:00:00 2001 From: Ojesanmi Date: Fri, 6 Jan 2023 16:25:32 -0500 Subject: [PATCH 023/132] fix: changed the secret script name --- deploy/chp-api/Jenkinsfile | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/deploy/chp-api/Jenkinsfile b/deploy/chp-api/Jenkinsfile index c6d75de..c7c435e 100644 --- a/deploy/chp-api/Jenkinsfile +++ b/deploy/chp-api/Jenkinsfile @@ -66,16 +66,21 @@ pipeline { checkout scm configFileProvider([ configFile(fileId: 'values-ci.yaml', targetLocation: 'deploy/chp-api/values-ncats.yaml'), - configFile(fileId: 'chp-secrets.sh', targetLocation: 'deploy/chp-api/chp-secrets.sh') + configFile(fileId: 'prepare.sh', targetLocation: 'deploy/chp-api/prepare.sh') ]){ script { sh ''' aws --region ${AWS_REGION} eks update-kubeconfig --name ${KUBERNETES_CLUSTER_NAME} - cd deploy/chp-api && /bin/bash chp-secrets.sh && /bin/bash deploy.sh + cd deploy/chp-api && /bin/bash prepare.sh && /bin/bash deploy.sh ''' } } - cleanWs() + } + post { + always { + echo " Clean up the workspace in deploy node!" + cleanWs() + } } } } From 861d88ad30ee64ce4be67d2dce55891f348534c7 Mon Sep 17 00:00:00 2001 From: Ojesanmi Date: Fri, 6 Jan 2023 16:28:00 -0500 Subject: [PATCH 024/132] fix: modified deploy.sh file --- deploy/chp-api/deploy.sh | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/deploy/chp-api/deploy.sh b/deploy/chp-api/deploy.sh index 2ea4f84..9c86fe6 100644 --- a/deploy/chp-api/deploy.sh +++ b/deploy/chp-api/deploy.sh @@ -23,12 +23,10 @@ do done sed -i.bak \ - -e "s/SECRET_KEY_VALUE/$SECRET_KEY/g" \ - -e "s/ENGINE_VALUE/$ENGINE/g;s/DBNAME_VALUE/$DBNAME/g" \ - -e "s/USERNAME_VALUE/$USERNAME/g;s/PASSWORD_VALUE/$PASSWORD/g" \ - -e "s/HOST_VALUE/$HOST/g;s/PORT_VALUE/$PORT/g" \ - configs/settings.py -rm configs/settings.py.bak + -e "s/APP_SECRET_KEY_VALUE/$APP_SECRET_KEY/g;s/DB_DATABASE_VALUE/$DB_DATABASE/g" \ + -e "s/DB_USERNAME_VALUE/$DB_USERNAME/g;s/DB_PASSWORD_VALUE/$DB_PASSWORD/g" \ + values-ncats.yaml +rm values-ncats.yaml.bak kubectl apply -f namespace.yaml From aa8aaf91e76c589357df8066e4e372ea9351fb9d Mon Sep 17 00:00:00 2001 From: Ojesanmi Date: Fri, 6 Jan 2023 18:35:30 -0500 Subject: [PATCH 025/132] fix: modified deploy.sh file --- deploy/chp-api/deploy.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/deploy/chp-api/deploy.sh b/deploy/chp-api/deploy.sh index 9c86fe6..99cc6c3 100644 --- a/deploy/chp-api/deploy.sh +++ b/deploy/chp-api/deploy.sh @@ -22,11 +22,11 @@ do rm values.yaml.bak done -sed -i.bak \ - -e "s/APP_SECRET_KEY_VALUE/$APP_SECRET_KEY/g;s/DB_DATABASE_VALUE/$DB_DATABASE/g" \ - -e "s/DB_USERNAME_VALUE/$DB_USERNAME/g;s/DB_PASSWORD_VALUE/$DB_PASSWORD/g" \ - values-ncats.yaml -rm values-ncats.yaml.bak +#sed -i.bak \ + # -e "s/APP_SECRET_KEY_VALUE/$APP_SECRET_KEY/g;s/DB_DATABASE_VALUE/$DB_DATABASE/g" \ +# -e "s/DB_USERNAME_VALUE/$DB_USERNAME/g;s/DB_PASSWORD_VALUE/$DB_PASSWORD/g" \ +# values-ncats.yaml +#rm values-ncats.yaml.bak kubectl apply -f namespace.yaml From 7d0d31394613dff8561d0da2dfecbd667be4935a Mon Sep 17 00:00:00 2001 From: Ojesanmi Date: Fri, 6 Jan 2023 18:49:59 -0500 Subject: [PATCH 026/132] fix: modified deploy.sh file --- deploy/chp-api/deploy.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/deploy/chp-api/deploy.sh b/deploy/chp-api/deploy.sh index 99cc6c3..9c86fe6 100644 --- a/deploy/chp-api/deploy.sh +++ b/deploy/chp-api/deploy.sh @@ -22,11 +22,11 @@ do rm values.yaml.bak done -#sed -i.bak \ - # -e "s/APP_SECRET_KEY_VALUE/$APP_SECRET_KEY/g;s/DB_DATABASE_VALUE/$DB_DATABASE/g" \ -# -e "s/DB_USERNAME_VALUE/$DB_USERNAME/g;s/DB_PASSWORD_VALUE/$DB_PASSWORD/g" \ -# values-ncats.yaml -#rm values-ncats.yaml.bak +sed -i.bak \ + -e "s/APP_SECRET_KEY_VALUE/$APP_SECRET_KEY/g;s/DB_DATABASE_VALUE/$DB_DATABASE/g" \ + -e "s/DB_USERNAME_VALUE/$DB_USERNAME/g;s/DB_PASSWORD_VALUE/$DB_PASSWORD/g" \ + values-ncats.yaml +rm values-ncats.yaml.bak kubectl apply -f namespace.yaml From a5b655912784a987c286ccf16dbd60733ada6229 Mon Sep 17 00:00:00 2001 From: Ojesanmi Date: Fri, 6 Jan 2023 19:56:11 -0500 Subject: [PATCH 027/132] fix: modified deploy.sh script --- deploy/chp-api/deploy.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/chp-api/deploy.sh b/deploy/chp-api/deploy.sh index 9c86fe6..f3e54eb 100644 --- a/deploy/chp-api/deploy.sh +++ b/deploy/chp-api/deploy.sh @@ -23,7 +23,7 @@ do done sed -i.bak \ - -e "s/APP_SECRET_KEY_VALUE/$APP_SECRET_KEY/g;s/DB_DATABASE_VALUE/$DB_DATABASE/g" \ + -e "s/APP_SECRET_KEY_VALUE/$APP_SECRET_KEY/g" \ -e "s/DB_USERNAME_VALUE/$DB_USERNAME/g;s/DB_PASSWORD_VALUE/$DB_PASSWORD/g" \ values-ncats.yaml rm values-ncats.yaml.bak From e263fc4d304ac69e15a90a4d5d8a18a9322dd62f Mon Sep 17 00:00:00 2001 From: Ojesanmi Date: Fri, 6 Jan 2023 21:07:15 -0500 Subject: [PATCH 028/132] fix: modified deploy.sh file --- deploy/chp-api/Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/deploy/chp-api/Jenkinsfile b/deploy/chp-api/Jenkinsfile index c7c435e..44dcaf4 100644 --- a/deploy/chp-api/Jenkinsfile +++ b/deploy/chp-api/Jenkinsfile @@ -50,6 +50,7 @@ pipeline { docker.image(env.DOCKER_REPO_NAME).push("${BUILD_VERSION}") sh ''' cp deploy/chp-api/configs/nginx.conf deploy/chp-api/nginx/ + pwd cd deploy/chp-api/nginx ''' docker.build(env.DOCKER_REPO_NAME, "--no-cache -f ./Dockerfile .") From e7f8e0a92ff55c0f21a342b8d2e97465f4d30be9 Mon Sep 17 00:00:00 2001 From: Ojesanmi Date: Sat, 7 Jan 2023 09:01:07 -0500 Subject: [PATCH 029/132] fix: modified docker build script --- deploy/chp-api/Jenkinsfile | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/deploy/chp-api/Jenkinsfile b/deploy/chp-api/Jenkinsfile index 44dcaf4..643110c 100644 --- a/deploy/chp-api/Jenkinsfile +++ b/deploy/chp-api/Jenkinsfile @@ -44,16 +44,10 @@ pipeline { steps { script { docker.build(env.DOCKER_REPO_NAME, "--no-cache -f ./Dockerfile ./") - sh ''' - docker login -u AWS -p $(aws ecr get-login-password --region us-east-1) 853771734544.dkr.ecr.us-east-1.amazonaws.com - ''' + sh 'docker login -u AWS -p $(aws ecr get-login-password --region us-east-1) 853771734544.dkr.ecr.us-east-1.amazonaws.com' docker.image(env.DOCKER_REPO_NAME).push("${BUILD_VERSION}") - sh ''' - cp deploy/chp-api/configs/nginx.conf deploy/chp-api/nginx/ - pwd - cd deploy/chp-api/nginx - ''' - docker.build(env.DOCKER_REPO_NAME, "--no-cache -f ./Dockerfile .") + sh 'cp deploy/chp-api/configs/nginx.conf deploy/chp-api/nginx/' + docker.build(env.DOCKER_REPO_NAME, "--no-cache ./deploy/chp-api/nginx") sh ''' docker login -u AWS -p $(aws ecr get-login-password --region us-east-1) 853771734544.dkr.ecr.us-east-1.amazonaws.com ''' From 6b1fa358220ee104b18b81b68be67a5b7a51a312 Mon Sep 17 00:00:00 2001 From: Ojesanmi Date: Tue, 17 Jan 2023 11:38:47 -0500 Subject: [PATCH 030/132] fix: modified ingress rules --- deploy/chp-api/templates/ingress.yaml | 28 ++++++++++++++------------- deploy/chp-api/values.yaml | 9 +++------ 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/deploy/chp-api/templates/ingress.yaml b/deploy/chp-api/templates/ingress.yaml index f495b20..29a8b99 100644 --- a/deploy/chp-api/templates/ingress.yaml +++ b/deploy/chp-api/templates/ingress.yaml @@ -13,18 +13,20 @@ metadata: {{- end }} spec: rules: - {{- range .Values.ingress.hosts }} - - host: {{ .host | quote }} + - host: {{ .Values.ingress.host | quote }} http: paths: - {{- range .paths }} - - path: {{ .path }} - pathType: {{ .pathType }} - backend: - service: - name: {{ $fullName }} - port: - number: {{ $svcPort }} - {{- end }} - {{- end }} -{{- end }} + - path: /* + pathType: ImplementationSpecific + backend: + service: + name: ssl-redirect + port: + name: use-annotation + - path: /* + pathType: ImplementationSpecific + backend: + service: + name: {{ .Values.appname }} + port: + number: {{ .Values.service.port }} \ No newline at end of file diff --git a/deploy/chp-api/values.yaml b/deploy/chp-api/values.yaml index 4635dd8..945c6fa 100644 --- a/deploy/chp-api/values.yaml +++ b/deploy/chp-api/values.yaml @@ -3,6 +3,7 @@ # Declare variables to be passed into your templates. replicaCount: 1 +appname: chp-api image: repository: 853771734544.dkr.ecr.us-east-1.amazonaws.com/translator-ea-chp-api @@ -60,12 +61,8 @@ ingress: nginx.ingress.kubernetes.io/proxy-connect-timeout: "360" nginx.ingress.kubernetes.io/proxy-read-timeout: "360" nginx.ingress.kubernetes.io/proxy-send-timeout: "360" - hosts: - - host: chp-api.ci.transltr.io - paths: - - path: / - pathType: ImplementationSpecific - + hosts: chp-api.ci.transltr.io + tolerations: - key: "transltr" value: "chp-api" From ed78f9aee4134536200c099f9be78c0c840a7abd Mon Sep 17 00:00:00 2001 From: Ojesanmi Date: Tue, 17 Jan 2023 11:52:12 -0500 Subject: [PATCH 031/132] fix: modified ingress rules --- deploy/chp-api/templates/ingress.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deploy/chp-api/templates/ingress.yaml b/deploy/chp-api/templates/ingress.yaml index 29a8b99..4061e20 100644 --- a/deploy/chp-api/templates/ingress.yaml +++ b/deploy/chp-api/templates/ingress.yaml @@ -29,4 +29,5 @@ spec: service: name: {{ .Values.appname }} port: - number: {{ .Values.service.port }} \ No newline at end of file + number: {{ .Values.service.port }} +{{- end }} \ No newline at end of file From 71d4b443fa271e7fe0895fb93cbbf3c00db84072 Mon Sep 17 00:00:00 2001 From: Ojesanmi Date: Tue, 17 Jan 2023 12:12:50 -0500 Subject: [PATCH 032/132] fix: modified ingress rules --- deploy/chp-api/templates/ingress.yaml | 29 ++++++++++++--------------- deploy/chp-api/values.yaml | 6 +++++- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/deploy/chp-api/templates/ingress.yaml b/deploy/chp-api/templates/ingress.yaml index 4061e20..f46ba08 100644 --- a/deploy/chp-api/templates/ingress.yaml +++ b/deploy/chp-api/templates/ingress.yaml @@ -13,21 +13,18 @@ metadata: {{- end }} spec: rules: - - host: {{ .Values.ingress.host | quote }} + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} http: paths: - - path: /* - pathType: ImplementationSpecific - backend: - service: - name: ssl-redirect - port: - name: use-annotation - - path: /* - pathType: ImplementationSpecific - backend: - service: - name: {{ .Values.appname }} - port: - number: {{ .Values.service.port }} -{{- end }} \ No newline at end of file + {{- range .paths }} + - path: {{ .path }} + pathType: {{ .pathType }} + backend: + service: + name: {{ $fullName }} + port: + number: {{ $svcPort }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/deploy/chp-api/values.yaml b/deploy/chp-api/values.yaml index 945c6fa..94bdeef 100644 --- a/deploy/chp-api/values.yaml +++ b/deploy/chp-api/values.yaml @@ -61,7 +61,11 @@ ingress: nginx.ingress.kubernetes.io/proxy-connect-timeout: "360" nginx.ingress.kubernetes.io/proxy-read-timeout: "360" nginx.ingress.kubernetes.io/proxy-send-timeout: "360" - hosts: chp-api.ci.transltr.io + hosts: + - host: chp-api.ci.transltr.io + paths: + - path: / + pathType: ImplementationSpecific tolerations: - key: "transltr" From 669fb0c959fcb740e61295a876a6b59bca57411e Mon Sep 17 00:00:00 2001 From: Ojesanmi Date: Tue, 17 Jan 2023 12:45:56 -0500 Subject: [PATCH 033/132] fix: removed second docker login command --- deploy/chp-api/Jenkinsfile | 3 --- 1 file changed, 3 deletions(-) diff --git a/deploy/chp-api/Jenkinsfile b/deploy/chp-api/Jenkinsfile index 643110c..001126d 100644 --- a/deploy/chp-api/Jenkinsfile +++ b/deploy/chp-api/Jenkinsfile @@ -48,9 +48,6 @@ pipeline { docker.image(env.DOCKER_REPO_NAME).push("${BUILD_VERSION}") sh 'cp deploy/chp-api/configs/nginx.conf deploy/chp-api/nginx/' docker.build(env.DOCKER_REPO_NAME, "--no-cache ./deploy/chp-api/nginx") - sh ''' - docker login -u AWS -p $(aws ecr get-login-password --region us-east-1) 853771734544.dkr.ecr.us-east-1.amazonaws.com - ''' docker.image(env.DOCKER_REPO_NAME).push("${BUILD_VERSION}-nginx") } } From 184b8f8a89a158bfe3586ef1ac9cef7ed9364887 Mon Sep 17 00:00:00 2001 From: Ojesanmi Date: Tue, 17 Jan 2023 15:13:44 -0500 Subject: [PATCH 034/132] fix: changed target pot --- deploy/chp-api/templates/service.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/chp-api/templates/service.yaml b/deploy/chp-api/templates/service.yaml index 7e951c6..a525a81 100644 --- a/deploy/chp-api/templates/service.yaml +++ b/deploy/chp-api/templates/service.yaml @@ -8,7 +8,7 @@ spec: type: {{ .Values.service.type }} ports: - port: {{ .Values.service.port }} - targetPort: http-nginx + targetPort: 8000 protocol: TCP name: http selector: From f5f46f49e4a14fb13441004df8389a697c480fd7 Mon Sep 17 00:00:00 2001 From: Ojesanmi Date: Tue, 17 Jan 2023 16:31:35 -0500 Subject: [PATCH 035/132] fix: changed target pot --- deploy/chp-api/templates/deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/chp-api/templates/deployment.yaml b/deploy/chp-api/templates/deployment.yaml index 1cee24b..f1d6f77 100644 --- a/deploy/chp-api/templates/deployment.yaml +++ b/deploy/chp-api/templates/deployment.yaml @@ -76,7 +76,7 @@ spec: imagePullPolicy: {{ .Values.image.pullPolicy }} ports: - name: http-nginx - containerPort: 80 + containerPort: 8000 protocol: TCP volumeMounts: - name: {{ include "chp-api.fullname" . }}-pvc From 03150b7c4fec24d5348a16a9fbde76a75a3a9aa7 Mon Sep 17 00:00:00 2001 From: Ojesanmi Date: Tue, 17 Jan 2023 21:57:44 -0500 Subject: [PATCH 036/132] fix: changed target port --- deploy/chp-api/templates/deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/chp-api/templates/deployment.yaml b/deploy/chp-api/templates/deployment.yaml index f1d6f77..1cee24b 100644 --- a/deploy/chp-api/templates/deployment.yaml +++ b/deploy/chp-api/templates/deployment.yaml @@ -76,7 +76,7 @@ spec: imagePullPolicy: {{ .Values.image.pullPolicy }} ports: - name: http-nginx - containerPort: 8000 + containerPort: 80 protocol: TCP volumeMounts: - name: {{ include "chp-api.fullname" . }}-pvc From 037631ab81d5b3a9bf8adac7f97cd0798f965dd2 Mon Sep 17 00:00:00 2001 From: Ojesanmi Date: Tue, 17 Jan 2023 22:12:46 -0500 Subject: [PATCH 037/132] fix: changed target port --- deploy/chp-api/templates/service.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/chp-api/templates/service.yaml b/deploy/chp-api/templates/service.yaml index a525a81..7e951c6 100644 --- a/deploy/chp-api/templates/service.yaml +++ b/deploy/chp-api/templates/service.yaml @@ -8,7 +8,7 @@ spec: type: {{ .Values.service.type }} ports: - port: {{ .Values.service.port }} - targetPort: 8000 + targetPort: http-nginx protocol: TCP name: http selector: From bd779548171705157aea6492f9b7e45d3262d532 Mon Sep 17 00:00:00 2001 From: Ojesanmi Date: Tue, 31 Jan 2023 12:48:33 -0500 Subject: [PATCH 038/132] fix added chp label --- deploy/chp-api/Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/chp-api/Jenkinsfile b/deploy/chp-api/Jenkinsfile index 001126d..0b9b119 100644 --- a/deploy/chp-api/Jenkinsfile +++ b/deploy/chp-api/Jenkinsfile @@ -5,7 +5,7 @@ pipeline { disableConcurrentBuilds() } agent { - node { label 'translator && aws && build' } + node { label 'translator && aws && build && chp' } } parameters { string(name: 'BUILD_VERSION', defaultValue: '', description: 'The build version to deploy (optional)') From d1f3540a7701b95aec4a6597369819da5a8a503c Mon Sep 17 00:00:00 2001 From: Chase Yakaboski Date: Wed, 15 Mar 2023 13:06:36 -0400 Subject: [PATCH 039/132] added some dev deployment scripts for local builds to dev machine. Also updated some required packages to accomodate changes in biolink/trapi models and relaxed thresholding on gene_specificity app. Also corrected some provenance terms in gene_specificity app. --- Dockerfile | 10 +- Dockerfile.new | 7 +- chp_api/chp_api/settings.py | 4 +- chp_api/chp_api/settings_build.py | 161 ++++++++++++++++++++++++++++++ chp_api/dispatcher/views.py | 16 +-- dev-deployment-script | 12 +++ dev-docker-compose.yml | 25 +++++ docker-compose.yml | 2 +- requirements-dev.txt | 2 - 9 files changed, 214 insertions(+), 25 deletions(-) create mode 100644 chp_api/chp_api/settings_build.py create mode 100755 dev-deployment-script create mode 100644 dev-docker-compose.yml diff --git a/Dockerfile b/Dockerfile index 3bade97..e22f0d3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,10 +13,8 @@ RUN apt-get update \ && apt-get install -y git python3-pip python3-dev dos2unix RUN git clone --single-branch --branch master https://github.com/di2ag/trapi_model.git -RUN git clone --single-branch --branch master https://github.com/di2ag/reasoner-validator.git RUN git clone --single-branch --branch master https://github.com/di2ag/chp_utils.git RUN git clone --single-branch --branch master https://github.com/di2ag/chp_look_up.git -RUN git clone --single-branch --branch master https://github.com/di2ag/chp_learn.git RUN git clone --single-branch --branch master https://github.com/di2ag/gene-specificity.git # lint @@ -31,18 +29,12 @@ RUN pip3 wheel --no-cache-dir --no-deps --wheel-dir /usr/src/chp_api/wheels -r r # gather trapi model wheel RUN cd trapi_model && python3 setup.py bdist_wheel && cd dist && cp trapi_model-*-py3-none-any.whl /usr/src/chp_api/wheels -# gather reasoner-validator wheel -RUN cd reasoner-validator && python3 setup.py bdist_wheel && cd dist && cp reasoner_validator-*-py3-none-any.whl /usr/src/chp_api/wheels - # gather chp-utils wheel RUN cd chp_utils && python3 setup.py bdist_wheel && cd dist && cp chp_utils-*-py3-none-any.whl /usr/src/chp_api/wheels #gather chp_look_up wheel RUN cd chp_look_up && python3 setup.py bdist_wheel && cd dist && cp chp_look_up-*-py3-none-any.whl /usr/src/chp_api/wheels -#gather chp_learn wheel -RUN cd chp_learn && python3 setup.py bdist_wheel && cd dist && cp chp_learn-*-py3-none-any.whl /usr/src/chp_api/wheels - #gather gene specificity wheel RUN cd gene-specificity && python3 setup.py bdist_wheel && cd dist && cp gene_specificity-*-py3-none-any.whl /usr/src/chp_api/wheels @@ -101,4 +93,4 @@ RUN chown -R chp_api:chp_api $APP_HOME USER chp_api # run entrypoint.sh -ENTRYPOINT ["/home/chp_api/web/entrypoint.sh"] \ No newline at end of file +ENTRYPOINT ["/home/chp_api/web/entrypoint.sh"] diff --git a/Dockerfile.new b/Dockerfile.new index 4a3a293..0a825d4 100644 --- a/Dockerfile.new +++ b/Dockerfile.new @@ -4,7 +4,9 @@ FROM python:3.8.3 as venv_builder COPY requirements.txt . +COPY requirements-dev.txt . RUN python3 -m venv /opt/venv +RUN /opt/venv/bin/pip install --upgrade pip RUN /opt/venv/bin/pip install -r requirements.txt RUN /opt/venv/bin/pip install -r requirements-dev.txt ########### @@ -27,12 +29,11 @@ COPY ./chp_api $SERVER_DIR # copy entry point COPY ./entrypoint.sh ${SERVER_DIR} +COPY ./gunicorn.config.py ${SERVER_DIR} +COPY ./chp_db_fixture.json.gz ${SERVER_DIR} # enter app directory WORKDIR $SERVER_DIR # Enable venv ENV PATH="/opt/venv/bin:$PATH" -# run server -# ENTRYPOINT ["./entrypoint.sh"] -CMD ["python", "manage.py", "runserver"] \ No newline at end of file diff --git a/chp_api/chp_api/settings.py b/chp_api/chp_api/settings.py index b2234c6..34059d5 100644 --- a/chp_api/chp_api/settings.py +++ b/chp_api/chp_api/settings.py @@ -46,7 +46,7 @@ INSTALLED_CHP_APPS = [ 'chp_look_up', - 'chp_learn', +# 'chp_learn', 'gene_specificity', ] @@ -159,4 +159,4 @@ ALLOWED_HOSTS = env("DJANGO_ALLOWED_HOSTS").split(" ") # SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = env("SECRET_KEY") \ No newline at end of file +SECRET_KEY = env("SECRET_KEY") diff --git a/chp_api/chp_api/settings_build.py b/chp_api/chp_api/settings_build.py new file mode 100644 index 0000000..b63e550 --- /dev/null +++ b/chp_api/chp_api/settings_build.py @@ -0,0 +1,161 @@ +""" +Base Django settings for chp_api project. + +Generated by 'django-admin startproject' using Django 3.0.7. + +For more information on this file, see +https://docs.djangoproject.com/en/3.0/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/3.0/ref/settings/ +""" +import os +from importlib import import_module +import environ as environ # type: ignore + +# Initialise environment variables +env = environ.Env() +environ.Env.read_env() + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = int(env("DEBUG", default=0)) + +# Build paths inside the project like this: os.path.join(BASE_DIR, ...) +BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +DATA_UPLOAD_MAX_MEMORY_SIZE = None +REST_FRAMEWORK = { + 'DEFAULT_PARSER_CLASSES': [ + 'rest_framework.parsers.JSONParser', + ] +} + +# Application definition +INSTALLED_BASE_APPS = [ + 'django.contrib.admin', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.messages', + 'django.contrib.staticfiles', + 'rest_framework', + 'dispatcher.apps.DispatcherConfig', + 'chp_utils', + 'django_extensions', +] + +INSTALLED_CHP_APPS = [ + 'chp_look_up', +# 'chp_learn', + 'gene_specificity', + ] + +OTHER_APPS = [ + 'chp_utils' + ] + +# CHP Versions +VERSIONS = {app_name: app.__version__ for app_name, app in [(app_name, import_module(app_name)) for app_name in INSTALLED_CHP_APPS + OTHER_APPS]} + +# Sets up installed apps relevent to django +INSTALLED_APPS = INSTALLED_BASE_APPS + INSTALLED_CHP_APPS + +MIDDLEWARE = [ + 'django.middleware.security.SecurityMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', +] + +ROOT_URLCONF = 'dispatcher.urls' + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [], + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': [ + 'django.template.context_processors.debug', + 'django.template.context_processors.request', + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + ], + }, + }, +] + +# Logging +LOGGING = { + 'version': 1, + 'disable_existing_loggers': False, + 'handlers': { + 'console': { + 'class': 'logging.StreamHandler', + }, + }, + 'root': { + 'handlers': ['console'], + 'level': 'WARNING', + }, +} + +WSGI_APPLICATION = 'chp_api.wsgi.application' + +# Password validation +# https://docs.djangoproject.com/en/3.0/ref/settings/#auth-password-validators + +AUTH_PASSWORD_VALIDATORS = [ + { + 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', + }, +] + +# Internationalization +# https://docs.djangoproject.com/en/3.0/topics/i18n/ +LANGUAGE_CODE = 'en-us' + +TIME_ZONE = 'UTC' + +USE_I18N = True + +USE_L10N = True + +USE_TZ = True + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/3.0/howto/static-files/ + +STATIC_URL = '/staticfiles/' +STATIC_ROOT = os.path.join(BASE_DIR, "staticfiles") + +# Hosts Configuration +#ROOT_HOSTCONF = 'chp_api.hosts' + +# Database +# https://docs.djangoproject.com/en/3.0/ref/settings/#databases +DATABASES = { + 'default': { + "ENGINE": os.environ.get("SQL_ENGINE", "django.db.backends.sqlite3"), + "NAME": os.environ.get("SQL_DATABASE", os.path.join(BASE_DIR, "db.sqlite3")), + "USER": os.environ.get("SQL_USER", "user"), + "PASSWORD": os.environ.get("SQL_PASSWORD", "password"), + "HOST": os.environ.get("SQL_HOST", "localhost"), + "PORT": os.environ.get("SQL_PORT", "5432"), + } +} +ALLOWED_HOSTS = env("DJANGO_ALLOWED_HOSTS").split(" ") +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = env("SECRET_KEY") diff --git a/chp_api/dispatcher/views.py b/chp_api/dispatcher/views.py index 3a26e38..70560de 100644 --- a/chp_api/dispatcher/views.py +++ b/chp_api/dispatcher/views.py @@ -18,8 +18,8 @@ class query(APIView): - trapi_version = '1.2' - def __init__(self, trapi_version='1.2', **kwargs): + trapi_version = '1.3' + def __init__(self, trapi_version='1.3', **kwargs): self.trapi_version = trapi_version super(query, self).__init__(**kwargs) @@ -40,8 +40,8 @@ def post(self, request): return dispatcher.get_response(query) class curies(APIView): - trapi_version = '1.2' - def __init__(self, trapi_version='1.2', **kwargs): + trapi_version = '1.3' + def __init__(self, trapi_version='1.3', **kwargs): self.trapi_version = trapi_version super(curies, self).__init__(**kwargs) @@ -55,8 +55,8 @@ def get(self, request): return JsonResponse(curies_db) class meta_knowledge_graph(APIView): - trapi_version = '1.2' - def __init__(self, trapi_version='1.2', **kwargs): + trapi_version = '1.3' + def __init__(self, trapi_version='1.3', **kwargs): self.trapi_version = trapi_version super(meta_knowledge_graph, self).__init__(**kwargs) @@ -70,8 +70,8 @@ def get(self, request): return JsonResponse(meta_knowledge_graph.to_dict()) class versions(APIView): - trapi_version = '1.2' - def __init__(self, trapi_version='1.2', **kwargs): + trapi_version = '1.3' + def __init__(self, trapi_version='1.3', **kwargs): self.trapi_version = trapi_version super(version, self).__init__(**kwargs) diff --git a/dev-deployment-script b/dev-deployment-script new file mode 100755 index 0000000..94affa4 --- /dev/null +++ b/dev-deployment-script @@ -0,0 +1,12 @@ +#!/bin/bash +# Only to be run when building on dev machine +echo "Taking down the NCATS Server." +docker-compose -f dev-docker-compose.yml stop + +docker-compose -f dev-docker-compose.yml build --no-cache api + +echo "Bringing up server." +docker-compose -f dev-docker-compose.yml up -d + +echo "Server should now be up." +echo "Check logs with: docker-compose -f dev-docker-compose.yml logs -f" diff --git a/dev-docker-compose.yml b/dev-docker-compose.yml new file mode 100644 index 0000000..ca599c0 --- /dev/null +++ b/dev-docker-compose.yml @@ -0,0 +1,25 @@ +version: '3.8' + +services: + api: + build: + context: . + dockerfile: Dockerfile.new + container_name: chp-api + command: python manage.py runserver 0.0.0.0:8000 + volumes: + - static_volume:/chp_api/staticfiles + expose: + - 80 + env_file: + - ./.dev.env + db: + image: postgres:latest + restart: unless-stopped + volumes: + - ../chp.sql:/var/lib/postgresql/data + env_file: + - .dev-db.env + +volumes: + static_volume: diff --git a/docker-compose.yml b/docker-compose.yml index 227b45e..5f9f6ec 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -22,4 +22,4 @@ services: - .dev-db.env volumes: - static_volume: \ No newline at end of file + static_volume: diff --git a/requirements-dev.txt b/requirements-dev.txt index 39c2337..a873696 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,6 +1,4 @@ -chp_learn @ git+https://github.com/di2ag/chp_learn.git@master chp_utils @ git+https://github.com/di2ag/chp_utils.git@master trapi_model @ git+https://github.com/di2ag/trapi_model.git@master chp_look_up @ git+https://github.com/di2ag/chp_look_up.git@master gene-specificity @ git+https://github.com/di2ag/gene-specificity.git@master -reasoner-validator @ git+https://github.com/di2ag/reasoner-validator.git@master \ No newline at end of file From 90efd61431250479fdcf2478d751e81c36360269 Mon Sep 17 00:00:00 2001 From: Chase Yakaboski Date: Mon, 17 Apr 2023 22:22:15 -0400 Subject: [PATCH 040/132] Rework in progress. --- Dockerfile | 32 ++++++----- chp_api/chp_api/settings.py | 30 ++++++---- chp_api/chp_api/urls.py | 7 ++- chp_api/dispatcher/models.py | 11 +++- docker-compose.yml | 105 ++++++++++++++++++++++++++++++----- nginx/Dockerfile | 34 +++++++++++- nginx/default.conf | 43 ++++++++++++++ nginx/nginx.conf | 69 ++++++++++++++++------- nginx/start.sh | 2 + 9 files changed, 266 insertions(+), 67 deletions(-) create mode 100644 nginx/default.conf create mode 100644 nginx/start.sh diff --git a/Dockerfile b/Dockerfile index e22f0d3..1232415 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,14 +3,15 @@ ########### # first stage of build to pull repos -FROM ubuntu:20.04 as intermediate +FROM python:3.8 as intermediate # set work directory WORKDIR /usr/src/chp_api # install git -RUN apt-get update \ - && apt-get install -y git python3-pip python3-dev dos2unix +#RUN apt-get update \ +# && apt-get install -y git python3-pip python3-dev +#dos2unix RUN git clone --single-branch --branch master https://github.com/di2ag/trapi_model.git RUN git clone --single-branch --branch master https://github.com/di2ag/chp_utils.git @@ -18,9 +19,9 @@ RUN git clone --single-branch --branch master https://github.com/di2ag/chp_look_ RUN git clone --single-branch --branch master https://github.com/di2ag/gene-specificity.git # lint -RUN pip3 install --upgrade pip -RUN pip3 install flake8 wheel -COPY . . +#RUN pip install --upgrade pip +#RUN pip3 install flake8 wheel +#COPY . . # install dependencies COPY ./requirements.txt . @@ -43,7 +44,7 @@ RUN cd gene-specificity && python3 setup.py bdist_wheel && cd dist && cp gene_sp ######### #pull official base image -FROM ubuntu:20.04 +FROM python:3.8 # add app user RUN groupadd chp_api && useradd -ms /bin/bash -g chp_api chp_api @@ -53,6 +54,7 @@ ENV HOME=/home/chp_api ENV APP_HOME=/home/chp_api/web RUN mkdir $APP_HOME RUN mkdir $APP_HOME/staticfiles +RUN mkdir $APP_HOME/mediafiles WORKDIR $APP_HOME # set environment variables @@ -64,17 +66,17 @@ ENV TZ=America/New_York ARG DEBIAN_FRONTEND=noninterative # install dependencies -RUN apt-get update \ - && apt-get install -y python3-pip graphviz openmpi-bin libopenmpi-dev build-essential libssl-dev libffi-dev python3-dev -RUN apt-get install -y libgraphviz-dev python3-pygraphviz -RUN apt-get install -y libpq-dev -RUN apt-get install -y netcat +#RUN apt-get update \ +# && apt-get install -y python3-pip graphviz openmpi-bin libopenmpi-dev build-essential libssl-dev libffi-dev python3-dev +#RUN apt-get install -y libgraphviz-dev python3-pygraphviz +#RUN apt-get install -y libpq-dev +#RUN apt-get install -y netcat # copy repo to new image COPY --from=intermediate /usr/src/chp_api/wheels /wheels COPY --from=intermediate /usr/src/chp_api/requirements.txt . -RUN pip3 install --upgrade pip -RUN python3 -m pip install --upgrade pip +#RUN pip3 install --upgrade pip +#RUN python3 -m pip install --upgrade pip RUN pip3 install --no-cache /wheels/* # copy entry point @@ -93,4 +95,4 @@ RUN chown -R chp_api:chp_api $APP_HOME USER chp_api # run entrypoint.sh -ENTRYPOINT ["/home/chp_api/web/entrypoint.sh"] +#ENTRYPOINT ["/home/chp_api/web/entrypoint.sh"] diff --git a/chp_api/chp_api/settings.py b/chp_api/chp_api/settings.py index 34059d5..0087dc0 100644 --- a/chp_api/chp_api/settings.py +++ b/chp_api/chp_api/settings.py @@ -55,7 +55,7 @@ ] # CHP Versions -VERSIONS = {app_name: app.__version__ for app_name, app in [(app_name, import_module(app_name)) for app_name in INSTALLED_CHP_APPS + OTHER_APPS]} +#VERSIONS = {app_name: app.__version__ for app_name, app in [(app_name, import_module(app_name)) for app_name in INSTALLED_CHP_APPS + OTHER_APPS]} # Sets up installed apps relevent to django INSTALLED_APPS = INSTALLED_BASE_APPS + INSTALLED_CHP_APPS @@ -70,7 +70,7 @@ 'django.middleware.clickjacking.XFrameOptionsMiddleware', ] -ROOT_URLCONF = 'dispatcher.urls' +ROOT_URLCONF = 'chp_api.urls' TEMPLATES = [ { @@ -138,25 +138,35 @@ # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/3.0/howto/static-files/ -STATIC_URL = '/staticfiles/' +STATIC_URL = '/static/' STATIC_ROOT = os.path.join(BASE_DIR, "staticfiles") +STATIC_URL = '/media/' +STATIC_ROOT = os.path.join(BASE_DIR, "mediafiles") + # Hosts Configuration #ROOT_HOSTCONF = 'chp_api.hosts' +with open(env("POSTGRES_PASSWORD_FILE"), 'r') as db_pwd: + DB_PASSWORD = db_pwd.readline().strip() + # Database # https://docs.djangoproject.com/en/3.0/ref/settings/#databases DATABASES = { 'default': { "ENGINE": "django.db.backends.postgresql_psycopg2", - "NAME": env("SQL_DATABASE"), - "USER": env("SQL_USER"), - "PASSWORD": env("SQL_PASSWORD"), - "HOST": env("SQL_HOST"), - "PORT": env("SQL_PORT"), + "NAME": env("POSTGRES_DB"), + "USER": env("POSTGRES_USER"), + "PASSWORD": DB_PASSWORD, + "HOST": env("POSTGRES_HOST"), + "PORT": env("POSTGRES_PORT"), } } -ALLOWED_HOSTS = env("DJANGO_ALLOWED_HOSTS").split(" ") +with open(env("DJANGO_ALLOWED_HOSTS_FILE"), 'r') as ah_file: + ALLOWED_HOSTS = ah_file.readline().strip().split(" ") + # SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = env("SECRET_KEY") + # Read the secret key from file +with open(env("SECRET_KEY_FILE"), 'r') as sk_file: + SECRET_KEY = sk_file.readline().strip() diff --git a/chp_api/chp_api/urls.py b/chp_api/chp_api/urls.py index 148c33e..e6a0057 100644 --- a/chp_api/chp_api/urls.py +++ b/chp_api/chp_api/urls.py @@ -14,11 +14,12 @@ 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) """ from django.contrib import admin -from django.urls import path +from django.urls import path, include from rest_framework.urlpatterns import format_suffix_patterns -from dispatcher import views - urlpatterns = [ + path('admin/', admin.site.urls), path('', include('dispatcher.urls')), ] + + diff --git a/chp_api/dispatcher/models.py b/chp_api/dispatcher/models.py index 28e69ac..a303344 100644 --- a/chp_api/dispatcher/models.py +++ b/chp_api/dispatcher/models.py @@ -6,5 +6,14 @@ class Transaction(models.Model): query = models.JSONField(default=dict) status = models.CharField(max_length=100, default="", null=True) versions = models.JSONField(default=dict) - chp_app = models.CharField(max_length=128, null=True) + chp_app = models.ForeignKey(App, on_delete=models.CASCADE) + +class App(models.Model): + name = models.CharField(max_length=128) + curies_file = models.FileField(upload_to='curies_files/', null=True, blank=True) + meta_knowledge_graph_file = models.FileField(upload_to='meta_knowledge_graph_files', null=True, blank=True) + + def __str__(self): + return self.name + diff --git a/docker-compose.yml b/docker-compose.yml index 5f9f6ec..2606028 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,25 +1,102 @@ version: '3.8' services: - api: + + nginx-proxy: + build: nginx + restart: always + volumes: + - ./nginx/default.conf:/tmp/default.conf + environment: + - DJANGO_SERVER_ADDR=chp-api:8000 + - STATIC_SERVER_ADDR=static-fs:8080 + ports: + - "80:80" + depends_on: + - chp-api + healthcheck: + test: ["CMD-SHELL", "curl --silent --fail localhost:80/health-check || exit 1"] + interval: 10s + timeout: 10s + retries: 3 + command: /app/start.sh + + chp-api: build: context: . dockerfile: Dockerfile - container_name: chp-api - command: python manage.py runserver 0.0.0.0:8000 + restart: always + ports: + - '8000:8000' + secrets: + - db-password + - django-key + - allowed-hosts + environment: + - POSTGRES_DB=chp_db + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD_FILE=/run/secrets/db-password + - POSTGRES_HOST=db + - POSTGRES_PORT=5432 + - SECRET_KEY_FILE=/run/secrets/django-key + - DEBUG=1 + - DJANGO_ALLOWED_HOSTS_FILE=/run/secrets/allowed-hosts + # Uncomment this for production + #- DJANGO_SETTINGS_MODULE=mysite.settings.production + # Comment this for development + #- DJANGO_SETTINGS_MODULE=mysite.settings.base + depends_on: + db: + condition: service_healthy + depends_on: + - static-fs + healthcheck: + #test: ["CMD-SHELL", "curl --silent --fail localhost:8000/flask-health-check || exit 1"] + interval: 10s + timeout: 10s + retries: 3 volumes: - - static_volume:/chp_api/staticfiles - expose: - - 80 - env_file: - - ./.dev.env + - ./static-files:/home/chp_api/staticfiles + #command: gunicorn -c gunicorn.config.py -b 0.0.0.0:8000 chp_api.wsgi:application + command: python3 manage.py runserver 0.0.0.0:8000 + db: - image: postgres:latest - restart: unless-stopped + image: postgres + restart: always + user: postgres + secrets: + - db-password volumes: - - ../chp.sql:/var/lib/postgresql/data - env_file: - - .dev-db.env + - db-data:/var/lib/postgresql/data + environment: + - POSTGRES_DB=chp_db + - POSTGRES_PASSWORD_FILE=/run/secrets/db-password + expose: + - 5432 + healthcheck: + test: [ "CMD", "pg_isready" ] + interval: 10s + timeout: 5s + retries: 5 + + static-fs: + image: halverneus/static-file-server:latest + environment: + - FOLDER=/var/www + - DEBUG=true + expose: + - 8080 + volumes: + - ./static-files:/var/www/static + - ./media-files:/var/www/media volumes: - static_volume: + db-data: + +secrets: + allowed-hosts: + file: secrets/chp_api/allowed_hosts.txt + db-password: + file: secrets/db/password.txt + django-key: + file: secrets/chp_api/secret_key.txt diff --git a/nginx/Dockerfile b/nginx/Dockerfile index 179179a..bfb3338 100644 --- a/nginx/Dockerfile +++ b/nginx/Dockerfile @@ -1,4 +1,32 @@ -FROM nginx:1.19.0-alpine +FROM nginx:1.19.7-alpine -RUN rm /etc/nginx/conf.d/default.conf -COPY nginx.conf /etc/nginx/conf.d/default.conf \ No newline at end of file +# Add bash for boot cmd +RUN apk add bash + +# Add nginx.conf to container +COPY --chown=nginx:nginx nginx.conf /etc/nginx/nginx.conf +COPY --chown=nginx:nginx start.sh /app/start.sh + +# set workdir +WORKDIR /app + +# permissions and nginx user for tightened security +RUN chown -R nginx:nginx /app && chmod -R 755 /app && \ + chown -R nginx:nginx /var/cache/nginx && \ + chown -R nginx:nginx /var/log/nginx && \ + chmod -R 755 /var/log/nginx; \ + chown -R nginx:nginx /etc/nginx/conf.d +RUN touch /var/run/nginx.pid && chown -R nginx:nginx /var/run/nginx.pid + +# # Uncomment to keep the nginx logs inside the container - Leave commented for logging to stdout and stderr +# RUN mkdir -p /var/log/nginx +# RUN unlink /var/log/nginx/access.log \ +# && unlink /var/log/nginx/error.log \ +# && touch /var/log/nginx/access.log \ +# && touch /var/log/nginx/error.log \ +# && chown nginx /var/log/nginx/*log \ +# && chmod 644 /var/log/nginx/*log + +USER nginx + +CMD ["nginx", "-g", "'daemon off;'"] diff --git a/nginx/default.conf b/nginx/default.conf new file mode 100644 index 0000000..a483bf9 --- /dev/null +++ b/nginx/default.conf @@ -0,0 +1,43 @@ +proxy_cache_path /tmp/cache levels=1:2 keys_zone=cache:10m max_size=500m inactive=60m use_temp_path=off; + +server { + listen 80; + + location / { + proxy_pass http://$DJANGO_SERVER_ADDR; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + + location /cache-me { + proxy_pass http://$DJANGO_SERVER_ADDR; + proxy_cache cache; + proxy_cache_lock on; + proxy_cache_valid 200 30s; + proxy_cache_use_stale error timeout updating http_500 http_502 http_503 http_504; + proxy_cache_revalidate on; + proxy_cache_background_update on; + expires 20s; + } + + location /health-check { + add_header Content-Type text/plain; + return 200 "success"; + } + + location /static { + proxy_pass http://$STATIC_SERVER_ADDR; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + + location /media { + proxy_pass http://$STATIC_SERVER_ADDR; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + +} diff --git a/nginx/nginx.conf b/nginx/nginx.conf index 3df1b0f..e6700a3 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -1,23 +1,50 @@ -upstream chp_api { - server web:8000; +worker_processes auto; +pid /var/run/nginx.pid; + +events { + worker_connections 1024; } -server { - - listen 80; - client_max_body_size 100M; - - location / { - proxy_pass http://chp_api; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header Host $host; - proxy_redirect off; - proxy_read_timeout 360; - proxy_send_timeout 360; - proxy_connect_timeout 360; - } - - location /staticfiles/ { - alias /home/chp_api/web/staticfiles/; - } -} \ No newline at end of file +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + # Define the format of log messages. + log_format main_ext '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for" ' + '"$host" sn="$server_name" ' + 'rt=$request_time ' + 'ua="$upstream_addr" us="$upstream_status" ' + 'ut="$upstream_response_time" ul="$upstream_response_length" ' + 'cs=$upstream_cache_status' ; + + access_log /var/log/nginx/access.log main_ext; + error_log /var/log/nginx/error.log warn; + + sendfile on; + + keepalive_timeout 65; + + # Enable Compression + gzip on; + + # Disable Display of NGINX Version + server_tokens off; + + # Size Limits + client_body_buffer_size 10K; + client_header_buffer_size 1k; + client_max_body_size 8m; + large_client_header_buffers 2 1k; + + # # SSL / TLS Settings - Suggested for Security + # ssl_protocols TLSv1.2 TLSv1.3; + # ssl_session_timeout 15m; + # ssl_ciphers ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256; + # ssl_prefer_server_ciphers on; + # ssl_session_tickets off; + + include /etc/nginx/conf.d/*.conf; + +} diff --git a/nginx/start.sh b/nginx/start.sh new file mode 100644 index 0000000..b7384d6 --- /dev/null +++ b/nginx/start.sh @@ -0,0 +1,2 @@ +#!/bin/bash +envsubst '$DJANGO_SERVER_ADDR,$STATIC_SERVER_ADDR' < /tmp/default.conf > /etc/nginx/conf.d/default.conf && nginx -g 'daemon off;' From 3090bc8f83736be1fe7762836653fb8f21fab349 Mon Sep 17 00:00:00 2001 From: Chase Yakaboski Date: Tue, 18 Apr 2023 16:34:56 -0400 Subject: [PATCH 041/132] Working App Admin portal with Zenodo file registation support. --- Dockerfile | 6 +- chp_api/chp_api/settings.py | 3 - chp_api/dispatcher/admin.py | 5 +- chp_api/dispatcher/apps.py | 7 +++ chp_api/dispatcher/base.py | 35 ++++++++--- chp_api/dispatcher/curie_database.py | 60 +++++++++++++++++++ .../0004_app_alter_transaction_chp_app.py | 28 +++++++++ ...odofile_remove_app_curies_file_and_more.py | 40 +++++++++++++ chp_api/dispatcher/models.py | 41 ++++++++++--- chp_api/dispatcher/scripts/__init__.py | 0 chp_api/dispatcher/scripts/load_db_apps.py | 10 ++++ chp_api/dispatcher/zenodo.py | 29 +++++++++ dev-deployment-script | 14 ++--- docker-compose.yml | 7 ++- nginx/default.conf | 7 --- requirements.txt | 4 +- 16 files changed, 253 insertions(+), 43 deletions(-) create mode 100644 chp_api/dispatcher/curie_database.py create mode 100755 chp_api/dispatcher/migrations/0004_app_alter_transaction_chp_app.py create mode 100755 chp_api/dispatcher/migrations/0005_zenodofile_remove_app_curies_file_and_more.py create mode 100644 chp_api/dispatcher/scripts/__init__.py create mode 100644 chp_api/dispatcher/scripts/load_db_apps.py create mode 100644 chp_api/dispatcher/zenodo.py diff --git a/Dockerfile b/Dockerfile index 1232415..ff22ebe 100644 --- a/Dockerfile +++ b/Dockerfile @@ -54,7 +54,6 @@ ENV HOME=/home/chp_api ENV APP_HOME=/home/chp_api/web RUN mkdir $APP_HOME RUN mkdir $APP_HOME/staticfiles -RUN mkdir $APP_HOME/mediafiles WORKDIR $APP_HOME # set environment variables @@ -80,7 +79,7 @@ COPY --from=intermediate /usr/src/chp_api/requirements.txt . RUN pip3 install --no-cache /wheels/* # copy entry point -COPY ./entrypoint.sh $APP_HOME +#COPY ./entrypoint.sh $APP_HOME # copy project COPY ./chp_api/chp_api $APP_HOME/chp_api @@ -89,7 +88,8 @@ COPY ./chp_api/dispatcher $APP_HOME/dispatcher COPY ./gunicorn.config.py $APP_HOME # chown all the files to the app user -RUN chown -R chp_api:chp_api $APP_HOME +RUN chown -R chp_api:chp_api $APP_HOME \ + && chmod 700 $APP_HOME/staticfiles # change to the app user USER chp_api diff --git a/chp_api/chp_api/settings.py b/chp_api/chp_api/settings.py index 0087dc0..1e1ad55 100644 --- a/chp_api/chp_api/settings.py +++ b/chp_api/chp_api/settings.py @@ -141,9 +141,6 @@ STATIC_URL = '/static/' STATIC_ROOT = os.path.join(BASE_DIR, "staticfiles") -STATIC_URL = '/media/' -STATIC_ROOT = os.path.join(BASE_DIR, "mediafiles") - # Hosts Configuration #ROOT_HOSTCONF = 'chp_api.hosts' diff --git a/chp_api/dispatcher/admin.py b/chp_api/dispatcher/admin.py index 8c38f3f..f3422e0 100644 --- a/chp_api/dispatcher/admin.py +++ b/chp_api/dispatcher/admin.py @@ -1,3 +1,6 @@ from django.contrib import admin -# Register your models here. +from .models import App, ZenodoFile + +admin.site.register(App) +admin.site.register(ZenodoFile) diff --git a/chp_api/dispatcher/apps.py b/chp_api/dispatcher/apps.py index 4fe2ce2..a4df4e0 100644 --- a/chp_api/dispatcher/apps.py +++ b/chp_api/dispatcher/apps.py @@ -1,5 +1,12 @@ +import requests_cache + from django.apps import AppConfig + class DispatcherConfig(AppConfig): default_auto_field = 'django.db.models.BigAutoField' name = 'dispatcher' + + # Install a requests cache + requests_cache.install_cache('dispatcher_cache') + diff --git a/chp_api/dispatcher/base.py b/chp_api/dispatcher/base.py index ce85926..fd1f7ba 100644 --- a/chp_api/dispatcher/base.py +++ b/chp_api/dispatcher/base.py @@ -8,13 +8,14 @@ from importlib import import_module from collections import defaultdict +from .curie_database import merge_curies_databases, CurieDatabase +from .models import Transaction, App + from chp_utils.trapi_query_processor import BaseQueryProcessor -from chp_utils.curie_database import merge_curies_databases -from trapi_model.meta_knowledge_graph import merge_meta_knowledge_graphs +from trapi_model.meta_knowledge_graph import MetaKnowledgeGraph, merge_meta_knowledge_graphs from trapi_model.query import Query from trapi_model.biolink import TOOLKIT -from .models import Transaction # Setup logging logging.addLevelName(25, "NOTE") @@ -50,16 +51,32 @@ def __init__(self, request, trapi_version): def get_curies(self): curies_dbs = [] - for app in APPS: - get_app_curies_fn = getattr(app, 'get_curies') - curies_dbs.append(get_app_curies_fn()) + for app, app_name in zip(APPS, settings.INSTALLED_CHP_APPS): + app_db_obj = App.objects.get(name=app_name) + # Load location from uploaded Zenodo files + if app_db_obj.curies_zenodo_file: + curies = app_db_obj.curies_zenodo_file.load_file(base_url="https://sandbox.zenodo.org/api/records") + curies_db = CurieDatabase(curies=curies) + # Load default location + else: + get_app_curies_fn = getattr(app, 'get_curies') + curies_db = get_app_curies_fn() + curies_dbs.append(curies_db) return merge_curies_databases(curies_dbs) def get_meta_knowledge_graph(self): meta_kgs = [] - for app in APPS: - get_app_meta_kg_fn = getattr(app, 'get_meta_knowledge_graph') - meta_kgs.append(get_app_meta_kg_fn()) + for app, app_name in zip(APPS, settings.INSTALLED_CHP_APPS): + app_db_obj = App.objects.get(name=app_name) + # Load location from uploaded Zenodo files + if app_db_obj.meta_knowledge_graph_zenodo_file: + meta_kg = app_db_obj.meta_knowledge_graph_zenodo_file.load_file(base_url="https://sandbox.zenodo.org/api/records") + meta_kg = MetaKnowledgeGraph.load('1.3', None, meta_knowledge_graph=meta_kg) + # Load default location + else: + get_app_meta_kg_fn = getattr(app, 'get_meta_knowledge_graph') + meta_kg = get_app_meta_kg_fn() + meta_kgs.append(meta_kg) return merge_meta_knowledge_graphs(meta_kgs) def process_invalid_trapi(self, request): diff --git a/chp_api/dispatcher/curie_database.py b/chp_api/dispatcher/curie_database.py new file mode 100644 index 0000000..1ea7cac --- /dev/null +++ b/chp_api/dispatcher/curie_database.py @@ -0,0 +1,60 @@ +""" A helper class to handle CHP supported curies. +""" +import json + +from trapi_model.biolink.constants import * + +def merge_curies_databases(list_of_curies_dbs): + if len(list_of_curies_dbs) == 1: + return list_of_curies_dbs[0].to_dict() + merged = list_of_curies_dbs[0].to_dict() + for curies_db in list_of_curies_dbs[1:]: + for biolink_entity, curies_info_dict in curies_db.to_dict().items(): + if biolink_entity not in merged: + merged[biolink_entity] = curies_info_dict + continue + for curie, info in curies_info_dict.items(): + if curie not in merged[biolink_entity]: + merged[biolink_entity][curie] = info + continue + new_info = set.union( + *[ + set(merged[biolink_entity][curie]), + set(info), + ] + ) + merged[biolink_entity][curie] = [info for info in new_info if info] + return merged + + +class CurieDatabase: + def __init__(self, curies=None, curies_filename=None): + if curies is None and curies_filename is None: + raise ValueError('Must pass in either conflation map or filename.') + elif curies is not None and curies_filename is not None: + raise ValueError('Must pass in either conflation map or filename, not both.') + self.curies = self.load_curies(curies, curies_filename) + + @staticmethod + def load_curies(curies, curies_filename): + _curies = {} + if curies_filename is not None: + with open(curies_filename) as f_: + curies = json.load(f_) + for biolink_entity, curies_list in curies.items(): + _curies[get_biolink_entity(biolink_entity)] = curies_list + return _curies + + def to_dict(self): + curies_dict = {} + for biolink_entity, curies_list in self.curies.items(): + curies_dict[biolink_entity.get_curie()] = curies_list + return curies_dict + + + def json(self, filename=None): + if filename is None: + return json.dumps(self.to_dict(), indent=2) + with open(filename, 'w') as f_: + json.dump(self.to_dict(), f_, indent=2) + diff --git a/chp_api/dispatcher/migrations/0004_app_alter_transaction_chp_app.py b/chp_api/dispatcher/migrations/0004_app_alter_transaction_chp_app.py new file mode 100755 index 0000000..c54df21 --- /dev/null +++ b/chp_api/dispatcher/migrations/0004_app_alter_transaction_chp_app.py @@ -0,0 +1,28 @@ +# Generated by Django 4.2 on 2023-04-18 04:28 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('dispatcher', '0003_transaction_chp_app'), + ] + + operations = [ + migrations.CreateModel( + name='App', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=128)), + ('curies_file', models.FileField(blank=True, null=True, upload_to='curies_files/')), + ('meta_knowledge_graph_file', models.FileField(blank=True, null=True, upload_to='meta_knowledge_graph_files')), + ], + ), + migrations.AlterField( + model_name='transaction', + name='chp_app', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='dispatcher.app'), + ), + ] diff --git a/chp_api/dispatcher/migrations/0005_zenodofile_remove_app_curies_file_and_more.py b/chp_api/dispatcher/migrations/0005_zenodofile_remove_app_curies_file_and_more.py new file mode 100755 index 0000000..41fd227 --- /dev/null +++ b/chp_api/dispatcher/migrations/0005_zenodofile_remove_app_curies_file_and_more.py @@ -0,0 +1,40 @@ +# Generated by Django 4.2 on 2023-04-18 19:28 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('dispatcher', '0004_app_alter_transaction_chp_app'), + ] + + operations = [ + migrations.CreateModel( + name='ZenodoFile', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('zenodo_id', models.CharField(max_length=128)), + ('file_key', models.CharField(max_length=128)), + ], + ), + migrations.RemoveField( + model_name='app', + name='curies_file', + ), + migrations.RemoveField( + model_name='app', + name='meta_knowledge_graph_file', + ), + migrations.AddField( + model_name='app', + name='curies_zenodo_file', + field=models.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='curies_zenodo_file', to='dispatcher.zenodofile'), + ), + migrations.AddField( + model_name='app', + name='meta_knowledge_graph_zenodo_file', + field=models.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='meta_knowledge_graph_zenodo_file', to='dispatcher.zenodofile'), + ), + ] diff --git a/chp_api/dispatcher/models.py b/chp_api/dispatcher/models.py index a303344..6a9238b 100644 --- a/chp_api/dispatcher/models.py +++ b/chp_api/dispatcher/models.py @@ -1,19 +1,42 @@ +import os +import json +import requests from django.db import models -class Transaction(models.Model): - id = models.CharField(max_length=100, primary_key=True) - date_time = models.DateTimeField(auto_now=True) - query = models.JSONField(default=dict) - status = models.CharField(max_length=100, default="", null=True) - versions = models.JSONField(default=dict) - chp_app = models.ForeignKey(App, on_delete=models.CASCADE) + +class ZenodoFile(models.Model): + zenodo_id = models.CharField(max_length=128) + file_key = models.CharField(max_length=128) + + def __str__(self): + return f'{self.zenodo_id}/{self.file_key}' + + def get_record(self): + return requests.get(f"https://zenodo.org/api/records/{self.zenodo_id}").json() + + def load_file(self, base_url="https://zenodo.org/api/records"): + r = requests.get(f"{base_url}/{self.zenodo_id}").json() + files = {f["key"]: f for f in r["files"]} + f = files[self.file_key] + download_link = f["links"]["self"] + file_type = f["type"] + if file_type == 'json': + return requests.get(download_link).json() + raise NotImplementedError(f'File type of: {ext} is not implemented.') class App(models.Model): name = models.CharField(max_length=128) - curies_file = models.FileField(upload_to='curies_files/', null=True, blank=True) - meta_knowledge_graph_file = models.FileField(upload_to='meta_knowledge_graph_files', null=True, blank=True) + curies_zenodo_file = models.OneToOneField(ZenodoFile, on_delete=models.CASCADE, null=True, blank=True, related_name='curies_zenodo_file') + meta_knowledge_graph_zenodo_file = models.OneToOneField(ZenodoFile, on_delete=models.CASCADE, null=True, blank=True, related_name='meta_knowledge_graph_zenodo_file') def __str__(self): return self.name +class Transaction(models.Model): + id = models.CharField(max_length=100, primary_key=True) + date_time = models.DateTimeField(auto_now=True) + query = models.JSONField(default=dict) + status = models.CharField(max_length=100, default="", null=True) + versions = models.JSONField(default=dict) + chp_app = models.ForeignKey(App, on_delete=models.CASCADE, null=True, blank=True) diff --git a/chp_api/dispatcher/scripts/__init__.py b/chp_api/dispatcher/scripts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/chp_api/dispatcher/scripts/load_db_apps.py b/chp_api/dispatcher/scripts/load_db_apps.py new file mode 100644 index 0000000..faf2bf6 --- /dev/null +++ b/chp_api/dispatcher/scripts/load_db_apps.py @@ -0,0 +1,10 @@ +from django.conf import settings + +from ..models import App + + +def run(): + for app_name in settings.INSTALLED_CHP_APPS: + app_db_obj, created = App.objects.get_or_create(name=app_name) + if created: + app_db_obj.save() diff --git a/chp_api/dispatcher/zenodo.py b/chp_api/dispatcher/zenodo.py new file mode 100644 index 0000000..705d7f6 --- /dev/null +++ b/chp_api/dispatcher/zenodo.py @@ -0,0 +1,29 @@ +import os +import json +import requests + +def zenodo_get(zenodo_id, file_key, file_type='infer'): + """ This function will download the requested Zenodo file into memory. + + Args: + :param zenodo_id: The string id for the Zenodo file. For example, if the Zenodo url is: https://zenodo.org/record/1184524#.ZD7aF_bML-g, + then the zenodo_id is: 1184524. + :type zenodo_id: str + :param file_key: This is a string to the file_key in the Zenodo bucket or in the zenodo record. + :type file_key: string + + Kwargs: + :param file_type: The file type of the hosted Zenodo file. If inferred, will try to infer type from file extension. + """ + r = requests.get(f"https://zenodo.org/api/records/{zenodo_id}").json() + files = {f[key]: f for f in r["files"]} + f = files[file_key] + download_link = f["links"]["self"] + if file_type == 'infer': + file_type = f["type"] + if file_type == 'json': + return requests.get(download_link).json() + raise NotImplementedError(f'File type of: {ext} is not implemented.') + + + diff --git a/dev-deployment-script b/dev-deployment-script index 94affa4..39ed58e 100755 --- a/dev-deployment-script +++ b/dev-deployment-script @@ -1,12 +1,12 @@ #!/bin/bash # Only to be run when building on dev machine -echo "Taking down the NCATS Server." -docker-compose -f dev-docker-compose.yml stop +docker compose build -docker-compose -f dev-docker-compose.yml build --no-cache api +docker compose up -d -echo "Bringing up server." -docker-compose -f dev-docker-compose.yml up -d +docker compose run chp-api python3 manage.py migrate -echo "Server should now be up." -echo "Check logs with: docker-compose -f dev-docker-compose.yml logs -f" +docker compose run --user root chp-api python3 manage.py collectstatic --noinput + + +echo "Check logs with: docker compose logs -f" diff --git a/docker-compose.yml b/docker-compose.yml index 2606028..d22f314 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -26,6 +26,7 @@ services: context: . dockerfile: Dockerfile restart: always + user: chp_api ports: - '8000:8000' secrets: @@ -56,7 +57,7 @@ services: timeout: 10s retries: 3 volumes: - - ./static-files:/home/chp_api/staticfiles + - static-files:/home/chp_api/staticfiles #command: gunicorn -c gunicorn.config.py -b 0.0.0.0:8000 chp_api.wsgi:application command: python3 manage.py runserver 0.0.0.0:8000 @@ -87,11 +88,11 @@ services: expose: - 8080 volumes: - - ./static-files:/var/www/static - - ./media-files:/var/www/media + - static-files:/var/www/static volumes: db-data: + static-files: secrets: allowed-hosts: diff --git a/nginx/default.conf b/nginx/default.conf index a483bf9..8db6a79 100644 --- a/nginx/default.conf +++ b/nginx/default.conf @@ -33,11 +33,4 @@ server { proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; } - location /media { - proxy_pass http://$STATIC_SERVER_ADDR; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - } - } diff --git a/requirements.txt b/requirements.txt index 21db109..081b490 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,6 @@ psycopg2-binary django-environ django-hosts gunicorn -django \ No newline at end of file +django +requests +requests-cache From 466a4c22c830cf054c9c8e33f121a13657a4d14f Mon Sep 17 00:00:00 2001 From: Chase Yakaboski Date: Mon, 1 May 2023 13:42:57 -0400 Subject: [PATCH 042/132] Working trapi 1.4 verision with legacy trapi_model. --- Dockerfile | 2 +- chp_api/chp_api/settings.py | 4 +- chp_api/dispatcher/admin.py | 3 +- chp_api/dispatcher/base.py | 11 ++-- .../migrations/0006_dispatchersettings.py | 23 ++++++++ chp_api/dispatcher/models.py | 23 ++++++++ chp_api/dispatcher/scripts/load_db_apps.py | 5 ++ chp_api/dispatcher/urls.py | 8 --- chp_api/dispatcher/views.py | 59 +++++++++++-------- dev-deployment-script | 3 + gs-sample.json | 31 ++++++++++ 11 files changed, 131 insertions(+), 41 deletions(-) create mode 100755 chp_api/dispatcher/migrations/0006_dispatchersettings.py create mode 100644 gs-sample.json diff --git a/Dockerfile b/Dockerfile index ff22ebe..f728166 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,7 @@ WORKDIR /usr/src/chp_api # && apt-get install -y git python3-pip python3-dev #dos2unix -RUN git clone --single-branch --branch master https://github.com/di2ag/trapi_model.git +RUN git clone --single-branch --branch pydantic-integration-yakaboskic https://github.com/di2ag/trapi_model.git RUN git clone --single-branch --branch master https://github.com/di2ag/chp_utils.git RUN git clone --single-branch --branch master https://github.com/di2ag/chp_look_up.git RUN git clone --single-branch --branch master https://github.com/di2ag/gene-specificity.git diff --git a/chp_api/chp_api/settings.py b/chp_api/chp_api/settings.py index 1e1ad55..0dd528c 100644 --- a/chp_api/chp_api/settings.py +++ b/chp_api/chp_api/settings.py @@ -45,8 +45,6 @@ ] INSTALLED_CHP_APPS = [ - 'chp_look_up', -# 'chp_learn', 'gene_specificity', ] @@ -55,7 +53,7 @@ ] # CHP Versions -#VERSIONS = {app_name: app.__version__ for app_name, app in [(app_name, import_module(app_name)) for app_name in INSTALLED_CHP_APPS + OTHER_APPS]} +VERSIONS = {app_name: app.__version__ for app_name, app in [(app_name, import_module(app_name)) for app_name in INSTALLED_CHP_APPS + OTHER_APPS]} # Sets up installed apps relevent to django INSTALLED_APPS = INSTALLED_BASE_APPS + INSTALLED_CHP_APPS diff --git a/chp_api/dispatcher/admin.py b/chp_api/dispatcher/admin.py index f3422e0..9db96c8 100644 --- a/chp_api/dispatcher/admin.py +++ b/chp_api/dispatcher/admin.py @@ -1,6 +1,7 @@ from django.contrib import admin -from .models import App, ZenodoFile +from .models import App, ZenodoFile, DispatcherSettings admin.site.register(App) admin.site.register(ZenodoFile) +admin.site.register(DispatcherSettings) diff --git a/chp_api/dispatcher/base.py b/chp_api/dispatcher/base.py index fd1f7ba..ad7f631 100644 --- a/chp_api/dispatcher/base.py +++ b/chp_api/dispatcher/base.py @@ -9,7 +9,7 @@ from collections import defaultdict from .curie_database import merge_curies_databases, CurieDatabase -from .models import Transaction, App +from .models import Transaction, App, DispatcherSettings from chp_utils.trapi_query_processor import BaseQueryProcessor from trapi_model.meta_knowledge_graph import MetaKnowledgeGraph, merge_meta_knowledge_graphs @@ -65,13 +65,15 @@ def get_curies(self): return merge_curies_databases(curies_dbs) def get_meta_knowledge_graph(self): + # Get current trapi and biolink versions + dispatcher_settings = DispatcherSettings.load() meta_kgs = [] for app, app_name in zip(APPS, settings.INSTALLED_CHP_APPS): app_db_obj = App.objects.get(name=app_name) # Load location from uploaded Zenodo files if app_db_obj.meta_knowledge_graph_zenodo_file: meta_kg = app_db_obj.meta_knowledge_graph_zenodo_file.load_file(base_url="https://sandbox.zenodo.org/api/records") - meta_kg = MetaKnowledgeGraph.load('1.3', None, meta_knowledge_graph=meta_kg) + meta_kg = MetaKnowledgeGraph.load(dispatcher_settings.trapi_version, None, meta_knowledge_graph=meta_kg) # Load default location else: get_app_meta_kg_fn = getattr(app, 'get_meta_knowledge_graph') @@ -275,14 +277,15 @@ def add_logs_from_query_list(self, target_query, query_list): target_query.logger.add_logs(query.logger.to_dict()) return target_query - def add_transaction(self, response, chp_app='dispatcher'): + def add_transaction(self, response, app_name='dispatcher'): + app_db_obj = App.objects.get(name=app_name) # Save the transaction transaction = Transaction( id = response.id, status = response.status, query = response.to_dict(), versions = settings.VERSIONS, - chp_app = chp_app, + chp_app = app_db_obj, ) transaction.save() diff --git a/chp_api/dispatcher/migrations/0006_dispatchersettings.py b/chp_api/dispatcher/migrations/0006_dispatchersettings.py new file mode 100755 index 0000000..7b57b18 --- /dev/null +++ b/chp_api/dispatcher/migrations/0006_dispatchersettings.py @@ -0,0 +1,23 @@ +# Generated by Django 4.2 on 2023-05-01 16:47 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('dispatcher', '0005_zenodofile_remove_app_curies_file_and_more'), + ] + + operations = [ + migrations.CreateModel( + name='DispatcherSettings', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('trapi_version', models.CharField(default='1.4', max_length=28)), + ], + options={ + 'abstract': False, + }, + ), + ] diff --git a/chp_api/dispatcher/models.py b/chp_api/dispatcher/models.py index 6a9238b..88d28a0 100644 --- a/chp_api/dispatcher/models.py +++ b/chp_api/dispatcher/models.py @@ -40,3 +40,26 @@ class Transaction(models.Model): versions = models.JSONField(default=dict) chp_app = models.ForeignKey(App, on_delete=models.CASCADE, null=True, blank=True) + +class Singleton(models.Model): + + class Meta: + abstract = True + + def save(self, *args, **kwargs): + self.pk = 1 + super(Singleton, self).save(*args, **kwargs) + + def delete(self, *args, **kwargs): + pass + + @classmethod + def load(cls): + obj, _ = cls.objects.get_or_create(pk=1) + return obj + +class DispatcherSettings(Singleton): + trapi_version = models.CharField(max_length=28, default='1.4') + + def __str__(self): + return 'settings' diff --git a/chp_api/dispatcher/scripts/load_db_apps.py b/chp_api/dispatcher/scripts/load_db_apps.py index faf2bf6..a340d3e 100644 --- a/chp_api/dispatcher/scripts/load_db_apps.py +++ b/chp_api/dispatcher/scripts/load_db_apps.py @@ -8,3 +8,8 @@ def run(): app_db_obj, created = App.objects.get_or_create(name=app_name) if created: app_db_obj.save() + + # Create a dummy app for the dispatcher + app_db_obj, created = App.objects.get_or_create(name='dispatcher') + if created: + app_db_obj.save() diff --git a/chp_api/dispatcher/urls.py b/chp_api/dispatcher/urls.py index f339597..e508e06 100644 --- a/chp_api/dispatcher/urls.py +++ b/chp_api/dispatcher/urls.py @@ -25,14 +25,6 @@ path('meta_knowledge_graph/', views.meta_knowledge_graph.as_view()), path('curies/', views.curies.as_view()), path('versions/', views.versions.as_view()), - path('v1.1/query/', views.query.as_view(trapi_version='1.1')), - path('v1.1/meta_knowledge_graph/', views.meta_knowledge_graph.as_view()), - path('v1.1/curies/', views.curies.as_view(trapi_version='1.1')), - path('v1.1/versions/', views.versions.as_view(trapi_version='1.1')), - path('v1.2/query/', views.query.as_view(trapi_version='1.2')), - path('v1.2/meta_knowledge_graph/', views.meta_knowledge_graph.as_view(trapi_version='1.2')), - path('v1.2/curies/', views.curies.as_view(trapi_version='1.2')), - path('v1.2/versions/', views.versions.as_view(trapi_version='1.2')), path('transactions/', views.TransactionList.as_view(), name='transaction-list'), path('recent/', views.RecentTransactionList.as_view(), name='recent-transaction-list'), path('transactions//', views.TransactionDetail.as_view(), name='transactions-detail') diff --git a/chp_api/dispatcher/views.py b/chp_api/dispatcher/views.py index 70560de..a96c775 100644 --- a/chp_api/dispatcher/views.py +++ b/chp_api/dispatcher/views.py @@ -5,7 +5,7 @@ from datetime import datetime, timedelta from .base import Dispatcher -from .models import Transaction +from .models import Transaction, DispatcherSettings from .serializers import TransactionListSerializer, TransactionDetailSerializer from django.http import HttpResponse, JsonResponse @@ -18,19 +18,24 @@ class query(APIView): - trapi_version = '1.3' - def __init__(self, trapi_version='1.3', **kwargs): - self.trapi_version = trapi_version - super(query, self).__init__(**kwargs) - + def post(self, request): + # Get current trapi and biolink versions + dispatcher_settings = DispatcherSettings.load() + if request.method == 'POST': # Initialize Dispatcher - dispatcher = Dispatcher(request, self.trapi_version) + dispatcher = Dispatcher( + request, + dispatcher_settings.trapi_version, + ) # Process Query query = None try: - query = dispatcher.process_request(request, trapi_version=self.trapi_version) + query = dispatcher.process_request( + request, + trapi_version=dispatcher_settings.trapi_version, + ) except Exception as e: if 'Workflow Error' in str(e): return dispatcher.process_invalid_workflow(request, str(e)) @@ -40,45 +45,51 @@ def post(self, request): return dispatcher.get_response(query) class curies(APIView): - trapi_version = '1.3' - def __init__(self, trapi_version='1.3', **kwargs): - self.trapi_version = trapi_version - super(curies, self).__init__(**kwargs) def get(self, request): + # Get current trapi and biolink versions + dispatcher_settings = DispatcherSettings.load() + if request.method == 'GET': # Initialize dispatcher - dispatcher = Dispatcher(request, self.trapi_version) + dispatcher = Dispatcher( + request, + dispatcher_settings.trapi_version, + ) # Get all chp app curies curies_db = dispatcher.get_curies() return JsonResponse(curies_db) class meta_knowledge_graph(APIView): - trapi_version = '1.3' - def __init__(self, trapi_version='1.3', **kwargs): - self.trapi_version = trapi_version - super(meta_knowledge_graph, self).__init__(**kwargs) - + def get(self, request): + # Get current trapi and biolink versions + dispatcher_settings = DispatcherSettings.load() + if request.method == 'GET': # Initialize Dispatcher - dispatcher = Dispatcher(request, self.trapi_version) + dispatcher = Dispatcher( + request, + dispatcher_settings.trapi_version, + ) # Get merged meta KG meta_knowledge_graph = dispatcher.get_meta_knowledge_graph() return JsonResponse(meta_knowledge_graph.to_dict()) class versions(APIView): - trapi_version = '1.3' - def __init__(self, trapi_version='1.3', **kwargs): - self.trapi_version = trapi_version - super(version, self).__init__(**kwargs) def get(self, request): + # Get current trapi and biolink versions + dispatcher_settings = DispatcherSettings.load() + if request.method == 'GET': # Initialize Dispatcher - dispatcher = Dispatcher(request, self.trapi_version) + dispatcher = Dispatcher( + request, + dispatcher_settings.trapi_version, + ) return JsonResponse(dispatcher.get_versions()) class TransactionList(mixins.ListModelMixin, generics.GenericAPIView): diff --git a/dev-deployment-script b/dev-deployment-script index 39ed58e..d15fe75 100755 --- a/dev-deployment-script +++ b/dev-deployment-script @@ -6,6 +6,9 @@ docker compose up -d docker compose run chp-api python3 manage.py migrate +# Load apps +docker compose run chp-api python3 manage.py runscript load_db_apps + docker compose run --user root chp-api python3 manage.py collectstatic --noinput diff --git a/gs-sample.json b/gs-sample.json new file mode 100644 index 0000000..28ca984 --- /dev/null +++ b/gs-sample.json @@ -0,0 +1,31 @@ +{ + "message": { + "query_graph": { + "edges": { + "e0": { + "object": "n1", + "predicates": [ + "biolink:expressed_in" + ], + "subject": "n0" + } + }, + "nodes": { + "n0": { + "categories": [ + "biolink:Gene", + "biolink:Protein" + ] + }, + "n1": { + "categories": [ + "biolink:GrossAnatomicalStructure" + ], + "ids": [ + "UBERON:0000458" + ] + } + } + } + } +} From a7424afdf68b5d7299b8f5866e0ec6d8ad4bef83 Mon Sep 17 00:00:00 2001 From: Chase Yakaboski Date: Tue, 2 May 2023 00:22:05 -0400 Subject: [PATCH 043/132] Added secrets and functionality for django admin website. --- chp_api/chp_api/settings.py | 8 ++++++++ dev-deployment-script | 8 ++++++++ docker-compose.yml | 16 ++++++++++++++-- 3 files changed, 30 insertions(+), 2 deletions(-) diff --git a/chp_api/chp_api/settings.py b/chp_api/chp_api/settings.py index 0dd528c..8135cc4 100644 --- a/chp_api/chp_api/settings.py +++ b/chp_api/chp_api/settings.py @@ -165,3 +165,11 @@ # Read the secret key from file with open(env("SECRET_KEY_FILE"), 'r') as sk_file: SECRET_KEY = sk_file.readline().strip() + +# Set UN, Email and Password for superuser +with open(env("DJANGO_SUPERUSER_USERNAME_FILE"), 'r') as dsu_file: + os.environ["DJANGO_SUPERUSER_USERNAME"] = dsu_file.readline().strip() +with open(env("DJANGO_SUPERUSER_EMAIL_FILE"), 'r') as dse_file: + os.environ["DJANGO_SUPERUSER_EMAIL"] = dse_file.readline().strip() +with open(env("DJANGO_SUPERUSER_PASSWORD_FILE"), 'r') as dsp_file: + os.environ["DJANGO_SUPERUSER_PASSWORD"] = dsp_file.readline().strip() diff --git a/dev-deployment-script b/dev-deployment-script index d15fe75..34ce831 100755 --- a/dev-deployment-script +++ b/dev-deployment-script @@ -1,4 +1,9 @@ #!/bin/bash + +# Variables +django_superuser_username='cat secrets/chp_api/django_superuser_username.txt' +django_superuser_email='cat secrets/chp_api/django_superuser_email.txt' + # Only to be run when building on dev machine docker compose build @@ -6,6 +11,9 @@ docker compose up -d docker compose run chp-api python3 manage.py migrate +# Create a database superuser +docker compose run --user root chp-api python3 manage.py createsuperuser --no-input #--username $django_superuser_username --email $django_superuser_email + # Load apps docker compose run chp-api python3 manage.py runscript load_db_apps diff --git a/docker-compose.yml b/docker-compose.yml index d22f314..c70d624 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -33,6 +33,9 @@ services: - db-password - django-key - allowed-hosts + - django-superuser-username + - django-superuser-email + - django-superuser-password environment: - POSTGRES_DB=chp_db - POSTGRES_USER=postgres @@ -42,6 +45,9 @@ services: - SECRET_KEY_FILE=/run/secrets/django-key - DEBUG=1 - DJANGO_ALLOWED_HOSTS_FILE=/run/secrets/allowed-hosts + - DJANGO_SUPERUSER_USERNAME_FILE=/run/secrets/django-superuser-username + - DJANGO_SUPERUSER_EMAIL_FILE=/run/secrets/django-superuser-email + - DJANGO_SUPERUSER_PASSWORD_FILE=/run/secrets/django-superuser-password # Uncomment this for production #- DJANGO_SETTINGS_MODULE=mysite.settings.production # Comment this for development @@ -58,8 +64,8 @@ services: retries: 3 volumes: - static-files:/home/chp_api/staticfiles - #command: gunicorn -c gunicorn.config.py -b 0.0.0.0:8000 chp_api.wsgi:application - command: python3 manage.py runserver 0.0.0.0:8000 + command: gunicorn -c gunicorn.config.py -b 0.0.0.0:8000 chp_api.wsgi:application + #command: python3 manage.py runserver 0.0.0.0:8000 db: image: postgres @@ -101,3 +107,9 @@ secrets: file: secrets/db/password.txt django-key: file: secrets/chp_api/secret_key.txt + django-superuser-username: + file: secrets/chp_api/django_superuser_username.txt + django-superuser-email: + file: secrets/chp_api/django_superuser_email.txt + django-superuser-password: + file: secrets/chp_api/django_superuser_password.txt From f93b339a90a0de6fd4bea2ec4083627c887ca204 Mon Sep 17 00:00:00 2001 From: Chase Yakaboski Date: Wed, 10 May 2023 11:47:29 -0400 Subject: [PATCH 044/132] trapi-1.4 fixes --- Dockerfile | 2 +- dev-deployment-script | 2 +- docker-compose.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index f728166..ff22ebe 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,7 @@ WORKDIR /usr/src/chp_api # && apt-get install -y git python3-pip python3-dev #dos2unix -RUN git clone --single-branch --branch pydantic-integration-yakaboskic https://github.com/di2ag/trapi_model.git +RUN git clone --single-branch --branch master https://github.com/di2ag/trapi_model.git RUN git clone --single-branch --branch master https://github.com/di2ag/chp_utils.git RUN git clone --single-branch --branch master https://github.com/di2ag/chp_look_up.git RUN git clone --single-branch --branch master https://github.com/di2ag/gene-specificity.git diff --git a/dev-deployment-script b/dev-deployment-script index 34ce831..3bc83ca 100755 --- a/dev-deployment-script +++ b/dev-deployment-script @@ -5,7 +5,7 @@ django_superuser_username='cat secrets/chp_api/django_superuser_username.txt' django_superuser_email='cat secrets/chp_api/django_superuser_email.txt' # Only to be run when building on dev machine -docker compose build +docker compose build --no-cache docker compose up -d diff --git a/docker-compose.yml b/docker-compose.yml index c70d624..0253de9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -65,7 +65,7 @@ services: volumes: - static-files:/home/chp_api/staticfiles command: gunicorn -c gunicorn.config.py -b 0.0.0.0:8000 chp_api.wsgi:application - #command: python3 manage.py runserver 0.0.0.0:8000 + #command: python3 manage.py runserver 0.0.0.0:8000 db: image: postgres From 6ea7e8b9c47aae3c48101ff94dc54b2365b548f8 Mon Sep 17 00:00:00 2001 From: Chase Yakaboski Date: Tue, 23 May 2023 00:11:34 -0400 Subject: [PATCH 045/132] Added initial gennifer integration with pidc and updated docker files. --- .gitmodules | 3 ++ Dockerfile.new | 39 ------------------- Dockerfile => chp_api/Dockerfile | 31 ++------------- .../gunicorn.config.py | 0 requirements.txt => chp_api/requirements.txt | 0 docker-compose.yml => compose.yaml | 23 +++++++++-- dev-deployment-script => deployment-script | 0 dev-docker-compose.yml | 25 ------------ entrypoint.sh | 25 ------------ gennifer/pidc | 1 + 10 files changed, 27 insertions(+), 120 deletions(-) create mode 100644 .gitmodules delete mode 100644 Dockerfile.new rename Dockerfile => chp_api/Dockerfile (72%) rename gunicorn.config.py => chp_api/gunicorn.config.py (100%) rename requirements.txt => chp_api/requirements.txt (100%) rename docker-compose.yml => compose.yaml (82%) rename dev-deployment-script => deployment-script (100%) delete mode 100644 dev-docker-compose.yml delete mode 100644 entrypoint.sh create mode 160000 gennifer/pidc diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..e37eb1a --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "gennifer/pidc"] + path = gennifer/pidc + url = git@github.com:di2ag/gennifer-pidc.git diff --git a/Dockerfile.new b/Dockerfile.new deleted file mode 100644 index 0a825d4..0000000 --- a/Dockerfile.new +++ /dev/null @@ -1,39 +0,0 @@ -################ -# venv builder # -################ -FROM python:3.8.3 as venv_builder - -COPY requirements.txt . -COPY requirements-dev.txt . -RUN python3 -m venv /opt/venv -RUN /opt/venv/bin/pip install --upgrade pip -RUN /opt/venv/bin/pip install -r requirements.txt -RUN /opt/venv/bin/pip install -r requirements-dev.txt -########### -# CHP API # -########### -FROM python:3.8.3-slim as chp-api - -# set environment variables -ENV PYTHONDONTWRITEBYTECODE 1 -ENV PYTHONUNBUFFERED 1 -ENV TZ=America/New_York -ENV SERVER_DIR=/chp_api/ -ENV VIRTUAL_ENV_PATH=/opt/venv - -# copy venv from venv builder image -COPY --from=venv_builder ${VIRTUAL_ENV_PATH} ${VIRTUAL_ENV_PATH} - -# copy project -COPY ./chp_api $SERVER_DIR - -# copy entry point -COPY ./entrypoint.sh ${SERVER_DIR} -COPY ./gunicorn.config.py ${SERVER_DIR} -COPY ./chp_db_fixture.json.gz ${SERVER_DIR} - -# enter app directory -WORKDIR $SERVER_DIR - -# Enable venv -ENV PATH="/opt/venv/bin:$PATH" diff --git a/Dockerfile b/chp_api/Dockerfile similarity index 72% rename from Dockerfile rename to chp_api/Dockerfile index ff22ebe..d555bfa 100644 --- a/Dockerfile +++ b/chp_api/Dockerfile @@ -8,21 +8,11 @@ FROM python:3.8 as intermediate # set work directory WORKDIR /usr/src/chp_api -# install git -#RUN apt-get update \ -# && apt-get install -y git python3-pip python3-dev -#dos2unix - RUN git clone --single-branch --branch master https://github.com/di2ag/trapi_model.git RUN git clone --single-branch --branch master https://github.com/di2ag/chp_utils.git RUN git clone --single-branch --branch master https://github.com/di2ag/chp_look_up.git RUN git clone --single-branch --branch master https://github.com/di2ag/gene-specificity.git -# lint -#RUN pip install --upgrade pip -#RUN pip3 install flake8 wheel -#COPY . . - # install dependencies COPY ./requirements.txt . RUN pip3 wheel --no-cache-dir --no-deps --wheel-dir /usr/src/chp_api/wheels -r requirements.txt @@ -64,27 +54,15 @@ ENV TZ=America/New_York # set ARGs ARG DEBIAN_FRONTEND=noninterative -# install dependencies -#RUN apt-get update \ -# && apt-get install -y python3-pip graphviz openmpi-bin libopenmpi-dev build-essential libssl-dev libffi-dev python3-dev -#RUN apt-get install -y libgraphviz-dev python3-pygraphviz -#RUN apt-get install -y libpq-dev -#RUN apt-get install -y netcat - # copy repo to new image COPY --from=intermediate /usr/src/chp_api/wheels /wheels COPY --from=intermediate /usr/src/chp_api/requirements.txt . -#RUN pip3 install --upgrade pip -#RUN python3 -m pip install --upgrade pip RUN pip3 install --no-cache /wheels/* -# copy entry point -#COPY ./entrypoint.sh $APP_HOME - # copy project -COPY ./chp_api/chp_api $APP_HOME/chp_api -COPY ./chp_api/manage.py $APP_HOME -COPY ./chp_api/dispatcher $APP_HOME/dispatcher +COPY ./chp_api $APP_HOME/chp_api +COPY ./manage.py $APP_HOME +COPY ./dispatcher $APP_HOME/dispatcher COPY ./gunicorn.config.py $APP_HOME # chown all the files to the app user @@ -93,6 +71,3 @@ RUN chown -R chp_api:chp_api $APP_HOME \ # change to the app user USER chp_api - -# run entrypoint.sh -#ENTRYPOINT ["/home/chp_api/web/entrypoint.sh"] diff --git a/gunicorn.config.py b/chp_api/gunicorn.config.py similarity index 100% rename from gunicorn.config.py rename to chp_api/gunicorn.config.py diff --git a/requirements.txt b/chp_api/requirements.txt similarity index 100% rename from requirements.txt rename to chp_api/requirements.txt diff --git a/docker-compose.yml b/compose.yaml similarity index 82% rename from docker-compose.yml rename to compose.yaml index 0253de9..45b9dc9 100644 --- a/docker-compose.yml +++ b/compose.yaml @@ -23,7 +23,7 @@ services: chp-api: build: - context: . + context: ./chp_api dockerfile: Dockerfile restart: always user: chp_api @@ -64,8 +64,8 @@ services: retries: 3 volumes: - static-files:/home/chp_api/staticfiles - command: gunicorn -c gunicorn.config.py -b 0.0.0.0:8000 chp_api.wsgi:application - #command: python3 manage.py runserver 0.0.0.0:8000 + #command: gunicorn -c gunicorn.config.py -b 0.0.0.0:8000 chp_api.wsgi:application + command: python3 manage.py runserver 0.0.0.0:8000 db: image: postgres @@ -95,6 +95,21 @@ services: - 8080 volumes: - static-files:/var/www/static + + gennifer-pidc: + build: + context: ./gennifer/pidc + dockerfile: Dockerfile + restart: always + user: gennifer_user + expose: + - 5000 + secrets: + - gennifer_key + environment: + - SECRET_KEY_FILE=/run/secrets/gennifer_key + #command: gunicorn -c gunicorn.config.py -b 0.0.0.0:5000 'pidc:create_app()' + command: flask --app pidc run --debug volumes: db-data: @@ -113,3 +128,5 @@ secrets: file: secrets/chp_api/django_superuser_email.txt django-superuser-password: file: secrets/chp_api/django_superuser_password.txt + gennifer_key: + file: secrets/gennifer/secret_key.txt diff --git a/dev-deployment-script b/deployment-script similarity index 100% rename from dev-deployment-script rename to deployment-script diff --git a/dev-docker-compose.yml b/dev-docker-compose.yml deleted file mode 100644 index ca599c0..0000000 --- a/dev-docker-compose.yml +++ /dev/null @@ -1,25 +0,0 @@ -version: '3.8' - -services: - api: - build: - context: . - dockerfile: Dockerfile.new - container_name: chp-api - command: python manage.py runserver 0.0.0.0:8000 - volumes: - - static_volume:/chp_api/staticfiles - expose: - - 80 - env_file: - - ./.dev.env - db: - image: postgres:latest - restart: unless-stopped - volumes: - - ../chp.sql:/var/lib/postgresql/data - env_file: - - .dev-db.env - -volumes: - static_volume: diff --git a/entrypoint.sh b/entrypoint.sh deleted file mode 100644 index 40e5a12..0000000 --- a/entrypoint.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/sh - -# Wait for Database image to start -if [ "$DATABASE" = "postgres" ] -then - echo "Waiting for postgres..." - - while ! nc -z $SQL_HOST $SQL_PORT; do - sleep 0.1 - done - - echo "PostgreSQL started" -fi - -# Run django migrations and collect static -echo "Collect static files" -python3 manage.py collectstatic --noinput - -echo "Make database migrations" -python3 manage.py makemigrations - -echo "Apply database migrations" -python3 manage.py migrate - -exec "$@" \ No newline at end of file diff --git a/gennifer/pidc b/gennifer/pidc new file mode 160000 index 0000000..39646c6 --- /dev/null +++ b/gennifer/pidc @@ -0,0 +1 @@ +Subproject commit 39646c6953f4f5cdb3191ef36c02d47e350269d7 From e747aa3dd7ba8e151a0e2a3a99d473d98f323fd5 Mon Sep 17 00:00:00 2001 From: Chase Yakaboski Date: Thu, 25 May 2023 19:40:07 -0400 Subject: [PATCH 046/132] meta_kg is working with pydantic reasoner, also implemented templating. Need to work on processing --- Dockerfile | 13 +- chp_api/chp_api/settings.py | 5 +- chp_api/dispatcher/base.py | 49 +++---- chp_api/dispatcher/curie_database.py | 60 --------- .../migrations/0007_template_templatematch.py | 34 +++++ chp_api/dispatcher/models.py | 12 ++ chp_api/dispatcher/scripts/templater.py | 126 ++++++++++++++++++ chp_api/dispatcher/urls.py | 1 - chp_api/dispatcher/views.py | 37 ++--- dev-deployment-script | 5 +- requirements-dev.txt | 4 - requirements.txt | 1 + 12 files changed, 205 insertions(+), 142 deletions(-) delete mode 100644 chp_api/dispatcher/curie_database.py create mode 100755 chp_api/dispatcher/migrations/0007_template_templatematch.py create mode 100644 chp_api/dispatcher/scripts/templater.py delete mode 100644 requirements-dev.txt diff --git a/Dockerfile b/Dockerfile index ff22ebe..6c5ef86 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,9 +13,6 @@ WORKDIR /usr/src/chp_api # && apt-get install -y git python3-pip python3-dev #dos2unix -RUN git clone --single-branch --branch master https://github.com/di2ag/trapi_model.git -RUN git clone --single-branch --branch master https://github.com/di2ag/chp_utils.git -RUN git clone --single-branch --branch master https://github.com/di2ag/chp_look_up.git RUN git clone --single-branch --branch master https://github.com/di2ag/gene-specificity.git # lint @@ -27,15 +24,6 @@ RUN git clone --single-branch --branch master https://github.com/di2ag/gene-spec COPY ./requirements.txt . RUN pip3 wheel --no-cache-dir --no-deps --wheel-dir /usr/src/chp_api/wheels -r requirements.txt -# gather trapi model wheel -RUN cd trapi_model && python3 setup.py bdist_wheel && cd dist && cp trapi_model-*-py3-none-any.whl /usr/src/chp_api/wheels - -# gather chp-utils wheel -RUN cd chp_utils && python3 setup.py bdist_wheel && cd dist && cp chp_utils-*-py3-none-any.whl /usr/src/chp_api/wheels - -#gather chp_look_up wheel -RUN cd chp_look_up && python3 setup.py bdist_wheel && cd dist && cp chp_look_up-*-py3-none-any.whl /usr/src/chp_api/wheels - #gather gene specificity wheel RUN cd gene-specificity && python3 setup.py bdist_wheel && cd dist && cp gene_specificity-*-py3-none-any.whl /usr/src/chp_api/wheels @@ -85,6 +73,7 @@ RUN pip3 install --no-cache /wheels/* COPY ./chp_api/chp_api $APP_HOME/chp_api COPY ./chp_api/manage.py $APP_HOME COPY ./chp_api/dispatcher $APP_HOME/dispatcher +COPY ./chp_db_fixture.json.gz $APP_HOME COPY ./gunicorn.config.py $APP_HOME # chown all the files to the app user diff --git a/chp_api/chp_api/settings.py b/chp_api/chp_api/settings.py index 8135cc4..a130fac 100644 --- a/chp_api/chp_api/settings.py +++ b/chp_api/chp_api/settings.py @@ -48,12 +48,9 @@ 'gene_specificity', ] -OTHER_APPS = [ - 'chp_utils' - ] # CHP Versions -VERSIONS = {app_name: app.__version__ for app_name, app in [(app_name, import_module(app_name)) for app_name in INSTALLED_CHP_APPS + OTHER_APPS]} +VERSIONS = {app_name: app.__version__ for app_name, app in [(app_name, import_module(app_name)) for app_name in INSTALLED_CHP_APPS]} # Sets up installed apps relevent to django INSTALLED_APPS = INSTALLED_BASE_APPS + INSTALLED_CHP_APPS diff --git a/chp_api/dispatcher/base.py b/chp_api/dispatcher/base.py index ad7f631..f679696 100644 --- a/chp_api/dispatcher/base.py +++ b/chp_api/dispatcher/base.py @@ -8,13 +8,14 @@ from importlib import import_module from collections import defaultdict -from .curie_database import merge_curies_databases, CurieDatabase +#from .curie_database import merge_curies_databases, CurieDatabase ## NEED TO REMOVE from .models import Transaction, App, DispatcherSettings +from reasoner_pydantic import MetaKnowledgeGraph -from chp_utils.trapi_query_processor import BaseQueryProcessor -from trapi_model.meta_knowledge_graph import MetaKnowledgeGraph, merge_meta_knowledge_graphs -from trapi_model.query import Query -from trapi_model.biolink import TOOLKIT +from chp_utils.trapi_query_processor import BaseQueryProcessor ## NEED TO REMOVE +#from trapi_model.meta_knowledge_graph import MetaKnowledgeGraph, merge_meta_knowledge_graphs ## NEED TO REMOVE +from trapi_model.query import Query ## NEED TO REMOVE +from trapi_model.biolink import TOOLKIT ## NEED TO REMOVE # Setup logging @@ -25,12 +26,6 @@ def note(self, message, *args, **kwargs): logging.Logger.note = note logger = logging.getLogger(__name__) -# Installed CHP Apps -#CHP_APPS = [ -# "chp.app", -# "chp_look_up.app", -# ] - # Import CHP Apps APPS = [import_module(app+'.app_interface') for app in settings.INSTALLED_CHP_APPS] @@ -49,37 +44,27 @@ def __init__(self, request, trapi_version): self.trapi_version = trapi_version super().__init__(None) - def get_curies(self): - curies_dbs = [] - for app, app_name in zip(APPS, settings.INSTALLED_CHP_APPS): - app_db_obj = App.objects.get(name=app_name) - # Load location from uploaded Zenodo files - if app_db_obj.curies_zenodo_file: - curies = app_db_obj.curies_zenodo_file.load_file(base_url="https://sandbox.zenodo.org/api/records") - curies_db = CurieDatabase(curies=curies) - # Load default location - else: - get_app_curies_fn = getattr(app, 'get_curies') - curies_db = get_app_curies_fn() - curies_dbs.append(curies_db) - return merge_curies_databases(curies_dbs) - def get_meta_knowledge_graph(self): # Get current trapi and biolink versions dispatcher_settings = DispatcherSettings.load() - meta_kgs = [] + merged_meta_kg = None for app, app_name in zip(APPS, settings.INSTALLED_CHP_APPS): app_db_obj = App.objects.get(name=app_name) # Load location from uploaded Zenodo files if app_db_obj.meta_knowledge_graph_zenodo_file: meta_kg = app_db_obj.meta_knowledge_graph_zenodo_file.load_file(base_url="https://sandbox.zenodo.org/api/records") - meta_kg = MetaKnowledgeGraph.load(dispatcher_settings.trapi_version, None, meta_knowledge_graph=meta_kg) - # Load default location + # Load default location else: get_app_meta_kg_fn = getattr(app, 'get_meta_knowledge_graph') - meta_kg = get_app_meta_kg_fn() - meta_kgs.append(meta_kg) - return merge_meta_knowledge_graphs(meta_kgs) + #TODO, when apps are set up correctly, they should return a pydantic_reasoner metakg. So I will have to remove .to_dict() + # from deprecated trapi_model + meta_kg = get_app_meta_kg_fn().to_dict() + meta_kg = MetaKnowledgeGraph.parse_obj(meta_kg) + if merged_meta_kg is None: + merged_meta_kg = meta_kg + else: + merged_meta_kg.update(meta_kg) + return merged_meta_kg def process_invalid_trapi(self, request): invalid_query_json = request.data diff --git a/chp_api/dispatcher/curie_database.py b/chp_api/dispatcher/curie_database.py deleted file mode 100644 index 1ea7cac..0000000 --- a/chp_api/dispatcher/curie_database.py +++ /dev/null @@ -1,60 +0,0 @@ -""" A helper class to handle CHP supported curies. -""" -import json - -from trapi_model.biolink.constants import * - -def merge_curies_databases(list_of_curies_dbs): - if len(list_of_curies_dbs) == 1: - return list_of_curies_dbs[0].to_dict() - merged = list_of_curies_dbs[0].to_dict() - for curies_db in list_of_curies_dbs[1:]: - for biolink_entity, curies_info_dict in curies_db.to_dict().items(): - if biolink_entity not in merged: - merged[biolink_entity] = curies_info_dict - continue - for curie, info in curies_info_dict.items(): - if curie not in merged[biolink_entity]: - merged[biolink_entity][curie] = info - continue - new_info = set.union( - *[ - set(merged[biolink_entity][curie]), - set(info), - ] - ) - merged[biolink_entity][curie] = [info for info in new_info if info] - return merged - - -class CurieDatabase: - def __init__(self, curies=None, curies_filename=None): - if curies is None and curies_filename is None: - raise ValueError('Must pass in either conflation map or filename.') - elif curies is not None and curies_filename is not None: - raise ValueError('Must pass in either conflation map or filename, not both.') - self.curies = self.load_curies(curies, curies_filename) - - @staticmethod - def load_curies(curies, curies_filename): - _curies = {} - if curies_filename is not None: - with open(curies_filename) as f_: - curies = json.load(f_) - for biolink_entity, curies_list in curies.items(): - _curies[get_biolink_entity(biolink_entity)] = curies_list - return _curies - - def to_dict(self): - curies_dict = {} - for biolink_entity, curies_list in self.curies.items(): - curies_dict[biolink_entity.get_curie()] = curies_list - return curies_dict - - - def json(self, filename=None): - if filename is None: - return json.dumps(self.to_dict(), indent=2) - with open(filename, 'w') as f_: - json.dump(self.to_dict(), f_, indent=2) - diff --git a/chp_api/dispatcher/migrations/0007_template_templatematch.py b/chp_api/dispatcher/migrations/0007_template_templatematch.py new file mode 100755 index 0000000..f67e43f --- /dev/null +++ b/chp_api/dispatcher/migrations/0007_template_templatematch.py @@ -0,0 +1,34 @@ +# Generated by Django 4.2.1 on 2023-05-25 19:53 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('dispatcher', '0006_dispatchersettings'), + ] + + operations = [ + migrations.CreateModel( + name='Template', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('app_name', models.CharField(max_length=128)), + ('subject', models.CharField(max_length=128)), + ('object', models.CharField(max_length=128)), + ('predicate', models.CharField(max_length=128)), + ], + ), + migrations.CreateModel( + name='TemplateMatch', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('subject', models.CharField(max_length=128)), + ('object', models.CharField(max_length=128)), + ('predicate', models.CharField(max_length=128)), + ('template', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='dispatcher.template')), + ], + ), + ] diff --git a/chp_api/dispatcher/models.py b/chp_api/dispatcher/models.py index 88d28a0..8da796f 100644 --- a/chp_api/dispatcher/models.py +++ b/chp_api/dispatcher/models.py @@ -32,6 +32,18 @@ class App(models.Model): def __str__(self): return self.name +class Template(models.Model): + app_name = models.CharField(max_length=128) + subject = models.CharField(max_length=128) + object = models.CharField(max_length=128) + predicate = models.CharField(max_length=128) + +class TemplateMatch(models.Model): + template = models.ForeignKey(Template, on_delete=models.CASCADE) + subject = models.CharField(max_length=128) + object = models.CharField(max_length=128) + predicate = models.CharField(max_length=128) + class Transaction(models.Model): id = models.CharField(max_length=100, primary_key=True) date_time = models.DateTimeField(auto_now=True) diff --git a/chp_api/dispatcher/scripts/templater.py b/chp_api/dispatcher/scripts/templater.py new file mode 100644 index 0000000..8d5feee --- /dev/null +++ b/chp_api/dispatcher/scripts/templater.py @@ -0,0 +1,126 @@ +import itertools +from bmt import Toolkit +from importlib import import_module +from collections import defaultdict +from django.core.management.base import BaseCommand +from reasoner_pydantic import MetaKnowledgeGraph, MetaEdge +from django.conf import settings +from ..models import App, Template, TemplateMatch + +APPS = [import_module(app+'.app_interface') for app in settings.INSTALLED_CHP_APPS] +TK = Toolkit() + + +def _collect_metakgs_by_app(): + # Collect each app's meta kg + app_to_meta_kg = dict() + for app, app_name in zip(APPS, settings.INSTALLED_CHP_APPS): + app_db_obj = App.objects.get(name=app_name) + if app_db_obj.meta_knowledge_graph_zenodo_file: + meta_kg = app_db_obj.meta_knowledge_graph_zenodo_file.load_file(base_url="https://sandbox.zenodo.org/api/records") + # Load default location + else: + get_app_meta_kg_fn = getattr(app, 'get_meta_knowledge_graph') + meta_kg = get_app_meta_kg_fn() + meta_kg = MetaKnowledgeGraph.parse_obj(meta_kg.to_dict()) + app_to_meta_kg[app_name] = meta_kg + return app_to_meta_kg + +def _build_app_templates(meta_kg): + matcher = defaultdict(set) + for meta_edge in meta_kg.edges: + subject_ancestors = TK.get_ancestors(meta_edge.subject, reflexive=True, mixin=False, formatted=True) + predicate_ancestors = TK.get_ancestors(meta_edge.predicate, reflexive=True, mixin=False, formatted=True) + object_ancestors = TK.get_ancestors(meta_edge.object, reflexive=True, mixin=False, formatted=True) + for edge in itertools.product(*[subject_ancestors, predicate_ancestors, object_ancestors]): + template_meta_edge = MetaEdge(subject=edge[0], predicate=edge[1], object=edge[2]) + matcher[template_meta_edge].add(meta_edge) + return dict(matcher) + +def run(): + app_to_meta_kg = _collect_metakgs_by_app() + app_to_templates = dict() + for app_name, meta_kg in app_to_meta_kg.items(): + app_to_templates[app_name] = _build_app_templates(meta_kg) + #Template.objects.all().delete() + #TemplateMatch.objects.all().delete() + + # Populate Templater + for app_name, app_templates in app_to_templates.items(): + for app_template, app_template_matches in app_templates.items(): + template = Template(app_name = app_name, + subject = app_template.subject, + object = app_template.object, + predicate = app_template.predicate) + template.save() + for app_template_match in app_template_matches: + template_match = TemplateMatch(template=template, + subject = app_template_match.subject, + object = app_template_match.object, + predicate = app_template_match.predicate) + + +''' +class Templater(BaseCommand): + + def _collect_metakgs_by_app(self): + # Collect each app's meta kg + app_to_meta_kg = dict() + for app, app_name in zip(APPS, settings.INSTALLED_CHP_APPS): + app_db_obj = App.objects.get(name=app_name) + if app_db_obj.meta_knowledge_graph_zenodo_file: + meta_kg = app_db_obj.meta_knowledge_graph_zenodo_file.load_file(base_url="https://sandbox.zenodo.org/api/records") + # Load default location + else: + get_app_meta_kg_fn = getattr(app, 'get_meta_knowledge_graph') + meta_kg = get_app_meta_kg_fn() + meta_kg = MetaKnowledgeGraph.parse_obj(meta_kg) + app_to_meta_kg[app_name] = meta_kg + + def _build_app_templates(self, meta_kg): + matcher = defaultdict(set) + for meta_edge in meta_knowledge_graph.edges: + subject_ancestors = TK.get_ancestors(meta_edge.subject, reflexive=True, mixin=False, formatted=True) + predicate_ancestors = TK.get_ancestors(meta_edge.predicate, reflexive=True, mixin=False, formatted=True) + object_ancestors = TK.get_ancestors(meta_edge.object, reflexive=True, mixin=False, formatted=True) + for edge in itertools.product(*[subject_ancestors, predicate_ancestors, object_ancestors]): + template_meta_edge = MetaEdge(subject=edge[0], predicate=edge[1], object=edge[2]) + matcher[template_meta_edge].add(meta_edge) + return dict(matcher) + + def handle(self, *args, **options): + app_to_meta_kg = self._collect_metakgs_by_app() + app_to_templates = dict() + for app_name, meta_kg in app_to_meta_kg.items(): + app_to_templates[app_name] = self._build_app_templates(meta_kg) + Templater.objects.all().delete() + Template.objects.all().delete() + TemplateMatch.objects.all().delete() + + # Populate Templater + for app_name, app_templates in app_to_templates.items(): + templater = Templater(app_name) + templater.save() + + for app_template, app_template_matches in app_template_matcher.items(): + template = Template(app_name = templater, + subject = app_template.subject, + object = app_template.object, + predicate = app_template.predicate) + template.save() + + for app_template_match in app_template_matches: + print("vals", template_match.subject, template_match.object, template_match.predicate) + template_match = TemplateMatch(template=template, + subject = app_template_match.subject, + object = app_template_match.object, + predicate = app_template_match.predicate) + self.stdout.write(self.style.SUCCESS('Data loaded successfully')) + +''' + + + + + + diff --git a/chp_api/dispatcher/urls.py b/chp_api/dispatcher/urls.py index e508e06..a5d1588 100644 --- a/chp_api/dispatcher/urls.py +++ b/chp_api/dispatcher/urls.py @@ -23,7 +23,6 @@ path('query/', views.query.as_view()), path('query', views.query.as_view()), path('meta_knowledge_graph/', views.meta_knowledge_graph.as_view()), - path('curies/', views.curies.as_view()), path('versions/', views.versions.as_view()), path('transactions/', views.TransactionList.as_view(), name='transaction-list'), path('recent/', views.RecentTransactionList.as_view(), name='recent-transaction-list'), diff --git a/chp_api/dispatcher/views.py b/chp_api/dispatcher/views.py index a96c775..46cf960 100644 --- a/chp_api/dispatcher/views.py +++ b/chp_api/dispatcher/views.py @@ -31,35 +31,18 @@ def post(self, request): ) # Process Query query = None - try: - query = dispatcher.process_request( - request, - trapi_version=dispatcher_settings.trapi_version, - ) - except Exception as e: - if 'Workflow Error' in str(e): - return dispatcher.process_invalid_workflow(request, str(e)) - else: - return dispatcher.process_invalid_trapi(request) - # Return responses - return dispatcher.get_response(query) - -class curies(APIView): - - def get(self, request): - # Get current trapi and biolink versions - dispatcher_settings = DispatcherSettings.load() - - if request.method == 'GET': - # Initialize dispatcher - dispatcher = Dispatcher( + #try: + query = dispatcher.process_request( request, - dispatcher_settings.trapi_version, + trapi_version=dispatcher_settings.trapi_version, ) - - # Get all chp app curies - curies_db = dispatcher.get_curies() - return JsonResponse(curies_db) + #except Exception as e: + # if 'Workflow Error' in str(e): + # return dispatcher.process_invalid_workflow(request, str(e)) + # else: + # return dispatcher.process_invalid_trapi(request) + # Return responses + return dispatcher.get_response(query) class meta_knowledge_graph(APIView): diff --git a/dev-deployment-script b/dev-deployment-script index 3bc83ca..d7e5daf 100755 --- a/dev-deployment-script +++ b/dev-deployment-script @@ -5,7 +5,8 @@ django_superuser_username='cat secrets/chp_api/django_superuser_username.txt' django_superuser_email='cat secrets/chp_api/django_superuser_email.txt' # Only to be run when building on dev machine -docker compose build --no-cache +# use --no-cache if need to rebuild submodules +docker compose build #--no-cache docker compose up -d @@ -16,8 +17,8 @@ docker compose run --user root chp-api python3 manage.py createsuperuser --no-in # Load apps docker compose run chp-api python3 manage.py runscript load_db_apps +docker compose run chp-api python3 manage.py runscript templater docker compose run --user root chp-api python3 manage.py collectstatic --noinput - echo "Check logs with: docker compose logs -f" diff --git a/requirements-dev.txt b/requirements-dev.txt deleted file mode 100644 index a873696..0000000 --- a/requirements-dev.txt +++ /dev/null @@ -1,4 +0,0 @@ -chp_utils @ git+https://github.com/di2ag/chp_utils.git@master -trapi_model @ git+https://github.com/di2ag/trapi_model.git@master -chp_look_up @ git+https://github.com/di2ag/chp_look_up.git@master -gene-specificity @ git+https://github.com/di2ag/gene-specificity.git@master diff --git a/requirements.txt b/requirements.txt index 081b490..1afe81e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ djangorestframework psycopg2-binary +reasoner_pydantic django-environ django-hosts gunicorn From dad6790473a876a60ade708076c4c449fa17bddd Mon Sep 17 00:00:00 2001 From: Chase Yakaboski Date: Fri, 26 May 2023 17:46:26 -0400 Subject: [PATCH 047/132] initial build with working templating. Removed all references to trapi_model and chp_utils. Ultimately this will be completely captured using pydantic_reasoner. Currently gene-spec app wont work because it needs trapi-model, so next step is to rewrite that app using pydantic. I will also need to work on merging resutls after, trimming any unnecesary fat and exception handling --- Dockerfile.new | 39 ----- chp_api/chp_api/settings.py | 2 - chp_api/chp_api/settings_build.py | 161 ------------------- chp_api/dispatcher/base.py | 195 +++++++++++------------- chp_api/dispatcher/scripts/templater.py | 65 +------- chp_api/dispatcher/views.py | 6 +- 6 files changed, 90 insertions(+), 378 deletions(-) delete mode 100644 Dockerfile.new delete mode 100644 chp_api/chp_api/settings_build.py diff --git a/Dockerfile.new b/Dockerfile.new deleted file mode 100644 index 0a825d4..0000000 --- a/Dockerfile.new +++ /dev/null @@ -1,39 +0,0 @@ -################ -# venv builder # -################ -FROM python:3.8.3 as venv_builder - -COPY requirements.txt . -COPY requirements-dev.txt . -RUN python3 -m venv /opt/venv -RUN /opt/venv/bin/pip install --upgrade pip -RUN /opt/venv/bin/pip install -r requirements.txt -RUN /opt/venv/bin/pip install -r requirements-dev.txt -########### -# CHP API # -########### -FROM python:3.8.3-slim as chp-api - -# set environment variables -ENV PYTHONDONTWRITEBYTECODE 1 -ENV PYTHONUNBUFFERED 1 -ENV TZ=America/New_York -ENV SERVER_DIR=/chp_api/ -ENV VIRTUAL_ENV_PATH=/opt/venv - -# copy venv from venv builder image -COPY --from=venv_builder ${VIRTUAL_ENV_PATH} ${VIRTUAL_ENV_PATH} - -# copy project -COPY ./chp_api $SERVER_DIR - -# copy entry point -COPY ./entrypoint.sh ${SERVER_DIR} -COPY ./gunicorn.config.py ${SERVER_DIR} -COPY ./chp_db_fixture.json.gz ${SERVER_DIR} - -# enter app directory -WORKDIR $SERVER_DIR - -# Enable venv -ENV PATH="/opt/venv/bin:$PATH" diff --git a/chp_api/chp_api/settings.py b/chp_api/chp_api/settings.py index a130fac..180461f 100644 --- a/chp_api/chp_api/settings.py +++ b/chp_api/chp_api/settings.py @@ -40,7 +40,6 @@ 'django.contrib.staticfiles', 'rest_framework', 'dispatcher.apps.DispatcherConfig', - 'chp_utils', 'django_extensions', ] @@ -48,7 +47,6 @@ 'gene_specificity', ] - # CHP Versions VERSIONS = {app_name: app.__version__ for app_name, app in [(app_name, import_module(app_name)) for app_name in INSTALLED_CHP_APPS]} diff --git a/chp_api/chp_api/settings_build.py b/chp_api/chp_api/settings_build.py deleted file mode 100644 index b63e550..0000000 --- a/chp_api/chp_api/settings_build.py +++ /dev/null @@ -1,161 +0,0 @@ -""" -Base Django settings for chp_api project. - -Generated by 'django-admin startproject' using Django 3.0.7. - -For more information on this file, see -https://docs.djangoproject.com/en/3.0/topics/settings/ - -For the full list of settings and their values, see -https://docs.djangoproject.com/en/3.0/ref/settings/ -""" -import os -from importlib import import_module -import environ as environ # type: ignore - -# Initialise environment variables -env = environ.Env() -environ.Env.read_env() - -# SECURITY WARNING: don't run with debug turned on in production! -DEBUG = int(env("DEBUG", default=0)) - -# Build paths inside the project like this: os.path.join(BASE_DIR, ...) -BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - -DATA_UPLOAD_MAX_MEMORY_SIZE = None -REST_FRAMEWORK = { - 'DEFAULT_PARSER_CLASSES': [ - 'rest_framework.parsers.JSONParser', - ] -} - -# Application definition -INSTALLED_BASE_APPS = [ - 'django.contrib.admin', - 'django.contrib.auth', - 'django.contrib.contenttypes', - 'django.contrib.sessions', - 'django.contrib.messages', - 'django.contrib.staticfiles', - 'rest_framework', - 'dispatcher.apps.DispatcherConfig', - 'chp_utils', - 'django_extensions', -] - -INSTALLED_CHP_APPS = [ - 'chp_look_up', -# 'chp_learn', - 'gene_specificity', - ] - -OTHER_APPS = [ - 'chp_utils' - ] - -# CHP Versions -VERSIONS = {app_name: app.__version__ for app_name, app in [(app_name, import_module(app_name)) for app_name in INSTALLED_CHP_APPS + OTHER_APPS]} - -# Sets up installed apps relevent to django -INSTALLED_APPS = INSTALLED_BASE_APPS + INSTALLED_CHP_APPS - -MIDDLEWARE = [ - 'django.middleware.security.SecurityMiddleware', - 'django.contrib.sessions.middleware.SessionMiddleware', - 'django.middleware.common.CommonMiddleware', - 'django.middleware.csrf.CsrfViewMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.messages.middleware.MessageMiddleware', - 'django.middleware.clickjacking.XFrameOptionsMiddleware', -] - -ROOT_URLCONF = 'dispatcher.urls' - -TEMPLATES = [ - { - 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'DIRS': [], - 'APP_DIRS': True, - 'OPTIONS': { - 'context_processors': [ - 'django.template.context_processors.debug', - 'django.template.context_processors.request', - 'django.contrib.auth.context_processors.auth', - 'django.contrib.messages.context_processors.messages', - ], - }, - }, -] - -# Logging -LOGGING = { - 'version': 1, - 'disable_existing_loggers': False, - 'handlers': { - 'console': { - 'class': 'logging.StreamHandler', - }, - }, - 'root': { - 'handlers': ['console'], - 'level': 'WARNING', - }, -} - -WSGI_APPLICATION = 'chp_api.wsgi.application' - -# Password validation -# https://docs.djangoproject.com/en/3.0/ref/settings/#auth-password-validators - -AUTH_PASSWORD_VALIDATORS = [ - { - 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', - }, - { - 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', - }, - { - 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', - }, - { - 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', - }, -] - -# Internationalization -# https://docs.djangoproject.com/en/3.0/topics/i18n/ -LANGUAGE_CODE = 'en-us' - -TIME_ZONE = 'UTC' - -USE_I18N = True - -USE_L10N = True - -USE_TZ = True - -# Static files (CSS, JavaScript, Images) -# https://docs.djangoproject.com/en/3.0/howto/static-files/ - -STATIC_URL = '/staticfiles/' -STATIC_ROOT = os.path.join(BASE_DIR, "staticfiles") - -# Hosts Configuration -#ROOT_HOSTCONF = 'chp_api.hosts' - -# Database -# https://docs.djangoproject.com/en/3.0/ref/settings/#databases -DATABASES = { - 'default': { - "ENGINE": os.environ.get("SQL_ENGINE", "django.db.backends.sqlite3"), - "NAME": os.environ.get("SQL_DATABASE", os.path.join(BASE_DIR, "db.sqlite3")), - "USER": os.environ.get("SQL_USER", "user"), - "PASSWORD": os.environ.get("SQL_PASSWORD", "password"), - "HOST": os.environ.get("SQL_HOST", "localhost"), - "PORT": os.environ.get("SQL_PORT", "5432"), - } -} -ALLOWED_HOSTS = env("DJANGO_ALLOWED_HOSTS").split(" ") -# SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = env("SECRET_KEY") diff --git a/chp_api/dispatcher/base.py b/chp_api/dispatcher/base.py index f679696..6e9dbdb 100644 --- a/chp_api/dispatcher/base.py +++ b/chp_api/dispatcher/base.py @@ -1,5 +1,6 @@ import logging import time +import itertools from copy import deepcopy from re import A from django.http import JsonResponse @@ -9,13 +10,14 @@ from collections import defaultdict #from .curie_database import merge_curies_databases, CurieDatabase ## NEED TO REMOVE -from .models import Transaction, App, DispatcherSettings -from reasoner_pydantic import MetaKnowledgeGraph +from .models import Transaction, App, DispatcherSettings, Template, TemplateMatch +from reasoner_pydantic import MetaKnowledgeGraph, Message, MetaEdge +from reasoner_pydantic.qgraph import QNode, QEdge -from chp_utils.trapi_query_processor import BaseQueryProcessor ## NEED TO REMOVE +#from chp_utils.trapi_query_processor import BaseQueryProcessor ## NEED TO REMOVE #from trapi_model.meta_knowledge_graph import MetaKnowledgeGraph, merge_meta_knowledge_graphs ## NEED TO REMOVE -from trapi_model.query import Query ## NEED TO REMOVE -from trapi_model.biolink import TOOLKIT ## NEED TO REMOVE +#from trapi_model.query import Query ## NEED TO REMOVE +#from trapi_model.biolink import TOOLKIT ## NEED TO REMOVE # Setup logging @@ -29,7 +31,7 @@ def note(self, message, *args, **kwargs): # Import CHP Apps APPS = [import_module(app+'.app_interface') for app in settings.INSTALLED_CHP_APPS] -class Dispatcher(BaseQueryProcessor): +class Dispatcher(): def __init__(self, request, trapi_version): """ Base API Query Processor class used to abstract the processing infrastructure from the views. Inherits from the CHP Utilities Trapi Query Processor which handles @@ -40,9 +42,7 @@ def __init__(self, request, trapi_version): """ self.request_data = deepcopy(request.data) - #self.chp_config, self.passed_subdomain = self.get_app_config(request) self.trapi_version = trapi_version - super().__init__(None) def get_meta_knowledge_graph(self): # Get current trapi and biolink versions @@ -77,21 +77,12 @@ def process_invalid_workflow(self, request, status_msg): return JsonResponse(invalid_query_json, status=400) def process_request(self, request, trapi_version): - """ Helper function that extracts the query from the message. + """ Helper function that extracts the message from the request data. """ - logger.info('Starting query.') - query = Query.load( - self.trapi_version, - biolink_version=None, - query=request.data - ) - - # Setup query in Base Processor - self.setup_query(query) - - logger.info('Query loaded') - - return query + logger.info('Starting query') + message = Message.parse_obj(request.data['message']) + logger.info('Message loaded') + return message def get_app_configs(self, query): """ Should get a base app configuration for your app or nothing. @@ -113,102 +104,85 @@ def get_trapi_interfaces(self, app_configs): base_interfaces.append(get_trapi_interface_fn(app_config)) return base_interfaces - def collect_app_queries(self, queries_list_of_lists): - all_queries = [] - for queries in queries_list_of_lists: - if type(queries) == list: - all_queries.extend(queries) - else: - all_queries.append(queries) - return all_queries - - def get_response(self, query): + def extract_message_templates(self, Message): + assert len(Message.query_graph.edges) == 1, 'CHP apps do not support multihop queries' + subject = None + predicates = [] + for edge_id, q_edge in Message.query_graph.edges.items(): + subject = q_edge.subject + if q_edge.predicates is None: + q_edge = QEdge(subject = q_edge.subject, predicates=['biolink:related_to'], object = q_edge.object) + for predicate in q_edge.predicates: + predicates.append(predicate) + subject_categories = [] + object_categories = [] + for node_id, q_node in Message.query_graph.nodes.items(): + if q_node.categories is None: + q_node = QNode(categories=['biolink:Entity']) + for category in q_node.categories: + if node_id == subject: + subject_categories.append(category) + else: + object_categories.append(category) + templates = [] + for edge in itertools.product(*[subject_categories, predicates, object_categories]): + meta_edge = MetaEdge(subject=edge[0], predicate=edge[1], object=edge[2]) + templates.append(meta_edge) + return templates + + def get_app_template_matches(self, app_name, templates): + template_matches = [] + for template in templates: + matches = TemplateMatch.objects.filter(template__app_name=app_name, + template__subject = template.subject, + template__object = template.object, + template__predicate = template.predicate) + template_matches.extend(matches) + return template_matches + + def apply_templates_to_message(self, message, matching_templates): + consistent_queries = [] + for template in matching_templates: + consistent_query = message.copy(deep=True) + for edge_id, edge in consistent_query.query_graph.edges.items(): + edge.predicate = template.predicate + subject_id = edge.subject + object_id = edge.object + consistent_query.query_graph.nodes[subject_id].categories = template.subject + consistent_query.query_graph.nodes[object_id].categories = template.object + consistent_queries.append(consistent_query) + return consistent_queries + + def get_response(self, message): """ Main function of the processor that handles primary logic for obtaining a cached or calculated query response. """ - query_copy = query.get_copy() - start_time = time.time() - logger.info('Running query.') - - base_app_configs = self.get_app_configs(query_copy) - base_interfaces = self.get_trapi_interfaces(base_app_configs) - - # Expand - expand_queries = self.expand_batch_query(query) - - # For each app run the normalization and semops pipline - # Make a copy of the expanded queries for each app - app_queries = [[q.get_copy() for q in expand_queries] for _ in range(len(base_interfaces))] - consistent_app_queries = [] - inconsistent_app_queries = [] - app_normalization_maps = [] - for interface, _expand_queries in zip(base_interfaces, app_queries): - _ex_copy = [] - # Normalize to Preferred Curies - normalization_time = time.time() - normalize_queries, normalization_map = self.normalize_to_preferred( - _expand_queries, - meta_knowledge_graph=interface.get_meta_knowledge_graph(), - with_normalization_map=True, - ) - app_normalization_maps.append(normalization_map) - logger.info('Normalizaion time: {} seconds.'.format(time.time() - normalization_time)) - # Conflate - conflation_time = time.time() - - conflate_queries = self.conflate_categories( - normalize_queries, - conflation_map=interface.get_conflation_map(), - ) - logger.info('Conflation time: {} seconds.'.format(time.time() - conflation_time)) - # Onto Expand - onto_time = time.time() - onto_queries = self.expand_supported_ontological_descendants( - conflate_queries, - curies_database=interface.get_curies(), - ) - logger.info('Ontological expansion time: {} seconds.'.format(time.time() - onto_time)) - # Semantic Ops Expand - semops_time = time.time() - semops_queries = self.expand_with_semantic_ops( - onto_queries, - meta_knowledge_graph=interface.get_meta_knowledge_graph(), - ) - logger.info('Sem ops time: {} seconds.'.format(time.time() - semops_time)) - # Filter out inconsistent queries - filter_time = time.time() - consistent_queries, inconsistent_queries = self.filter_queries_inconsistent_with_meta_knowledge_graph( - semops_queries, - meta_knowledge_graph=interface.get_meta_knowledge_graph(), - with_inconsistent_queries=True - ) - logger.info('Consistency filter time: {} seconds.'.format(time.time() - filter_time)) + logger.info('Running message.') + start_time = time.time() + logger.info('Getting message templates.') + message_templates = self.extract_message_templates(message) - logger.info('Number of consistent queries derived from passed query: {}.'.format(len(consistent_queries))) - consistent_app_queries.append(consistent_queries) - inconsistent_app_queries.append(inconsistent_queries) - # Ensure that there are actually consistent queries that have been extracted - if sum([len(_qs) for _qs in consistent_app_queries]) == 0: - # Add all logs from inconsistent queries of all apps - all_inconsistent_queries = self.collect_app_queries(inconsistent_queries) - query_copy = self.add_logs_from_query_list(query_copy, all_inconsistent_queries) - query_copy.set_status('Bad request. See description.') - query_copy.set_description('Could not extract any supported queries from query graph.') - self.add_transaction(query_copy) - return JsonResponse(query_copy.to_dict()) - # Collect responses from each CHP app app_responses = [] app_logs = [] - app_status = [] + app_statuses = [] app_descriptions = [] - for app, consistent_queries in zip(APPS, consistent_app_queries): - get_app_response_fn = getattr(app, 'get_response') - responses, logs, status, description = get_app_response_fn(consistent_queries) - app_responses.extend(responses) - app_logs.extend(logs) - app_status.append(status) - app_descriptions.append(description) + for app, app_name in zip(APPS, settings.INSTALLED_CHP_APPS): + logger.info('Checking template matches for {}'.format(app_name)) + matching_templates = self.get_app_template_matches(app_name, message_templates) + logger.info('Detected {} matches for {}'.format(len(matching_templates), app_name)) + if len(matching_templates) > 0: + logger.info('Constructing queries on matching templates') + consistent_app_queries = self.apply_templates_to_message(message, matching_templates) + logger.info('Sending {} consistent queries') + get_app_response_fn = getattr(app, 'get_response') + responses, logs, status, description = get_app_response_fn(consistent_app_queries) + app_responses.extend(responses) + app_logs.extend(logs) + app_statuses.append(status) + app_descriptions.append(description) + print(app_responses) + ''' # Check if any responses came back from any apps if len(app_responses) == 0: # Add logs from consistent queries of all apps @@ -256,6 +230,7 @@ def get_response(self, query): self.add_transaction(unnormalized_response) return JsonResponse(unnormalized_response.to_dict()) + ''' def add_logs_from_query_list(self, target_query, query_list): for query in query_list: diff --git a/chp_api/dispatcher/scripts/templater.py b/chp_api/dispatcher/scripts/templater.py index 8d5feee..3c96f42 100644 --- a/chp_api/dispatcher/scripts/templater.py +++ b/chp_api/dispatcher/scripts/templater.py @@ -42,8 +42,8 @@ def run(): app_to_templates = dict() for app_name, meta_kg in app_to_meta_kg.items(): app_to_templates[app_name] = _build_app_templates(meta_kg) - #Template.objects.all().delete() - #TemplateMatch.objects.all().delete() + Template.objects.all().delete() + TemplateMatch.objects.all().delete() # Populate Templater for app_name, app_templates in app_to_templates.items(): @@ -58,66 +58,7 @@ def run(): subject = app_template_match.subject, object = app_template_match.object, predicate = app_template_match.predicate) - - -''' -class Templater(BaseCommand): - - def _collect_metakgs_by_app(self): - # Collect each app's meta kg - app_to_meta_kg = dict() - for app, app_name in zip(APPS, settings.INSTALLED_CHP_APPS): - app_db_obj = App.objects.get(name=app_name) - if app_db_obj.meta_knowledge_graph_zenodo_file: - meta_kg = app_db_obj.meta_knowledge_graph_zenodo_file.load_file(base_url="https://sandbox.zenodo.org/api/records") - # Load default location - else: - get_app_meta_kg_fn = getattr(app, 'get_meta_knowledge_graph') - meta_kg = get_app_meta_kg_fn() - meta_kg = MetaKnowledgeGraph.parse_obj(meta_kg) - app_to_meta_kg[app_name] = meta_kg - - def _build_app_templates(self, meta_kg): - matcher = defaultdict(set) - for meta_edge in meta_knowledge_graph.edges: - subject_ancestors = TK.get_ancestors(meta_edge.subject, reflexive=True, mixin=False, formatted=True) - predicate_ancestors = TK.get_ancestors(meta_edge.predicate, reflexive=True, mixin=False, formatted=True) - object_ancestors = TK.get_ancestors(meta_edge.object, reflexive=True, mixin=False, formatted=True) - for edge in itertools.product(*[subject_ancestors, predicate_ancestors, object_ancestors]): - template_meta_edge = MetaEdge(subject=edge[0], predicate=edge[1], object=edge[2]) - matcher[template_meta_edge].add(meta_edge) - return dict(matcher) - - def handle(self, *args, **options): - app_to_meta_kg = self._collect_metakgs_by_app() - app_to_templates = dict() - for app_name, meta_kg in app_to_meta_kg.items(): - app_to_templates[app_name] = self._build_app_templates(meta_kg) - Templater.objects.all().delete() - Template.objects.all().delete() - TemplateMatch.objects.all().delete() - - # Populate Templater - for app_name, app_templates in app_to_templates.items(): - templater = Templater(app_name) - templater.save() - - for app_template, app_template_matches in app_template_matcher.items(): - template = Template(app_name = templater, - subject = app_template.subject, - object = app_template.object, - predicate = app_template.predicate) - template.save() - - for app_template_match in app_template_matches: - print("vals", template_match.subject, template_match.object, template_match.predicate) - template_match = TemplateMatch(template=template, - subject = app_template_match.subject, - object = app_template_match.object, - predicate = app_template_match.predicate) - self.stdout.write(self.style.SUCCESS('Data loaded successfully')) - -''' + template_match.save() diff --git a/chp_api/dispatcher/views.py b/chp_api/dispatcher/views.py index 46cf960..4e9147a 100644 --- a/chp_api/dispatcher/views.py +++ b/chp_api/dispatcher/views.py @@ -1,6 +1,5 @@ """ CHP Core API Views """ -from jsonschema import ValidationError from copy import deepcopy from datetime import datetime, timedelta @@ -30,9 +29,8 @@ def post(self, request): dispatcher_settings.trapi_version, ) # Process Query - query = None #try: - query = dispatcher.process_request( + Message = dispatcher.process_request( request, trapi_version=dispatcher_settings.trapi_version, ) @@ -42,7 +40,7 @@ def post(self, request): # else: # return dispatcher.process_invalid_trapi(request) # Return responses - return dispatcher.get_response(query) + return dispatcher.get_response(Message) class meta_knowledge_graph(APIView): From 379a618b06bc580b166ea3b9889174cffae438f5 Mon Sep 17 00:00:00 2001 From: Chase Yakaboski Date: Sat, 27 May 2023 14:46:48 -0400 Subject: [PATCH 048/132] Reasonable working version using templating for the dispatcher. Will need to eventually update to python 3.9 to resolve some requirement conflicts, but is functional --- Dockerfile | 2 +- chp_api/dispatcher/base.py | 150 +++++++++++------------------------ chp_api/dispatcher/logger.py | 66 +++++++++++++++ chp_api/dispatcher/views.py | 28 ++++--- dev-deployment-script | 2 +- dev-docker-compose.yml | 25 ------ requirements.txt | 2 + 7 files changed, 132 insertions(+), 143 deletions(-) create mode 100644 chp_api/dispatcher/logger.py delete mode 100644 dev-docker-compose.yml diff --git a/Dockerfile b/Dockerfile index 6c5ef86..3b3a1a8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,7 @@ WORKDIR /usr/src/chp_api # && apt-get install -y git python3-pip python3-dev #dos2unix -RUN git clone --single-branch --branch master https://github.com/di2ag/gene-specificity.git +RUN git clone --single-branch --branch gene_spec_pydantic-ghyde https://github.com/di2ag/gene-specificity.git # lint #RUN pip install --upgrade pip diff --git a/chp_api/dispatcher/base.py b/chp_api/dispatcher/base.py index 6e9dbdb..f62af5d 100644 --- a/chp_api/dispatcher/base.py +++ b/chp_api/dispatcher/base.py @@ -1,25 +1,21 @@ -import logging +import uuid import time +import logging +from .logger import Logger import itertools from copy import deepcopy from re import A +#from reasoner_validator import TRAPISchemaValidator from django.http import JsonResponse from django.apps import apps from django.conf import settings from importlib import import_module from collections import defaultdict -#from .curie_database import merge_curies_databases, CurieDatabase ## NEED TO REMOVE from .models import Transaction, App, DispatcherSettings, Template, TemplateMatch from reasoner_pydantic import MetaKnowledgeGraph, Message, MetaEdge from reasoner_pydantic.qgraph import QNode, QEdge -#from chp_utils.trapi_query_processor import BaseQueryProcessor ## NEED TO REMOVE -#from trapi_model.meta_knowledge_graph import MetaKnowledgeGraph, merge_meta_knowledge_graphs ## NEED TO REMOVE -#from trapi_model.query import Query ## NEED TO REMOVE -#from trapi_model.biolink import TOOLKIT ## NEED TO REMOVE - - # Setup logging logging.addLevelName(25, "NOTE") # Add a special logging function @@ -32,7 +28,7 @@ def note(self, message, *args, **kwargs): APPS = [import_module(app+'.app_interface') for app in settings.INSTALLED_CHP_APPS] class Dispatcher(): - def __init__(self, request, trapi_version): + def __init__(self, request, trapi_version, biolink_version): """ Base API Query Processor class used to abstract the processing infrastructure from the views. Inherits from the CHP Utilities Trapi Query Processor which handles node normalization, curie ontology expansion, and semantic operations. @@ -41,8 +37,10 @@ def __init__(self, request, trapi_version): :type request: requests.request """ self.request_data = deepcopy(request.data) - + self.biolink_version = biolink_version self.trapi_version = trapi_version + #self.validator = TRAPISchemaValidator(self.trapi_version) + self.logger = Logger() def get_meta_knowledge_graph(self): # Get current trapi and biolink versions @@ -56,8 +54,6 @@ def get_meta_knowledge_graph(self): # Load default location else: get_app_meta_kg_fn = getattr(app, 'get_meta_knowledge_graph') - #TODO, when apps are set up correctly, they should return a pydantic_reasoner metakg. So I will have to remove .to_dict() - # from deprecated trapi_model meta_kg = get_app_meta_kg_fn().to_dict() meta_kg = MetaKnowledgeGraph.parse_obj(meta_kg) if merged_meta_kg is None: @@ -68,12 +64,7 @@ def get_meta_knowledge_graph(self): def process_invalid_trapi(self, request): invalid_query_json = request.data - invalid_query_json['status'] = 'Bad TRAPI.' - return JsonResponse(invalid_query_json, status=400) - - def process_invalid_workflow(self, request, status_msg): - invalid_query_json = request.data - invalid_query_json['status'] = status_msg + invalid_query_json['status'] = 'Malformed Query' return JsonResponse(invalid_query_json, status=400) def process_request(self, request, trapi_version): @@ -81,16 +72,18 @@ def process_request(self, request, trapi_version): """ logger.info('Starting query') message = Message.parse_obj(request.data['message']) + #logger.info('Validating query') + #self.validator.validate(message.to_dict(), 'Message') logger.info('Message loaded') return message - def get_app_configs(self, query): + def get_app_configs(self, message): """ Should get a base app configuration for your app or nothing. """ app_configs = [] for app in APPS: get_app_config_fn = getattr(app, 'get_app_config') - app_configs.append(get_app_config_fn(query)) + app_configs.append(get_app_config_fn(message)) return app_configs def get_trapi_interfaces(self, app_configs): @@ -104,11 +97,11 @@ def get_trapi_interfaces(self, app_configs): base_interfaces.append(get_trapi_interface_fn(app_config)) return base_interfaces - def extract_message_templates(self, Message): - assert len(Message.query_graph.edges) == 1, 'CHP apps do not support multihop queries' + def extract_message_templates(self, message): + assert len(message.query_graph.edges) == 1, 'CHP apps do not support multihop queries' subject = None predicates = [] - for edge_id, q_edge in Message.query_graph.edges.items(): + for edge_id, q_edge in message.query_graph.edges.items(): subject = q_edge.subject if q_edge.predicates is None: q_edge = QEdge(subject = q_edge.subject, predicates=['biolink:related_to'], object = q_edge.object) @@ -116,7 +109,7 @@ def extract_message_templates(self, Message): predicates.append(predicate) subject_categories = [] object_categories = [] - for node_id, q_node in Message.query_graph.nodes.items(): + for node_id, q_node in message.query_graph.nodes.items(): if q_node.categories is None: q_node = QNode(categories=['biolink:Entity']) for category in q_node.categories: @@ -157,98 +150,47 @@ def get_response(self, message): """ Main function of the processor that handles primary logic for obtaining a cached or calculated query response. """ - - logger.info('Running message.') + self.logger.info('Running message.') start_time = time.time() - logger.info('Getting message templates.') + self.logger.info('Getting message templates.') message_templates = self.extract_message_templates(message) - app_responses = [] - app_logs = [] - app_statuses = [] - app_descriptions = [] for app, app_name in zip(APPS, settings.INSTALLED_CHP_APPS): - logger.info('Checking template matches for {}'.format(app_name)) + self.logger.info('Checking template matches for {}'.format(app_name)) matching_templates = self.get_app_template_matches(app_name, message_templates) - logger.info('Detected {} matches for {}'.format(len(matching_templates), app_name)) + self.logger.info('Detected {} matches for {}'.format(len(matching_templates), app_name)) if len(matching_templates) > 0: - logger.info('Constructing queries on matching templates') + self.logger.info('Constructing queries on matching templates') consistent_app_queries = self.apply_templates_to_message(message, matching_templates) - logger.info('Sending {} consistent queries') + self.logger.info('Sending {} consistent queries'.format(len(consistent_app_queries))) get_app_response_fn = getattr(app, 'get_response') - responses, logs, status, description = get_app_response_fn(consistent_app_queries) - app_responses.extend(responses) - app_logs.extend(logs) - app_statuses.append(status) - app_descriptions.append(description) - print(app_responses) - ''' - # Check if any responses came back from any apps - if len(app_responses) == 0: - # Add logs from consistent queries of all apps - all_consistent_queries = self.collect_app_queries(consistent_app_queries) - query_copy = self.add_logs_from_query_list(query_copy, all_consistent_queries) - # Add app level logs - query_copy.logger.add_logs(app_logs) - query_copy.set_status('No results.') - self.add_transaction(query_copy) - return JsonResponse(query_copy.to_dict()) - - # Add responses into database - self.add_transactions(app_responses, app_names=[interface.get_name() for interface in base_interfaces]) - - # Construct merged response - response = self.merge_responses(query_copy, app_responses) - - # Now merge all app level log messages from each app - response.logger.add_logs(app_logs) - - # Log any error messages for apps - for app_name, status, description in zip(APPS, app_status, app_descriptions): - if status != 'Success': - response.warning('CHP App: {} reported a unsuccessful status: {} with description: {}'.format( - app_name, status, description) - ) - - # Unnormalize with each apps normalization map - unnormalized_response = response - for normalization_map in app_normalization_maps: - unnormalized_response = self.undo_normalization(unnormalized_response, normalization_map) - - logger.info('Constructed TRAPI response.') - - logger.info('Responded in {} seconds'.format(time.time() - start_time)) - unnormalized_response.set_status('Success') - - # Add workflow - unnormalized_response.add_workflow("lookup") - - # Set the used biolink version - unnormalized_response.biolink_version = TOOLKIT.get_model_version() - - # Add response to database - self.add_transaction(unnormalized_response) - - return JsonResponse(unnormalized_response.to_dict()) - ''' - - def add_logs_from_query_list(self, target_query, query_list): - for query in query_list: - target_query.logger.add_logs(query.logger.to_dict()) - return target_query - - def add_transaction(self, response, app_name='dispatcher'): + responses = get_app_response_fn(consistent_app_queries, self.logger) + self.logger.info('Received responses from {}'.format(app_name)) + for response in responses: + response.query_graph = message.query_graph + self.add_transaction(response.to_dict(), str(uuid.uuid4()), 'Success', app_name) + message.update(response) + + message = message.to_dict() + message['logs'] = self.logger.to_dict() + message['trapi_version'] = self.trapi_version + message['biolink_version'] = self.biolink_version + message['status'] = 'Success' + message['id'] = str(uuid.uuid4()) + message['workflow'] = [{"id": "lookup"}] + + self.add_transaction(message, message['id'], 'Success', 'dispatcher') + + return JsonResponse(message) + + def add_transaction(self, response, _id, status, app_name): app_db_obj = App.objects.get(name=app_name) # Save the transaction transaction = Transaction( - id = response.id, - status = response.status, - query = response.to_dict(), + id = _id, + status = status, + query = response, versions = settings.VERSIONS, chp_app = app_db_obj, ) transaction.save() - - def add_transactions(self, responses, app_names): - for response, chp_app in zip(responses, app_names): - self.add_transaction(response, chp_app) diff --git a/chp_api/dispatcher/logger.py b/chp_api/dispatcher/logger.py new file mode 100644 index 0000000..fdd7369 --- /dev/null +++ b/chp_api/dispatcher/logger.py @@ -0,0 +1,66 @@ +import datetime + +class LogEntry(): + def __init__(self, level, message, code=None, timestamp=None): + if timestamp is None: + self.timestamp = datetime.datetime.utcnow().isoformat() + else: + self.timestamp = timestamp + self.level = level + self.message = message + self.code = code + + def to_dict(self): + return { + "timestamp": self.timestamp, + "level": self.level, + "message": self.message, + "code": self.code, + } + + @staticmethod + def load_log(log_dict): + timestamp = log_dict.pop("timestamp") + level = log_dict.pop("level") + message = log_dict.pop("message") + code = log_dict.pop("code") + return LogEntry( + level, + message, + code=code, + timestamp=timestamp, + ) + + +class Logger(): + def __init__(self): + self.logs = [] + + def add_log(self, level, message, code=None): + self.logs.append( + LogEntry( + level, + message, + code, + ) + ) + + def add_logs(self, logs): + for log in logs: + self.logs.append(LogEntry.load_log(log)) + + def info(self, message, code=None): + self.add_log('INFO', message, code) + + def debug(self, message, code=None): + self.add_log('DEBUG', message, code) + + def warning(self, message, code=None): + self.add_log('WARNING', message, code) + + def error(self, message, code=None): + self.add_log('ERROR', message, code) + + def to_dict(self): + logs = [log.to_dict() for log in self.logs] + return logs diff --git a/chp_api/dispatcher/views.py b/chp_api/dispatcher/views.py index 4e9147a..d9f74eb 100644 --- a/chp_api/dispatcher/views.py +++ b/chp_api/dispatcher/views.py @@ -3,6 +3,7 @@ from copy import deepcopy from datetime import datetime, timedelta +from bmt import Toolkit from .base import Dispatcher from .models import Transaction, DispatcherSettings from .serializers import TransactionListSerializer, TransactionDetailSerializer @@ -15,6 +16,7 @@ from rest_framework import mixins from rest_framework import generics +TOOLKIT = Toolkit() class query(APIView): @@ -27,20 +29,20 @@ def post(self, request): dispatcher = Dispatcher( request, dispatcher_settings.trapi_version, + TOOLKIT.get_model_version() ) # Process Query - #try: - Message = dispatcher.process_request( - request, - trapi_version=dispatcher_settings.trapi_version, - ) - #except Exception as e: - # if 'Workflow Error' in str(e): - # return dispatcher.process_invalid_workflow(request, str(e)) - # else: - # return dispatcher.process_invalid_trapi(request) - # Return responses - return dispatcher.get_response(Message) + try: + message = dispatcher.process_request( + request, + trapi_version=dispatcher_settings.trapi_version, + ) + except Exception as e: + if 'Workflow Error' in str(e): + return dispatcher.process_invalid_workflow(request, str(e)) + else: + return dispatcher.process_invalid_trapi(request) + return dispatcher.get_response(message) class meta_knowledge_graph(APIView): @@ -53,6 +55,7 @@ def get(self, request): dispatcher = Dispatcher( request, dispatcher_settings.trapi_version, + TOOLKIT.get_model_version() ) # Get merged meta KG @@ -70,6 +73,7 @@ def get(self, request): dispatcher = Dispatcher( request, dispatcher_settings.trapi_version, + TOOLKIT.get_model_version() ) return JsonResponse(dispatcher.get_versions()) diff --git a/dev-deployment-script b/dev-deployment-script index d7e5daf..ab3cca8 100755 --- a/dev-deployment-script +++ b/dev-deployment-script @@ -6,7 +6,7 @@ django_superuser_email='cat secrets/chp_api/django_superuser_email.txt' # Only to be run when building on dev machine # use --no-cache if need to rebuild submodules -docker compose build #--no-cache +docker compose build --no-cache docker compose up -d diff --git a/dev-docker-compose.yml b/dev-docker-compose.yml deleted file mode 100644 index ca599c0..0000000 --- a/dev-docker-compose.yml +++ /dev/null @@ -1,25 +0,0 @@ -version: '3.8' - -services: - api: - build: - context: . - dockerfile: Dockerfile.new - container_name: chp-api - command: python manage.py runserver 0.0.0.0:8000 - volumes: - - static_volume:/chp_api/staticfiles - expose: - - 80 - env_file: - - ./.dev.env - db: - image: postgres:latest - restart: unless-stopped - volumes: - - ../chp.sql:/var/lib/postgresql/data - env_file: - - .dev-db.env - -volumes: - static_volume: diff --git a/requirements.txt b/requirements.txt index 1afe81e..fc5d72b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,8 @@ djangorestframework psycopg2-binary +bmt reasoner_pydantic +#reasoner-validator django-environ django-hosts gunicorn From 266fb77c3a536fbfe952ccca092676dfab364d48 Mon Sep 17 00:00:00 2001 From: Chase Yakaboski Date: Sat, 27 May 2023 16:57:41 -0400 Subject: [PATCH 049/132] Removed old pidc submodule. --- .gitmodules | 3 --- gennifer/pidc | 1 - 2 files changed, 4 deletions(-) delete mode 160000 gennifer/pidc diff --git a/.gitmodules b/.gitmodules index e37eb1a..e69de29 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +0,0 @@ -[submodule "gennifer/pidc"] - path = gennifer/pidc - url = git@github.com:di2ag/gennifer-pidc.git diff --git a/gennifer/pidc b/gennifer/pidc deleted file mode 160000 index 39646c6..0000000 --- a/gennifer/pidc +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 39646c6953f4f5cdb3191ef36c02d47e350269d7 From 173deca38844889701b46b9b46a148e7a3c2c66c Mon Sep 17 00:00:00 2001 From: Chase Yakaboski Date: Mon, 29 May 2023 12:47:58 -0400 Subject: [PATCH 050/132] added script to populate curie template for gene_spec app. Overall much more efficient querying --- chp_api/dispatcher/base.py | 10 +-- .../scripts/gene_spec_curie_templater.py | 76 +++++++++++++++++++ dev-deployment-script | 3 +- requirements.txt | 1 + 4 files changed, 84 insertions(+), 6 deletions(-) create mode 100644 chp_api/dispatcher/scripts/gene_spec_curie_templater.py diff --git a/chp_api/dispatcher/base.py b/chp_api/dispatcher/base.py index f62af5d..7327d3f 100644 --- a/chp_api/dispatcher/base.py +++ b/chp_api/dispatcher/base.py @@ -138,11 +138,11 @@ def apply_templates_to_message(self, message, matching_templates): for template in matching_templates: consistent_query = message.copy(deep=True) for edge_id, edge in consistent_query.query_graph.edges.items(): - edge.predicate = template.predicate + edge.predicates = [template.predicate] subject_id = edge.subject object_id = edge.object - consistent_query.query_graph.nodes[subject_id].categories = template.subject - consistent_query.query_graph.nodes[object_id].categories = template.object + consistent_query.query_graph.nodes[subject_id].categories = [template.subject] + consistent_query.query_graph.nodes[object_id].categories = [template.object] consistent_queries.append(consistent_query) return consistent_queries @@ -168,17 +168,17 @@ def get_response(self, message): self.logger.info('Received responses from {}'.format(app_name)) for response in responses: response.query_graph = message.query_graph - self.add_transaction(response.to_dict(), str(uuid.uuid4()), 'Success', app_name) + self.add_transaction({'message':response.to_dict()}, str(uuid.uuid4()), 'Success', app_name) message.update(response) message = message.to_dict() + message = {'message':message} message['logs'] = self.logger.to_dict() message['trapi_version'] = self.trapi_version message['biolink_version'] = self.biolink_version message['status'] = 'Success' message['id'] = str(uuid.uuid4()) message['workflow'] = [{"id": "lookup"}] - self.add_transaction(message, message['id'], 'Success', 'dispatcher') return JsonResponse(message) diff --git a/chp_api/dispatcher/scripts/gene_spec_curie_templater.py b/chp_api/dispatcher/scripts/gene_spec_curie_templater.py new file mode 100644 index 0000000..2e6e78b --- /dev/null +++ b/chp_api/dispatcher/scripts/gene_spec_curie_templater.py @@ -0,0 +1,76 @@ +import tqdm +import json +import requests +from collections import defaultdict +from gene_specificity.models import CurieTemplate, CurieTemplateMatch, SpecificityMeanGene, SpecificityMeanTissue + +CHUNK_SIZE = 500 + +def _get_ascendants(curies, category): + mapping = defaultdict(set) + # map curie with curie + for curie in curies: + mapping[curie].add(curie) + if category == 'biolink:Gene': + return dict(mapping) + for i in tqdm.tqdm(range(0, len(curies), CHUNK_SIZE), desc='Getting ancestors in chunks of size {}'.format(CHUNK_SIZE)): + curie_subset = curies[i:i+CHUNK_SIZE] + query_graph = { + "nodes": { + "n0": { + "categories":[category] + }, + "n1": { + "ids": curie_subset + }, + }, + "edges": { + "e0": { + "subject": "n1", + "object": "n0", + "predicates": ["biolink:part_of", "biolink:subclass_of"], + } + } + } + query = { + "message": { + "query_graph": query_graph, + } + } + url = 'https://ontology-kp.apps.renci.org/query' + r = requests.post(url, json=query, timeout=1000) + answer = json.loads(r.content) + for edge_id, edge in answer['message']['knowledge_graph']['edges'].items(): + subject = edge['subject'] + object = edge['object'] + mapping[object].add(subject) + return dict(mapping) + + +def run(): + objects = SpecificityMeanGene.objects.all() + gene_curies = set() + tissue_curies = set() + for object in objects: + gene_curies.add(object.gene_curie) + tissue_curies.add(object.tissue_curie) + gene_ascendants = _get_ascendants(list(gene_curies), 'biolink:Gene') + tissue_ascendants = _get_ascendants(list(tissue_curies), 'biolink:GrossAnatomicalStructure') + + CurieTemplate.objects.all().delete() + CurieTemplateMatch.objects.all().delete() + + for gene_ancestor, handled_gene_descendants in gene_ascendants.items(): + curie_template = CurieTemplate(curie=gene_ancestor) + curie_template.save() + for handled_gene_descendant in handled_gene_descendants: + curie_template_match = CurieTemplateMatch(curie_template=curie_template, + curie=handled_gene_descendant) + curie_template_match.save() + for tissue_ancestor, handled_tissue_descendants in tissue_ascendants.items(): + curie_template = CurieTemplate(curie=tissue_ancestor) + curie_template.save() + for handled_tissue_descendant in handled_tissue_descendants: + curie_template_match = CurieTemplateMatch(curie_template=curie_template, + curie=handled_tissue_descendant) + curie_template_match.save() diff --git a/dev-deployment-script b/dev-deployment-script index ab3cca8..a9f454c 100755 --- a/dev-deployment-script +++ b/dev-deployment-script @@ -6,7 +6,7 @@ django_superuser_email='cat secrets/chp_api/django_superuser_email.txt' # Only to be run when building on dev machine # use --no-cache if need to rebuild submodules -docker compose build --no-cache +docker compose build #--no-cache docker compose up -d @@ -18,6 +18,7 @@ docker compose run --user root chp-api python3 manage.py createsuperuser --no-in # Load apps docker compose run chp-api python3 manage.py runscript load_db_apps docker compose run chp-api python3 manage.py runscript templater +docker compose run chp_api Python3 manage.py runscript gene_spec_curie_templater docker compose run --user root chp-api python3 manage.py collectstatic --noinput diff --git a/requirements.txt b/requirements.txt index fc5d72b..f0fea9a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +tqdm djangorestframework psycopg2-binary bmt From 52a2a4c4f162a4c3f755f6cc36578d3b91bfd433 Mon Sep 17 00:00:00 2001 From: Chase Yakaboski Date: Mon, 29 May 2023 17:44:29 -0400 Subject: [PATCH 051/132] Initial integration to be merged with production. --- .gitmodules | 3 + chp_api/Dockerfile | 1 + chp_api/chp_api/__init__.py | 3 + chp_api/chp_api/celery.py | 7 ++ chp_api/chp_api/settings.py | 5 + chp_api/chp_api/urls.py | 1 + chp_api/dispatcher/models.py | 1 + chp_api/gennifer/__init__.py | 0 chp_api/gennifer/admin.py | 3 + chp_api/gennifer/apps.py | 6 ++ chp_api/gennifer/migrations/__init__.py | 0 chp_api/gennifer/models.py | 64 ++++++++++++ chp_api/gennifer/serializers.py | 35 +++++++ chp_api/gennifer/tasks.py | 129 ++++++++++++++++++++++++ chp_api/gennifer/tests.py | 3 + chp_api/gennifer/urls.py | 15 +++ chp_api/gennifer/views.py | 76 ++++++++++++++ chp_api/requirements.txt | 5 + compose.yaml => compose.chp-api.yaml | 92 +++++++++++++---- compose.gennifer.yaml | 53 ++++++++++ deployment-script | 2 +- gennifer | 1 + nginx/default.conf | 7 ++ nginx/start.sh | 2 +- 24 files changed, 492 insertions(+), 22 deletions(-) create mode 100644 chp_api/chp_api/celery.py create mode 100644 chp_api/gennifer/__init__.py create mode 100644 chp_api/gennifer/admin.py create mode 100644 chp_api/gennifer/apps.py create mode 100644 chp_api/gennifer/migrations/__init__.py create mode 100644 chp_api/gennifer/models.py create mode 100644 chp_api/gennifer/serializers.py create mode 100644 chp_api/gennifer/tasks.py create mode 100644 chp_api/gennifer/tests.py create mode 100644 chp_api/gennifer/urls.py create mode 100644 chp_api/gennifer/views.py rename compose.yaml => compose.chp-api.yaml (58%) create mode 100644 compose.gennifer.yaml create mode 160000 gennifer diff --git a/.gitmodules b/.gitmodules index e69de29..7f8ab1d 100644 --- a/.gitmodules +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "gennifer"] + path = gennifer + url = git@github.com:di2ag/gennifer.git diff --git a/chp_api/Dockerfile b/chp_api/Dockerfile index d555bfa..7a4ab4d 100644 --- a/chp_api/Dockerfile +++ b/chp_api/Dockerfile @@ -63,6 +63,7 @@ RUN pip3 install --no-cache /wheels/* COPY ./chp_api $APP_HOME/chp_api COPY ./manage.py $APP_HOME COPY ./dispatcher $APP_HOME/dispatcher +COPY ./gennifer $APP_HOME/gennifer COPY ./gunicorn.config.py $APP_HOME # chown all the files to the app user diff --git a/chp_api/chp_api/__init__.py b/chp_api/chp_api/__init__.py index e69de29..53f4ccb 100644 --- a/chp_api/chp_api/__init__.py +++ b/chp_api/chp_api/__init__.py @@ -0,0 +1,3 @@ +from .celery import app as celery_app + +__all__ = ("celery_app",) diff --git a/chp_api/chp_api/celery.py b/chp_api/chp_api/celery.py new file mode 100644 index 0000000..4b910d6 --- /dev/null +++ b/chp_api/chp_api/celery.py @@ -0,0 +1,7 @@ +import os +from celery import Celery + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "chp_api.settings") +app = Celery("chp_api") +app.config_from_object("django.conf:settings", namespace="CELERY") +app.autodiscover_tasks() diff --git a/chp_api/chp_api/settings.py b/chp_api/chp_api/settings.py index 8135cc4..367b792 100644 --- a/chp_api/chp_api/settings.py +++ b/chp_api/chp_api/settings.py @@ -39,6 +39,7 @@ 'django.contrib.messages', 'django.contrib.staticfiles', 'rest_framework', + 'django_filters', 'dispatcher.apps.DispatcherConfig', 'chp_utils', 'django_extensions', @@ -173,3 +174,7 @@ os.environ["DJANGO_SUPERUSER_EMAIL"] = dse_file.readline().strip() with open(env("DJANGO_SUPERUSER_PASSWORD_FILE"), 'r') as dsp_file: os.environ["DJANGO_SUPERUSER_PASSWORD"] = dsp_file.readline().strip() + +# Celery Settings +CELERY_BROKER_URL = os.environ.get("CELERY_BROKER_URL", "redis://localhost:6379") +CELERY_RESULT_BACKEND = os.environ.get("CELERY_RESULT_BACKEND", "redis://localhost:6379") diff --git a/chp_api/chp_api/urls.py b/chp_api/chp_api/urls.py index e6a0057..24d1feb 100644 --- a/chp_api/chp_api/urls.py +++ b/chp_api/chp_api/urls.py @@ -20,6 +20,7 @@ urlpatterns = [ path('admin/', admin.site.urls), path('', include('dispatcher.urls')), + path('gennifer/', include('gennifer.urls')), ] diff --git a/chp_api/dispatcher/models.py b/chp_api/dispatcher/models.py index 88d28a0..3d52fdc 100644 --- a/chp_api/dispatcher/models.py +++ b/chp_api/dispatcher/models.py @@ -60,6 +60,7 @@ def load(cls): class DispatcherSettings(Singleton): trapi_version = models.CharField(max_length=28, default='1.4') + sri_node_normalizer_baseurl = models.URLField(max_length=128, default='https://nodenormalization-sri.renci.org') def __str__(self): return 'settings' diff --git a/chp_api/gennifer/__init__.py b/chp_api/gennifer/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/chp_api/gennifer/admin.py b/chp_api/gennifer/admin.py new file mode 100644 index 0000000..8c38f3f --- /dev/null +++ b/chp_api/gennifer/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/chp_api/gennifer/apps.py b/chp_api/gennifer/apps.py new file mode 100644 index 0000000..2967f54 --- /dev/null +++ b/chp_api/gennifer/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class GenniferConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'gennifer' diff --git a/chp_api/gennifer/migrations/__init__.py b/chp_api/gennifer/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/chp_api/gennifer/models.py b/chp_api/gennifer/models.py new file mode 100644 index 0000000..aa27fde --- /dev/null +++ b/chp_api/gennifer/models.py @@ -0,0 +1,64 @@ +from django.db import models +from django.contrib.auth.models import User + + +class Algorithm(models.Model): + name = models.CharField(max_length=128) + run_url = models.URLField(max_length=128) + + def __str__(self): + return self.name + + +class Dataset(models.Model): + title = models.CharField(max_length=128) + zenodo_id = models.CharField(max_length=128, primary_key=True) + doi = models.CharField(max_length=128) + description = models.TextField(null=True, blank=True) + upload_user = models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True) + + def save(self, *args, **kwargs): + import re + + CLEANR = re.compile('<.*?>') + + info = self.get_record() + self.doi = info["doi"] + self.description = re.sub(CLEANR, '', infoi["metadata"]["description"]) + self.title = re.sub(CLEANR, '', infoi["metadata"]["title"]) + + def get_record(self): + return requests.get(f"https://zenodo.org/api/records/{self.zenodo_id}").json() + + +class Gene(models.Model): + name = models.CharField(max_length=128) + curie = models.CharField(max_length=128) + variant = models.TextField(null=True, blank=True) + + def __str__(self): + return self.name + + +class InferenceStudy(models.Model): + algorithm = models.ForeignKey(Algorithm, on_delete=models.CASCADE, related_name='studies') + user = models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True, related_name='studies') + dataset = models.ForeignKey(Dataset, on_delete=models.CASCADE, related_name='studies') + timestamp = models.DateTimeField(auto_now_add=True) + # Study characteristics for all edge weights in a given study over a dataset + max_study_edge_weight = models.FloatField(null=True) + min_study_edge_weight = models.FloatField(null=True) + avg_study_edge_weight = models.FloatField(null=True) + std_study_edge_weight = models.FloatField(null=True) + is_public = models.BooleanField(default=False) + status = models.CharField(max_length=10) + error_message = models.TextField(null=True, blank=True) + +class InferenceResult(models.Model): + # Stands for transcription factor + tf = models.ForeignKey(Gene, on_delete=models.CASCADE) + # Target is the gene that is regulated by the transcription factor + target = models.ForeignKey(Gene, on_delete=models.CASCADE) + edge_weight = models.FloatField() + study = models.ForeignKey(InferenceStudy, on_delete=models.CASCADE, related_name='results') + is_public = models.BooleanField(default=False) diff --git a/chp_api/gennifer/serializers.py b/chp_api/gennifer/serializers.py new file mode 100644 index 0000000..0ce4ecb --- /dev/null +++ b/chp_api/gennifer/serializers.py @@ -0,0 +1,35 @@ +from rest_framework import serializers + +from .models import Dataset, InferenceStudy, InferenceResult + +class DatasetSerializer(serializers.ModelSerializer): + class Meta: + model = Dataset + fields = ['name', 'zenodo_id', 'doi', 'description'] + + +class InferenceStudySerializer(serializers.ModelSerializer): + class Meta: + model = InferenceStudy + fields = [ + 'algorithm', + 'user', + 'dataset', + 'timestamp', + 'max_study_edge_weight', + 'min_study_edge_weight', + 'avg_study_edge_weight', + 'std_study_edge_weight', + 'is_public', + ] + +class InferenceResultSerializer(serializers.ModelSerializer): + class Meta: + model = InferenceResult + fields = [ + 'tf', + 'target', + 'edge_weight', + 'study', + 'is_public', + ] diff --git a/chp_api/gennifer/tasks.py b/chp_api/gennifer/tasks.py new file mode 100644 index 0000000..e9cfc48 --- /dev/null +++ b/chp_api/gennifer/tasks.py @@ -0,0 +1,129 @@ +import os +import time +import pandas as pd +import requests + +from celery import shared_task + +from .models import Dataset, Gene, InferenceStudy, InferenceResult +from dispacter.models import DispatcherSettings + +def normalize_nodes(curies): + dispatcher_settings = DispatcherSettings.load() + base_url = dispatcher_settings.sri_node_normalizer_baseurl + return requests.post( + f'{base_url}/get_normalized_nodes', + json={"curies": curies} + ) + +def extract_variant_info(gene_id): + split = gene_id.split('(') + gene_id = split[0] + if len(split) > 1: + variant_info = split[1][:-1] + else: + variant_info = None + return gene_id, variant_info + +def save_inference_study(study, status, failed=False): + study.status = status["task_status"] + if failed: + study.message = status["task_result"] + else: + # Construct Dataframe from result + df = pd.DataFrame.from_records(status["task_result"]) + + # Add study edge weight features + stats = df["EdgeWeight"].astype(float).describe() + study.max_study_edge_weight = stats["max"] + study.min_study_edge_weight = stats["min"] + study.avg_study_edge_weight = stats["mean"] + study.std_study_edge_weight = stats["std"] + + # Collect all genes + genes = set() + for _, row in df.iterrows(): + gene1, _ = extract_variant_info(row["Gene1"]) + gene2, _ = extract_variant_info(row["Gene2"]) + genes.add(gene1) + genes.add(gene2) + + # Normalize + res = normalize_nodes(list(genes)) + + # Now Extract results + for _, row in df.iterrows(): + # Construct Gene Objects + gene1, variant_info1 = extract_variant_info(row["Gene1"]) + gene2, variant_info2 = extract_variant_info(row["Gene2"]) + gene1_obj, created = Gene.objects.get_or_create( + name=res[gene1]["id"]["label"], + curie=gene1, + variant=variant_info1, + ) + if created: + gene1_obj.save() + gene2_obj, created = Gene.objects.get_or_create( + name=res[gene2]["id"]["label"], + curie=gene2, + variant=variant_info2, + ) + if created: + gene2_obj.save() + # Construct and save Result + result = InferenceResult.objects.create( + tf=gene1_obj, + target=gene2_obj, + edge_weight=row["EdgeWeight"], + study=study, + ) + result.save() + study.save() + return True + +def get_status(algo, task_id): + return requests.get(f'{algo.url}/status/{task_id}').json() + +@shared_task(name="create_gennifer_task") +def create_task(algorithm, zenodo_id, hyperparameters, user): + # Initialize dataset instance + dataset, created = Dataset.objects.get_or_create( + zenodo_id=zenodo_id, + upload_user=user, + ) + if created: + dataset.save() + + # Send to gennifer app + gennifer_request = { + "zenodo_id": zenodo_id, + "hyperparameters": hyperparameters, + } + task["task_id"] = requests.post(f'{algo.url}/run', data=gennifer_request) + + # Get initial status + status = get_status(algo, task["task_id"]) + + # Create Inference Study + study = InferenceStudy.objects.create( + algorithm=algo, + user=user, + dataset=dataset, + status=status["task_status"], + ) + # Save initial study + study.save() + + # Enter a loop to keep checking back in and populate the study once it has completed. + #TODO: Not sure if this is best practice + while True: + # Check in every 2 seconds + time.sleep(2) + status = get_status(algo, task["task_id"]) + if status["task_status"] == 'SUCCESS': + return save_inference_study(study, status) + if status["task_status"] == "FAILURE": + return save_inference_study(study, status, failed=True) + if status["task_status"] != study.status: + study.status = status["task_status"] + study.save() diff --git a/chp_api/gennifer/tests.py b/chp_api/gennifer/tests.py new file mode 100644 index 0000000..7ce503c --- /dev/null +++ b/chp_api/gennifer/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/chp_api/gennifer/urls.py b/chp_api/gennifer/urls.py new file mode 100644 index 0000000..17860e4 --- /dev/null +++ b/chp_api/gennifer/urls.py @@ -0,0 +1,15 @@ +from django.urls import path, include +from rest_framework.routers import DefaultRouter + +from . import views + +# Create router and register viewsets +router = DefaultRouter() +router.register(r'datasets', views.DatasetViewSet, basename='dataset') +router.register(r'inference_studies', views.InferenceStudyViewSet, basename='inference_study') +router.register(r'inference_results', views.InferenceResultViewSet, basename='inference_result') + +urlpatterns = [ + path('', include(router.urls)), + path('run', views.run.as_view()), + ] diff --git a/chp_api/gennifer/views.py b/chp_api/gennifer/views.py new file mode 100644 index 0000000..aca73bb --- /dev/null +++ b/chp_api/gennifer/views.py @@ -0,0 +1,76 @@ +import requests + +from django.http import HttpResponse, JsonResponse +from django.shortcuts import get_object_or_404 +from django.core.exceptions import ObjectDoesNotExist + +from rest_framework import viewsets +from rest_framework.views import APIView +from rest_framework.requests import Response +from rest_framework.permissions import IsAuthenticated +from django_filters.rest_framework import DjangoFilterBackend + +from .models import Dataset, InferenceStudy, InferenceResult, Algorithm +from .serializers import DatasetSerializer, InferenceStudySerializer, InferenceResultSerializer +from .tasks import create_task + +class DatasetViewSet(viewsets.ModelViewSet): + queryset = Dataset.objects.all() + serializer_class = DatasetSerializer + filter_backend = [DjangoFilterBackend] + filterset_fields = ['user', 'zenodo_id'] + permission_classes = [IsAuthenticated] + + +class InferenceStudyViewSet(viewsets.ModelViewSet): + serializer_class = InferenceStudySerializer + filter_backend = [DjangoFilterBackend] + filterset_fields = ['is_public', 'dataset', 'algorithm'] + permission_classes = [IsAuthenticated] + + def get_queryset(self): + user = self.request.user + return InferenceStudy.objects.filter(user=user) + + +class InferenceResultViewSet(viewsets.ModelViewSet): + serializer_class = InferenceResultSerializer + filter_backend = [DjangoFilterBackend] + filterset_fields = ['is_public', 'study'] + permission_classes = [IsAuthenticated] + + def get_queryset(self): + user = self.request.user + return InferenceResult.objects.filter(user=user) + + +class run(APIView): + + def post(self, request): + """ Request comes in as a list of algorithms to run. + """ + # Build gennifer requests + tasks = request.data['tasks'] + response = [] + for task in tasks: + algorithm_name = task.get("algorithm_name", None) + zenodo_id = task.get("zenodo_id", None) + hyperparameters = task.get("hyperparameters", None) + if not algorithm_name: + task["error"] = "No algorithm name provided." + response.append(task) + continue + if not zenodo_id: + task["error"] = "No dataset Zenodo identifer provided." + response.append(task) + continue + try: + algo = Algorithm.objects.get(name=algorithm_name) + except ObjectDoesNotExist: + task["error"] = f"The algorithm: {algorithm_name} is not supported in Gennifer." + response.append(task) + continue + # If all pass, now send to gennifer services + task["task_id"] = create_task.delay(algo, zenodo_id, hyperparameters, request.user).id + response.append(task) + return Response(response) diff --git a/chp_api/requirements.txt b/chp_api/requirements.txt index 081b490..af88c4d 100644 --- a/chp_api/requirements.txt +++ b/chp_api/requirements.txt @@ -6,3 +6,8 @@ gunicorn django requests requests-cache +django-filter +celery +flower +redis +pandas diff --git a/compose.yaml b/compose.chp-api.yaml similarity index 58% rename from compose.yaml rename to compose.chp-api.yaml index 45b9dc9..cb74564 100644 --- a/compose.yaml +++ b/compose.chp-api.yaml @@ -8,12 +8,13 @@ services: volumes: - ./nginx/default.conf:/tmp/default.conf environment: - - DJANGO_SERVER_ADDR=chp-api:8000 + - DJANGO_SERVER_ADDR=api:8000 - STATIC_SERVER_ADDR=static-fs:8080 + - FLOWER_DASHBOARD_ADDR=dashboard:5556 ports: - "80:80" depends_on: - - chp-api + - api healthcheck: test: ["CMD-SHELL", "curl --silent --fail localhost:80/health-check || exit 1"] interval: 10s @@ -21,7 +22,7 @@ services: retries: 3 command: /app/start.sh - chp-api: + api: build: context: ./chp_api dockerfile: Dockerfile @@ -67,6 +68,74 @@ services: #command: gunicorn -c gunicorn.config.py -b 0.0.0.0:8000 chp_api.wsgi:application command: python3 manage.py runserver 0.0.0.0:8000 + worker-api: + build: + context: ./chp_api + dockerfile: Dockerfile + secrets: + - db-password + - django-key + - allowed-hosts + - django-superuser-username + - django-superuser-email + - django-superuser-password + environment: + - POSTGRES_DB=chp_db + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD_FILE=/run/secrets/db-password + - POSTGRES_HOST=db + - POSTGRES_PORT=5432 + - SECRET_KEY_FILE=/run/secrets/django-key + - DEBUG=1 + - DJANGO_ALLOWED_HOSTS_FILE=/run/secrets/allowed-hosts + - DJANGO_SUPERUSER_USERNAME_FILE=/run/secrets/django-superuser-username + - DJANGO_SUPERUSER_EMAIL_FILE=/run/secrets/django-superuser-email + - DJANGO_SUPERUSER_PASSWORD_FILE=/run/secrets/django-superuser-password + command: celery -A chp_api worker --loglevel=info + environment: + - CELERY_BROKER_URL=redis://redis:6379/0 + - CELERY_RESULT_BACKEND=redis://redis:6379/0 + depends_on: + - api + - redis + + dashboard: + build: + context: ./chp_api + dockerfile: Dockerfile + secrets: + - db-password + - django-key + - allowed-hosts + - django-superuser-username + - django-superuser-email + - django-superuser-password + environment: + - POSTGRES_DB=chp_db + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD_FILE=/run/secrets/db-password + - POSTGRES_HOST=db + - POSTGRES_PORT=5432 + - SECRET_KEY_FILE=/run/secrets/django-key + - DEBUG=1 + - DJANGO_ALLOWED_HOSTS_FILE=/run/secrets/allowed-hosts + - DJANGO_SUPERUSER_USERNAME_FILE=/run/secrets/django-superuser-username + - DJANGO_SUPERUSER_EMAIL_FILE=/run/secrets/django-superuser-email + - DJANGO_SUPERUSER_PASSWORD_FILE=/run/secrets/django-superuser-password + command: celery -A chp_api flower --port=5555 --broker=redis://redis:6379/0 + ports: + - 5556:5555 + environment: + - CELERY_BROKER_URL=redis://redis:6379/0 + - CELERY_RESULT_BACKEND=redis://redis:6379/0 + depends_on: + - api + - redis + - worker-api + + redis: + image: redis:6-alpine + db: image: postgres restart: always @@ -96,21 +165,6 @@ services: volumes: - static-files:/var/www/static - gennifer-pidc: - build: - context: ./gennifer/pidc - dockerfile: Dockerfile - restart: always - user: gennifer_user - expose: - - 5000 - secrets: - - gennifer_key - environment: - - SECRET_KEY_FILE=/run/secrets/gennifer_key - #command: gunicorn -c gunicorn.config.py -b 0.0.0.0:5000 'pidc:create_app()' - command: flask --app pidc run --debug - volumes: db-data: static-files: @@ -128,5 +182,3 @@ secrets: file: secrets/chp_api/django_superuser_email.txt django-superuser-password: file: secrets/chp_api/django_superuser_password.txt - gennifer_key: - file: secrets/gennifer/secret_key.txt diff --git a/compose.gennifer.yaml b/compose.gennifer.yaml new file mode 100644 index 0000000..0f91baf --- /dev/null +++ b/compose.gennifer.yaml @@ -0,0 +1,53 @@ +version: '3.8' + +services: + + pidc: + build: + context: ./gennifer/pidc + dockerfile: Dockerfile + restart: always + user: gennifer_user + ports: + - 5004:5000 + secrets: + - gennifer_key + environment: + - SECRET_KEY_FILE=/run/secrets/gennifer_key + - CELERY_BROKER_URL=redis://redis:6379/0 + - CELERY_RESULT_BACKEND=redis://redis:6379/0 + #command: gunicorn -c gunicorn.config.py -b 0.0.0.0:5000 'pidc:create_app()' + command: flask --app pidc run --debug --host 0.0.0.0 + depends_on: + - redis + + worker-pidc: + build: + context: ./gennifer/pidc + dockerfile: Dockerfile + command: celery --app pidc.tasks.celery worker --loglevel=info + environment: + - CELERY_BROKER_URL=redis://redis:6379/0 + - CELERY_RESULT_BACKEND=redis://redis:6379/0 + depends_on: + - pidc + - redis + + dashboard-pidc: + build: + context: ./gennifer/pidc + dockerfile: Dockerfile + command: celery --app pidc.tasks.celery flower --port=5555 --broker=redis://redis:6379/0 + ports: + - 5557:5555 + environment: + - CELERY_BROKER_URL=redis://redis:6379/0 + - CELERY_RESULT_BACKEND=redis://redis:6379/0 + depends_on: + - pidc + - redis + - worker-pidc + +secrets: + gennifer_key: + file: secrets/gennifer/secret_key.txt diff --git a/deployment-script b/deployment-script index 3bc83ca..276aa23 100755 --- a/deployment-script +++ b/deployment-script @@ -5,7 +5,7 @@ django_superuser_username='cat secrets/chp_api/django_superuser_username.txt' django_superuser_email='cat secrets/chp_api/django_superuser_email.txt' # Only to be run when building on dev machine -docker compose build --no-cache +docker compose -f compose.chp-api.yaml -f compose.gennifer.yaml build docker compose up -d diff --git a/gennifer b/gennifer new file mode 160000 index 0000000..ea5019c --- /dev/null +++ b/gennifer @@ -0,0 +1 @@ +Subproject commit ea5019c09a019fe8ccb4e5d2d80d1fe1befc5382 diff --git a/nginx/default.conf b/nginx/default.conf index 8db6a79..8d113d4 100644 --- a/nginx/default.conf +++ b/nginx/default.conf @@ -33,4 +33,11 @@ server { proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; } + location /flower-dashboard { + proxy_pass http://$FLOWER_DASHBOARD_ADDR; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + } diff --git a/nginx/start.sh b/nginx/start.sh index b7384d6..eeb5d3a 100644 --- a/nginx/start.sh +++ b/nginx/start.sh @@ -1,2 +1,2 @@ #!/bin/bash -envsubst '$DJANGO_SERVER_ADDR,$STATIC_SERVER_ADDR' < /tmp/default.conf > /etc/nginx/conf.d/default.conf && nginx -g 'daemon off;' +envsubst '$DJANGO_SERVER_ADDR,$STATIC_SERVER_ADDR,$FLOWER_DASHBOARD_ADDR' < /tmp/default.conf > /etc/nginx/conf.d/default.conf && nginx -g 'daemon off;' From cd17ee12a4688bba6520c9047f02c60648fb695f Mon Sep 17 00:00:00 2001 From: Chase Yakaboski Date: Mon, 29 May 2023 22:58:42 -0400 Subject: [PATCH 052/132] Working API of gennifer integrated with pidc. --- chp_api/Dockerfile | 8 +-- chp_api/chp_api/celery.py | 10 ++- chp_api/chp_api/settings.py | 5 +- chp_api/dispatcher/admin.py | 4 +- chp_api/dispatcher/base.py | 4 +- ...hersettings_sri_node_normalizer_baseurl.py | 18 +++++ ...me_dispatchersettings_dispatchersetting.py | 17 +++++ chp_api/dispatcher/models.py | 2 +- chp_api/dispatcher/views.py | 8 +-- chp_api/gennifer/__init__.py | 1 + chp_api/gennifer/admin.py | 6 +- chp_api/gennifer/migrations/0001_initial.py | 72 +++++++++++++++++++ .../migrations/0002_inferenceresult_user.py | 21 ++++++ ..._remove_algorithm_run_url_algorithm_url.py | 23 ++++++ chp_api/gennifer/models.py | 15 ++-- chp_api/gennifer/serializers.py | 3 +- chp_api/gennifer/tasks.py | 49 +++++++++---- chp_api/gennifer/views.py | 26 +++---- compose.chp-api.yaml | 16 ++--- compose.gennifer.yaml | 4 +- copy-migrations | 5 ++ deployment-script | 16 ++--- gennifer | 2 +- gennifer-sample.json | 9 +++ 24 files changed, 274 insertions(+), 70 deletions(-) create mode 100755 chp_api/dispatcher/migrations/0008_dispatchersettings_sri_node_normalizer_baseurl.py create mode 100755 chp_api/dispatcher/migrations/0009_rename_dispatchersettings_dispatchersetting.py create mode 100755 chp_api/gennifer/migrations/0001_initial.py create mode 100755 chp_api/gennifer/migrations/0002_inferenceresult_user.py create mode 100755 chp_api/gennifer/migrations/0003_remove_algorithm_run_url_algorithm_url.py create mode 100755 copy-migrations create mode 100644 gennifer-sample.json diff --git a/chp_api/Dockerfile b/chp_api/Dockerfile index f973451..10c0b1f 100644 --- a/chp_api/Dockerfile +++ b/chp_api/Dockerfile @@ -48,11 +48,11 @@ COPY --from=intermediate /usr/src/chp_api/requirements.txt . RUN pip3 install --no-cache /wheels/* # copy project -COPY ./chp_api/chp_api $APP_HOME/chp_api -COPY ./chp_api/manage.py $APP_HOME -COPY ./chp_api/dispatcher $APP_HOME/dispatcher +COPY ./chp_api $APP_HOME/chp_api +COPY ./manage.py $APP_HOME +COPY ./dispatcher $APP_HOME/dispatcher COPY ./gennifer $APP_HOME/gennifer -COPY ./chp_db_fixture.json.gz $APP_HOME +#COPY ./chp_db_fixture.json.gz $APP_HOME COPY ./gunicorn.config.py $APP_HOME # chown all the files to the app user diff --git a/chp_api/chp_api/celery.py b/chp_api/chp_api/celery.py index 4b910d6..79b8df3 100644 --- a/chp_api/chp_api/celery.py +++ b/chp_api/chp_api/celery.py @@ -2,6 +2,14 @@ from celery import Celery os.environ.setdefault("DJANGO_SETTINGS_MODULE", "chp_api.settings") -app = Celery("chp_api") +app = Celery( + "chp_api", + include=['gennifer.tasks'], + ) app.config_from_object("django.conf:settings", namespace="CELERY") app.autodiscover_tasks() +app.conf.update({ + "task_routes": { + "create_gennifer_task": {"queue": 'chp_api'} + } + }) diff --git a/chp_api/chp_api/settings.py b/chp_api/chp_api/settings.py index 24edd07..cebb02a 100644 --- a/chp_api/chp_api/settings.py +++ b/chp_api/chp_api/settings.py @@ -10,9 +10,11 @@ https://docs.djangoproject.com/en/3.0/ref/settings/ """ import os -from importlib import import_module import environ as environ # type: ignore +from importlib import import_module + + # Initialise environment variables env = environ.Env() environ.Env.read_env() @@ -42,6 +44,7 @@ 'django_filters', 'dispatcher.apps.DispatcherConfig', 'django_extensions', + 'gennifer', # Need to make into CHP app ] INSTALLED_CHP_APPS = [ diff --git a/chp_api/dispatcher/admin.py b/chp_api/dispatcher/admin.py index 9db96c8..06e6480 100644 --- a/chp_api/dispatcher/admin.py +++ b/chp_api/dispatcher/admin.py @@ -1,7 +1,7 @@ from django.contrib import admin -from .models import App, ZenodoFile, DispatcherSettings +from .models import App, ZenodoFile, DispatcherSetting admin.site.register(App) admin.site.register(ZenodoFile) -admin.site.register(DispatcherSettings) +admin.site.register(DispatcherSetting) diff --git a/chp_api/dispatcher/base.py b/chp_api/dispatcher/base.py index 7327d3f..c659a4a 100644 --- a/chp_api/dispatcher/base.py +++ b/chp_api/dispatcher/base.py @@ -12,7 +12,7 @@ from importlib import import_module from collections import defaultdict -from .models import Transaction, App, DispatcherSettings, Template, TemplateMatch +from .models import Transaction, App, DispatcherSetting, Template, TemplateMatch from reasoner_pydantic import MetaKnowledgeGraph, Message, MetaEdge from reasoner_pydantic.qgraph import QNode, QEdge @@ -44,7 +44,7 @@ def __init__(self, request, trapi_version, biolink_version): def get_meta_knowledge_graph(self): # Get current trapi and biolink versions - dispatcher_settings = DispatcherSettings.load() + dispatcher_settings = DispatcherSetting.load() merged_meta_kg = None for app, app_name in zip(APPS, settings.INSTALLED_CHP_APPS): app_db_obj = App.objects.get(name=app_name) diff --git a/chp_api/dispatcher/migrations/0008_dispatchersettings_sri_node_normalizer_baseurl.py b/chp_api/dispatcher/migrations/0008_dispatchersettings_sri_node_normalizer_baseurl.py new file mode 100755 index 0000000..9a69512 --- /dev/null +++ b/chp_api/dispatcher/migrations/0008_dispatchersettings_sri_node_normalizer_baseurl.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.1 on 2023-05-29 22:26 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('dispatcher', '0007_template_templatematch'), + ] + + operations = [ + migrations.AddField( + model_name='dispatchersettings', + name='sri_node_normalizer_baseurl', + field=models.URLField(default='https://nodenormalization-sri.renci.org', max_length=128), + ), + ] diff --git a/chp_api/dispatcher/migrations/0009_rename_dispatchersettings_dispatchersetting.py b/chp_api/dispatcher/migrations/0009_rename_dispatchersettings_dispatchersetting.py new file mode 100755 index 0000000..8f5d55f --- /dev/null +++ b/chp_api/dispatcher/migrations/0009_rename_dispatchersettings_dispatchersetting.py @@ -0,0 +1,17 @@ +# Generated by Django 4.2.1 on 2023-05-29 23:32 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('dispatcher', '0008_dispatchersettings_sri_node_normalizer_baseurl'), + ] + + operations = [ + migrations.RenameModel( + old_name='DispatcherSettings', + new_name='DispatcherSetting', + ), + ] diff --git a/chp_api/dispatcher/models.py b/chp_api/dispatcher/models.py index 457fec9..22d49cd 100644 --- a/chp_api/dispatcher/models.py +++ b/chp_api/dispatcher/models.py @@ -70,7 +70,7 @@ def load(cls): obj, _ = cls.objects.get_or_create(pk=1) return obj -class DispatcherSettings(Singleton): +class DispatcherSetting(Singleton): trapi_version = models.CharField(max_length=28, default='1.4') sri_node_normalizer_baseurl = models.URLField(max_length=128, default='https://nodenormalization-sri.renci.org') diff --git a/chp_api/dispatcher/views.py b/chp_api/dispatcher/views.py index d9f74eb..afddf72 100644 --- a/chp_api/dispatcher/views.py +++ b/chp_api/dispatcher/views.py @@ -5,7 +5,7 @@ from bmt import Toolkit from .base import Dispatcher -from .models import Transaction, DispatcherSettings +from .models import Transaction, DispatcherSetting from .serializers import TransactionListSerializer, TransactionDetailSerializer from django.http import HttpResponse, JsonResponse @@ -22,7 +22,7 @@ class query(APIView): def post(self, request): # Get current trapi and biolink versions - dispatcher_settings = DispatcherSettings.load() + dispatcher_settings = DispatcherSetting.load() if request.method == 'POST': # Initialize Dispatcher @@ -48,7 +48,7 @@ class meta_knowledge_graph(APIView): def get(self, request): # Get current trapi and biolink versions - dispatcher_settings = DispatcherSettings.load() + dispatcher_settings = DispatcherSetting.load() if request.method == 'GET': # Initialize Dispatcher @@ -66,7 +66,7 @@ class versions(APIView): def get(self, request): # Get current trapi and biolink versions - dispatcher_settings = DispatcherSettings.load() + dispatcher_settings = DispatcherSetting.load() if request.method == 'GET': # Initialize Dispatcher diff --git a/chp_api/gennifer/__init__.py b/chp_api/gennifer/__init__.py index e69de29..b8023d8 100644 --- a/chp_api/gennifer/__init__.py +++ b/chp_api/gennifer/__init__.py @@ -0,0 +1 @@ +__version__ = '0.0.1' diff --git a/chp_api/gennifer/admin.py b/chp_api/gennifer/admin.py index 8c38f3f..b4a7ca6 100644 --- a/chp_api/gennifer/admin.py +++ b/chp_api/gennifer/admin.py @@ -1,3 +1,7 @@ from django.contrib import admin -# Register your models here. +from .models import Algorithm, Dataset, InferenceStudy + +admin.site.register(Algorithm) +admin.site.register(Dataset) +admin.site.register(InferenceStudy) diff --git a/chp_api/gennifer/migrations/0001_initial.py b/chp_api/gennifer/migrations/0001_initial.py new file mode 100755 index 0000000..b48d24a --- /dev/null +++ b/chp_api/gennifer/migrations/0001_initial.py @@ -0,0 +1,72 @@ +# Generated by Django 4.2.1 on 2023-05-29 22:28 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='Algorithm', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=128)), + ('run_url', models.URLField(max_length=128)), + ], + ), + migrations.CreateModel( + name='Dataset', + fields=[ + ('title', models.CharField(max_length=128)), + ('zenodo_id', models.CharField(max_length=128, primary_key=True, serialize=False)), + ('doi', models.CharField(max_length=128)), + ('description', models.TextField(blank=True, null=True)), + ('upload_user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + ), + migrations.CreateModel( + name='Gene', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=128)), + ('curie', models.CharField(max_length=128)), + ('variant', models.TextField(blank=True, null=True)), + ], + ), + migrations.CreateModel( + name='InferenceStudy', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('timestamp', models.DateTimeField(auto_now_add=True)), + ('max_study_edge_weight', models.FloatField(null=True)), + ('min_study_edge_weight', models.FloatField(null=True)), + ('avg_study_edge_weight', models.FloatField(null=True)), + ('std_study_edge_weight', models.FloatField(null=True)), + ('is_public', models.BooleanField(default=False)), + ('status', models.CharField(max_length=10)), + ('error_message', models.TextField(blank=True, null=True)), + ('algorithm', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='studies', to='gennifer.algorithm')), + ('dataset', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='studies', to='gennifer.dataset')), + ('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='studies', to=settings.AUTH_USER_MODEL)), + ], + ), + migrations.CreateModel( + name='InferenceResult', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('edge_weight', models.FloatField()), + ('is_public', models.BooleanField(default=False)), + ('study', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='results', to='gennifer.inferencestudy')), + ('target', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='inference_result_target', to='gennifer.gene')), + ('tf', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='inference_result_tf', to='gennifer.gene')), + ], + ), + ] diff --git a/chp_api/gennifer/migrations/0002_inferenceresult_user.py b/chp_api/gennifer/migrations/0002_inferenceresult_user.py new file mode 100755 index 0000000..4fa44e3 --- /dev/null +++ b/chp_api/gennifer/migrations/0002_inferenceresult_user.py @@ -0,0 +1,21 @@ +# Generated by Django 4.2.1 on 2023-05-29 23:15 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('gennifer', '0001_initial'), + ] + + operations = [ + migrations.AddField( + model_name='inferenceresult', + name='user', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='results', to=settings.AUTH_USER_MODEL), + ), + ] diff --git a/chp_api/gennifer/migrations/0003_remove_algorithm_run_url_algorithm_url.py b/chp_api/gennifer/migrations/0003_remove_algorithm_run_url_algorithm_url.py new file mode 100755 index 0000000..c4184e4 --- /dev/null +++ b/chp_api/gennifer/migrations/0003_remove_algorithm_run_url_algorithm_url.py @@ -0,0 +1,23 @@ +# Generated by Django 4.2.1 on 2023-05-30 00:32 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('gennifer', '0002_inferenceresult_user'), + ] + + operations = [ + migrations.RemoveField( + model_name='algorithm', + name='run_url', + ), + migrations.AddField( + model_name='algorithm', + name='url', + field=models.CharField(default='localhost', max_length=128), + preserve_default=False, + ), + ] diff --git a/chp_api/gennifer/models.py b/chp_api/gennifer/models.py index aa27fde..dd2793a 100644 --- a/chp_api/gennifer/models.py +++ b/chp_api/gennifer/models.py @@ -1,10 +1,12 @@ +import requests + from django.db import models from django.contrib.auth.models import User class Algorithm(models.Model): name = models.CharField(max_length=128) - run_url = models.URLField(max_length=128) + url = models.CharField(max_length=128) def __str__(self): return self.name @@ -24,8 +26,10 @@ def save(self, *args, **kwargs): info = self.get_record() self.doi = info["doi"] - self.description = re.sub(CLEANR, '', infoi["metadata"]["description"]) - self.title = re.sub(CLEANR, '', infoi["metadata"]["title"]) + self.description = re.sub(CLEANR, '', info["metadata"]["description"]) + self.title = re.sub(CLEANR, '', info["metadata"]["title"]) + + super(Dataset, self).save(*args, **kwargs) def get_record(self): return requests.get(f"https://zenodo.org/api/records/{self.zenodo_id}").json() @@ -56,9 +60,10 @@ class InferenceStudy(models.Model): class InferenceResult(models.Model): # Stands for transcription factor - tf = models.ForeignKey(Gene, on_delete=models.CASCADE) + tf = models.ForeignKey(Gene, on_delete=models.CASCADE, related_name='inference_result_tf') # Target is the gene that is regulated by the transcription factor - target = models.ForeignKey(Gene, on_delete=models.CASCADE) + target = models.ForeignKey(Gene, on_delete=models.CASCADE, related_name='inference_result_target') edge_weight = models.FloatField() study = models.ForeignKey(InferenceStudy, on_delete=models.CASCADE, related_name='results') is_public = models.BooleanField(default=False) + user = models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True, related_name='results') diff --git a/chp_api/gennifer/serializers.py b/chp_api/gennifer/serializers.py index 0ce4ecb..9acc819 100644 --- a/chp_api/gennifer/serializers.py +++ b/chp_api/gennifer/serializers.py @@ -5,7 +5,7 @@ class DatasetSerializer(serializers.ModelSerializer): class Meta: model = Dataset - fields = ['name', 'zenodo_id', 'doi', 'description'] + fields = ['upload_user', 'title', 'zenodo_id', 'doi', 'description'] class InferenceStudySerializer(serializers.ModelSerializer): @@ -32,4 +32,5 @@ class Meta: 'edge_weight', 'study', 'is_public', + 'user', ] diff --git a/chp_api/gennifer/tasks.py b/chp_api/gennifer/tasks.py index e9cfc48..dc00a91 100644 --- a/chp_api/gennifer/tasks.py +++ b/chp_api/gennifer/tasks.py @@ -3,18 +3,21 @@ import pandas as pd import requests +from django.db import transaction +from django.contrib.auth.models import User from celery import shared_task +from celery.utils.log import get_task_logger -from .models import Dataset, Gene, InferenceStudy, InferenceResult -from dispacter.models import DispatcherSettings +from .models import Dataset, Gene, InferenceStudy, InferenceResult, Algorithm +from dispatcher.models import DispatcherSetting + +logger = get_task_logger(__name__) def normalize_nodes(curies): - dispatcher_settings = DispatcherSettings.load() + dispatcher_settings = DispatcherSetting.load() base_url = dispatcher_settings.sri_node_normalizer_baseurl - return requests.post( - f'{base_url}/get_normalized_nodes', - json={"curies": curies} - ) + res = requests.post(f'{base_url}/get_normalized_nodes', json={"curies": curies}) + return res.json() def extract_variant_info(gene_id): split = gene_id.split('(') @@ -56,15 +59,23 @@ def save_inference_study(study, status, failed=False): # Construct Gene Objects gene1, variant_info1 = extract_variant_info(row["Gene1"]) gene2, variant_info2 = extract_variant_info(row["Gene2"]) + try: + gene1_name = res[gene1]["id"]["label"] + except TypeError: + gene1_name = 'Not found in SRI Node Normalizer.' + try: + gene2_name = res[gene2]["id"]["label"] + except TypeError: + gene2_name = 'Not found in SRI Node Normalizer.' gene1_obj, created = Gene.objects.get_or_create( - name=res[gene1]["id"]["label"], + name=gene1_name, curie=gene1, variant=variant_info1, ) if created: gene1_obj.save() gene2_obj, created = Gene.objects.get_or_create( - name=res[gene2]["id"]["label"], + name=gene2_name, curie=gene2, variant=variant_info2, ) @@ -76,16 +87,22 @@ def save_inference_study(study, status, failed=False): target=gene2_obj, edge_weight=row["EdgeWeight"], study=study, + user=study.user, ) result.save() study.save() return True def get_status(algo, task_id): - return requests.get(f'{algo.url}/status/{task_id}').json() + return requests.get(f'{algo.url}/status/{task_id}', headers={'Cache-Control': 'no-cache'}).json() @shared_task(name="create_gennifer_task") -def create_task(algorithm, zenodo_id, hyperparameters, user): +def create_task(algorithm_name, zenodo_id, hyperparameters, user_pk): + # Get algorithm obj + algo = Algorithm.objects.get(name=algorithm_name) + # Get User obj + user = User.objects.get(pk=user_pk) + # Initialize dataset instance dataset, created = Dataset.objects.get_or_create( zenodo_id=zenodo_id, @@ -99,10 +116,12 @@ def create_task(algorithm, zenodo_id, hyperparameters, user): "zenodo_id": zenodo_id, "hyperparameters": hyperparameters, } - task["task_id"] = requests.post(f'{algo.url}/run', data=gennifer_request) + task_id = requests.post(f'{algo.url}/run', json=gennifer_request).json()["task_id"] + + logger.info(f'TASK_ID: {task_id}') # Get initial status - status = get_status(algo, task["task_id"]) + status = get_status(algo, task_id) # Create Inference Study study = InferenceStudy.objects.create( @@ -118,8 +137,8 @@ def create_task(algorithm, zenodo_id, hyperparameters, user): #TODO: Not sure if this is best practice while True: # Check in every 2 seconds - time.sleep(2) - status = get_status(algo, task["task_id"]) + time.sleep(5) + status = get_status(algo, task_id) if status["task_status"] == 'SUCCESS': return save_inference_study(study, status) if status["task_status"] == "FAILURE": diff --git a/chp_api/gennifer/views.py b/chp_api/gennifer/views.py index aca73bb..04f9f6b 100644 --- a/chp_api/gennifer/views.py +++ b/chp_api/gennifer/views.py @@ -6,7 +6,7 @@ from rest_framework import viewsets from rest_framework.views import APIView -from rest_framework.requests import Response +from rest_framework.response import Response from rest_framework.permissions import IsAuthenticated from django_filters.rest_framework import DjangoFilterBackend @@ -17,14 +17,14 @@ class DatasetViewSet(viewsets.ModelViewSet): queryset = Dataset.objects.all() serializer_class = DatasetSerializer - filter_backend = [DjangoFilterBackend] - filterset_fields = ['user', 'zenodo_id'] + filter_backends = [DjangoFilterBackend] + filterset_fields = ['upload_user', 'zenodo_id'] permission_classes = [IsAuthenticated] class InferenceStudyViewSet(viewsets.ModelViewSet): serializer_class = InferenceStudySerializer - filter_backend = [DjangoFilterBackend] + filter_backends = [DjangoFilterBackend] filterset_fields = ['is_public', 'dataset', 'algorithm'] permission_classes = [IsAuthenticated] @@ -35,8 +35,8 @@ def get_queryset(self): class InferenceResultViewSet(viewsets.ModelViewSet): serializer_class = InferenceResultSerializer - filter_backend = [DjangoFilterBackend] - filterset_fields = ['is_public', 'study'] + filter_backends = [DjangoFilterBackend] + filterset_fields = ['is_public', 'study', 'tf', 'target'] permission_classes = [IsAuthenticated] def get_queryset(self): @@ -51,26 +51,26 @@ def post(self, request): """ # Build gennifer requests tasks = request.data['tasks'] - response = [] + response = {"tasks": []} for task in tasks: algorithm_name = task.get("algorithm_name", None) zenodo_id = task.get("zenodo_id", None) hyperparameters = task.get("hyperparameters", None) if not algorithm_name: task["error"] = "No algorithm name provided." - response.append(task) + response["tasks"].append(task) continue if not zenodo_id: task["error"] = "No dataset Zenodo identifer provided." - response.append(task) + response["tasks"].append(task) continue try: algo = Algorithm.objects.get(name=algorithm_name) except ObjectDoesNotExist: task["error"] = f"The algorithm: {algorithm_name} is not supported in Gennifer." - response.append(task) + response["tasks"].append(task) continue # If all pass, now send to gennifer services - task["task_id"] = create_task.delay(algo, zenodo_id, hyperparameters, request.user).id - response.append(task) - return Response(response) + task["task_id"] = create_task.delay(algo.name, zenodo_id, hyperparameters, request.user.pk).id + response["tasks"].append(task) + return JsonResponse(response) diff --git a/compose.chp-api.yaml b/compose.chp-api.yaml index cb74564..f4c19f5 100644 --- a/compose.chp-api.yaml +++ b/compose.chp-api.yaml @@ -49,6 +49,8 @@ services: - DJANGO_SUPERUSER_USERNAME_FILE=/run/secrets/django-superuser-username - DJANGO_SUPERUSER_EMAIL_FILE=/run/secrets/django-superuser-email - DJANGO_SUPERUSER_PASSWORD_FILE=/run/secrets/django-superuser-password + - CELERY_BROKER_URL=redis://redis:6379/0 + - CELERY_RESULT_BACKEND=redis://redis:6379/0 # Uncomment this for production #- DJANGO_SETTINGS_MODULE=mysite.settings.production # Comment this for development @@ -91,10 +93,9 @@ services: - DJANGO_SUPERUSER_USERNAME_FILE=/run/secrets/django-superuser-username - DJANGO_SUPERUSER_EMAIL_FILE=/run/secrets/django-superuser-email - DJANGO_SUPERUSER_PASSWORD_FILE=/run/secrets/django-superuser-password - command: celery -A chp_api worker --loglevel=info - environment: - - CELERY_BROKER_URL=redis://redis:6379/0 - - CELERY_RESULT_BACKEND=redis://redis:6379/0 + - CELERY_BROKER_URL=redis://redis:6379/0 + - CELERY_RESULT_BACKEND=redis://redis:6379/0 + command: celery -A chp_api worker -Q chp_api --loglevel=info depends_on: - api - redis @@ -122,12 +123,11 @@ services: - DJANGO_SUPERUSER_USERNAME_FILE=/run/secrets/django-superuser-username - DJANGO_SUPERUSER_EMAIL_FILE=/run/secrets/django-superuser-email - DJANGO_SUPERUSER_PASSWORD_FILE=/run/secrets/django-superuser-password - command: celery -A chp_api flower --port=5555 --broker=redis://redis:6379/0 + - CELERY_BROKER_URL=redis://redis:6379/0 + - CELERY_RESULT_BACKEND=redis://redis:6379/0 + command: celery -A chp_api --broker=redis://redis:6379/0 flower --port=5555 ports: - 5556:5555 - environment: - - CELERY_BROKER_URL=redis://redis:6379/0 - - CELERY_RESULT_BACKEND=redis://redis:6379/0 depends_on: - api - redis diff --git a/compose.gennifer.yaml b/compose.gennifer.yaml index 0f91baf..bbb6a7e 100644 --- a/compose.gennifer.yaml +++ b/compose.gennifer.yaml @@ -25,7 +25,7 @@ services: build: context: ./gennifer/pidc dockerfile: Dockerfile - command: celery --app pidc.tasks.celery worker --loglevel=info + command: celery --app pidc.tasks.celery worker -Q pidc --loglevel=info environment: - CELERY_BROKER_URL=redis://redis:6379/0 - CELERY_RESULT_BACKEND=redis://redis:6379/0 @@ -37,7 +37,7 @@ services: build: context: ./gennifer/pidc dockerfile: Dockerfile - command: celery --app pidc.tasks.celery flower --port=5555 --broker=redis://redis:6379/0 + command: celery --app pidc.tasks.celery --broker=redis://redis:6379/0 flower --port=5555 ports: - 5557:5555 environment: diff --git a/copy-migrations b/copy-migrations new file mode 100755 index 0000000..c362e86 --- /dev/null +++ b/copy-migrations @@ -0,0 +1,5 @@ +#!/bin/bash + +docker compose -f compose.chp-api.yaml run -v migrations:/home/migrations \ + --user root api \ + bash -c "python3 manage.py makemigrations && cp -r /home/chp_api/web/dispatcher/migrations /home/migrations/dispatcher && cp -r /home/chp_api/web/gennifer/migrations /home/migrations/gennifer" diff --git a/deployment-script b/deployment-script index eacc8d8..0d572bb 100755 --- a/deployment-script +++ b/deployment-script @@ -5,20 +5,18 @@ django_superuser_username='cat secrets/chp_api/django_superuser_username.txt' django_superuser_email='cat secrets/chp_api/django_superuser_email.txt' # Only to be run when building on dev machine -docker compose -f compose.chp-api.yaml -f compose.gennifer.yaml build +docker compose -f compose.chp-api.yaml -f compose.gennifer.yaml up -d --build --remove-orphans -docker compose up -d - -docker compose run chp-api python3 manage.py migrate +docker compose -f compose.chp-api.yaml run api python3 manage.py migrate # Create a database superuser -docker compose run --user root chp-api python3 manage.py createsuperuser --no-input #--username $django_superuser_username --email $django_superuser_email +docker compose -f compose.chp-api.yaml run --user root api python3 manage.py createsuperuser --no-input #--username $django_superuser_username --email $django_superuser_email # Load apps -docker compose run chp-api python3 manage.py runscript load_db_apps -docker compose run chp-api python3 manage.py runscript templater -docker compose run chp_api Python3 manage.py runscript gene_spec_curie_templater +docker compose -f compose.chp-api.yaml run api python3 manage.py runscript load_db_apps +docker compose -f compose.chp-api.yaml run api python3 manage.py runscript templater +docker compose -f compose.chp-api.yaml run api python3 manage.py runscript gene_spec_curie_templater -docker compose run --user root chp-api python3 manage.py collectstatic --noinput +docker compose -f compose.chp-api.yaml run --user root api python3 manage.py collectstatic --noinput echo "Check logs with: docker compose logs -f" diff --git a/gennifer b/gennifer index ea5019c..2ad99ae 160000 --- a/gennifer +++ b/gennifer @@ -1 +1 @@ -Subproject commit ea5019c09a019fe8ccb4e5d2d80d1fe1befc5382 +Subproject commit 2ad99ae704fd3ee4b43b7f9ef9690b831da8f670 diff --git a/gennifer-sample.json b/gennifer-sample.json new file mode 100644 index 0000000..fad8894 --- /dev/null +++ b/gennifer-sample.json @@ -0,0 +1,9 @@ +{ + "tasks": [ + { + "algorithm_name": "pidc", + "zenodo_id":"7982629", + "hyperparameters": null + } + ] +} From c850bc54e764221564bfbba47047c64c0d29a534 Mon Sep 17 00:00:00 2001 From: Chase Yakaboski Date: Tue, 30 May 2023 22:17:49 -0400 Subject: [PATCH 053/132] Updated to working gennifer API with PIDC support and working TRAPI interface that is integrated with dispatcher. --- chp_api/chp_api/settings.py | 8 +- chp_api/chp_api/urls.py | 2 +- chp_api/dispatcher/base.py | 22 +- chp_api/gennifer/_version.py | 1 + chp_api/gennifer/admin.py | 4 +- chp_api/gennifer/algorithm-loader.py | 0 chp_api/gennifer/app_interface.py | 27 +++ .../app_meta_data/conflation_map.json | 1 + chp_api/gennifer/app_meta_data/epc.json | 11 + .../app_meta_data/meta_knowledge_graph.json | 21 ++ ...emove_inferencestudy_algorithm_and_more.py | 47 ++++ .../0005_gene_chp_preferred_curie.py | 18 ++ chp_api/gennifer/models.py | 25 +- chp_api/gennifer/scripts/algorithm_loader.py | 18 ++ chp_api/gennifer/serializers.py | 2 +- chp_api/gennifer/tasks.py | 65 +++++- chp_api/gennifer/trapi_interface.py | 219 ++++++++++++++++++ chp_api/gennifer/views.py | 5 +- compose.gennifer.yaml | 15 -- deployment-script | 1 + gennifer | 2 +- gennifer-sample.json | 2 +- 22 files changed, 487 insertions(+), 29 deletions(-) create mode 100644 chp_api/gennifer/_version.py create mode 100644 chp_api/gennifer/algorithm-loader.py create mode 100644 chp_api/gennifer/app_interface.py create mode 100644 chp_api/gennifer/app_meta_data/conflation_map.json create mode 100644 chp_api/gennifer/app_meta_data/epc.json create mode 100644 chp_api/gennifer/app_meta_data/meta_knowledge_graph.json create mode 100755 chp_api/gennifer/migrations/0004_remove_inferencestudy_algorithm_and_more.py create mode 100755 chp_api/gennifer/migrations/0005_gene_chp_preferred_curie.py create mode 100644 chp_api/gennifer/scripts/algorithm_loader.py create mode 100644 chp_api/gennifer/trapi_interface.py diff --git a/chp_api/chp_api/settings.py b/chp_api/chp_api/settings.py index cebb02a..99c3ed9 100644 --- a/chp_api/chp_api/settings.py +++ b/chp_api/chp_api/settings.py @@ -44,11 +44,12 @@ 'django_filters', 'dispatcher.apps.DispatcherConfig', 'django_extensions', - 'gennifer', # Need to make into CHP app + #'gennifer', # Need to make into CHP app ] INSTALLED_CHP_APPS = [ 'gene_specificity', + 'gennifer', ] # CHP Versions @@ -176,3 +177,8 @@ # Celery Settings CELERY_BROKER_URL = os.environ.get("CELERY_BROKER_URL", "redis://localhost:6379") CELERY_RESULT_BACKEND = os.environ.get("CELERY_RESULT_BACKEND", "redis://localhost:6379") + +# Gennifer settings +GENNIFER_ALGORITHM_URLS = [ + "http://pidc:5000" + ] diff --git a/chp_api/chp_api/urls.py b/chp_api/chp_api/urls.py index 24d1feb..4ba98dc 100644 --- a/chp_api/chp_api/urls.py +++ b/chp_api/chp_api/urls.py @@ -20,7 +20,7 @@ urlpatterns = [ path('admin/', admin.site.urls), path('', include('dispatcher.urls')), - path('gennifer/', include('gennifer.urls')), + path('gennifer/api/', include('gennifer.urls')), ] diff --git a/chp_api/dispatcher/base.py b/chp_api/dispatcher/base.py index c659a4a..92e7108 100644 --- a/chp_api/dispatcher/base.py +++ b/chp_api/dispatcher/base.py @@ -13,7 +13,7 @@ from collections import defaultdict from .models import Transaction, App, DispatcherSetting, Template, TemplateMatch -from reasoner_pydantic import MetaKnowledgeGraph, Message, MetaEdge +from reasoner_pydantic import MetaKnowledgeGraph, Message, MetaEdge, MetaNode from reasoner_pydantic.qgraph import QNode, QEdge # Setup logging @@ -42,6 +42,24 @@ def __init__(self, request, trapi_version, biolink_version): #self.validator = TRAPISchemaValidator(self.trapi_version) self.logger = Logger() + def merge_meta_kg(self, metakg1, metakg2): + new_metakg = MetaKnowledgeGraph.parse_obj({"nodes": [], "edges": []}) + # Merge nodes + new_metakg.nodes = metakg1.nodes + for n, v in metakg2.nodes.items(): + if n in new_metakg.nodes: + id_prefixes = list(set.union(*[set(list(metakg1.nodes[n].id_prefixes)), set(list(metakg2.nodes[n].id_prefixes))])) + new_node = MetaNode.parse_obj({"id_prefixes": id_prefixes}) + new_metakg.nodes[n] = new_node + else: + new_metakg.nodes[n] = MetaNode.parse_obj(v) + # Merge edges + for e in metakg1.edges: + new_metakg.edges.append(e) + for e in metakg2.edges: + new_metakg.edges.append(e) + return new_metakg + def get_meta_knowledge_graph(self): # Get current trapi and biolink versions dispatcher_settings = DispatcherSetting.load() @@ -59,7 +77,7 @@ def get_meta_knowledge_graph(self): if merged_meta_kg is None: merged_meta_kg = meta_kg else: - merged_meta_kg.update(meta_kg) + merged_meta_kg = self.merge_meta_kg(merged_meta_kg, meta_kg) return merged_meta_kg def process_invalid_trapi(self, request): diff --git a/chp_api/gennifer/_version.py b/chp_api/gennifer/_version.py new file mode 100644 index 0000000..1f356cc --- /dev/null +++ b/chp_api/gennifer/_version.py @@ -0,0 +1 @@ +__version__ = '1.0.0' diff --git a/chp_api/gennifer/admin.py b/chp_api/gennifer/admin.py index b4a7ca6..d2ab304 100644 --- a/chp_api/gennifer/admin.py +++ b/chp_api/gennifer/admin.py @@ -1,7 +1,9 @@ from django.contrib import admin -from .models import Algorithm, Dataset, InferenceStudy +from .models import Algorithm, Dataset, InferenceStudy, InferenceResult, Gene admin.site.register(Algorithm) admin.site.register(Dataset) admin.site.register(InferenceStudy) +admin.site.register(InferenceResult) +admin.site.register(Gene) diff --git a/chp_api/gennifer/algorithm-loader.py b/chp_api/gennifer/algorithm-loader.py new file mode 100644 index 0000000..e69de29 diff --git a/chp_api/gennifer/app_interface.py b/chp_api/gennifer/app_interface.py new file mode 100644 index 0000000..3e9a717 --- /dev/null +++ b/chp_api/gennifer/app_interface.py @@ -0,0 +1,27 @@ +from asyncio import constants +from .trapi_interface import TrapiInterface +from .apps import GenniferConfig +from reasoner_pydantic import MetaKnowledgeGraph, Message +from typing import TYPE_CHECKING, Union, List + +def get_app_config(message: Union[Message, None]) -> GenniferConfig: + return GenniferConfig + + +def get_trapi_interface(get_app_config: GenniferConfig = get_app_config(None)): + return TrapiInterface(trapi_version='1.4') + + +def get_meta_knowledge_graph() -> MetaKnowledgeGraph: + interface: TrapiInterface = get_trapi_interface() + return interface.get_meta_knowledge_graph() + + +def get_response(consistent_queries: List[Message], logger): + """ Should return app responses plus app_logs, status, and description information.""" + responses = [] + interface = get_trapi_interface() + for consistent_query in consistent_queries: + response = interface.get_response(consistent_query, logger) + responses.append(response) + return responses diff --git a/chp_api/gennifer/app_meta_data/conflation_map.json b/chp_api/gennifer/app_meta_data/conflation_map.json new file mode 100644 index 0000000..9e26dfe --- /dev/null +++ b/chp_api/gennifer/app_meta_data/conflation_map.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/chp_api/gennifer/app_meta_data/epc.json b/chp_api/gennifer/app_meta_data/epc.json new file mode 100644 index 0000000..63a13b9 --- /dev/null +++ b/chp_api/gennifer/app_meta_data/epc.json @@ -0,0 +1,11 @@ +[ + { + "attribute_type_id": "biolink:primary_knowledge_source", + "original_attribute_name": null, + "value": "infores:connections-hypothesis", + "value_type_id": "biolink:InformationResource", + "attribute_source": "infores:connections-hypothesis", + "value_url": "http://chp.thayer.dartmouth.edu", + "description": "The Connections Hypothesis Provider from NCATS Translator." + } +] diff --git a/chp_api/gennifer/app_meta_data/meta_knowledge_graph.json b/chp_api/gennifer/app_meta_data/meta_knowledge_graph.json new file mode 100644 index 0000000..41ffac5 --- /dev/null +++ b/chp_api/gennifer/app_meta_data/meta_knowledge_graph.json @@ -0,0 +1,21 @@ +{ + "nodes": { + "biolink:Gene": { + "id_prefixes": [ + "ENSEMBL" + ] + } + }, + "edges": [ + { + "subject": "biolink:Gene", + "object": "biolink:Gene", + "predicate": "biolink:regulates" + }, + { + "subject": "biolink:Gene", + "object": "biolink:Gene", + "predicate": "biolink:regulated_by" + } + ] +} diff --git a/chp_api/gennifer/migrations/0004_remove_inferencestudy_algorithm_and_more.py b/chp_api/gennifer/migrations/0004_remove_inferencestudy_algorithm_and_more.py new file mode 100755 index 0000000..f15b3db --- /dev/null +++ b/chp_api/gennifer/migrations/0004_remove_inferencestudy_algorithm_and_more.py @@ -0,0 +1,47 @@ +# Generated by Django 4.2.1 on 2023-05-30 22:18 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('gennifer', '0003_remove_algorithm_run_url_algorithm_url'), + ] + + operations = [ + migrations.RemoveField( + model_name='inferencestudy', + name='algorithm', + ), + migrations.AddField( + model_name='algorithm', + name='description', + field=models.TextField(blank=True, null=True), + ), + migrations.AddField( + model_name='algorithm', + name='edge_weight_description', + field=models.TextField(blank=True, null=True), + ), + migrations.AddField( + model_name='algorithm', + name='edge_weight_type', + field=models.CharField(blank=True, max_length=128, null=True), + ), + migrations.CreateModel( + name='AlgorithmInstance', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('hyperparameters', models.JSONField(null=True)), + ('algorithm', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='instances', to='gennifer.algorithm')), + ], + ), + migrations.AddField( + model_name='inferencestudy', + name='algorithm_instance', + field=models.ForeignKey(default=1, on_delete=django.db.models.deletion.CASCADE, related_name='studies', to='gennifer.algorithminstance'), + preserve_default=False, + ), + ] diff --git a/chp_api/gennifer/migrations/0005_gene_chp_preferred_curie.py b/chp_api/gennifer/migrations/0005_gene_chp_preferred_curie.py new file mode 100755 index 0000000..3b1955b --- /dev/null +++ b/chp_api/gennifer/migrations/0005_gene_chp_preferred_curie.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.1 on 2023-05-31 01:29 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('gennifer', '0004_remove_inferencestudy_algorithm_and_more'), + ] + + operations = [ + migrations.AddField( + model_name='gene', + name='chp_preferred_curie', + field=models.CharField(blank=True, max_length=128, null=True), + ), + ] diff --git a/chp_api/gennifer/models.py b/chp_api/gennifer/models.py index dd2793a..8abde76 100644 --- a/chp_api/gennifer/models.py +++ b/chp_api/gennifer/models.py @@ -7,11 +7,26 @@ class Algorithm(models.Model): name = models.CharField(max_length=128) url = models.CharField(max_length=128) + edge_weight_description = models.TextField(null=True, blank=True) + edge_weight_type = models.CharField(max_length=128, null=True, blank=True) + description = models.TextField(null=True, blank=True) def __str__(self): return self.name +class AlgorithmInstance(models.Model): + algorithm = models.ForeignKey(Algorithm, on_delete=models.CASCADE, related_name='instances') + hyperparameters = models.JSONField(null=True) + + def __str__(self): + if self.hyperparameters: + hypers = tuple([f'{k}={v}' for k, v in self.hyperparameters.items()]) + else: + hypers = '()' + return f'{self.algorithm.name}{hypers}' + + class Dataset(models.Model): title = models.CharField(max_length=128) zenodo_id = models.CharField(max_length=128, primary_key=True) @@ -39,13 +54,14 @@ class Gene(models.Model): name = models.CharField(max_length=128) curie = models.CharField(max_length=128) variant = models.TextField(null=True, blank=True) + chp_preferred_curie = models.CharField(max_length=128, null=True, blank=True) def __str__(self): return self.name class InferenceStudy(models.Model): - algorithm = models.ForeignKey(Algorithm, on_delete=models.CASCADE, related_name='studies') + algorithm_instance = models.ForeignKey(AlgorithmInstance, on_delete=models.CASCADE, related_name='studies') user = models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True, related_name='studies') dataset = models.ForeignKey(Dataset, on_delete=models.CASCADE, related_name='studies') timestamp = models.DateTimeField(auto_now_add=True) @@ -58,6 +74,10 @@ class InferenceStudy(models.Model): status = models.CharField(max_length=10) error_message = models.TextField(null=True, blank=True) + def __str__(self): + return f'{self.algorithm_instance} on {self.dataset.zenodo_id}' + + class InferenceResult(models.Model): # Stands for transcription factor tf = models.ForeignKey(Gene, on_delete=models.CASCADE, related_name='inference_result_tf') @@ -67,3 +87,6 @@ class InferenceResult(models.Model): study = models.ForeignKey(InferenceStudy, on_delete=models.CASCADE, related_name='results') is_public = models.BooleanField(default=False) user = models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True, related_name='results') + + def __str__(self): + return f'{self.tf}:{self.tf.curie} -> regulates -> {self.target}:{self.target.curie}' diff --git a/chp_api/gennifer/scripts/algorithm_loader.py b/chp_api/gennifer/scripts/algorithm_loader.py new file mode 100644 index 0000000..af2893f --- /dev/null +++ b/chp_api/gennifer/scripts/algorithm_loader.py @@ -0,0 +1,18 @@ +import requests +from django.conf import settings + +from ..models import Algorithm + + +def run(): + Algorithm.objects.all().delete() + for url in settings.GENNIFER_ALGORITHM_URLS: + algo_info = requests.get(f'{url}/info').json() + algo = Algorithm.objects.create( + name=algo_info["name"], + url=url, + edge_weight_description=algo_info["edge_weight_description"], + edge_weight_type=algo_info["edge_weight_type"], + description=algo_info["description"], + ) + algo.save() diff --git a/chp_api/gennifer/serializers.py b/chp_api/gennifer/serializers.py index 9acc819..dea0750 100644 --- a/chp_api/gennifer/serializers.py +++ b/chp_api/gennifer/serializers.py @@ -12,7 +12,7 @@ class InferenceStudySerializer(serializers.ModelSerializer): class Meta: model = InferenceStudy fields = [ - 'algorithm', + 'algorithm_instance', 'user', 'dataset', 'timestamp', diff --git a/chp_api/gennifer/tasks.py b/chp_api/gennifer/tasks.py index dc00a91..70dd1dc 100644 --- a/chp_api/gennifer/tasks.py +++ b/chp_api/gennifer/tasks.py @@ -7,8 +7,9 @@ from django.contrib.auth.models import User from celery import shared_task from celery.utils.log import get_task_logger +from copy import deepcopy -from .models import Dataset, Gene, InferenceStudy, InferenceResult, Algorithm +from .models import Dataset, Gene, InferenceStudy, InferenceResult, Algorithm, AlgorithmInstance from dispatcher.models import DispatcherSetting logger = get_task_logger(__name__) @@ -28,6 +29,12 @@ def extract_variant_info(gene_id): variant_info = None return gene_id, variant_info +def get_chp_preferred_curie(info): + for _id in info['equivalent_identifiers']: + if 'ENSEMBL' in _id['identifier']: + return _id['identifier'] + return None + def save_inference_study(study, status, failed=False): study.status = status["task_status"] if failed: @@ -61,16 +68,21 @@ def save_inference_study(study, status, failed=False): gene2, variant_info2 = extract_variant_info(row["Gene2"]) try: gene1_name = res[gene1]["id"]["label"] + gene1_chp_preferred_curie = get_chp_preferred_curie(res[gene1]) except TypeError: gene1_name = 'Not found in SRI Node Normalizer.' + gene1_chp_preferred_curie = None try: gene2_name = res[gene2]["id"]["label"] + gene2_chp_preferred_curie = get_chp_preferred_curie(res[gene2]) except TypeError: gene2_name = 'Not found in SRI Node Normalizer.' + gene2_chp_preferred_curie = None gene1_obj, created = Gene.objects.get_or_create( name=gene1_name, curie=gene1, variant=variant_info1, + chp_preferred_curie=gene1_chp_preferred_curie, ) if created: gene1_obj.save() @@ -78,6 +90,7 @@ def save_inference_study(study, status, failed=False): name=gene2_name, curie=gene2, variant=variant_info2, + chp_preferred_curie=gene2_chp_preferred_curie, ) if created: gene2_obj.save() @@ -96,21 +109,65 @@ def save_inference_study(study, status, failed=False): def get_status(algo, task_id): return requests.get(f'{algo.url}/status/{task_id}', headers={'Cache-Control': 'no-cache'}).json() + +def return_saved_study(studies, user): + study = studies[0] + # Copy study results + results = deepcopy(study.results) + # Create a new study that is a duplicate but assign to this user. + study.pk = None + study.results = None + study.save() + + # Now go through and assign all results to this study and user. + for result in results: + result.pk = None + result.study = study + result.user = user + result.save() + return True + + @shared_task(name="create_gennifer_task") def create_task(algorithm_name, zenodo_id, hyperparameters, user_pk): # Get algorithm obj algo = Algorithm.objects.get(name=algorithm_name) + + # Get or create a new algorithm instance based on the hyperparameters + if not hyperparameters: + algo_instance, algo_instance_created = AlgorithmInstance.objects.get_or_create( + algorithm=algo, + hyperparameters__isnull=True, + ) + else: + algo_instance, algo_instance_created = AlgorithmInstance.objects.get_or_create( + algorithm=algo, + hyperparameters=hyperparameters, + ) + # Get User obj user = User.objects.get(pk=user_pk) # Initialize dataset instance - dataset, created = Dataset.objects.get_or_create( + dataset, dataset_created = Dataset.objects.get_or_create( zenodo_id=zenodo_id, upload_user=user, ) - if created: + + if dataset_created: dataset.save() + if not algo_instance_created and not dataset_created: + # This means we've already run the study. So let's just return that and not bother our workers. + studies = InferenceStudy.objects.filter( + algorithm_instance=algo_instance, + dataset=dataset, + status='SUCCESS', + ) + #TODO: Probably should add some timestamp handling here + if len(studies) > 0: + return_saved_study(studies, user) + # Send to gennifer app gennifer_request = { "zenodo_id": zenodo_id, @@ -125,7 +182,7 @@ def create_task(algorithm_name, zenodo_id, hyperparameters, user_pk): # Create Inference Study study = InferenceStudy.objects.create( - algorithm=algo, + algorithm_instance=algo_instance, user=user, dataset=dataset, status=status["task_status"], diff --git a/chp_api/gennifer/trapi_interface.py b/chp_api/gennifer/trapi_interface.py new file mode 100644 index 0000000..cf83e75 --- /dev/null +++ b/chp_api/gennifer/trapi_interface.py @@ -0,0 +1,219 @@ +'''trapi interface''' +import os +import uuid +import json +import pkgutil +import logging + +from typing import Tuple, Union +from django.db.models import QuerySet +from django.core.exceptions import ObjectDoesNotExist +from reasoner_pydantic import MetaKnowledgeGraph, Message, KnowledgeGraph +from reasoner_pydantic.kgraph import RetrievalSource, Attribute +from reasoner_pydantic.results import NodeBinding, EdgeBinding, Result, Results, Analysis + +from .models import InferenceResult, Gene + +# Setup logging +logging.addLevelName(25, "NOTE") +# Add a special logging function +def note(self, message, *args, **kwargs): + self._log(25, message, args, kwargs) +logging.Logger.note = note +internal_logger = logging.getLogger(__name__) + +APP_PATH = os.path.dirname(os.path.abspath(__file__)) + +class TrapiInterface: + def __init__(self, trapi_version: str = '1.4'): + self.trapi_version = trapi_version + + def get_meta_knowledge_graph(self) -> MetaKnowledgeGraph: + return self._read_meta_knowledge_graph() + + def _read_meta_knowledge_graph(self) -> MetaKnowledgeGraph: + with open(os.path.join(APP_PATH, 'app_meta_data', 'meta_knowledge_graph.json'), 'r') as mkg_file: + mkg_json = json.load(mkg_file) + return MetaKnowledgeGraph.parse_obj(mkg_json) + + def get_name(self) -> str: + return 'gennifer' + + def _get_sources(self): + source_1 = RetrievalSource(resource_id = "infores:connections-hypothesis", + resource_role="primary_knowledge_source") + return {source_1} + + def _get_attributes(self, val, algorithm_instance, dataset): + att_1 = Attribute( + attribute_type_id = algorithm_instance.algorithm.edge_weight_type, + value_type_id='biolink:has_evidence', + value=val, + description=algorithm_instance.algorithm.edge_weight_description, + ) + att_2 = Attribute( + attribute_type_id='grn_inference_algorithm', + value_type_id='biolink:supporting_study_method_type', + value=str(algorithm_instance), + description=algorithm_instance.algorithm.description, + ) + att_3 = Attribute( + attribute_type_id='inferenced_dataset', + value_type_id='biolink:supporting_data_set', + value=f'zenodo:{dataset.zenodo_id}', + description=f'{dataset.title}: {dataset.description}', + ) + att_4 = Attribute( + attribute_type_id = 'primary_knowledge_source', + value='infores:connections-hypothesis', + value_url='https://github.com/di2ag/gennifer', + description='The Connections Hypothesis Provider from NCATS Translator.' + ) + return {att_1, att_2, att_3, att_4} + + def _add_results( + self, + message, + node_bindings, + edge_bindings, + qg_subject_id, + subject_curies, + subject_category, + predicate, + qg_edge_id, + qg_object_id, + object_curies, + object_category, + vals, + algorithms, + datasets, + ): + nodes = dict() + edges = dict() + val_id = 0 + for subject_curie in subject_curies: + for object_curie in object_curies: + nodes[subject_curie] = {"categories": [subject_category]} + nodes[object_curie] = {"categories": [object_category]} + kg_edge_id = str(uuid.uuid4()) + edges[kg_edge_id] = {"predicate": predicate, + "subject": subject_curie, + "object": object_curie, + "sources": self._get_sources(), + "attributes": self._get_attributes( + vals[val_id], + algorithms[val_id], + datasets[val_id], + )} + val_id += 1 + node_bindings[qg_subject_id].add(NodeBinding(id = subject_curie)) + node_bindings[qg_object_id].add(NodeBinding(id = object_curie)) + edge_bindings[qg_edge_id].add(EdgeBinding(id = kg_edge_id)) + kgraph = KnowledgeGraph(nodes=nodes, edges=edges) + if message.knowledge_graph is not None: + message.knowledge_graph.update(kgraph) + else: + message.knowledge_graph = kgraph + + def _extract_qnode_info(self, qnode): + return qnode.ids, qnode.categories[0] + + def get_response(self, message: Message, logger): + for edge_id, edge in message.query_graph.edges.items(): + predicate = edge.predicates[0] + qg_edge_id = edge_id + qg_subject_id = edge.subject + qg_object_id = edge.object + subject_curies, subject_category = self._extract_qnode_info(message.query_graph.nodes[qg_subject_id]) + object_curies, object_category = self._extract_qnode_info(message.query_graph.nodes[qg_object_id]) + # annotation + node_bindings = {qg_subject_id: set(), qg_object_id: set()} + edge_bindings = {qg_edge_id : set()} + #TODO: Should probably offer support to return all results + if subject_curies is not None and object_curies is not None: + logger.info('Annotation edges detected') + logger.info('Annotate edge not currently supported') + return message + elif object_curies is not None: + logger.info('Wildcard detected') + for curie in object_curies: + # Get object gene, if we don't have then continue + obj_genes = Gene.objects.filter(chp_preferred_curie=curie) + if len(obj_genes) == 0: + continue + if predicate == 'biolink:regulates': + results = [] + for obj_gene in obj_genes: + results.extend(InferenceResult.objects.filter(target=obj_gene, is_public=True)) + subject_curies = [r.tf.chp_preferred_curie for r in results] + elif predicate == 'biolink:regulated_by': + results = [] + for obj_gene in obj_genes: + results.extend(InferenceResult.objects.filter(tf=obj_gene, is_public=True)) + subject_curies = [r.target.chp_preferred_curie for r in results] + else: + raise ValueError(f'Unknown predicate: {predicate}.') + vals = [r.edge_weight for r in results] + algorithms = [r.study.algorithm_instance for r in results] + datasets = [r.study.dataset for r in results] + self._add_results( + message, + node_bindings, + edge_bindings, + qg_subject_id, + subject_curies, + subject_category, + predicate, + qg_edge_id, + qg_object_id, + [curie], + object_category, + vals, + algorithms, + datasets, + ) + elif subject_curies is not None: + logger.info('Wildcard detected') + for curie in subject_curies: + # Get object gene, if we don't have then continue + sub_genes = Gene.objects.filter(chp_preferred_curie=curie) + if len(sub_genes) == 0: + continue + if predicate == 'biolink:regulates': + results = [] + for sub_gene in sub_genes: + results.extend(InferenceResult.objects.filter(tf=sub_gene, is_public=True)) + object_curies = [r.target.chp_preferred_curie for r in results] + elif predicate == 'biolink:regulated_by': + results = [] + for sub_gene in sub_genes: + results.extend(InferenceResult.objects.filter(target=sub_gene, is_public=True)) + object_curies = [r.tf.chp_preferred_curie for r in results] + else: + raise ValueError(f'Unknown predicate: {predicate}.') + vals = [r.edge_weight for r in results] + algorithms = [r.study.algorithm_instance for r in results] + datasets = [r.study.dataset for r in results] + self._add_results( + message, + node_bindings, + edge_bindings, + qg_subject_id, + subject_curies, + subject_category, + predicate, + qg_edge_id, + qg_object_id, + [curie], + object_category, + vals, + algorithms, + datasets, + ) + else: + logger.info('No curies detected. Returning no results') + return message + analysis = Analysis(resource_id='infores:connections-hypothesis', edge_bindings=edge_bindings) + result = Result(node_bindings=node_bindings, analyses=[analysis]) + message.results = Results(__root__ = {result}) + return message diff --git a/chp_api/gennifer/views.py b/chp_api/gennifer/views.py index 04f9f6b..bac9ee7 100644 --- a/chp_api/gennifer/views.py +++ b/chp_api/gennifer/views.py @@ -25,7 +25,7 @@ class DatasetViewSet(viewsets.ModelViewSet): class InferenceStudyViewSet(viewsets.ModelViewSet): serializer_class = InferenceStudySerializer filter_backends = [DjangoFilterBackend] - filterset_fields = ['is_public', 'dataset', 'algorithm'] + filterset_fields = ['is_public', 'dataset', 'algorithm_instance'] permission_classes = [IsAuthenticated] def get_queryset(self): @@ -56,6 +56,9 @@ def post(self, request): algorithm_name = task.get("algorithm_name", None) zenodo_id = task.get("zenodo_id", None) hyperparameters = task.get("hyperparameters", None) + if hyperparameters: + if len(hyperparameters) == 0: + hyperparameters = None if not algorithm_name: task["error"] = "No algorithm name provided." response["tasks"].append(task) diff --git a/compose.gennifer.yaml b/compose.gennifer.yaml index bbb6a7e..fac4a43 100644 --- a/compose.gennifer.yaml +++ b/compose.gennifer.yaml @@ -33,21 +33,6 @@ services: - pidc - redis - dashboard-pidc: - build: - context: ./gennifer/pidc - dockerfile: Dockerfile - command: celery --app pidc.tasks.celery --broker=redis://redis:6379/0 flower --port=5555 - ports: - - 5557:5555 - environment: - - CELERY_BROKER_URL=redis://redis:6379/0 - - CELERY_RESULT_BACKEND=redis://redis:6379/0 - depends_on: - - pidc - - redis - - worker-pidc - secrets: gennifer_key: file: secrets/gennifer/secret_key.txt diff --git a/deployment-script b/deployment-script index 0d572bb..c077fa1 100755 --- a/deployment-script +++ b/deployment-script @@ -16,6 +16,7 @@ docker compose -f compose.chp-api.yaml run --user root api python3 manage.py cre docker compose -f compose.chp-api.yaml run api python3 manage.py runscript load_db_apps docker compose -f compose.chp-api.yaml run api python3 manage.py runscript templater docker compose -f compose.chp-api.yaml run api python3 manage.py runscript gene_spec_curie_templater +docker compose -f compose.chp-api.yaml run api python3 manage.py runscript algorithm_loader docker compose -f compose.chp-api.yaml run --user root api python3 manage.py collectstatic --noinput diff --git a/gennifer b/gennifer index 2ad99ae..ea758cf 160000 --- a/gennifer +++ b/gennifer @@ -1 +1 @@ -Subproject commit 2ad99ae704fd3ee4b43b7f9ef9690b831da8f670 +Subproject commit ea758cfec58c88b69fd8ff029b03b4a66012c472 diff --git a/gennifer-sample.json b/gennifer-sample.json index fad8894..803f0f0 100644 --- a/gennifer-sample.json +++ b/gennifer-sample.json @@ -2,7 +2,7 @@ "tasks": [ { "algorithm_name": "pidc", - "zenodo_id":"7982629", + "zenodo_id":"7988181", "hyperparameters": null } ] From c1a5763c743f8f8f65768ca88f64ec78354968f0 Mon Sep 17 00:00:00 2001 From: Chase Yakaboski Date: Thu, 1 Jun 2023 10:50:30 -0400 Subject: [PATCH 054/132] Added new model viewsets. --- chp_api/gennifer/permissions.py | 29 +++++++++++++++++++++++++++++ chp_api/gennifer/serializers.py | 11 ++++++++++- chp_api/gennifer/urls.py | 1 + chp_api/gennifer/views.py | 30 +++++++++++++++++++----------- 4 files changed, 59 insertions(+), 12 deletions(-) create mode 100644 chp_api/gennifer/permissions.py diff --git a/chp_api/gennifer/permissions.py b/chp_api/gennifer/permissions.py new file mode 100644 index 0000000..6001dfe --- /dev/null +++ b/chp_api/gennifer/permissions.py @@ -0,0 +1,29 @@ +from rest_framework import permissions + + +class IsOwnerOrReadOnly(permissions.BasePermission): + """ + Object-level permission to only allow owners of an object to edit it. + Assumes the model instance has an `owner` attribute. + """ + + def has_object_permission(self, request, view, obj): + # Read permissions are allowed to any request, + # so we'll always allow GET, HEAD or OPTIONS requests. + if request.method in permissions.SAFE_METHODS: + return True + + # Instance must have an attribute named `owner`. + return obj.user == request.user + +class IsAdminOrReadOnly(permissions.BasePermission): + + def has_object_permission(self, request, view, obj): + # Read permissions are allowed to any request, + # so we'll always allow GET, HEAD or OPTIONS requests. + if request.method in permissions.SAFE_METHODS: + return True + + # Instance must have an attribute named `owner`. + return request.user.is_staff + diff --git a/chp_api/gennifer/serializers.py b/chp_api/gennifer/serializers.py index dea0750..64b85da 100644 --- a/chp_api/gennifer/serializers.py +++ b/chp_api/gennifer/serializers.py @@ -1,6 +1,6 @@ from rest_framework import serializers -from .models import Dataset, InferenceStudy, InferenceResult +from .models import Dataset, InferenceStudy, InferenceResult, Algorithm class DatasetSerializer(serializers.ModelSerializer): class Meta: @@ -34,3 +34,12 @@ class Meta: 'is_public', 'user', ] + +class AlgorithmSerializer(serializers.ModelSerializer): + class Meta: + model = Algorithm + fields = [ + 'name', + 'description', + 'edge_weight_type', + ] diff --git a/chp_api/gennifer/urls.py b/chp_api/gennifer/urls.py index 17860e4..621ea9a 100644 --- a/chp_api/gennifer/urls.py +++ b/chp_api/gennifer/urls.py @@ -8,6 +8,7 @@ router.register(r'datasets', views.DatasetViewSet, basename='dataset') router.register(r'inference_studies', views.InferenceStudyViewSet, basename='inference_study') router.register(r'inference_results', views.InferenceResultViewSet, basename='inference_result') +router.register(r'algorithms', views.AlgorithmViewSet, basename='algorithm') urlpatterns = [ path('', include(router.urls)), diff --git a/chp_api/gennifer/views.py b/chp_api/gennifer/views.py index bac9ee7..486b53e 100644 --- a/chp_api/gennifer/views.py +++ b/chp_api/gennifer/views.py @@ -7,41 +7,49 @@ from rest_framework import viewsets from rest_framework.views import APIView from rest_framework.response import Response -from rest_framework.permissions import IsAuthenticated +from rest_framework.permissions import IsAuthenticated, IsAuthenticatedOrReadOnly from django_filters.rest_framework import DjangoFilterBackend from .models import Dataset, InferenceStudy, InferenceResult, Algorithm -from .serializers import DatasetSerializer, InferenceStudySerializer, InferenceResultSerializer +from .serializers import DatasetSerializer, InferenceStudySerializer, InferenceResultSerializer, AlgorithmSerializer from .tasks import create_task +from .permissions import IsOwnerOrReadOnly, IsAdminOrReadOnly class DatasetViewSet(viewsets.ModelViewSet): queryset = Dataset.objects.all() serializer_class = DatasetSerializer filter_backends = [DjangoFilterBackend] filterset_fields = ['upload_user', 'zenodo_id'] - permission_classes = [IsAuthenticated] + permission_classes = [IsAuthenticatedOrReadOnly] class InferenceStudyViewSet(viewsets.ModelViewSet): + queryset = InferenceStudy.objects.all() serializer_class = InferenceStudySerializer filter_backends = [DjangoFilterBackend] filterset_fields = ['is_public', 'dataset', 'algorithm_instance'] - permission_classes = [IsAuthenticated] + permission_classes = [IsOwnerOrReadOnly] - def get_queryset(self): - user = self.request.user - return InferenceStudy.objects.filter(user=user) + #def get_queryset(self): + # user = self.request.user + # return InferenceStudy.objects.filter(user=user) class InferenceResultViewSet(viewsets.ModelViewSet): + queryset = InferenceResult.objects.all() serializer_class = InferenceResultSerializer filter_backends = [DjangoFilterBackend] filterset_fields = ['is_public', 'study', 'tf', 'target'] - permission_classes = [IsAuthenticated] + permission_classes = [IsOwnerOrReadOnly] - def get_queryset(self): - user = self.request.user - return InferenceResult.objects.filter(user=user) + #def get_queryset(self): + # user = self.request.user + # return InferenceResult.objects.filter(user=user) + +class AlgorithmViewSet(viewsets.ModelViewSet): + serializer_class = AlgorithmSerializer + queryset = Algorithm.objects.all() + permissions = [IsAdminOrReadOnly] class run(APIView): From 3470a43cee1ac949f9847e4fa1077a101f9bb53f Mon Sep 17 00:00:00 2001 From: Chase Yakaboski Date: Thu, 1 Jun 2023 17:35:40 -0400 Subject: [PATCH 055/132] Made necessary changes to helm deployment and chp settings. Probably need to add new secret for django user. --- chp_api/chp_api/settings.py | 42 ++++++++++++++------ deploy/chp-api/Jenkinsfile | 2 + deploy/chp-api/configs/nginx.conf | 15 +++++-- deploy/chp-api/static-file-server/Dockerfile | 41 +++++++++++++++++++ deploy/chp-api/templates/deployment.yaml | 39 ++++++++++++++---- deploy/chp-api/templates/secret.yaml | 3 +- deploy/chp-api/values.yaml | 6 +++ docker-compose.yml | 17 +++++--- 8 files changed, 135 insertions(+), 30 deletions(-) create mode 100755 deploy/chp-api/static-file-server/Dockerfile diff --git a/chp_api/chp_api/settings.py b/chp_api/chp_api/settings.py index 180461f..c8e3018 100644 --- a/chp_api/chp_api/settings.py +++ b/chp_api/chp_api/settings.py @@ -137,8 +137,11 @@ # Hosts Configuration #ROOT_HOSTCONF = 'chp_api.hosts' -with open(env("POSTGRES_PASSWORD_FILE"), 'r') as db_pwd: - DB_PASSWORD = db_pwd.readline().strip() +DB_PASSWORD = env("POSTGRES_PASSWORD", default=None) + +if not DB_PASSWORD: + with open(env("POSTGRES_PASSWORD_FILE"), 'r') as db_pwd: + DB_PASSWORD = db_pwd.readline().strip() # Database # https://docs.djangoproject.com/en/3.0/ref/settings/#databases @@ -153,18 +156,33 @@ } } -with open(env("DJANGO_ALLOWED_HOSTS_FILE"), 'r') as ah_file: - ALLOWED_HOSTS = ah_file.readline().strip().split(" ") + +ALLOWED_HOSTS = env("DJANGO_ALLOWED_HOSTS", default=None) +if not ALLOWED_HOSTS: + with open(env("DJANGO_ALLOWED_HOSTS_FILE"), 'r') as ah_file: + ALLOWED_HOSTS = ah_file.readline().strip().split(" ") +else: + ALLOWED_HOSTS = ALLOWED_HOSTS.split(',') # SECURITY WARNING: keep the secret key used in production secret! # Read the secret key from file -with open(env("SECRET_KEY_FILE"), 'r') as sk_file: - SECRET_KEY = sk_file.readline().strip() +SECRET_KEY = env("SECRET_KEY", default=None) +if not SECRET_KEY: + with open(env("SECRET_KEY_FILE"), 'r') as sk_file: + SECRET_KEY = sk_file.readline().strip() # Set UN, Email and Password for superuser -with open(env("DJANGO_SUPERUSER_USERNAME_FILE"), 'r') as dsu_file: - os.environ["DJANGO_SUPERUSER_USERNAME"] = dsu_file.readline().strip() -with open(env("DJANGO_SUPERUSER_EMAIL_FILE"), 'r') as dse_file: - os.environ["DJANGO_SUPERUSER_EMAIL"] = dse_file.readline().strip() -with open(env("DJANGO_SUPERUSER_PASSWORD_FILE"), 'r') as dsp_file: - os.environ["DJANGO_SUPERUSER_PASSWORD"] = dsp_file.readline().strip() +DJANGO_SUPERUSER_USERNAME = env("DJANGO_SUPERUSER_USERNAME", default=None) +if not DJANGO_SUPERUSER_USERNAME: + with open(env("DJANGO_SUPERUSER_USERNAME_FILE"), 'r') as dsu_file: + os.environ["DJANGO_SUPERUSER_USERNAME"] = dsu_file.readline().strip() + +DJANGO_SUPERUSER_EMAIL = env("DJANGO_SUPERUSER_EMAIL", default=None) +if not DJANGO_SUPERUSER_EMAIL: + with open(env("DJANGO_SUPERUSER_EMAIL_FILE"), 'r') as dse_file: + os.environ["DJANGO_SUPERUSER_EMAIL"] = dse_file.readline().strip() + +DJANGO_SUPERUSER_PASSWORD = env("DJANGO_SUPERUSER_PASSWORD", default=None) +if not DJANGO_SUPERUSER_PASSWORD: + with open(env("DJANGO_SUPERUSER_PASSWORD_FILE"), 'r') as dsp_file: + os.environ["DJANGO_SUPERUSER_PASSWORD"] = dsp_file.readline().strip() diff --git a/deploy/chp-api/Jenkinsfile b/deploy/chp-api/Jenkinsfile index 0b9b119..7050624 100644 --- a/deploy/chp-api/Jenkinsfile +++ b/deploy/chp-api/Jenkinsfile @@ -49,6 +49,8 @@ pipeline { sh 'cp deploy/chp-api/configs/nginx.conf deploy/chp-api/nginx/' docker.build(env.DOCKER_REPO_NAME, "--no-cache ./deploy/chp-api/nginx") docker.image(env.DOCKER_REPO_NAME).push("${BUILD_VERSION}-nginx") + docker.build(env.DOCKER_REPO_NAME, "--no-cache ./deploy/chp-api/static-file-server") + docker.image(env.DOCKER_REPO_NAME).push("${BUILD_VERSION}-staticfs") } } } diff --git a/deploy/chp-api/configs/nginx.conf b/deploy/chp-api/configs/nginx.conf index 2a2e900..3efc112 100644 --- a/deploy/chp-api/configs/nginx.conf +++ b/deploy/chp-api/configs/nginx.conf @@ -2,6 +2,10 @@ upstream chp_api_app { server localhost:8000; } +upstream chp_staticfs { + server localhost:8080; +} + server { listen 80; @@ -16,8 +20,11 @@ server { proxy_send_timeout 360; proxy_connect_timeout 360; } - - location /staticfiles/ { - alias /home/chp_api/web/staticfiles/; - } + + location /static { + proxy_pass http://chp_staticfs; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } } diff --git a/deploy/chp-api/static-file-server/Dockerfile b/deploy/chp-api/static-file-server/Dockerfile new file mode 100755 index 0000000..ff58c95 --- /dev/null +++ b/deploy/chp-api/static-file-server/Dockerfile @@ -0,0 +1,41 @@ +################################################################################ +## GO BUILDER +################################################################################ +FROM golang:1.20.2 as builder + +ENV VERSION 1.8.8 +ENV CGO_ENABLED 0 +ENV BUILD_DIR /build + +RUN mkdir -p ${BUILD_DIR} +WORKDIR ${BUILD_DIR} + +COPY go.* ./ +RUN go mod download +COPY . . + +RUN go test -cover ./... +RUN go build -a -tags netgo -installsuffix netgo -ldflags "-s -w -X github.com/halverneus/static-file-server/cli/version.version=${VERSION}" -o /serve /build/bin/serve + +RUN adduser --system --no-create-home --uid 1000 --shell /usr/sbin/nologin static + +################################################################################ +## DEPLOYMENT CONTAINER +################################################################################ +FROM scratch + +EXPOSE 8080 +COPY --from=builder /serve / +COPY --from=builder /etc/passwd /etc/passwd + +USER static +ENTRYPOINT ["/serve"] +CMD [] + +# Metadata +LABEL life.apets.vendor="Halverneus" \ + life.apets.url="https://github.com/halverneus/static-file-server" \ + life.apets.name="Static File Server" \ + life.apets.description="A tiny static file server" \ + life.apets.version="v1.8.8" \ + life.apets.schema-version="1.0" diff --git a/deploy/chp-api/templates/deployment.yaml b/deploy/chp-api/templates/deployment.yaml index 1cee24b..d97d8ed 100644 --- a/deploy/chp-api/templates/deployment.yaml +++ b/deploy/chp-api/templates/deployment.yaml @@ -26,7 +26,7 @@ spec: image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" imagePullPolicy: {{ .Values.image.pullPolicy }} command: ["/bin/sh"] - args: ["-c", "/bin/bash /home/chp_api/web/entrypoint.sh && gunicorn -c gunicorn.config.py --log-file=- --env DJANGO_SETTINGS_MODULE=chp_api.settings chp_api.wsgi:application --bind 0.0.0.0:8000"] + args: ["-c", "gunicorn -c gunicorn.config.py --log-file=- --env DJANGO_SETTINGS_MODULE=chp_api.settings chp_api.wsgi:application --bind 0.0.0.0:8000"] ports: - name: http-app containerPort: 8000 @@ -40,26 +40,30 @@ spec: secretKeyRef: name: {{ include "chp-api.fullname" . }}-secret key: secret_key - - name: SQL_DATABASE + - name: POSTGRES_DB valueFrom: secretKeyRef: name: {{ include "chp-api.fullname" . }}-secret key: sql_database - - name: SQL_USER + - name: POSTGRES_USER valueFrom: secretKeyRef: name: {{ include "chp-api.fullname" . }}-secret key: sql_username - - name: SQL_PASSWORD + - name: POSTGRES_PASSWORD valueFrom: secretKeyRef: name: {{ include "chp-api.fullname" . }}-secret key: sql_password + - name: DJANGO_SUPERUSER_PASSWORD + secretKeyRef: + name: {{ include "chp-api.fullname" . }}-secret + key: django_superuser_password - name: SQL_ENGINE value: "{{ .Values.db.engine }}" - - name: SQL_HOST + - name: POSTGRES_HOST value: "{{ .Values.db.host }}" - - name: SQL_PORT + - name: POSTGRES_PORT value: "{{ .Values.db.port }}" - name: DATABASE value: "{{ .Values.db.type }}" @@ -69,6 +73,10 @@ spec: value: "{{ .Values.app.djangoAllowedHosts }}" - name: DJANGO_SETTINGS_MODULE value: "{{ .Values.app.djangoSettingsModule }}" + - name: DJANGO_SUPERUSER_USERNAME=chp_admin + value: "{{ .Values.app.djangoSuperuserUsername }}" + - name: DJANGO_SUPERUSER_EMAIL + value: "{{ .Values.app.djangoSuperuserEmail }}" - name: {{ .Chart.Name }}-nginx securityContext: {{- toYaml .Values.securityContextNginx | nindent 12 }} @@ -84,6 +92,23 @@ spec: - name: config-vol mountPath: /etc/nginx/conf.d/default.conf subPath: nginx.conf + - name: {{ .Chart.Name }}-staticfs + securityContext: + {{- toYaml .Values.securityContextStaticfs | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.staticfsTag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: http-staticfs + containerPort: 8080 + protocol: TCP + volumeMounts: + - name: {{ include "chp-api.fullname" . }}-pvc + mountPath: /var/www/static + env: + - name: FOLDER + value: "{{ .Values.app.staticfsFolder }}" + - name: DEBUG + value: "{{ .Values.app.staticfsDebug }}" volumes: - name: config-vol configMap: @@ -103,4 +128,4 @@ spec: accessModes: [ "ReadWriteOnce" ] resources: requests: - storage: 1Gi \ No newline at end of file + storage: 1Gi diff --git a/deploy/chp-api/templates/secret.yaml b/deploy/chp-api/templates/secret.yaml index e141878..c435a39 100644 --- a/deploy/chp-api/templates/secret.yaml +++ b/deploy/chp-api/templates/secret.yaml @@ -8,4 +8,5 @@ stringData: sql_database: {{ .Values.db.database }} sql_username: {{ .Values.db.username }} sql_password: {{ .Values.db.password }} - secret_key: {{ .Values.app.secret_key }} \ No newline at end of file + secret_key: {{ .Values.app.secret_key }} + django_superuser_password: {{ .Values.app.djangoSuperuserPassword }} diff --git a/deploy/chp-api/values.yaml b/deploy/chp-api/values.yaml index 94bdeef..0d415a0 100644 --- a/deploy/chp-api/values.yaml +++ b/deploy/chp-api/values.yaml @@ -11,6 +11,7 @@ image: # Overrides the image tag whose default is the chart appVersion. tag: "BUILD_VERSION" nginxTag: "BUILD_VERSION-nginx" + staticfsTag: "BUILD_VERSION-staticfs" nameOverride: "" fullnameOverride: "" @@ -21,6 +22,11 @@ app: secret_key: "" djangoAllowedHosts: "" djangoSettingsModule: "chp_api.settings" + djangoSuperuserUsername: "chp_admin" + djangoSuperuserPassword: "" + djangoSuperuserEmail: "chp_admin@chp.com" + staticfsFolder: "/var/www" + staticfsDebug: "0" # database connection information db: diff --git a/docker-compose.yml b/docker-compose.yml index 0253de9..37f77d3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -39,19 +39,22 @@ services: environment: - POSTGRES_DB=chp_db - POSTGRES_USER=postgres - - POSTGRES_PASSWORD_FILE=/run/secrets/db-password - POSTGRES_HOST=db - POSTGRES_PORT=5432 - - SECRET_KEY_FILE=/run/secrets/django-key - DEBUG=1 + # For Helm testing purposes + #- POSTGRES_PASSWORD=31173e51d8f78b56606d06dfb66a1b126630cdf4711bed9427025d8979976f31 + #- SECRET_KEY=e1743ca40af220389cd1165d213e3d677f2d59c00d7b0f94e7a302c91f95f029 + #- DJANGO_ALLOWED_HOSTS=localhost,chp.thayer.dartmouth.edu + #- DJANGO_SUPERUSER_USERNAME=chp_admin + #- DJANGO_SUPERUSER_EMAIL=chp_admin@chp.com + #- DJANGO_SUPERUSER_PASSWORD=e12ff26f070819d9a72e317898148679680e6b3976e464b4102bd6eb18357919 + - POSTGRES_PASSWORD_FILE=/run/secrets/db-password + - SECRET_KEY_FILE=/run/secrets/django-key - DJANGO_ALLOWED_HOSTS_FILE=/run/secrets/allowed-hosts - DJANGO_SUPERUSER_USERNAME_FILE=/run/secrets/django-superuser-username - DJANGO_SUPERUSER_EMAIL_FILE=/run/secrets/django-superuser-email - DJANGO_SUPERUSER_PASSWORD_FILE=/run/secrets/django-superuser-password - # Uncomment this for production - #- DJANGO_SETTINGS_MODULE=mysite.settings.production - # Comment this for development - #- DJANGO_SETTINGS_MODULE=mysite.settings.base depends_on: db: condition: service_healthy @@ -78,6 +81,8 @@ services: environment: - POSTGRES_DB=chp_db - POSTGRES_PASSWORD_FILE=/run/secrets/db-password + # For Helm testing purposes + #- POSTGRES_PASSWORD=31173e51d8f78b56606d06dfb66a1b126630cdf4711bed9427025d8979976f31 expose: - 5432 healthcheck: From b41d16c970778bc55366bd0ee2f18deb43c0ab94 Mon Sep 17 00:00:00 2001 From: Chase Yakaboski Date: Fri, 2 Jun 2023 14:13:13 -0400 Subject: [PATCH 056/132] Removed cp of chp_db fixture from Dockerfile as this can't be included due to ITRB. --- Dockerfile | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3b3a1a8..38965b0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,18 +8,8 @@ FROM python:3.8 as intermediate # set work directory WORKDIR /usr/src/chp_api -# install git -#RUN apt-get update \ -# && apt-get install -y git python3-pip python3-dev -#dos2unix - RUN git clone --single-branch --branch gene_spec_pydantic-ghyde https://github.com/di2ag/gene-specificity.git -# lint -#RUN pip install --upgrade pip -#RUN pip3 install flake8 wheel -#COPY . . - # install dependencies COPY ./requirements.txt . RUN pip3 wheel --no-cache-dir --no-deps --wheel-dir /usr/src/chp_api/wheels -r requirements.txt @@ -52,28 +42,15 @@ ENV TZ=America/New_York # set ARGs ARG DEBIAN_FRONTEND=noninterative -# install dependencies -#RUN apt-get update \ -# && apt-get install -y python3-pip graphviz openmpi-bin libopenmpi-dev build-essential libssl-dev libffi-dev python3-dev -#RUN apt-get install -y libgraphviz-dev python3-pygraphviz -#RUN apt-get install -y libpq-dev -#RUN apt-get install -y netcat - # copy repo to new image COPY --from=intermediate /usr/src/chp_api/wheels /wheels COPY --from=intermediate /usr/src/chp_api/requirements.txt . -#RUN pip3 install --upgrade pip -#RUN python3 -m pip install --upgrade pip RUN pip3 install --no-cache /wheels/* -# copy entry point -#COPY ./entrypoint.sh $APP_HOME - # copy project COPY ./chp_api/chp_api $APP_HOME/chp_api COPY ./chp_api/manage.py $APP_HOME COPY ./chp_api/dispatcher $APP_HOME/dispatcher -COPY ./chp_db_fixture.json.gz $APP_HOME COPY ./gunicorn.config.py $APP_HOME # chown all the files to the app user @@ -82,6 +59,3 @@ RUN chown -R chp_api:chp_api $APP_HOME \ # change to the app user USER chp_api - -# run entrypoint.sh -#ENTRYPOINT ["/home/chp_api/web/entrypoint.sh"] From bce52bd27da946a7fc443aeb2429d1fdf9e51da6 Mon Sep 17 00:00:00 2001 From: Chase Yakaboski Date: Fri, 2 Jun 2023 17:37:00 -0400 Subject: [PATCH 057/132] Added in all files for static file server. Should probably just add a DockerHub image to the Jenkins build config, instead of this but do not know how to do that. --- .../chp-api/static-file-server/Dockerfile.all | 26 + deploy/chp-api/static-file-server/LICENSE | 21 + deploy/chp-api/static-file-server/README.md | 153 ++++ .../static-file-server/bin/serve/main.go | 13 + deploy/chp-api/static-file-server/cli/args.go | 27 + .../static-file-server/cli/args_test.go | 81 ++ .../chp-api/static-file-server/cli/execute.go | 95 +++ .../static-file-server/cli/execute_test.go | 162 ++++ .../static-file-server/cli/help/help.go | 190 +++++ .../static-file-server/cli/help/help_test.go | 9 + .../static-file-server/cli/server/server.go | 96 +++ .../cli/server/server_test.go | 127 ++++ .../static-file-server/cli/version/version.go | 24 + .../cli/version/version_test.go | 9 + .../static-file-server/config/config.go | 301 ++++++++ .../static-file-server/config/config_test.go | 513 +++++++++++++ deploy/chp-api/static-file-server/go.mod | 5 + deploy/chp-api/static-file-server/go.sum | 4 + .../static-file-server/handle/handle.go | 252 +++++++ .../static-file-server/handle/handle_test.go | 703 ++++++++++++++++++ .../static-file-server/img/sponsor.svg | 147 ++++ deploy/chp-api/static-file-server/update.sh | 32 + 22 files changed, 2990 insertions(+) create mode 100644 deploy/chp-api/static-file-server/Dockerfile.all create mode 100644 deploy/chp-api/static-file-server/LICENSE create mode 100644 deploy/chp-api/static-file-server/README.md create mode 100644 deploy/chp-api/static-file-server/bin/serve/main.go create mode 100644 deploy/chp-api/static-file-server/cli/args.go create mode 100644 deploy/chp-api/static-file-server/cli/args_test.go create mode 100644 deploy/chp-api/static-file-server/cli/execute.go create mode 100644 deploy/chp-api/static-file-server/cli/execute_test.go create mode 100644 deploy/chp-api/static-file-server/cli/help/help.go create mode 100644 deploy/chp-api/static-file-server/cli/help/help_test.go create mode 100644 deploy/chp-api/static-file-server/cli/server/server.go create mode 100644 deploy/chp-api/static-file-server/cli/server/server_test.go create mode 100644 deploy/chp-api/static-file-server/cli/version/version.go create mode 100644 deploy/chp-api/static-file-server/cli/version/version_test.go create mode 100644 deploy/chp-api/static-file-server/config/config.go create mode 100644 deploy/chp-api/static-file-server/config/config_test.go create mode 100644 deploy/chp-api/static-file-server/go.mod create mode 100644 deploy/chp-api/static-file-server/go.sum create mode 100644 deploy/chp-api/static-file-server/handle/handle.go create mode 100644 deploy/chp-api/static-file-server/handle/handle_test.go create mode 100644 deploy/chp-api/static-file-server/img/sponsor.svg create mode 100755 deploy/chp-api/static-file-server/update.sh diff --git a/deploy/chp-api/static-file-server/Dockerfile.all b/deploy/chp-api/static-file-server/Dockerfile.all new file mode 100644 index 0000000..79d17c7 --- /dev/null +++ b/deploy/chp-api/static-file-server/Dockerfile.all @@ -0,0 +1,26 @@ +FROM golang:1.20.2 as builder + +ENV VERSION 1.8.8 +ENV BUILD_DIR /build +ENV CGO_ENABLED 0 + +RUN mkdir -p ${BUILD_DIR} +WORKDIR ${BUILD_DIR} + +COPY . . +RUN go test -cover ./... +RUN GOOS=linux GOARCH=amd64 go build -a -tags netgo -installsuffix netgo -ldflags "-s -w -X github.com/halverneus/static-file-server/cli/version.version=${VERSION}" -o pkg/linux-amd64/serve /build/bin/serve +RUN GOOS=linux GOARCH=386 go build -a -tags netgo -installsuffix netgo -ldflags "-s -w -X github.com/halverneus/static-file-server/cli/version.version=${VERSION}" -o pkg/linux-i386/serve /build/bin/serve +RUN GOOS=linux GOARCH=arm GOARM=6 go build -a -tags netgo -installsuffix netgo -ldflags "-s -w -X github.com/halverneus/static-file-server/cli/version.version=${VERSION}" -o pkg/linux-arm6/serve /build/bin/serve +RUN GOOS=linux GOARCH=arm GOARM=7 go build -a -tags netgo -installsuffix netgo -ldflags "-s -w -X github.com/halverneus/static-file-server/cli/version.version=${VERSION}" -o pkg/linux-arm7/serve /build/bin/serve +RUN GOOS=linux GOARCH=arm64 go build -a -tags netgo -installsuffix netgo -ldflags "-s -w -X github.com/halverneus/static-file-server/cli/version.version=${VERSION}" -o pkg/linux-arm64/serve /build/bin/serve +RUN GOOS=darwin GOARCH=amd64 go build -a -tags netgo -installsuffix netgo -ldflags "-s -w -X github.com/halverneus/static-file-server/cli/version.version=${VERSION}" -o pkg/darwin-amd64/serve /build/bin/serve +RUN GOOS=windows GOARCH=amd64 go build -a -tags netgo -installsuffix netgo -ldflags "-s -w -X github.com/halverneus/static-file-server/cli/version.version=${VERSION}" -o pkg/win-amd64/serve.exe /build/bin/serve + +# Metadata +LABEL life.apets.vendor="Halverneus" \ + life.apets.url="https://github.com/halverneus/static-file-server" \ + life.apets.name="Static File Server" \ + life.apets.description="A tiny static file server" \ + life.apets.version="v1.8.8" \ + life.apets.schema-version="1.0" diff --git a/deploy/chp-api/static-file-server/LICENSE b/deploy/chp-api/static-file-server/LICENSE new file mode 100644 index 0000000..dd8824a --- /dev/null +++ b/deploy/chp-api/static-file-server/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2017 Jeromy Streets + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/deploy/chp-api/static-file-server/README.md b/deploy/chp-api/static-file-server/README.md new file mode 100644 index 0000000..abd3757 --- /dev/null +++ b/deploy/chp-api/static-file-server/README.md @@ -0,0 +1,153 @@ +# static-file-server + + + + + +## Introduction + +Tiny, simple static file server using environment variables for configuration. +Install from any of the following locations: + +- Docker Hub: https://hub.docker.com/r/halverneus/static-file-server/ +- GitHub: https://github.com/halverneus/static-file-server + +## Configuration + +### Environment Variables + +Default values are shown with the associated environment variable. + +```bash +# Enables resource access from any domain. +CORS=false + +# Enable debugging for troubleshooting. If set to 'true' this prints extra +# information during execution. IMPORTANT NOTE: The configuration summary is +# printed to stdout while logs generated during execution are printed to stderr. +DEBUG=false + +# Optional Hostname for binding. Leave black to accept any incoming HTTP request +# on the prescribed port. +HOST= + +# If assigned, must be a valid port number. +PORT=8080 + +# When set to 'true' the index.html file in the folder will be served. And +# the file list will not be served. +ALLOW_INDEX=true + +# Automatically serve the index of file list for a given directory (default). +SHOW_LISTING=true + +# Folder with the content to serve. +FOLDER=/web + +# URL path prefix. If 'my.file' is in the root of $FOLDER and $URL_PREFIX is +# '/my/place' then file is retrieved with 'http://$HOST:$PORT/my/place/my.file'. +URL_PREFIX= + +# Paths to the TLS certificate and key. If one is set then both must be set. If +# both set then files are served using HTTPS. If neither are set then files are +# served using HTTP. +TLS_CERT= +TLS_KEY= + +# If TLS certificates are set then the minimum TLS version may also be set. If +# the value isn't set then the default minimum TLS version is 1.0. Allowed +# values include "TLS10", "TLS11", "TLS12" and "TLS13" for TLS1.0, TLS1.1, +# TLS1.2 and TLS1.3, respectively. The value is not case-sensitive. +TLS_MIN_VERS= + +# List of accepted HTTP referrers. Return 403 if HTTP header `Referer` does not +# match prefixes provided in the list. +# Examples: +# 'REFERRERS=http://localhost,https://...,https://another.name' +# To accept missing referrer header, add a blank entry (start comma): +# 'REFERRERS=,http://localhost,https://another.name' +REFERRERS= + +# Use key / code parameter in the request URL for access control. The code is +# computed by requested PATH and your key. +# Example: +# ACCESS_KEY=username +# To access your file, either access: +# http://$HOST:$PORT/my/place/my.file?key=username +# or access (md5sum of "/my/place/my.fileusername"): +# http://$HOST:$PORT/my/place/my.file?code=44356A355E89D9EE7B2D5687E48024B0 +ACCESS_KEY= +``` + +### YAML Configuration File + +YAML settings are individually overridden by the corresponding environment +variable. The following is an example configuration file with defaults. Pass in +the path to the configuration file using the command line option +('-c', '-config', '--config'). + +```yaml +cors: false +debug: false +folder: /web +host: "" +port: 8080 +referrers: [] +show-listing: true +tls-cert: "" +tls-key: "" +tls-min-vers: "" +url-prefix: "" +access-key: "" +``` + +Example configuration with possible alternative values: + +```yaml +debug: true +folder: /var/www +port: 80 +referrers: + - http://localhost + - https://mydomain.com +``` + +## Deployment + +### Without Docker + +```bash +PORT=8888 FOLDER=. ./serve +``` + +Files can then be accessed by going to http://localhost:8888/my/file.txt + +### With Docker + +```bash +docker run -d \ + -v /my/folder:/web \ + -p 8080:8080 \ + halverneus/static-file-server:latest +``` + +This will serve the folder "/my/folder" over http://localhost:8080/my/file.txt + +Any of the variables can also be modified: + +```bash +docker run -d \ + -v /home/me/dev/source:/content/html \ + -v /home/me/dev/files:/content/more/files \ + -e FOLDER=/content \ + -p 8080:8080 \ + halverneus/static-file-server:latest +``` + +### Getting Help + +```bash +./serve help +# OR +docker run -it halverneus/static-file-server:latest help +``` diff --git a/deploy/chp-api/static-file-server/bin/serve/main.go b/deploy/chp-api/static-file-server/bin/serve/main.go new file mode 100644 index 0000000..127b879 --- /dev/null +++ b/deploy/chp-api/static-file-server/bin/serve/main.go @@ -0,0 +1,13 @@ +package main + +import ( + "log" + + "github.com/halverneus/static-file-server/cli" +) + +func main() { + if err := cli.Execute(); nil != err { + log.Fatalf("Error: %v\n", err) + } +} diff --git a/deploy/chp-api/static-file-server/cli/args.go b/deploy/chp-api/static-file-server/cli/args.go new file mode 100644 index 0000000..3e160f9 --- /dev/null +++ b/deploy/chp-api/static-file-server/cli/args.go @@ -0,0 +1,27 @@ +package cli + +// Args parsed from the command-line. +type Args []string + +// Parse command-line arguments into Args. Value is returned to support daisy +// chaining. +func Parse(values []string) Args { + args := Args(values) + return args +} + +// Matches is used to determine if the arguments match the provided pattern. +func (args Args) Matches(pattern ...string) bool { + // If lengths don't match then nothing does. + if len(pattern) != len(args) { + return false + } + + // Compare slices using '*' as a wildcard. + for index, value := range pattern { + if "*" != value && value != args[index] { + return false + } + } + return true +} diff --git a/deploy/chp-api/static-file-server/cli/args_test.go b/deploy/chp-api/static-file-server/cli/args_test.go new file mode 100644 index 0000000..90e5679 --- /dev/null +++ b/deploy/chp-api/static-file-server/cli/args_test.go @@ -0,0 +1,81 @@ +package cli + +import ( + "testing" +) + +func TestParse(t *testing.T) { + matches := func(args Args, orig []string) bool { + if nil == orig { + return nil == args + } + if len(orig) != len(args) { + return false + } + for index, value := range args { + if orig[index] != value { + return false + } + } + return true + } + + testCases := []struct { + name string + value []string + }{ + {"Nil arguments", nil}, + {"No arguments", []string{}}, + {"Arguments", []string{"first", "second", "*"}}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + if args := Parse(tc.value); !matches(args, tc.value) { + t.Errorf("Expected [%v] but got [%v]", tc.value, args) + } + }) + } +} + +func TestMatches(t *testing.T) { + testCases := []struct { + name string + value []string + pattern []string + result bool + }{ + {"Nil args and nil pattern", nil, nil, true}, + {"No args and nil pattern", []string{}, nil, true}, + {"Nil args and no pattern", nil, []string{}, true}, + {"No args and no pattern", []string{}, []string{}, true}, + {"Nil args and pattern", nil, []string{"test"}, false}, + {"No args and pattern", []string{}, []string{"test"}, false}, + {"Args and nil pattern", []string{"test"}, nil, false}, + {"Args and no pattern", []string{"test"}, []string{}, false}, + {"Simple single compare", []string{"test"}, []string{"test"}, true}, + {"Simple double compare", []string{"one", "two"}, []string{"one", "two"}, true}, + {"Bad single", []string{"one"}, []string{"two"}, false}, + {"Bad double", []string{"one", "two"}, []string{"one", "owt"}, false}, + {"Count mismatch", []string{"one", "two"}, []string{"one"}, false}, + {"Nil args and wild", nil, []string{"*"}, false}, + {"No args and wild", []string{}, []string{"*"}, false}, + {"Single arg and wild", []string{"one"}, []string{"*"}, true}, + {"Double arg and first wild", []string{"one", "two"}, []string{"*", "two"}, true}, + {"Double arg and second wild", []string{"one", "two"}, []string{"one", "*"}, true}, + {"Double arg and first wild mismatched", []string{"one", "two"}, []string{"*", "owt"}, false}, + {"Double arg and second wild mismatched", []string{"one", "two"}, []string{"eno", "*"}, false}, + {"Double arg and double wild", []string{"one", "two"}, []string{"*", "*"}, true}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + args := Parse(tc.value) + if resp := args.Matches(tc.pattern...); tc.result != resp { + msg := "For arguments [%v] matched to pattern [%v] expected " + + "%b but got %b" + t.Errorf(msg, tc.value, tc.pattern, tc.result, resp) + } + }) + } +} diff --git a/deploy/chp-api/static-file-server/cli/execute.go b/deploy/chp-api/static-file-server/cli/execute.go new file mode 100644 index 0000000..cf54266 --- /dev/null +++ b/deploy/chp-api/static-file-server/cli/execute.go @@ -0,0 +1,95 @@ +package cli + +import ( + "flag" + "fmt" + + "github.com/halverneus/static-file-server/cli/help" + "github.com/halverneus/static-file-server/cli/server" + "github.com/halverneus/static-file-server/cli/version" + "github.com/halverneus/static-file-server/config" +) + +var ( + option struct { + configFile string + helpFlag bool + versionFlag bool + } +) + +// Assignments used to simplify testing. +var ( + selectRoutine = selectionRoutine + unknownArgsFunc = unknownArgs + runServerFunc = server.Run + runHelpFunc = help.Run + runVersionFunc = version.Run + loadConfig = config.Load +) + +func init() { + setupFlags() +} + +func setupFlags() { + flag.StringVar(&option.configFile, "config", "", "") + flag.StringVar(&option.configFile, "c", "", "") + flag.BoolVar(&option.helpFlag, "help", false, "") + flag.BoolVar(&option.helpFlag, "h", false, "") + flag.BoolVar(&option.versionFlag, "version", false, "") + flag.BoolVar(&option.versionFlag, "v", false, "") +} + +// Execute CLI arguments. +func Execute() (err error) { + // Parse flag options, then parse commands arguments. + flag.Parse() + args := Parse(flag.Args()) + + job := selectRoutine(args) + return job() +} + +func selectionRoutine(args Args) func() error { + switch { + + // serve help + // serve --help + // serve -h + case args.Matches("help") || option.helpFlag: + return runHelpFunc + + // serve version + // serve --version + // serve -v + case args.Matches("version") || option.versionFlag: + return runVersionFunc + + // serve + case args.Matches(): + return withConfig(runServerFunc) + + // Unknown arguments. + default: + return unknownArgsFunc(args) + } +} + +func unknownArgs(args Args) func() error { + return func() error { + return fmt.Errorf( + "unknown arguments provided [%v], try: 'help'", + args, + ) + } +} + +func withConfig(routine func() error) func() error { + return func() (err error) { + if err = loadConfig(option.configFile); nil != err { + return + } + return routine() + } +} diff --git a/deploy/chp-api/static-file-server/cli/execute_test.go b/deploy/chp-api/static-file-server/cli/execute_test.go new file mode 100644 index 0000000..e8ac885 --- /dev/null +++ b/deploy/chp-api/static-file-server/cli/execute_test.go @@ -0,0 +1,162 @@ +package cli + +import ( + "errors" + "flag" + "os" + "testing" +) + +func TestSetupFlags(t *testing.T) { + app := os.Args[0] + + file := "file.txt" + wConfig := "Config (file.txt)" + + testCases := []struct { + name string + args []string + config string + help bool + version bool + }{ + {"Empty args", []string{app}, "", false, false}, + {"Help (--help)", []string{app, "--help"}, "", true, false}, + {"Help (-help)", []string{app, "-help"}, "", true, false}, + {"Help (-h)", []string{app, "-h"}, "", true, false}, + {"Version (--version)", []string{app, "--version"}, "", false, true}, + {"Version (-version)", []string{app, "-version"}, "", false, true}, + {"Version (-v)", []string{app, "-v"}, "", false, true}, + {"Config ()", []string{app, "--config", ""}, "", false, false}, + {wConfig, []string{app, "--config", file}, file, false, false}, + {wConfig, []string{app, "--config=file.txt"}, file, false, false}, + {wConfig, []string{app, "-config", file}, file, false, false}, + {wConfig, []string{app, "-config=file.txt"}, file, false, false}, + {wConfig, []string{app, "-c", file}, file, false, false}, + {"All set", []string{app, "-h", "-v", "-c", file}, file, true, true}, + } + + reset := func() { + option.configFile = "" + option.helpFlag = false + option.versionFlag = false + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + reset() + os.Args = tc.args + flag.Parse() + + if option.configFile != tc.config { + t.Errorf( + "For options [%v] expected a config file of %s but got %s", + tc.args, tc.config, option.configFile, + ) + } + if option.helpFlag != tc.help { + t.Errorf( + "For options [%v] expected help flag of %t but got %t", + tc.args, tc.help, option.helpFlag, + ) + } + if option.versionFlag != tc.version { + t.Errorf( + "For options [%v] expected version flag of %t but got %t", + tc.args, tc.version, option.versionFlag, + ) + } + }) + } +} + +func TestExecuteAndSelection(t *testing.T) { + app := os.Args[0] + + runHelpFuncError := errors.New("help") + runHelpFunc = func() error { + return runHelpFuncError + } + runVersionFuncError := errors.New("version") + runVersionFunc = func() error { + return runVersionFuncError + } + runServerFuncError := errors.New("server") + runServerFunc = func() error { + return runServerFuncError + } + unknownArgsFuncError := errors.New("unknown") + unknownArgsFunc = func(Args) func() error { + return func() error { + return unknownArgsFuncError + } + } + + reset := func() { + option.configFile = "" + option.helpFlag = false + option.versionFlag = false + } + + testCases := []struct { + name string + args []string + result error + }{ + {"Help", []string{app, "help"}, runHelpFuncError}, + {"Help", []string{app, "--help"}, runHelpFuncError}, + {"Version", []string{app, "version"}, runVersionFuncError}, + {"Version", []string{app, "--version"}, runVersionFuncError}, + {"Serve", []string{app}, runServerFuncError}, + {"Unknown", []string{app, "unknown"}, unknownArgsFuncError}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + reset() + os.Args = tc.args + + if err := Execute(); tc.result != err { + t.Errorf( + "Expected error for %v but got %v", + tc.result, err, + ) + } + }) + } +} + +func TestUnknownArgs(t *testing.T) { + errFunc := unknownArgs(Args{"unknown"}) + if err := errFunc(); nil == err { + t.Errorf( + "Expected a given unknown argument error but got %v", + err, + ) + } +} + +func TestWithConfig(t *testing.T) { + configError := errors.New("config") + routineError := errors.New("routine") + routine := func() error { return routineError } + + testCases := []struct { + name string + loadConfig func(string) error + result error + }{ + {"Config error", func(string) error { return configError }, configError}, + {"Routine error", func(string) error { return nil }, routineError}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + loadConfig = tc.loadConfig + errFunc := withConfig(routine) + if err := errFunc(); tc.result != err { + t.Errorf("Expected error %v but got %v", tc.result, err) + } + }) + } +} diff --git a/deploy/chp-api/static-file-server/cli/help/help.go b/deploy/chp-api/static-file-server/cli/help/help.go new file mode 100644 index 0000000..9dee86e --- /dev/null +++ b/deploy/chp-api/static-file-server/cli/help/help.go @@ -0,0 +1,190 @@ +package help + +import ( + "fmt" +) + +// Run print operation. +func Run() error { + fmt.Println(Text) + return nil +} + +var ( + // Text for directly accessing help. + Text = ` +NAME + static-file-server + +SYNOPSIS + static-file-server + static-file-server [ -c | -config | --config ] /path/to/config.yml + static-file-server [ help | -help | --help ] + static-file-server [ version | -version | --version ] + +DESCRIPTION + The Static File Server is intended to be a tiny, fast and simple solution + for serving files over HTTP. The features included are limited to make to + binding to a host name and port, selecting a folder to serve, choosing a + URL path prefix and selecting TLS certificates. If you want really awesome + reverse proxy features, I recommend Nginx. + +DEPENDENCIES + None... not even libc! + +ENVIRONMENT VARIABLES + CORS + When set to 'true' it enables resource access from any domain. All + responses will include the headers 'Access-Control-Allow-Origin' and + 'Access-Control-Allow-Headers' with a wildcard value ('*'). + DEBUG + When set to 'true' enables additional logging, including the + configuration used and an access log for each request. IMPORTANT NOTE: + The configuration summary is printed to stdout while logs generated + during execution are printed to stderr. Default value is 'false'. + FOLDER + The path to the folder containing the contents to be served over + HTTP(s). If not supplied, defaults to '/web' (for Docker reasons). + HOST + The hostname used for binding. If not supplied, contents will be served + to a client without regard for the hostname. + PORT + The port used for binding. If not supplied, defaults to port '8080'. + REFERRERS + A comma-separated list of acceped Referrers based on the 'Referer' HTTP + header. If incoming header value is not in the list, a 403 HTTP error is + returned. To accept requests without a 'Referer' HTTP header in addition + to the whitelisted values, include an empty value (either with a leading + comma in the environment variable or with an empty list item in the YAML + configuration file) as demonstrated in the second example. If not + supplied the 'Referer' HTTP header is ignored. + Examples: + REFERRERS='http://localhost,https://some.site,http://other.site:8080' + REFERRERS=',http://localhost,https://some.site,http://other.site:8080' + ALLOW_INDEX + When set to 'true' the index.html file in the folder(not include the + sub folders) will be served. And the file list will not be served. + For example, if the client requests 'http://127.0.0.1/' the 'index.html' + file in the root of the directory being served is returned. Default value + is 'true'. + SHOW_LISTING + Automatically serve the index file for the directory if requested. For + example, if the client requests 'http://127.0.0.1/' the 'index.html' + file in the root of the directory being served is returned. If the value + is set to 'false', the same request will return a 'NOT FOUND'. Default + value is 'true'. + TLS_CERT + Path to the TLS certificate file to serve files using HTTPS. If supplied + then TLS_KEY must also be supplied. If not supplied, contents will be + served via HTTP. + TLS_KEY + Path to the TLS key file to serve files using HTTPS. If supplied then + TLS_CERT must also be supplied. If not supplied, contents will be served + via HTTPS + TLS_MIN_VERS + The minimum TLS version to use. If not supplied, defaults to TLS1.0. + Acceptable values are 'TLS10', 'TLS11', 'TLS12' and 'TLS13' for TLS1.0, + TLS1.1, TLS1.2 and TLS1.3, respectively. Values are not case-sensitive. + URL_PREFIX + The prefix to use in the URL path. If supplied, then the prefix must + start with a forward-slash and NOT end with a forward-slash. If not + supplied then no prefix is used. + +CONFIGURATION FILE + Configuration can also managed used a YAML configuration file. To select the + configuration values using the YAML file, pass in the path to the file using + the appropriate flags (-c, --config). Environment variables take priority + over the configuration file. The following is an example configuration using + the default values. + + Example config.yml with defaults: + ---------------------------------------------------------------------------- + cors: false + debug: false + folder: /web + host: "" + port: 8080 + referrers: [] + show-listing: true + tls-cert: "" + tls-key: "" + tls-min-vers: "" + url-prefix: "" + ---------------------------------------------------------------------------- + + Example config.yml with possible alternative values: + ---------------------------------------------------------------------------- + debug: true + folder: /var/www + port: 80 + referrers: + - http://localhost + - https://mydomain.com + ---------------------------------------------------------------------------- + +USAGE + FILE LAYOUT + /var/www/sub/my.file + /var/www/index.html + + COMMAND + export FOLDER=/var/www/sub + static-file-server + Retrieve with: wget http://localhost:8080/my.file + wget http://my.machine:8080/my.file + + export FOLDER=/var/www + export HOST=my.machine + export PORT=80 + static-file-server + Retrieve with: wget http://my.machine/sub/my.file + + export FOLDER=/var/www + static-file-server -c config.yml + Result: Runs with values from config.yml, but with the folder being + served overridden by the FOLDER environment variable. + + export FOLDER=/var/www/sub + export HOST=my.machine + export PORT=80 + export URL_PREFIX=/my/stuff + static-file-server + Retrieve with: wget http://my.machine/my/stuff/my.file + + export FOLDER=/var/www/sub + export TLS_CERT=/etc/server/my.machine.crt + export TLS_KEY=/etc/server/my.machine.key + static-file-server + Retrieve with: wget https://my.machine:8080/my.file + + export FOLDER=/var/www/sub + export PORT=443 + export TLS_CERT=/etc/server/my.machine.crt + export TLS_KEY=/etc/server/my.machine.key + export TLS_MIN_VERS=TLS12 + static-file-server + Retrieve with: wget https://my.machine/my.file + + export FOLDER=/var/www + export PORT=80 + export ALLOW_INDEX=true # Default behavior + export SHOW_LISTING=true # Default behavior + static-file-server + Retrieve 'index.html' with: wget http://my.machine/ + + export FOLDER=/var/www + export PORT=80 + export ALLOW_INDEX=true # Default behavior + export SHOW_LISTING=false + static-file-server + Retrieve 'index.html' with: wget http://my.machine/ + Returns 'NOT FOUND': wget http://my.machine/dir/ + + export FOLDER=/var/www + export PORT=80 + export ALLOW_INDEX=false + export SHOW_LISTING=false + static-file-server + Returns 'NOT FOUND': wget http://my.machine/ +` +) diff --git a/deploy/chp-api/static-file-server/cli/help/help_test.go b/deploy/chp-api/static-file-server/cli/help/help_test.go new file mode 100644 index 0000000..371673b --- /dev/null +++ b/deploy/chp-api/static-file-server/cli/help/help_test.go @@ -0,0 +1,9 @@ +package help + +import "testing" + +func TestRun(t *testing.T) { + if err := Run(); nil != err { + t.Errorf("While running help got %v", err) + } +} diff --git a/deploy/chp-api/static-file-server/cli/server/server.go b/deploy/chp-api/static-file-server/cli/server/server.go new file mode 100644 index 0000000..a1b008e --- /dev/null +++ b/deploy/chp-api/static-file-server/cli/server/server.go @@ -0,0 +1,96 @@ +package server + +import ( + "fmt" + "net/http" + + "github.com/halverneus/static-file-server/config" + "github.com/halverneus/static-file-server/handle" +) + +var ( + // Values to be overridden to simplify unit testing. + selectHandler = handlerSelector + selectListener = listenerSelector +) + +// Run server. +func Run() error { + if config.Get.Debug { + config.Log() + } + // Choose and set the appropriate, optimized static file serving function. + handler := selectHandler() + + // Serve files over HTTP or HTTPS based on paths to TLS files being + // provided. + listener := selectListener() + + binding := fmt.Sprintf("%s:%d", config.Get.Host, config.Get.Port) + return listener(binding, handler) +} + +// handlerSelector returns the appropriate request handler based on +// configuration. +func handlerSelector() (handler http.HandlerFunc) { + var serveFileHandler handle.FileServerFunc + + serveFileHandler = http.ServeFile + if config.Get.Debug { + serveFileHandler = handle.WithLogging(serveFileHandler) + } + + if 0 != len(config.Get.Referrers) { + serveFileHandler = handle.WithReferrers( + serveFileHandler, config.Get.Referrers, + ) + } + + // Choose and set the appropriate, optimized static file serving function. + if 0 == len(config.Get.URLPrefix) { + handler = handle.Basic(serveFileHandler, config.Get.Folder) + } else { + handler = handle.Prefix( + serveFileHandler, + config.Get.Folder, + config.Get.URLPrefix, + ) + } + + // Determine whether index files should hidden. + if !config.Get.ShowListing { + if config.Get.AllowIndex { + handler = handle.PreventListings(handler, config.Get.Folder, config.Get.URLPrefix) + } else { + handler = handle.IgnoreIndex(handler) + } + } + // If configured, apply wildcard CORS support. + if config.Get.Cors { + handler = handle.AddCorsWildcardHeaders(handler) + } + + // If configured, apply key code access control. + if "" != config.Get.AccessKey { + handler = handle.AddAccessKey(handler, config.Get.AccessKey) + } + + return +} + +// listenerSelector returns the appropriate listener handler based on +// configuration. +func listenerSelector() (listener handle.ListenerFunc) { + // Serve files over HTTP or HTTPS based on paths to TLS files being + // provided. + if 0 < len(config.Get.TLSCert) { + handle.SetMinimumTLSVersion(config.Get.TLSMinVers) + listener = handle.TLSListening( + config.Get.TLSCert, + config.Get.TLSKey, + ) + } else { + listener = handle.Listening() + } + return +} diff --git a/deploy/chp-api/static-file-server/cli/server/server_test.go b/deploy/chp-api/static-file-server/cli/server/server_test.go new file mode 100644 index 0000000..9f78554 --- /dev/null +++ b/deploy/chp-api/static-file-server/cli/server/server_test.go @@ -0,0 +1,127 @@ +package server + +import ( + "errors" + "net/http" + "testing" + + "github.com/halverneus/static-file-server/config" + "github.com/halverneus/static-file-server/handle" +) + +func TestRun(t *testing.T) { + listenerError := errors.New("listener") + selectListener = func() handle.ListenerFunc { + return func(string, http.HandlerFunc) error { + return listenerError + } + } + + config.Get.Debug = false + if err := Run(); listenerError != err { + t.Errorf("Without debug expected %v but got %v", listenerError, err) + } + + config.Get.Debug = true + if err := Run(); listenerError != err { + t.Errorf("With debug expected %v but got %v", listenerError, err) + } +} + +func TestHandlerSelector(t *testing.T) { + // This test only exercises function branches. + testFolder := "/web" + testPrefix := "/url/prefix" + var ignoreReferrer []string + testReferrer := []string{"http://localhost"} + testAccessKey := "access-key" + + testCases := []struct { + name string + folder string + prefix string + listing bool + debug bool + refer []string + cors bool + accessKey string + }{ + {"Basic handler w/o debug", testFolder, "", true, false, ignoreReferrer, false, ""}, + {"Prefix handler w/o debug", testFolder, testPrefix, true, false, ignoreReferrer, false, ""}, + {"Basic and hide listing handler w/o debug", testFolder, "", false, false, ignoreReferrer, false, ""}, + {"Prefix and hide listing handler w/o debug", testFolder, testPrefix, false, false, ignoreReferrer, false, ""}, + {"Basic handler w/debug", testFolder, "", true, true, ignoreReferrer, false, ""}, + {"Prefix handler w/debug", testFolder, testPrefix, true, true, ignoreReferrer, false, ""}, + {"Basic and hide listing handler w/debug", testFolder, "", false, true, ignoreReferrer, false, ""}, + {"Prefix and hide listing handler w/debug", testFolder, testPrefix, false, true, ignoreReferrer, false, ""}, + {"Basic handler w/o debug w/refer", testFolder, "", true, false, testReferrer, false, ""}, + {"Prefix handler w/o debug w/refer", testFolder, testPrefix, true, false, testReferrer, false, ""}, + {"Basic and hide listing handler w/o debug w/refer", testFolder, "", false, false, testReferrer, false, ""}, + {"Prefix and hide listing handler w/o debug w/refer", testFolder, testPrefix, false, false, testReferrer, false, ""}, + {"Basic handler w/debug w/refer w/o cors", testFolder, "", true, true, testReferrer, false, ""}, + {"Prefix handler w/debug w/refer w/o cors", testFolder, testPrefix, true, true, testReferrer, false, ""}, + {"Basic and hide listing handler w/debug w/refer w/o cors", testFolder, "", false, true, testReferrer, false, ""}, + {"Prefix and hide listing handler w/debug w/refer w/o cors", testFolder, testPrefix, false, true, testReferrer, false, ""}, + {"Basic handler w/debug w/refer w/cors", testFolder, "", true, true, testReferrer, true, ""}, + {"Prefix handler w/debug w/refer w/cors", testFolder, testPrefix, true, true, testReferrer, true, ""}, + {"Basic and hide listing handler w/debug w/refer w/cors", testFolder, "", false, true, testReferrer, true, ""}, + {"Prefix and hide listing handler w/debug w/refer w/cors", testFolder, testPrefix, false, true, testReferrer, true, ""}, + {"Access Key and Basic handler w/o debug", testFolder, "", true, false, ignoreReferrer, false, testAccessKey}, + {"Access Key and Prefix handler w/o debug", testFolder, testPrefix, true, false, ignoreReferrer, false, testAccessKey}, + {"Access Key and Basic and hide listing handler w/o debug", testFolder, "", false, false, ignoreReferrer, false, testAccessKey}, + {"Access Key and Prefix and hide listing handler w/o debug", testFolder, testPrefix, false, false, ignoreReferrer, false, testAccessKey}, + {"Access Key and Basic handler w/debug", testFolder, "", true, true, ignoreReferrer, false, testAccessKey}, + {"Access Key and Prefix handler w/debug", testFolder, testPrefix, true, true, ignoreReferrer, false, testAccessKey}, + {"Access Key and Basic and hide listing handler w/debug", testFolder, "", false, true, ignoreReferrer, false, testAccessKey}, + {"Access Key and Prefix and hide listing handler w/debug", testFolder, testPrefix, false, true, ignoreReferrer, false, testAccessKey}, + {"Access Key and Basic handler w/o debug w/refer", testFolder, "", true, false, testReferrer, false, testAccessKey}, + {"Access Key and Prefix handler w/o debug w/refer", testFolder, testPrefix, true, false, testReferrer, false, testAccessKey}, + {"Access Key and Basic and hide listing handler w/o debug w/refer", testFolder, "", false, false, testReferrer, false, testAccessKey}, + {"Access Key and Prefix and hide listing handler w/o debug w/refer", testFolder, testPrefix, false, false, testReferrer, false, testAccessKey}, + {"Access Key and Basic handler w/debug w/refer w/o cors", testFolder, "", true, true, testReferrer, false, testAccessKey}, + {"Access Key and Prefix handler w/debug w/refer w/o cors", testFolder, testPrefix, true, true, testReferrer, false, testAccessKey}, + {"Access Key and Basic and hide listing handler w/debug w/refer w/o cors", testFolder, "", false, true, testReferrer, false, testAccessKey}, + {"Access Key and Prefix and hide listing handler w/debug w/refer w/o cors", testFolder, testPrefix, false, true, testReferrer, false, testAccessKey}, + {"Access Key and Basic handler w/debug w/refer w/cors", testFolder, "", true, true, testReferrer, true, testAccessKey}, + {"Access Key and Prefix handler w/debug w/refer w/cors", testFolder, testPrefix, true, true, testReferrer, true, testAccessKey}, + {"Access Key and Basic and hide listing handler w/debug w/refer w/cors", testFolder, "", false, true, testReferrer, true, testAccessKey}, + {"Access Key and Prefix and hide listing handler w/debug w/refer w/cors", testFolder, testPrefix, false, true, testReferrer, true, testAccessKey}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + config.Get.Debug = tc.debug + config.Get.Folder = tc.folder + config.Get.ShowListing = tc.listing + config.Get.URLPrefix = tc.prefix + config.Get.Referrers = tc.refer + config.Get.Cors = tc.cors + config.Get.AccessKey = tc.accessKey + + handlerSelector() + }) + } +} + +func TestListenerSelector(t *testing.T) { + // This test only exercises function branches. + testCert := "file.crt" + testKey := "file.key" + + testCases := []struct { + name string + cert string + key string + }{ + {"HTTP", "", ""}, + {"HTTPS", testCert, testKey}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + config.Get.TLSCert = tc.cert + config.Get.TLSKey = tc.key + listenerSelector() + }) + } +} diff --git a/deploy/chp-api/static-file-server/cli/version/version.go b/deploy/chp-api/static-file-server/cli/version/version.go new file mode 100644 index 0000000..b1956c4 --- /dev/null +++ b/deploy/chp-api/static-file-server/cli/version/version.go @@ -0,0 +1,24 @@ +package version + +import ( + "fmt" + "runtime" +) + +// Run print operation. +func Run() error { + fmt.Printf("%s\n%s\n", VersionText, GoVersionText) + return nil +} + +var ( + // version is the application version set during build. + version string + + // VersionText for directly accessing the static-file-server version. + VersionText = fmt.Sprintf("v%s", version) + + // GoVersionText for directly accessing the version of the Go runtime + // compiled with the static-file-server. + GoVersionText = runtime.Version() +) diff --git a/deploy/chp-api/static-file-server/cli/version/version_test.go b/deploy/chp-api/static-file-server/cli/version/version_test.go new file mode 100644 index 0000000..2bc47d5 --- /dev/null +++ b/deploy/chp-api/static-file-server/cli/version/version_test.go @@ -0,0 +1,9 @@ +package version + +import "testing" + +func TestVersion(t *testing.T) { + if err := Run(); nil != err { + t.Errorf("While running version got %v", err) + } +} diff --git a/deploy/chp-api/static-file-server/config/config.go b/deploy/chp-api/static-file-server/config/config.go new file mode 100644 index 0000000..9b3ec59 --- /dev/null +++ b/deploy/chp-api/static-file-server/config/config.go @@ -0,0 +1,301 @@ +package config + +import ( + "crypto/tls" + "errors" + "fmt" + "io/ioutil" + "log" + "os" + "strconv" + "strings" + + yaml "gopkg.in/yaml.v3" +) + +var ( + // Get the desired configuration value. + Get struct { + Cors bool `yaml:"cors"` + Debug bool `yaml:"debug"` + Folder string `yaml:"folder"` + Host string `yaml:"host"` + Port uint16 `yaml:"port"` + AllowIndex bool `yaml:"allow-index"` + ShowListing bool `yaml:"show-listing"` + TLSCert string `yaml:"tls-cert"` + TLSKey string `yaml:"tls-key"` + TLSMinVers uint16 `yaml:"-"` + TLSMinVersStr string `yaml:"tls-min-vers"` + URLPrefix string `yaml:"url-prefix"` + Referrers []string `yaml:"referrers"` + AccessKey string `yaml:"access-key"` + } +) + +const ( + corsKey = "CORS" + debugKey = "DEBUG" + folderKey = "FOLDER" + hostKey = "HOST" + portKey = "PORT" + referrersKey = "REFERRERS" + allowIndexKey = "ALLOW_INDEX" + showListingKey = "SHOW_LISTING" + tlsCertKey = "TLS_CERT" + tlsKeyKey = "TLS_KEY" + tlsMinVersKey = "TLS_MIN_VERS" + urlPrefixKey = "URL_PREFIX" + accessKeyKey = "ACCESS_KEY" +) + +var ( + defaultDebug = false + defaultFolder = "/web" + defaultHost = "" + defaultPort = uint16(8080) + defaultReferrers = []string{} + defaultAllowIndex = true + defaultShowListing = true + defaultTLSCert = "" + defaultTLSKey = "" + defaultTLSMinVers = "" + defaultURLPrefix = "" + defaultCors = false + defaultAccessKey = "" +) + +func init() { + // init calls setDefaults to better support testing. + setDefaults() +} + +func setDefaults() { + Get.Debug = defaultDebug + Get.Folder = defaultFolder + Get.Host = defaultHost + Get.Port = defaultPort + Get.Referrers = defaultReferrers + Get.AllowIndex = defaultAllowIndex + Get.ShowListing = defaultShowListing + Get.TLSCert = defaultTLSCert + Get.TLSKey = defaultTLSKey + Get.TLSMinVersStr = defaultTLSMinVers + Get.URLPrefix = defaultURLPrefix + Get.Cors = defaultCors + Get.AccessKey = defaultAccessKey +} + +// Load the configuration file. +func Load(filename string) (err error) { + // If no filename provided, assign envvars. + if filename == "" { + overrideWithEnvVars() + return validate() + } + + // Read contents from configuration file. + var contents []byte + if contents, err = ioutil.ReadFile(filename); nil != err { + return + } + + // Parse contents into 'Get' configuration. + if err = yaml.Unmarshal(contents, &Get); nil != err { + return + } + + overrideWithEnvVars() + return validate() +} + +// Log the current configuration. +func Log() { + // YAML marshalling should never error, but if it could, the result is that + // the contents of the configuration are not logged. + contents, _ := yaml.Marshal(&Get) + + // Log the configuration. + fmt.Println("Using the following configuration:") + fmt.Println(string(contents)) +} + +// overrideWithEnvVars the default values and the configuration file values. +func overrideWithEnvVars() { + // Assign envvars, if set. + Get.Cors = envAsBool(corsKey, Get.Cors) + Get.Debug = envAsBool(debugKey, Get.Debug) + Get.Folder = envAsStr(folderKey, Get.Folder) + Get.Host = envAsStr(hostKey, Get.Host) + Get.Port = envAsUint16(portKey, Get.Port) + Get.AllowIndex = envAsBool(allowIndexKey, Get.AllowIndex) + Get.ShowListing = envAsBool(showListingKey, Get.ShowListing) + Get.TLSCert = envAsStr(tlsCertKey, Get.TLSCert) + Get.TLSKey = envAsStr(tlsKeyKey, Get.TLSKey) + Get.TLSMinVersStr = envAsStr(tlsMinVersKey, Get.TLSMinVersStr) + Get.URLPrefix = envAsStr(urlPrefixKey, Get.URLPrefix) + Get.Referrers = envAsStrSlice(referrersKey, Get.Referrers) + Get.AccessKey = envAsStr(accessKeyKey, Get.AccessKey) +} + +// validate the configuration. +func validate() error { + // If HTTPS is to be used, verify both TLS_* environment variables are set. + useTLS := false + if 0 < len(Get.TLSCert) || 0 < len(Get.TLSKey) { + if len(Get.TLSCert) == 0 || len(Get.TLSKey) == 0 { + msg := "if value for either 'TLS_CERT' or 'TLS_KEY' is set then " + + "then value for the other must also be set (values are " + + "currently '%s' and '%s', respectively)" + return fmt.Errorf(msg, Get.TLSCert, Get.TLSKey) + } + if _, err := os.Stat(Get.TLSCert); nil != err { + msg := "value of TLS_CERT is set with filename '%s' that returns %v" + return fmt.Errorf(msg, Get.TLSCert, err) + } + if _, err := os.Stat(Get.TLSKey); nil != err { + msg := "value of TLS_KEY is set with filename '%s' that returns %v" + return fmt.Errorf(msg, Get.TLSKey, err) + } + useTLS = true + } + + // Verify TLS_MIN_VERS is only (optionally) set if TLS is to be used. + Get.TLSMinVers = tls.VersionTLS10 + if useTLS { + if 0 < len(Get.TLSMinVersStr) { + var err error + if Get.TLSMinVers, err = tlsMinVersAsUint16( + Get.TLSMinVersStr, + ); nil != err { + return err + } + } + + // For logging minimum TLS version being used while debugging, backfill + // the TLSMinVersStr field. + switch Get.TLSMinVers { + case tls.VersionTLS10: + Get.TLSMinVersStr = "TLS1.0" + case tls.VersionTLS11: + Get.TLSMinVersStr = "TLS1.1" + case tls.VersionTLS12: + Get.TLSMinVersStr = "TLS1.2" + case tls.VersionTLS13: + Get.TLSMinVersStr = "TLS1.3" + } + } else { + if 0 < len(Get.TLSMinVersStr) { + msg := "value for 'TLS_MIN_VERS' is set but 'TLS_CERT' and 'TLS_KEY' are not" + return errors.New(msg) + } + } + + // If the URL path prefix is to be used, verify it is properly formatted. + if 0 < len(Get.URLPrefix) && + (!strings.HasPrefix(Get.URLPrefix, "/") || strings.HasSuffix(Get.URLPrefix, "/")) { + msg := "if value for 'URL_PREFIX' is set then the value must start " + + "with '/' and not end with '/' (current value of '%s' vs valid " + + "example of '/my/prefix'" + return fmt.Errorf(msg, Get.URLPrefix) + } + + return nil +} + +// envAsStr returns the value of the environment variable as a string if set. +func envAsStr(key, fallback string) string { + if value := os.Getenv(key); value != "" { + return value + } + return fallback +} + +// envAsStrSlice returns the value of the environment variable as a slice of +// strings if set. +func envAsStrSlice(key string, fallback []string) []string { + if value := os.Getenv(key); value != "" { + return strings.Split(value, ",") + } + return fallback +} + +// envAsUint16 returns the value of the environment variable as a uint16 if set. +func envAsUint16(key string, fallback uint16) uint16 { + // Retrieve the string value of the environment variable. If not set, + // fallback is used. + valueStr := os.Getenv(key) + if valueStr == "" { + return fallback + } + + // Parse the string into a uint16. + base := 10 + bitSize := 16 + valueAsUint64, err := strconv.ParseUint(valueStr, base, bitSize) + if nil != err { + log.Printf( + "Invalid value for '%s': %v\nUsing fallback: %d", + key, err, fallback, + ) + return fallback + } + return uint16(valueAsUint64) +} + +// envAsBool returns the value for an environment variable or, if not set, a +// fallback value as a boolean. +func envAsBool(key string, fallback bool) bool { + // Retrieve the string value of the environment variable. If not set, + // fallback is used. + valueStr := os.Getenv(key) + if valueStr == "" { + return fallback + } + + // Parse the string into a boolean. + value, err := strAsBool(valueStr) + if nil != err { + log.Printf( + "Invalid value for '%s': %v\nUsing fallback: %t", + key, err, fallback, + ) + return fallback + } + return value +} + +// strAsBool converts the intent of the passed value into a boolean +// representation. +func strAsBool(value string) (result bool, err error) { + lvalue := strings.ToLower(value) + switch lvalue { + case "0", "false", "f", "no", "n": + result = false + case "1", "true", "t", "yes", "y": + result = true + default: + result = false + msg := "unknown conversion from string to bool for value '%s'" + err = fmt.Errorf(msg, value) + } + return +} + +// tlsMinVersAsUint16 converts the intent of the passed value into an +// enumeration for the crypto/tls package. +func tlsMinVersAsUint16(value string) (result uint16, err error) { + switch strings.ToLower(value) { + case "tls10": + result = tls.VersionTLS10 + case "tls11": + result = tls.VersionTLS11 + case "tls12": + result = tls.VersionTLS12 + case "tls13": + result = tls.VersionTLS13 + default: + err = fmt.Errorf("unknown value for TLS_MIN_VERS: %s", value) + } + return +} diff --git a/deploy/chp-api/static-file-server/config/config_test.go b/deploy/chp-api/static-file-server/config/config_test.go new file mode 100644 index 0000000..a4c541b --- /dev/null +++ b/deploy/chp-api/static-file-server/config/config_test.go @@ -0,0 +1,513 @@ +package config + +import ( + "crypto/tls" + "fmt" + "io/ioutil" + "os" + "strconv" + "testing" + + yaml "gopkg.in/yaml.v3" +) + +func TestLoad(t *testing.T) { + // Verify envvars are set. + testFolder := "/my/directory" + os.Setenv(folderKey, testFolder) + if err := Load(""); nil != err { + t.Errorf( + "While loading an empty file name expected no error but got %v", + err, + ) + } + if Get.Folder != testFolder { + t.Errorf( + "While loading an empty file name expected folder %s but got %s", + testFolder, Get.Folder, + ) + } + + // Verify error if file doesn't exist. + if err := Load("/this/file/should/never/exist"); nil == err { + t.Error("While loading non-existing file expected error but got nil") + } + + // Verify bad YAML returns an error. + func(t *testing.T) { + filename := "testing.tmp" + contents := []byte("{") + defer os.Remove(filename) + + if err := ioutil.WriteFile(filename, contents, 0666); nil != err { + t.Errorf("Failed to save bad YAML file with: %v\n", err) + } + if err := Load(filename); nil == err { + t.Error("While loading bad YAML expected error but got nil") + } + }(t) + + // Verify good YAML returns no error and sets value. + func(t *testing.T) { + filename := "testing.tmp" + testFolder := "/test/folder" + contents := []byte(fmt.Sprintf( + `{"folder": "%s"}`, testFolder, + )) + defer os.Remove(filename) + + if err := ioutil.WriteFile(filename, contents, 0666); nil != err { + t.Errorf("Failed to save good YAML file with: %v\n", err) + } + if err := Load(filename); nil != err { + t.Errorf( + "While loading good YAML expected nil but got %v", + err, + ) + } + }(t) +} + +func TestLog(t *testing.T) { + // Test whether YAML marshalling works, as that is the only error case. + if _, err := yaml.Marshal(&Get); nil != err { + t.Errorf("While testing YAML marshalling for config Log() got %v", err) + } + Log() +} + +func TestOverrideWithEnvvars(t *testing.T) { + // Choose values that are different than defaults. + testDebug := true + testFolder := "/my/directory" + testHost := "apets.life" + testPort := uint16(666) + testAllowIndex := false + testShowListing := false + testTLSCert := "my.pem" + testTLSKey := "my.key" + testURLPrefix := "/url/prefix" + + // Set all environment variables with test values. + os.Setenv(debugKey, fmt.Sprintf("%t", testDebug)) + os.Setenv(folderKey, testFolder) + os.Setenv(hostKey, testHost) + os.Setenv(portKey, strconv.Itoa(int(testPort))) + os.Setenv(allowIndexKey, fmt.Sprintf("%t", testAllowIndex)) + os.Setenv(showListingKey, fmt.Sprintf("%t", testShowListing)) + os.Setenv(tlsCertKey, testTLSCert) + os.Setenv(tlsKeyKey, testTLSKey) + os.Setenv(urlPrefixKey, testURLPrefix) + + // Verification functions. + equalStrings := func(t *testing.T, name, key, expected, result string) { + if expected != result { + t.Errorf( + "While checking %s for '%s' expected '%s' but got '%s'", + name, key, expected, result, + ) + } + } + equalUint16 := func(t *testing.T, name, key string, expected, result uint16) { + if expected != result { + t.Errorf( + "While checking %s for '%s' expected %d but got %d", + name, key, expected, result, + ) + } + } + equalBool := func(t *testing.T, name, key string, expected, result bool) { + if expected != result { + t.Errorf( + "While checking %s for '%s' expected %t but got %t", + name, key, expected, result, + ) + } + } + + // Verify defaults. + setDefaults() + phase := "defaults" + equalBool(t, phase, debugKey, defaultDebug, Get.Debug) + equalStrings(t, phase, folderKey, defaultFolder, Get.Folder) + equalStrings(t, phase, hostKey, defaultHost, Get.Host) + equalUint16(t, phase, portKey, defaultPort, Get.Port) + equalBool(t, phase, showListingKey, defaultShowListing, Get.ShowListing) + equalStrings(t, phase, tlsCertKey, defaultTLSCert, Get.TLSCert) + equalStrings(t, phase, tlsKeyKey, defaultTLSKey, Get.TLSKey) + equalStrings(t, phase, urlPrefixKey, defaultURLPrefix, Get.URLPrefix) + + // Apply overrides. + overrideWithEnvVars() + + // Verify overrides. + phase = "overrides" + equalBool(t, phase, debugKey, testDebug, Get.Debug) + equalStrings(t, phase, folderKey, testFolder, Get.Folder) + equalStrings(t, phase, hostKey, testHost, Get.Host) + equalUint16(t, phase, portKey, testPort, Get.Port) + equalBool(t, phase, showListingKey, testShowListing, Get.ShowListing) + equalStrings(t, phase, tlsCertKey, testTLSCert, Get.TLSCert) + equalStrings(t, phase, tlsKeyKey, testTLSKey, Get.TLSKey) + equalStrings(t, phase, urlPrefixKey, testURLPrefix, Get.URLPrefix) +} + +func TestValidate(t *testing.T) { + validPath := "config.go" + invalidPath := "should/never/exist.txt" + empty := "" + prefix := "/my/prefix" + + testCases := []struct { + name string + cert string + key string + prefix string + minTLS string + isError bool + }{ + {"Valid paths w/prefix", validPath, validPath, prefix, "", false}, + {"Valid paths wo/prefix", validPath, validPath, empty, "", false}, + {"Empty paths w/prefix", empty, empty, prefix, "", false}, + {"Empty paths wo/prefix", empty, empty, empty, "", false}, + {"Mixed paths w/prefix", empty, validPath, prefix, "", true}, + {"Alt mixed paths w/prefix", validPath, empty, prefix, "", true}, + {"Mixed paths wo/prefix", empty, validPath, empty, "", true}, + {"Alt mixed paths wo/prefix", validPath, empty, empty, "", true}, + {"Invalid cert w/prefix", invalidPath, validPath, prefix, "", true}, + {"Invalid key w/prefix", validPath, invalidPath, prefix, "", true}, + {"Invalid cert & key w/prefix", invalidPath, invalidPath, prefix, "", true}, + {"Prefix missing leading /", empty, empty, "my/prefix", "", true}, + {"Prefix with trailing /", empty, empty, "/my/prefix/", "", true}, + {"Valid paths w/min ok TLS", validPath, validPath, prefix, "tls11", false}, + {"Valid paths w/min ok TLS", validPath, validPath, prefix, "tls12", false}, + {"Valid paths w/min ok TLS", validPath, validPath, prefix, "tls13", false}, + {"Valid paths w/min bad TLS", validPath, validPath, prefix, "bad", true}, + {"Empty paths w/min ok TLS", empty, empty, prefix, "tls11", true}, + {"Empty paths w/min bad TLS", empty, empty, prefix, "bad", true}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + Get.TLSCert = tc.cert + Get.TLSKey = tc.key + Get.TLSMinVersStr = tc.minTLS + Get.URLPrefix = tc.prefix + err := validate() + hasError := nil != err + if hasError && !tc.isError { + t.Errorf("Expected no error but got %v", err) + } + if !hasError && tc.isError { + t.Error("Expected an error but got no error") + } + }) + } +} + +func TestEnvAsStr(t *testing.T) { + sv := "STRING_VALUE" + fv := "FLOAT_VALUE" + iv := "INT_VALUE" + bv := "BOOL_VALUE" + ev := "EMPTY_VALUE" + uv := "UNSET_VALUE" + + sr := "String Cheese" // String result + fr := "123.456" // Float result + ir := "-123" // Int result + br := "true" // Bool result + er := "" // Empty result + fbr := "fallback result" // Fallback result + efbr := "" // Empty fallback result + + os.Setenv(sv, sr) + os.Setenv(fv, fr) + os.Setenv(iv, ir) + os.Setenv(bv, br) + os.Setenv(ev, er) + + testCases := []struct { + name string + key string + fallback string + result string + }{ + {"Good string", sv, fbr, sr}, + {"Float string", fv, fbr, fr}, + {"Int string", iv, fbr, ir}, + {"Bool string", bv, fbr, br}, + {"Empty string", ev, fbr, fbr}, + {"Unset", uv, fbr, fbr}, + {"Good string with empty fallback", sv, efbr, sr}, + {"Unset with empty fallback", uv, efbr, efbr}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + result := envAsStr(tc.key, tc.fallback) + if tc.result != result { + t.Errorf( + "For %s with a '%s' fallback expected '%s' but got '%s'", + tc.key, tc.fallback, tc.result, result, + ) + } + }) + } +} + +func TestEnvAsStrSlice(t *testing.T) { + oe := "ONE_ENTRY" + oewc := "ONE_ENTRY_WITH_COMMA" + oewtc := "ONE_ENTRY_WITH_TRAILING_COMMA" + te := "TWO_ENTRY" + tewc := "TWO_ENTRY_WITH_COMMA" + oc := "ONLY_COMMA" + ev := "EMPTY_VALUE" + uv := "UNSET_VALUE" + + fs := "http://my.site" + ts := "http://other.site" + fbr := []string{"one", "two"} + var efbr []string + + oes := fs + oer := []string{fs} + oewcs := "," + fs + oewcr := []string{"", fs} + oewtcs := fs + "," + oewtcr := []string{fs, ""} + tes := fs + "," + ts + ter := []string{fs, ts} + tewcs := "," + fs + "," + ts + tewcr := []string{"", fs, ts} + ocs := "," + ocr := []string{"", ""} + evs := "" + + os.Setenv(oe, oes) + os.Setenv(oewc, oewcs) + os.Setenv(oewtc, oewtcs) + os.Setenv(te, tes) + os.Setenv(tewc, tewcs) + os.Setenv(oc, ocs) + os.Setenv(ev, evs) + + testCases := []struct { + name string + key string + fallback []string + result []string + }{ + {"One entry", oe, fbr, oer}, + {"One entry w/comma", oewc, fbr, oewcr}, + {"One entry w/trailing comma", oewtc, fbr, oewtcr}, + {"Two entry", te, fbr, ter}, + {"Two entry w/comma", tewc, fbr, tewcr}, + {"Only comma", oc, fbr, ocr}, + {"Empty value w/fallback", ev, fbr, fbr}, + {"Empty value wo/fallback", ev, efbr, efbr}, + {"Unset w/fallback", uv, fbr, fbr}, + {"Unset wo/fallback", uv, efbr, efbr}, + } + + matches := func(a, b []string) bool { + if len(a) != len(b) { + return false + } + tally := make(map[int]bool) + for i := range a { + tally[i] = false + } + for _, val := range a { + for i, other := range b { + if other == val && !tally[i] { + tally[i] = true + break + } + } + } + for _, found := range tally { + if !found { + return false + } + } + return true + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + result := envAsStrSlice(tc.key, tc.fallback) + if !matches(tc.result, result) { + t.Errorf( + "For %s with a '%v' fallback expected '%v' but got '%v'", + tc.key, tc.fallback, tc.result, result, + ) + } + }) + } +} + +func TestEnvAsUint16(t *testing.T) { + ubv := "UPPER_BOUNDS_VALUE" + lbv := "LOWER_BOUNDS_VALUE" + hv := "HIGH_VALUE" + lv := "LOW_VALUE" + bv := "BOOL_VALUE" + sv := "STRING_VALUE" + uv := "UNSET_VALUE" + + fbr := uint16(666) // Fallback result + ubr := uint16(65535) // Upper bounds result + lbr := uint16(0) // Lower bounds result + + os.Setenv(ubv, "65535") + os.Setenv(lbv, "0") + os.Setenv(hv, "65536") + os.Setenv(lv, "-1") + os.Setenv(bv, "true") + os.Setenv(sv, "Cheese") + + testCases := []struct { + name string + key string + fallback uint16 + result uint16 + }{ + {"Upper bounds", ubv, fbr, ubr}, + {"Lower bounds", lbv, fbr, lbr}, + {"Out-of-bounds high", hv, fbr, fbr}, + {"Out-of-bounds low", lv, fbr, fbr}, + {"Boolean", bv, fbr, fbr}, + {"String", sv, fbr, fbr}, + {"Unset", uv, fbr, fbr}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + result := envAsUint16(tc.key, tc.fallback) + if tc.result != result { + t.Errorf( + "For %s with a %d fallback expected %d but got %d", + tc.key, tc.fallback, tc.result, result, + ) + } + }) + } +} + +func TestEnvAsBool(t *testing.T) { + tv := "TRUE_VALUE" + fv := "FALSE_VALUE" + bv := "BAD_VALUE" + uv := "UNSET_VALUE" + + os.Setenv(tv, "True") + os.Setenv(fv, "NO") + os.Setenv(bv, "BAD") + + testCases := []struct { + name string + key string + fallback bool + result bool + }{ + {"True with true fallback", tv, true, true}, + {"True with false fallback", tv, false, true}, + {"False with true fallback", fv, true, false}, + {"False with false fallback", fv, false, false}, + {"Bad with true fallback", bv, true, true}, + {"Bad with false fallback", bv, false, false}, + {"Unset with true fallback", uv, true, true}, + {"Unset with false fallback", uv, false, false}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + result := envAsBool(tc.key, tc.fallback) + if tc.result != result { + t.Errorf( + "For %s with a %t fallback expected %t but got %t", + tc.key, tc.fallback, tc.result, result, + ) + } + }) + } +} + +func TestStrAsBool(t *testing.T) { + testCases := []struct { + name string + value string + result bool + isError bool + }{ + {"Empty value", "", false, true}, + {"False value", "0", false, false}, + {"True value", "1", true, false}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + result, err := strAsBool(tc.value) + if result != tc.result { + t.Errorf( + "Expected %t for %s but got %t", + tc.result, tc.value, result, + ) + } + if tc.isError && nil == err { + t.Errorf( + "Expected error for %s but got no error", + tc.value, + ) + } + if !tc.isError && nil != err { + t.Errorf( + "Expected no error for %s but got %v", + tc.value, err, + ) + } + }) + + } +} + +func TestTlsMinVersAsUint16(t *testing.T) { + testCases := []struct { + name string + value string + result uint16 + isError bool + }{ + {"Empty value", "", 0, true}, + {"Valid TLS1.0", "TLS10", tls.VersionTLS10, false}, + {"Valid TLS1.1", "tls11", tls.VersionTLS11, false}, + {"Valid TLS1.2", "tls12", tls.VersionTLS12, false}, + {"Valid TLS1.3", "tLS13", tls.VersionTLS13, false}, + {"Invalid TLS1.4", "tls14", 0, true}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + result, err := tlsMinVersAsUint16(tc.value) + if result != tc.result { + t.Errorf( + "Expected %d for %s but got %d", + tc.result, tc.value, result, + ) + } + if tc.isError && nil == err { + t.Errorf( + "Expected error for %s but got no error", + tc.value, + ) + } else if !tc.isError && nil != err { + t.Errorf( + "Expected no error for %s but got %v", + tc.value, err, + ) + } + }) + } +} diff --git a/deploy/chp-api/static-file-server/go.mod b/deploy/chp-api/static-file-server/go.mod new file mode 100644 index 0000000..0225e99 --- /dev/null +++ b/deploy/chp-api/static-file-server/go.mod @@ -0,0 +1,5 @@ +module github.com/halverneus/static-file-server + +go 1.18 + +require gopkg.in/yaml.v3 v3.0.1 diff --git a/deploy/chp-api/static-file-server/go.sum b/deploy/chp-api/static-file-server/go.sum new file mode 100644 index 0000000..a62c313 --- /dev/null +++ b/deploy/chp-api/static-file-server/go.sum @@ -0,0 +1,4 @@ +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/deploy/chp-api/static-file-server/handle/handle.go b/deploy/chp-api/static-file-server/handle/handle.go new file mode 100644 index 0000000..6e85835 --- /dev/null +++ b/deploy/chp-api/static-file-server/handle/handle.go @@ -0,0 +1,252 @@ +package handle + +import ( + "crypto/md5" + "crypto/tls" + "fmt" + "log" + "net/http" + "os" + "path" + "strings" +) + +var ( + // These assignments are for unit testing. + listenAndServe = http.ListenAndServe + listenAndServeTLS = defaultListenAndServeTLS + setHandler = http.HandleFunc +) + +var ( + // Server options to be set prior to calling the listening function. + // minTLSVersion is the minimum allowed TLS version to be used by the + // server. + minTLSVersion uint16 = tls.VersionTLS10 +) + +// defaultListenAndServeTLS is the default implementation of the listening +// function for serving with TLS enabled. This is, effectively, a copy from +// the standard library but with the ability to set the minimum TLS version. +func defaultListenAndServeTLS( + binding, certFile, keyFile string, handler http.Handler, +) error { + if handler == nil { + handler = http.DefaultServeMux + } + server := &http.Server{ + Addr: binding, + Handler: handler, + TLSConfig: &tls.Config{ + MinVersion: minTLSVersion, + }, + } + return server.ListenAndServeTLS(certFile, keyFile) +} + +// SetMinimumTLSVersion to be used by the server. +func SetMinimumTLSVersion(version uint16) { + if version < tls.VersionTLS10 { + version = tls.VersionTLS10 + } else if version > tls.VersionTLS13 { + version = tls.VersionTLS13 + } + minTLSVersion = version +} + +// ListenerFunc accepts the {hostname:port} binding string required by HTTP +// listeners and the handler (router) function and returns any errors that +// occur. +type ListenerFunc func(string, http.HandlerFunc) error + +// FileServerFunc is used to serve the file from the local file system to the +// requesting client. +type FileServerFunc func(http.ResponseWriter, *http.Request, string) + +// WithReferrers returns a function that evaluates the HTTP 'Referer' header +// value and returns HTTP error 403 if the value is not found in the whitelist. +// If one of the whitelisted referrers are an empty string, then it is allowed +// for the 'Referer' HTTP header key to not be set. +func WithReferrers(serveFile FileServerFunc, referrers []string) FileServerFunc { + return func(w http.ResponseWriter, r *http.Request, name string) { + if !validReferrer(referrers, r.Referer()) { + http.Error( + w, + fmt.Sprintf("Invalid source '%s'", r.Referer()), + http.StatusForbidden, + ) + return + } + serveFile(w, r, name) + } +} + +// WithLogging returns a function that logs information about the request prior +// to serving the requested file. +func WithLogging(serveFile FileServerFunc) FileServerFunc { + return func(w http.ResponseWriter, r *http.Request, name string) { + referer := r.Referer() + if len(referer) == 0 { + log.Printf( + "REQ from '%s': %s %s %s%s -> %s\n", + r.RemoteAddr, + r.Method, + r.Proto, + r.Host, + r.URL.Path, + name, + ) + } else { + log.Printf( + "REQ from '%s' (REFERER: '%s'): %s %s %s%s -> %s\n", + r.RemoteAddr, + referer, + r.Method, + r.Proto, + r.Host, + r.URL.Path, + name, + ) + } + serveFile(w, r, name) + } +} + +// Basic file handler servers files from the passed folder. +func Basic(serveFile FileServerFunc, folder string) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + serveFile(w, r, folder+r.URL.Path) + } +} + +// Prefix file handler is an alternative to Basic where a URL prefix is removed +// prior to serving a file (http://my.machine/prefix/file.txt will serve +// file.txt from the root of the folder being served (ignoring 'prefix')). +func Prefix(serveFile FileServerFunc, folder, urlPrefix string) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + if !strings.HasPrefix(r.URL.Path, urlPrefix) { + http.NotFound(w, r) + return + } + serveFile(w, r, folder+strings.TrimPrefix(r.URL.Path, urlPrefix)) + } +} + +// PreventListings returns a function that prevents listing of directories but +// still allows index.html to be served. +func PreventListings(serve http.HandlerFunc, folder string, urlPrefix string) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "/") { + // If the directory does not contain an index.html file, then + // return 'NOT FOUND' to prevent listing of the directory. + stat, err := os.Stat(path.Join(folder, strings.TrimPrefix(r.URL.Path, urlPrefix), "index.html")) + if err != nil || (err == nil && !stat.Mode().IsRegular()) { + http.NotFound(w, r) + return + } + } + serve(w, r) + } +} + +// IgnoreIndex wraps an HTTP request. In the event of a folder root request, +// this function will automatically return 'NOT FOUND' as opposed to default +// behavior where the index file for that directory is retrieved. +func IgnoreIndex(serve http.HandlerFunc) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "/") { + http.NotFound(w, r) + return + } + serve(w, r) + } +} + +// AddCorsWildcardHeaders wraps an HTTP request to notify client browsers that +// resources should be allowed to be retrieved by any other domain. +func AddCorsWildcardHeaders(serve http.HandlerFunc) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Access-Control-Allow-Origin", "*") + w.Header().Set("Access-Control-Allow-Headers", "*") + serve(w, r) + } +} + +// AddAccessKey provides Access Control through url parameters. The access key +// is set by ACCESS_KEY. md5sum is computed by queried path + access key +// (e.g. "/my/file" + ACCESS_KEY) +func AddAccessKey(serve http.HandlerFunc, accessKey string) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + // Get key or md5sum from this access. + keys, keyOk := r.URL.Query()["key"] + var code string + if !keyOk || len(keys[0]) < 1 { + // In case a code is provided + codes, codeOk := r.URL.Query()["code"] + if !codeOk || len(codes[0]) < 1 { + http.NotFound(w, r) + return + } + code = strings.ToUpper(codes[0]) + } else { + // In case a key is provided, convert to code. + data := []byte(r.URL.Path + keys[0]) + hash := md5.Sum(data) + code = fmt.Sprintf("%X", hash) + } + + // Compute the correct md5sum of this access. + localData := []byte(r.URL.Path + accessKey) + hash := md5.Sum(localData) + localCode := fmt.Sprintf("%X", hash) + + // Compare the two. + if code != localCode { + http.NotFound(w, r) + return + } + serve(w, r) + } +} + +// Listening function for serving the handler function. +func Listening() ListenerFunc { + return func(binding string, handler http.HandlerFunc) error { + setHandler("/", handler) + return listenAndServe(binding, nil) + } +} + +// TLSListening function for serving the handler function with encryption. +func TLSListening(tlsCert, tlsKey string) ListenerFunc { + return func(binding string, handler http.HandlerFunc) error { + setHandler("/", handler) + return listenAndServeTLS(binding, tlsCert, tlsKey, nil) + } +} + +// validReferrer returns true if the passed referrer can be resolved by the +// passed list of referrers. +func validReferrer(s []string, e string) bool { + // Whitelisted referer list is empty. All requests are allowed. + if len(s) == 0 { + return true + } + + for _, a := range s { + // Handle blank HTTP Referer header, if configured + if a == "" { + if e == "" { + return true + } + // Continue loop (all strings start with "") + continue + } + + // Compare header with allowed prefixes + if strings.HasPrefix(e, a) { + return true + } + } + return false +} diff --git a/deploy/chp-api/static-file-server/handle/handle_test.go b/deploy/chp-api/static-file-server/handle/handle_test.go new file mode 100644 index 0000000..ef6158f --- /dev/null +++ b/deploy/chp-api/static-file-server/handle/handle_test.go @@ -0,0 +1,703 @@ +package handle + +import ( + "crypto/md5" + "crypto/tls" + "errors" + "fmt" + "io/ioutil" + "log" + "net/http" + "net/http/httptest" + "os" + "path" + "testing" +) + +var ( + baseDir = "tmp/" + subDir = "sub/" + subDeepDir = "sub/deep/" + tmpIndexName = "index.html" + tmpFileName = "file.txt" + tmpBadName = "bad.txt" + tmpSubIndexName = "sub/index.html" + tmpSubFileName = "sub/file.txt" + tmpSubBadName = "sub/bad.txt" + tmpSubDeepIndexName = "sub/deep/index.html" + tmpSubDeepFileName = "sub/deep/file.txt" + tmpSubDeepBadName = "sub/deep/bad.txt" + tmpNoIndexDir = "noindex/" + tmpNoIndexName = "noindex/noindex.txt" + + tmpIndex = "Space: the final frontier" + tmpFile = "These are the voyages of the starship Enterprise." + tmpSubIndex = "Its continuing mission:" + tmpSubFile = "To explore strange new worlds" + tmpSubDeepIndex = "To seek out new life and new civilizations" + tmpSubDeepFile = "To boldly go where no one has gone before" + + nothing = "" + ok = http.StatusOK + missing = http.StatusNotFound + redirect = http.StatusMovedPermanently + notFound = "404 page not found\n" + + files = map[string]string{ + baseDir + tmpIndexName: tmpIndex, + baseDir + tmpFileName: tmpFile, + baseDir + tmpSubIndexName: tmpSubIndex, + baseDir + tmpSubFileName: tmpSubFile, + baseDir + tmpSubDeepIndexName: tmpSubDeepIndex, + baseDir + tmpSubDeepFileName: tmpSubDeepFile, + baseDir + tmpNoIndexName: tmpSubDeepFile, + } + + serveFileFuncs = []FileServerFunc{ + http.ServeFile, + WithLogging(http.ServeFile), + } +) + +func TestMain(m *testing.M) { + code := func(m *testing.M) int { + if err := setup(); nil != err { + log.Fatalf("While setting up test got: %v\n", err) + } + defer teardown() + return m.Run() + }(m) + os.Exit(code) +} + +func setup() (err error) { + for filename, contents := range files { + if err = os.MkdirAll(path.Dir(filename), 0700); nil != err { + return + } + if err = ioutil.WriteFile( + filename, + []byte(contents), + 0600, + ); nil != err { + return + } + } + return +} + +func teardown() (err error) { + return os.RemoveAll("tmp") +} + +func TestSetMinimumTLSVersion(t *testing.T) { + testCases := []struct { + name string + value uint16 + expected uint16 + }{ + {"Too low", tls.VersionTLS10 - 1, tls.VersionTLS10}, + {"Lower bounds", tls.VersionTLS10, tls.VersionTLS10}, + {"Upper bounds", tls.VersionTLS13, tls.VersionTLS13}, + {"Too high", tls.VersionTLS13 + 1, tls.VersionTLS13}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + SetMinimumTLSVersion(tc.value) + if tc.expected != minTLSVersion { + t.Errorf("Expected %d but got %d", tc.expected, minTLSVersion) + } + }) + } +} + +func TestWithReferrers(t *testing.T) { + forbidden := http.StatusForbidden + + ok1 := "http://valid.com" + ok2 := "https://valid.com" + ok3 := "http://localhost" + bad := "http://other.pl" + + var noRefer []string + emptyRefer := []string{} + onlyNoRefer := []string{""} + refer := []string{ok1, ok2, ok3} + noWithRefer := []string{"", ok1, ok2, ok3} + + testCases := []struct { + name string + refers []string + refer string + code int + }{ + {"Nil refer list", noRefer, bad, ok}, + {"Empty refer list", emptyRefer, bad, ok}, + {"Unassigned allowed & unassigned", onlyNoRefer, "", ok}, + {"Unassigned allowed & assigned", onlyNoRefer, bad, forbidden}, + {"Whitelist with unassigned", refer, "", forbidden}, + {"Whitelist with bad", refer, bad, forbidden}, + {"Whitelist with ok1", refer, ok1, ok}, + {"Whitelist with ok2", refer, ok2, ok}, + {"Whitelist with ok3", refer, ok3, ok}, + {"Whitelist and none with unassigned", noWithRefer, "", ok}, + {"Whitelist with bad", noWithRefer, bad, forbidden}, + {"Whitelist with ok1", noWithRefer, ok1, ok}, + {"Whitelist with ok2", noWithRefer, ok2, ok}, + {"Whitelist with ok3", noWithRefer, ok3, ok}, + } + + success := func(w http.ResponseWriter, r *http.Request, name string) { + defer r.Body.Close() + w.WriteHeader(ok) + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + handler := WithReferrers(success, tc.refers) + + fullpath := "http://localhost/" + tmpIndexName + req := httptest.NewRequest("GET", fullpath, nil) + req.Header.Add("Referer", tc.refer) + w := httptest.NewRecorder() + + handler(w, req, "") + + resp := w.Result() + _, err := ioutil.ReadAll(resp.Body) + if nil != err { + t.Errorf("While reading body got %v", err) + } + if tc.code != resp.StatusCode { + t.Errorf( + "With referer '%s' in '%v' expected status code %d but got %d", + tc.refer, tc.refers, tc.code, resp.StatusCode, + ) + } + }) + } +} + +func TestBasicWithAndWithoutLogging(t *testing.T) { + referer := "http://localhost" + noReferer := "" + testCases := []struct { + name string + path string + code int + refer string + contents string + }{ + {"Good base dir", "", ok, referer, tmpIndex}, + {"Good base index", tmpIndexName, redirect, referer, nothing}, + {"Good base file", tmpFileName, ok, referer, tmpFile}, + {"Bad base file", tmpBadName, missing, referer, notFound}, + {"Good subdir dir", subDir, ok, referer, tmpSubIndex}, + {"Good subdir index", tmpSubIndexName, redirect, referer, nothing}, + {"Good subdir file", tmpSubFileName, ok, referer, tmpSubFile}, + {"Good base dir", "", ok, noReferer, tmpIndex}, + {"Good base index", tmpIndexName, redirect, noReferer, nothing}, + {"Good base file", tmpFileName, ok, noReferer, tmpFile}, + {"Bad base file", tmpBadName, missing, noReferer, notFound}, + {"Good subdir dir", subDir, ok, noReferer, tmpSubIndex}, + {"Good subdir index", tmpSubIndexName, redirect, noReferer, nothing}, + {"Good subdir file", tmpSubFileName, ok, noReferer, tmpSubFile}, + } + + for _, serveFile := range serveFileFuncs { + handler := Basic(serveFile, baseDir) + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + fullpath := "http://localhost/" + tc.path + req := httptest.NewRequest("GET", fullpath, nil) + req.Header.Add("Referer", tc.refer) + w := httptest.NewRecorder() + + handler(w, req) + + resp := w.Result() + body, err := ioutil.ReadAll(resp.Body) + if nil != err { + t.Errorf("While reading body got %v", err) + } + contents := string(body) + if tc.code != resp.StatusCode { + t.Errorf( + "While retrieving %s expected status code of %d but got %d", + fullpath, tc.code, resp.StatusCode, + ) + } + if tc.contents != contents { + t.Errorf( + "While retrieving %s expected contents '%s' but got '%s'", + fullpath, tc.contents, contents, + ) + } + }) + } + } +} + +func TestPrefix(t *testing.T) { + prefix := "/my/prefix/path/" + + testCases := []struct { + name string + path string + code int + contents string + }{ + {"Good base dir", prefix, ok, tmpIndex}, + {"Good base index", prefix + tmpIndexName, redirect, nothing}, + {"Good base file", prefix + tmpFileName, ok, tmpFile}, + {"Bad base file", prefix + tmpBadName, missing, notFound}, + {"Good subdir dir", prefix + subDir, ok, tmpSubIndex}, + {"Good subdir index", prefix + tmpSubIndexName, redirect, nothing}, + {"Good subdir file", prefix + tmpSubFileName, ok, tmpSubFile}, + {"Unknown prefix", tmpFileName, missing, notFound}, + } + + for _, serveFile := range serveFileFuncs { + handler := Prefix(serveFile, baseDir, prefix) + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + fullpath := "http://localhost" + tc.path + req := httptest.NewRequest("GET", fullpath, nil) + w := httptest.NewRecorder() + + handler(w, req) + + resp := w.Result() + body, err := ioutil.ReadAll(resp.Body) + if nil != err { + t.Errorf("While reading body got %v", err) + } + contents := string(body) + if tc.code != resp.StatusCode { + t.Errorf( + "While retrieving %s expected status code of %d but got %d", + fullpath, tc.code, resp.StatusCode, + ) + } + if tc.contents != contents { + t.Errorf( + "While retrieving %s expected contents '%s' but got '%s'", + fullpath, tc.contents, contents, + ) + } + }) + } + } +} + +func TestIgnoreIndex(t *testing.T) { + testCases := []struct { + name string + path string + code int + contents string + }{ + {"Good base dir", "", missing, notFound}, + {"Good base index", tmpIndexName, redirect, nothing}, + {"Good base file", tmpFileName, ok, tmpFile}, + {"Bad base file", tmpBadName, missing, notFound}, + {"Good subdir dir", subDir, missing, notFound}, + {"Good subdir index", tmpSubIndexName, redirect, nothing}, + {"Good subdir file", tmpSubFileName, ok, tmpSubFile}, + } + + for _, serveFile := range serveFileFuncs { + handler := IgnoreIndex(Basic(serveFile, baseDir)) + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + fullpath := "http://localhost/" + tc.path + req := httptest.NewRequest("GET", fullpath, nil) + w := httptest.NewRecorder() + + handler(w, req) + + resp := w.Result() + body, err := ioutil.ReadAll(resp.Body) + if nil != err { + t.Errorf("While reading body got %v", err) + } + contents := string(body) + if tc.code != resp.StatusCode { + t.Errorf( + "While retrieving %s expected status code of %d but got %d", + fullpath, tc.code, resp.StatusCode, + ) + } + if tc.contents != contents { + t.Errorf( + "While retrieving %s expected contents '%s' but got '%s'", + fullpath, tc.contents, contents, + ) + } + }) + } + } +} + +func TestPreventListings(t *testing.T) { + testCases := []struct { + name string + path string + code int + contents string + }{ + {"Good base dir", "", ok, tmpIndex}, + {"Good base index", tmpIndexName, redirect, nothing}, + {"Good base file", tmpFileName, ok, tmpFile}, + {"Bad base file", tmpBadName, missing, notFound}, + {"Good subdir dir", subDir, ok, tmpSubIndex}, + {"Good subdir index", tmpSubIndexName, redirect, nothing}, + {"Good subdir file", tmpSubFileName, ok, tmpSubFile}, + {"Dir without index", tmpNoIndexDir, missing, notFound}, + } + + for _, serveFile := range serveFileFuncs { + handler := PreventListings(Basic(serveFile, baseDir), baseDir, "") + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + fullpath := "http://localhost/" + tc.path + req := httptest.NewRequest("GET", fullpath, nil) + w := httptest.NewRecorder() + + handler(w, req) + + resp := w.Result() + body, err := ioutil.ReadAll(resp.Body) + if nil != err { + t.Errorf("While reading body got %v", err) + } + contents := string(body) + if tc.code != resp.StatusCode { + t.Errorf( + "While retrieving %s expected status code of %d but got %d", + fullpath, tc.code, resp.StatusCode, + ) + } + if tc.contents != contents { + t.Errorf( + "While retrieving %s expected contents '%s' but got '%s'", + fullpath, tc.contents, contents, + ) + } + }) + } + } +} + +func TestAddAccessKey(t *testing.T) { + // Prepare testing data. + accessKey := "my-access-key" + + code := func(path, key string) string { + data := []byte("/" + path + key) + fmt.Printf("TEST: '%s'\n", data) + return fmt.Sprintf("%X", md5.Sum(data)) + } + + // Define test cases. + testCases := []struct { + name string + path string + key string + value string + code int + contents string + }{ + { + "Good base file with code", tmpFileName, + "code", code(tmpFileName, accessKey), + ok, tmpFile, + }, + { + "Good base file with key", tmpFileName, + "key", accessKey, + ok, tmpFile, + }, + { + "Bad base file with code", tmpBadName, + "code", code(tmpBadName, accessKey), + missing, notFound, + }, + { + "Bad base file with key", tmpBadName, + "key", accessKey, + missing, notFound, + }, + { + "Good base file with no code or key", tmpFileName, + "my", "value", + missing, notFound, + }, + { + "Good base file with bad code", tmpFileName, + "code", code(tmpFileName, "bad-access-key"), + missing, notFound, + }, + { + "Good base file with bad key", tmpFileName, + "key", "bad-access-key", + missing, notFound, + }, + } + + for _, serveFile := range serveFileFuncs { + handler := AddAccessKey(Basic(serveFile, baseDir), accessKey) + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + fullpath := fmt.Sprintf( + "http://localhost/%s?%s=%s", + tc.path, tc.key, tc.value, + ) + req := httptest.NewRequest("GET", fullpath, nil) + w := httptest.NewRecorder() + + handler(w, req) + + resp := w.Result() + body, err := ioutil.ReadAll(resp.Body) + if nil != err { + t.Errorf("While reading body got %v", err) + } + contents := string(body) + if tc.code != resp.StatusCode { + t.Errorf( + "While retrieving %s expected status code of %d but got %d", + fullpath, tc.code, resp.StatusCode, + ) + } + if tc.contents != contents { + t.Errorf( + "While retrieving %s expected contents '%s' but got '%s'", + fullpath, tc.contents, contents, + ) + } + }) + } + } +} + +func TestListening(t *testing.T) { + // Choose values for testing. + called := false + testBinding := "host:port" + testError := errors.New("random problem") + + // Create an empty placeholder router function. + handler := func(http.ResponseWriter, *http.Request) {} + + // Override setHandler so that multiple calls to 'http.HandleFunc' doesn't + // panic. + setHandler = func(string, func(http.ResponseWriter, *http.Request)) {} + + // Override listenAndServe with a function with more introspection and + // control than 'http.ListenAndServe'. + listenAndServe = func( + binding string, handler http.Handler, + ) error { + if testBinding != binding { + t.Errorf( + "While serving expected binding of %s but got %s", + testBinding, binding, + ) + } + called = !called + if called { + return nil + } + return testError + } + + // Perform test. + listener := Listening() + if err := listener(testBinding, handler); nil != err { + t.Errorf("While serving first expected nil error but got %v", err) + } + if err := listener(testBinding, handler); nil == err { + t.Errorf( + "While serving second got nil while expecting %v", testError, + ) + } +} + +func TestTLSListening(t *testing.T) { + // Choose values for testing. + called := false + testBinding := "host:port" + testTLSCert := "test/file.pem" + testTLSKey := "test/file.key" + testError := errors.New("random problem") + + // Create an empty placeholder router function. + handler := func(http.ResponseWriter, *http.Request) {} + + // Override setHandler so that multiple calls to 'http.HandleFunc' doesn't + // panic. + setHandler = func(string, func(http.ResponseWriter, *http.Request)) {} + + // Override listenAndServeTLS with a function with more introspection and + // control than 'http.ListenAndServeTLS'. + listenAndServeTLS = func( + binding, tlsCert, tlsKey string, handler http.Handler, + ) error { + if testBinding != binding { + t.Errorf( + "While serving TLS expected binding of %s but got %s", + testBinding, binding, + ) + } + if testTLSCert != tlsCert { + t.Errorf( + "While serving TLS expected TLS cert of %s but got %s", + testTLSCert, tlsCert, + ) + } + if testTLSKey != tlsKey { + t.Errorf( + "While serving TLS expected TLS key of %s but got %s", + testTLSKey, tlsKey, + ) + } + called = !called + if called { + return nil + } + return testError + } + + // Perform test. + listener := TLSListening(testTLSCert, testTLSKey) + if err := listener(testBinding, handler); nil != err { + t.Errorf("While serving first TLS expected nil error but got %v", err) + } + if err := listener(testBinding, handler); nil == err { + t.Errorf( + "While serving second TLS got nil while expecting %v", testError, + ) + } +} + +func TestValidReferrer(t *testing.T) { + ok1 := "http://valid.com" + ok2 := "https://valid.com" + ok3 := "http://localhost" + bad := "http://other.pl" + + var noRefer []string + emptyRefer := []string{} + onlyNoRefer := []string{""} + refer := []string{ok1, ok2, ok3} + noWithRefer := []string{"", ok1, ok2, ok3} + + testCases := []struct { + name string + refers []string + refer string + result bool + }{ + {"Nil refer list", noRefer, bad, true}, + {"Empty refer list", emptyRefer, bad, true}, + {"Unassigned allowed & unassigned", onlyNoRefer, "", true}, + {"Unassigned allowed & assigned", onlyNoRefer, bad, false}, + {"Whitelist with unassigned", refer, "", false}, + {"Whitelist with bad", refer, bad, false}, + {"Whitelist with ok1", refer, ok1, true}, + {"Whitelist with ok2", refer, ok2, true}, + {"Whitelist with ok3", refer, ok3, true}, + {"Whitelist and none with unassigned", noWithRefer, "", true}, + {"Whitelist with bad", noWithRefer, bad, false}, + {"Whitelist with ok1", noWithRefer, ok1, true}, + {"Whitelist with ok2", noWithRefer, ok2, true}, + {"Whitelist with ok3", noWithRefer, ok3, true}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + result := validReferrer(tc.refers, tc.refer) + if result != tc.result { + t.Errorf( + "With referrers of '%v' and a value of '%s' expected %t but got %t", + tc.refers, tc.refer, tc.result, result, + ) + } + }) + } +} + +func TestAddCorsWildcardHeaders(t *testing.T) { + testCases := []struct { + name string + corsEnabled bool + }{ + {"CORS disabled", false}, + {"CORS enabled", true}, + } + + corsHeaders := map[string]string{ + "Access-Control-Allow-Origin": "*", + "Access-Control-Allow-Headers": "*", + } + + for _, serveFile := range serveFileFuncs { + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + var handler http.HandlerFunc + if tc.corsEnabled { + handler = AddCorsWildcardHeaders(Basic(serveFile, baseDir)) + } else { + handler = Basic(serveFile, baseDir) + } + + fullpath := "http://localhost/" + tmpFileName + req := httptest.NewRequest("GET", fullpath, nil) + w := httptest.NewRecorder() + + handler(w, req) + + resp := w.Result() + body, err := ioutil.ReadAll(resp.Body) + if nil != err { + t.Errorf("While reading body got %v", err) + } + contents := string(body) + if ok != resp.StatusCode { + t.Errorf( + "While retrieving %s expected status code of %d but got %d", + fullpath, ok, resp.StatusCode, + ) + } + if tmpFile != contents { + t.Errorf( + "While retrieving %s expected contents '%s' but got '%s'", + fullpath, tmpFile, contents, + ) + } + + if tc.corsEnabled { + for k, v := range corsHeaders { + if v != resp.Header.Get(k) { + t.Errorf( + "With CORS enabled expect header '%s' to return '%s' but got '%s'", + k, v, resp.Header.Get(k), + ) + } + } + } else { + for k := range corsHeaders { + if "" != resp.Header.Get(k) { + t.Errorf( + "With CORS disabled expected header '%s' to return '' but got '%s'", + k, resp.Header.Get(k), + ) + } + } + } + }) + } + } +} diff --git a/deploy/chp-api/static-file-server/img/sponsor.svg b/deploy/chp-api/static-file-server/img/sponsor.svg new file mode 100644 index 0000000..a10f598 --- /dev/null +++ b/deploy/chp-api/static-file-server/img/sponsor.svg @@ -0,0 +1,147 @@ + + diff --git a/deploy/chp-api/static-file-server/update.sh b/deploy/chp-api/static-file-server/update.sh new file mode 100755 index 0000000..a4fbb16 --- /dev/null +++ b/deploy/chp-api/static-file-server/update.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +set -e + +if [ $# -eq 0 ]; then + echo "Usage: ./update.sh v#.#.#" + exit +fi + +VERSION=$1 + +docker build -t sfs-builder -f ./Dockerfile.all . + +ID=$(docker create sfs-builder) + +rm -rf out +mkdir -p out +docker cp "${ID}:/build/pkg/linux-amd64/serve" "./out/static-file-server-${VERSION}-linux-amd64" +docker cp "${ID}:/build/pkg/linux-i386/serve" "./out/static-file-server-${VERSION}-linux-386" +docker cp "${ID}:/build/pkg/linux-arm6/serve" "./out/static-file-server-${VERSION}-linux-arm6" +docker cp "${ID}:/build/pkg/linux-arm7/serve" "./out/static-file-server-${VERSION}-linux-arm7" +docker cp "${ID}:/build/pkg/linux-arm64/serve" "./out/static-file-server-${VERSION}-linux-arm64" +docker cp "${ID}:/build/pkg/darwin-amd64/serve" "./out/static-file-server-${VERSION}-darwin-amd64" +docker cp "${ID}:/build/pkg/win-amd64/serve.exe" "./out/static-file-server-${VERSION}-windows-amd64.exe" + +docker rm -f "${ID}" +docker rmi sfs-builder + +docker buildx build --push --platform linux/arm/v7,linux/arm64/v8,linux/amd64 --tag "halverneus/static-file-server:${VERSION}" . +docker buildx build --push --platform linux/arm/v7,linux/arm64/v8,linux/amd64 --tag halverneus/static-file-server:latest . + +echo "Done" From 56acf0986cd37c737463c6d83439bde532cb354f Mon Sep 17 00:00:00 2001 From: bettyli037 Date: Tue, 6 Jun 2023 16:03:25 -0400 Subject: [PATCH 058/132] feat: update Jenkinsfile for chp-api --- deploy/chp-api/Jenkinsfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/deploy/chp-api/Jenkinsfile b/deploy/chp-api/Jenkinsfile index 7050624..bb3d18f 100644 --- a/deploy/chp-api/Jenkinsfile +++ b/deploy/chp-api/Jenkinsfile @@ -49,7 +49,10 @@ pipeline { sh 'cp deploy/chp-api/configs/nginx.conf deploy/chp-api/nginx/' docker.build(env.DOCKER_REPO_NAME, "--no-cache ./deploy/chp-api/nginx") docker.image(env.DOCKER_REPO_NAME).push("${BUILD_VERSION}-nginx") - docker.build(env.DOCKER_REPO_NAME, "--no-cache ./deploy/chp-api/static-file-server") + sh ``` + docker pull halverneus/static-file-server:latest + docker tag halverneus/static-file-server:latest env.DOCKER_REPO_NAME + ``` docker.image(env.DOCKER_REPO_NAME).push("${BUILD_VERSION}-staticfs") } } From 9a676bbdb5c77390fb07be2b0de5818240186545 Mon Sep 17 00:00:00 2001 From: bettyli037 Date: Tue, 6 Jun 2023 16:09:21 -0400 Subject: [PATCH 059/132] feat: update Jenkinsfile for chp-api --- deploy/chp-api/Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/chp-api/Jenkinsfile b/deploy/chp-api/Jenkinsfile index bb3d18f..1f8bc40 100644 --- a/deploy/chp-api/Jenkinsfile +++ b/deploy/chp-api/Jenkinsfile @@ -49,10 +49,10 @@ pipeline { sh 'cp deploy/chp-api/configs/nginx.conf deploy/chp-api/nginx/' docker.build(env.DOCKER_REPO_NAME, "--no-cache ./deploy/chp-api/nginx") docker.image(env.DOCKER_REPO_NAME).push("${BUILD_VERSION}-nginx") - sh ``` + sh ''' docker pull halverneus/static-file-server:latest docker tag halverneus/static-file-server:latest env.DOCKER_REPO_NAME - ``` + ''' docker.image(env.DOCKER_REPO_NAME).push("${BUILD_VERSION}-staticfs") } } From 949e1288659890475d89ece6f48d984cdc17743b Mon Sep 17 00:00:00 2001 From: bettyli037 Date: Tue, 6 Jun 2023 16:13:54 -0400 Subject: [PATCH 060/132] feat: update Jenkinsfile for chp-api --- deploy/chp-api/Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/chp-api/Jenkinsfile b/deploy/chp-api/Jenkinsfile index 1f8bc40..98a8e0b 100644 --- a/deploy/chp-api/Jenkinsfile +++ b/deploy/chp-api/Jenkinsfile @@ -51,7 +51,7 @@ pipeline { docker.image(env.DOCKER_REPO_NAME).push("${BUILD_VERSION}-nginx") sh ''' docker pull halverneus/static-file-server:latest - docker tag halverneus/static-file-server:latest env.DOCKER_REPO_NAME + docker tag halverneus/static-file-server:latest $DOCKER_REPO_NAME ''' docker.image(env.DOCKER_REPO_NAME).push("${BUILD_VERSION}-staticfs") } From 0a2e608f37918bbe79a6fe3c0fac87e5bcc72ece Mon Sep 17 00:00:00 2001 From: bettyli037 Date: Wed, 7 Jun 2023 10:40:16 -0400 Subject: [PATCH 061/132] feat: update deployment.yaml for secretKeyRef --- deploy/chp-api/templates/deployment.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deploy/chp-api/templates/deployment.yaml b/deploy/chp-api/templates/deployment.yaml index d97d8ed..3682fca 100644 --- a/deploy/chp-api/templates/deployment.yaml +++ b/deploy/chp-api/templates/deployment.yaml @@ -56,7 +56,8 @@ spec: name: {{ include "chp-api.fullname" . }}-secret key: sql_password - name: DJANGO_SUPERUSER_PASSWORD - secretKeyRef: + valueFrom: + secretKeyRef: name: {{ include "chp-api.fullname" . }}-secret key: django_superuser_password - name: SQL_ENGINE From 429323b1492041ee6feffb5e2b998affa4df7103 Mon Sep 17 00:00:00 2001 From: bettyli037 Date: Wed, 7 Jun 2023 10:54:30 -0400 Subject: [PATCH 062/132] feat: tmp change for Jenkinsfile and deploy.sh --- deploy/chp-api/Jenkinsfile | 12 ++++++------ deploy/chp-api/deploy.sh | 10 +++++----- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/deploy/chp-api/Jenkinsfile b/deploy/chp-api/Jenkinsfile index 98a8e0b..9f95957 100644 --- a/deploy/chp-api/Jenkinsfile +++ b/deploy/chp-api/Jenkinsfile @@ -73,12 +73,12 @@ pipeline { } } } - post { - always { - echo " Clean up the workspace in deploy node!" - cleanWs() - } - } + // post { + // always { + // echo " Clean up the workspace in deploy node!" + // cleanWs() + // } + // } } } } diff --git a/deploy/chp-api/deploy.sh b/deploy/chp-api/deploy.sh index f3e54eb..c84834d 100644 --- a/deploy/chp-api/deploy.sh +++ b/deploy/chp-api/deploy.sh @@ -22,11 +22,11 @@ do rm values.yaml.bak done -sed -i.bak \ - -e "s/APP_SECRET_KEY_VALUE/$APP_SECRET_KEY/g" \ - -e "s/DB_USERNAME_VALUE/$DB_USERNAME/g;s/DB_PASSWORD_VALUE/$DB_PASSWORD/g" \ - values-ncats.yaml -rm values-ncats.yaml.bak +# sed -i.bak \ +# -e "s/APP_SECRET_KEY_VALUE/$APP_SECRET_KEY/g" \ +# -e "s/DB_USERNAME_VALUE/$DB_USERNAME/g;s/DB_PASSWORD_VALUE/$DB_PASSWORD/g" \ +# values-ncats.yaml +# rm values-ncats.yaml.bak kubectl apply -f namespace.yaml From 96ce10c2cd0979b9e5dca44d6788a35e1767adb6 Mon Sep 17 00:00:00 2001 From: bettyli037 Date: Wed, 7 Jun 2023 15:52:02 -0400 Subject: [PATCH 063/132] feat: add Jenkinsfile --- deploy/chp-api/Jenkinsfile | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/deploy/chp-api/Jenkinsfile b/deploy/chp-api/Jenkinsfile index 9f95957..98a8e0b 100644 --- a/deploy/chp-api/Jenkinsfile +++ b/deploy/chp-api/Jenkinsfile @@ -73,12 +73,12 @@ pipeline { } } } - // post { - // always { - // echo " Clean up the workspace in deploy node!" - // cleanWs() - // } - // } + post { + always { + echo " Clean up the workspace in deploy node!" + cleanWs() + } + } } } } From 17bc2d0ed3281b1b05c86c23b28f8a8f0353e009 Mon Sep 17 00:00:00 2001 From: bettyli037 Date: Wed, 7 Jun 2023 16:01:30 -0400 Subject: [PATCH 064/132] feat: update deploy.sh and Jenkinsfile --- deploy/chp-api/Jenkinsfile | 6 ++++-- deploy/chp-api/deploy.sh | 12 +++++++----- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/deploy/chp-api/Jenkinsfile b/deploy/chp-api/Jenkinsfile index 98a8e0b..c04e6eb 100644 --- a/deploy/chp-api/Jenkinsfile +++ b/deploy/chp-api/Jenkinsfile @@ -66,9 +66,11 @@ pipeline { configFile(fileId: 'prepare.sh', targetLocation: 'deploy/chp-api/prepare.sh') ]){ script { - sh ''' + sh '''#!/bin/bash aws --region ${AWS_REGION} eks update-kubeconfig --name ${KUBERNETES_CLUSTER_NAME} - cd deploy/chp-api && /bin/bash prepare.sh && /bin/bash deploy.sh + cd deploy/chp-api + source prepare.sh + /bin/bash deploy.sh ''' } } diff --git a/deploy/chp-api/deploy.sh b/deploy/chp-api/deploy.sh index c84834d..5c062dc 100644 --- a/deploy/chp-api/deploy.sh +++ b/deploy/chp-api/deploy.sh @@ -22,11 +22,13 @@ do rm values.yaml.bak done -# sed -i.bak \ -# -e "s/APP_SECRET_KEY_VALUE/$APP_SECRET_KEY/g" \ -# -e "s/DB_USERNAME_VALUE/$DB_USERNAME/g;s/DB_PASSWORD_VALUE/$DB_PASSWORD/g" \ -# values-ncats.yaml -# rm values-ncats.yaml.bak +sed -i.bak \ + -e "s/APP_SECRET_KEY_VALUE/$APP_SECRET_KEY/g" \ + -e "s/DB_USERNAME_VALUE/$DB_USERNAME/g" \ + -e "s/DB_PASSWORD_VALUE/$DB_PASSWORD/g" \ + -e "s/DJANGO_SUPERUSER_PASSWORD_VALUE/$DJANGO_SUPERUSER_PASSWORD/g" \ + values-ncats.yaml +rm values-ncats.yaml.bak kubectl apply -f namespace.yaml From 1491aa7461daffad6f6ec08d70b1b811d26400cc Mon Sep 17 00:00:00 2001 From: bettyli037 Date: Wed, 7 Jun 2023 16:24:45 -0400 Subject: [PATCH 065/132] feat: update Jenkinsfile --- deploy/chp-api/Jenkinsfile | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/deploy/chp-api/Jenkinsfile b/deploy/chp-api/Jenkinsfile index c04e6eb..335e4f1 100644 --- a/deploy/chp-api/Jenkinsfile +++ b/deploy/chp-api/Jenkinsfile @@ -75,12 +75,12 @@ pipeline { } } } - post { - always { - echo " Clean up the workspace in deploy node!" - cleanWs() - } - } + // post { + // always { + // echo " Clean up the workspace in deploy node!" + // cleanWs() + // } + // } } } } From a5179a88616e1cfd61c20fe213c81ca37b96b885 Mon Sep 17 00:00:00 2001 From: bettyli037 Date: Wed, 7 Jun 2023 16:26:22 -0400 Subject: [PATCH 066/132] feat: update deploy.sh --- deploy/chp-api/deploy.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/deploy/chp-api/deploy.sh b/deploy/chp-api/deploy.sh index 5c062dc..0ab7c42 100644 --- a/deploy/chp-api/deploy.sh +++ b/deploy/chp-api/deploy.sh @@ -32,5 +32,6 @@ rm values-ncats.yaml.bak kubectl apply -f namespace.yaml +helm -n chp template ${projectName} -f values-ncats.yaml ./ # deploy helm chart helm -n ${namespace} upgrade --install ${projectName} -f values-ncats.yaml ./ \ No newline at end of file From b625d455822af2a964471262b712a247e5eeb7fc Mon Sep 17 00:00:00 2001 From: bettyli037 Date: Wed, 7 Jun 2023 17:11:14 -0400 Subject: [PATCH 067/132] feat: update deployment.yaml --- deploy/chp-api/templates/deployment.yaml | 25 +--- deploy/chp-api/templates/deployment.yaml-bck | 132 +++++++++++++++++++ 2 files changed, 137 insertions(+), 20 deletions(-) create mode 100644 deploy/chp-api/templates/deployment.yaml-bck diff --git a/deploy/chp-api/templates/deployment.yaml b/deploy/chp-api/templates/deployment.yaml index 3682fca..0d34349 100644 --- a/deploy/chp-api/templates/deployment.yaml +++ b/deploy/chp-api/templates/deployment.yaml @@ -36,30 +36,15 @@ spec: mountPath: /home/chp_api/web/chp_api/staticfiles env: - name: SECRET_KEY - valueFrom: - secretKeyRef: - name: {{ include "chp-api.fullname" . }}-secret - key: secret_key + value: "{{ .Values.app.secret_key }}" - name: POSTGRES_DB - valueFrom: - secretKeyRef: - name: {{ include "chp-api.fullname" . }}-secret - key: sql_database + value: "{{ .Values.db.database }}" - name: POSTGRES_USER - valueFrom: - secretKeyRef: - name: {{ include "chp-api.fullname" . }}-secret - key: sql_username + value: "{{ .Values.db.username }}" - name: POSTGRES_PASSWORD - valueFrom: - secretKeyRef: - name: {{ include "chp-api.fullname" . }}-secret - key: sql_password + value: "{{ .Values.db.password }}" - name: DJANGO_SUPERUSER_PASSWORD - valueFrom: - secretKeyRef: - name: {{ include "chp-api.fullname" . }}-secret - key: django_superuser_password + value: "{{ .Values.app.djangoSuperuserPassword }}" - name: SQL_ENGINE value: "{{ .Values.db.engine }}" - name: POSTGRES_HOST diff --git a/deploy/chp-api/templates/deployment.yaml-bck b/deploy/chp-api/templates/deployment.yaml-bck new file mode 100644 index 0000000..3682fca --- /dev/null +++ b/deploy/chp-api/templates/deployment.yaml-bck @@ -0,0 +1,132 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "chp-api.fullname" . }} + labels: + {{- include "chp-api.labels" . | nindent 4 }} +spec: + serviceName: {{ include "chp-api.fullname" . }} + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "chp-api.selectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + labels: + {{- include "chp-api.selectorLabels" . | nindent 8 }} + spec: + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: ["/bin/sh"] + args: ["-c", "gunicorn -c gunicorn.config.py --log-file=- --env DJANGO_SETTINGS_MODULE=chp_api.settings chp_api.wsgi:application --bind 0.0.0.0:8000"] + ports: + - name: http-app + containerPort: 8000 + protocol: TCP + volumeMounts: + - name: {{ include "chp-api.fullname" . }}-pvc + mountPath: /home/chp_api/web/chp_api/staticfiles + env: + - name: SECRET_KEY + valueFrom: + secretKeyRef: + name: {{ include "chp-api.fullname" . }}-secret + key: secret_key + - name: POSTGRES_DB + valueFrom: + secretKeyRef: + name: {{ include "chp-api.fullname" . }}-secret + key: sql_database + - name: POSTGRES_USER + valueFrom: + secretKeyRef: + name: {{ include "chp-api.fullname" . }}-secret + key: sql_username + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "chp-api.fullname" . }}-secret + key: sql_password + - name: DJANGO_SUPERUSER_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "chp-api.fullname" . }}-secret + key: django_superuser_password + - name: SQL_ENGINE + value: "{{ .Values.db.engine }}" + - name: POSTGRES_HOST + value: "{{ .Values.db.host }}" + - name: POSTGRES_PORT + value: "{{ .Values.db.port }}" + - name: DATABASE + value: "{{ .Values.db.type }}" + - name: DEBUG + value: "{{ .Values.app.debug }}" + - name: DJANGO_ALLOWED_HOSTS + value: "{{ .Values.app.djangoAllowedHosts }}" + - name: DJANGO_SETTINGS_MODULE + value: "{{ .Values.app.djangoSettingsModule }}" + - name: DJANGO_SUPERUSER_USERNAME=chp_admin + value: "{{ .Values.app.djangoSuperuserUsername }}" + - name: DJANGO_SUPERUSER_EMAIL + value: "{{ .Values.app.djangoSuperuserEmail }}" + - name: {{ .Chart.Name }}-nginx + securityContext: + {{- toYaml .Values.securityContextNginx | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.nginxTag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: http-nginx + containerPort: 80 + protocol: TCP + volumeMounts: + - name: {{ include "chp-api.fullname" . }}-pvc + mountPath: /home/chp_api/web/staticfiles + - name: config-vol + mountPath: /etc/nginx/conf.d/default.conf + subPath: nginx.conf + - name: {{ .Chart.Name }}-staticfs + securityContext: + {{- toYaml .Values.securityContextStaticfs | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.staticfsTag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: http-staticfs + containerPort: 8080 + protocol: TCP + volumeMounts: + - name: {{ include "chp-api.fullname" . }}-pvc + mountPath: /var/www/static + env: + - name: FOLDER + value: "{{ .Values.app.staticfsFolder }}" + - name: DEBUG + value: "{{ .Values.app.staticfsDebug }}" + volumes: + - name: config-vol + configMap: + name: {{ include "chp-api.fullname" . }}-configs + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumeClaimTemplates: + - metadata: + name: {{ include "chp-api.fullname" . }}-pvc + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 1Gi From 5d23d70f12f4a3440c11999747cceb1f742ca075 Mon Sep 17 00:00:00 2001 From: bettyli037 Date: Wed, 7 Jun 2023 17:15:55 -0400 Subject: [PATCH 068/132] feat: update deploy.sh --- deploy/chp-api/deploy.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/chp-api/deploy.sh b/deploy/chp-api/deploy.sh index 0ab7c42..fba9cff 100644 --- a/deploy/chp-api/deploy.sh +++ b/deploy/chp-api/deploy.sh @@ -32,6 +32,6 @@ rm values-ncats.yaml.bak kubectl apply -f namespace.yaml -helm -n chp template ${projectName} -f values-ncats.yaml ./ +# helm -n chp template ${projectName} -f values-ncats.yaml ./ # deploy helm chart helm -n ${namespace} upgrade --install ${projectName} -f values-ncats.yaml ./ \ No newline at end of file From b230113e11a4e50bc0eff95acf2e44c707ecedb1 Mon Sep 17 00:00:00 2001 From: bettyli037 Date: Thu, 8 Jun 2023 09:35:33 -0400 Subject: [PATCH 069/132] feat: update helm chart for chp-api --- deploy/chp-api/templates/deployment.yaml | 2 +- deploy/chp-api/templates/deployment.yaml-bck | 132 ------------------- deploy/chp-api/templates/secret.yaml | 12 -- 3 files changed, 1 insertion(+), 145 deletions(-) delete mode 100644 deploy/chp-api/templates/deployment.yaml-bck delete mode 100644 deploy/chp-api/templates/secret.yaml diff --git a/deploy/chp-api/templates/deployment.yaml b/deploy/chp-api/templates/deployment.yaml index 0d34349..f2686af 100644 --- a/deploy/chp-api/templates/deployment.yaml +++ b/deploy/chp-api/templates/deployment.yaml @@ -59,7 +59,7 @@ spec: value: "{{ .Values.app.djangoAllowedHosts }}" - name: DJANGO_SETTINGS_MODULE value: "{{ .Values.app.djangoSettingsModule }}" - - name: DJANGO_SUPERUSER_USERNAME=chp_admin + - name: DJANGO_SUPERUSER_USERNAME value: "{{ .Values.app.djangoSuperuserUsername }}" - name: DJANGO_SUPERUSER_EMAIL value: "{{ .Values.app.djangoSuperuserEmail }}" diff --git a/deploy/chp-api/templates/deployment.yaml-bck b/deploy/chp-api/templates/deployment.yaml-bck deleted file mode 100644 index 3682fca..0000000 --- a/deploy/chp-api/templates/deployment.yaml-bck +++ /dev/null @@ -1,132 +0,0 @@ -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: {{ include "chp-api.fullname" . }} - labels: - {{- include "chp-api.labels" . | nindent 4 }} -spec: - serviceName: {{ include "chp-api.fullname" . }} - replicas: {{ .Values.replicaCount }} - selector: - matchLabels: - {{- include "chp-api.selectorLabels" . | nindent 6 }} - template: - metadata: - annotations: - checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} - labels: - {{- include "chp-api.selectorLabels" . | nindent 8 }} - spec: - securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} - containers: - - name: {{ .Chart.Name }} - securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - command: ["/bin/sh"] - args: ["-c", "gunicorn -c gunicorn.config.py --log-file=- --env DJANGO_SETTINGS_MODULE=chp_api.settings chp_api.wsgi:application --bind 0.0.0.0:8000"] - ports: - - name: http-app - containerPort: 8000 - protocol: TCP - volumeMounts: - - name: {{ include "chp-api.fullname" . }}-pvc - mountPath: /home/chp_api/web/chp_api/staticfiles - env: - - name: SECRET_KEY - valueFrom: - secretKeyRef: - name: {{ include "chp-api.fullname" . }}-secret - key: secret_key - - name: POSTGRES_DB - valueFrom: - secretKeyRef: - name: {{ include "chp-api.fullname" . }}-secret - key: sql_database - - name: POSTGRES_USER - valueFrom: - secretKeyRef: - name: {{ include "chp-api.fullname" . }}-secret - key: sql_username - - name: POSTGRES_PASSWORD - valueFrom: - secretKeyRef: - name: {{ include "chp-api.fullname" . }}-secret - key: sql_password - - name: DJANGO_SUPERUSER_PASSWORD - valueFrom: - secretKeyRef: - name: {{ include "chp-api.fullname" . }}-secret - key: django_superuser_password - - name: SQL_ENGINE - value: "{{ .Values.db.engine }}" - - name: POSTGRES_HOST - value: "{{ .Values.db.host }}" - - name: POSTGRES_PORT - value: "{{ .Values.db.port }}" - - name: DATABASE - value: "{{ .Values.db.type }}" - - name: DEBUG - value: "{{ .Values.app.debug }}" - - name: DJANGO_ALLOWED_HOSTS - value: "{{ .Values.app.djangoAllowedHosts }}" - - name: DJANGO_SETTINGS_MODULE - value: "{{ .Values.app.djangoSettingsModule }}" - - name: DJANGO_SUPERUSER_USERNAME=chp_admin - value: "{{ .Values.app.djangoSuperuserUsername }}" - - name: DJANGO_SUPERUSER_EMAIL - value: "{{ .Values.app.djangoSuperuserEmail }}" - - name: {{ .Chart.Name }}-nginx - securityContext: - {{- toYaml .Values.securityContextNginx | nindent 12 }} - image: "{{ .Values.image.repository }}:{{ .Values.image.nginxTag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - ports: - - name: http-nginx - containerPort: 80 - protocol: TCP - volumeMounts: - - name: {{ include "chp-api.fullname" . }}-pvc - mountPath: /home/chp_api/web/staticfiles - - name: config-vol - mountPath: /etc/nginx/conf.d/default.conf - subPath: nginx.conf - - name: {{ .Chart.Name }}-staticfs - securityContext: - {{- toYaml .Values.securityContextStaticfs | nindent 12 }} - image: "{{ .Values.image.repository }}:{{ .Values.image.staticfsTag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - ports: - - name: http-staticfs - containerPort: 8080 - protocol: TCP - volumeMounts: - - name: {{ include "chp-api.fullname" . }}-pvc - mountPath: /var/www/static - env: - - name: FOLDER - value: "{{ .Values.app.staticfsFolder }}" - - name: DEBUG - value: "{{ .Values.app.staticfsDebug }}" - volumes: - - name: config-vol - configMap: - name: {{ include "chp-api.fullname" . }}-configs - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - volumeClaimTemplates: - - metadata: - name: {{ include "chp-api.fullname" . }}-pvc - spec: - accessModes: [ "ReadWriteOnce" ] - resources: - requests: - storage: 1Gi diff --git a/deploy/chp-api/templates/secret.yaml b/deploy/chp-api/templates/secret.yaml deleted file mode 100644 index c435a39..0000000 --- a/deploy/chp-api/templates/secret.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: {{ include "chp-api.fullname" . }}-secret - labels: - {{- include "chp-api.labels" . | nindent 4 }} -stringData: - sql_database: {{ .Values.db.database }} - sql_username: {{ .Values.db.username }} - sql_password: {{ .Values.db.password }} - secret_key: {{ .Values.app.secret_key }} - django_superuser_password: {{ .Values.app.djangoSuperuserPassword }} From 5552c2e3fb44b082591ecd159531d97b27eca48f Mon Sep 17 00:00:00 2001 From: bettyli037 Date: Thu, 8 Jun 2023 09:53:45 -0400 Subject: [PATCH 070/132] feat: update Jenkinsfile --- deploy/chp-api/Jenkinsfile | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/deploy/chp-api/Jenkinsfile b/deploy/chp-api/Jenkinsfile index 335e4f1..c04e6eb 100644 --- a/deploy/chp-api/Jenkinsfile +++ b/deploy/chp-api/Jenkinsfile @@ -75,12 +75,12 @@ pipeline { } } } - // post { - // always { - // echo " Clean up the workspace in deploy node!" - // cleanWs() - // } - // } + post { + always { + echo " Clean up the workspace in deploy node!" + cleanWs() + } + } } } } From acdb4d309418ea7afc84f47aecec7b5ffdb47ca7 Mon Sep 17 00:00:00 2001 From: bettyli037 Date: Thu, 8 Jun 2023 09:59:19 -0400 Subject: [PATCH 071/132] feat: update deploy.sh --- deploy/chp-api/deploy.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/deploy/chp-api/deploy.sh b/deploy/chp-api/deploy.sh index fba9cff..5c062dc 100644 --- a/deploy/chp-api/deploy.sh +++ b/deploy/chp-api/deploy.sh @@ -32,6 +32,5 @@ rm values-ncats.yaml.bak kubectl apply -f namespace.yaml -# helm -n chp template ${projectName} -f values-ncats.yaml ./ # deploy helm chart helm -n ${namespace} upgrade --install ${projectName} -f values-ncats.yaml ./ \ No newline at end of file From 5ad4745933abe676803eda66ee30a66803536607 Mon Sep 17 00:00:00 2001 From: bettyli037 Date: Thu, 8 Jun 2023 10:13:08 -0400 Subject: [PATCH 072/132] feat: update templates/deployment.yaml --- deploy/chp-api/templates/deployment.yaml | 25 +++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/deploy/chp-api/templates/deployment.yaml b/deploy/chp-api/templates/deployment.yaml index f2686af..06a4e8e 100644 --- a/deploy/chp-api/templates/deployment.yaml +++ b/deploy/chp-api/templates/deployment.yaml @@ -36,15 +36,30 @@ spec: mountPath: /home/chp_api/web/chp_api/staticfiles env: - name: SECRET_KEY - value: "{{ .Values.app.secret_key }}" + valueFrom: + secretKeyRef: + name: {{ include "chp-api.fullname" . }}-secret + key: secret_key - name: POSTGRES_DB - value: "{{ .Values.db.database }}" + valueFrom: + secretKeyRef: + name: {{ include "chp-api.fullname" . }}-secret + key: sql_database - name: POSTGRES_USER - value: "{{ .Values.db.username }}" + valueFrom: + secretKeyRef: + name: {{ include "chp-api.fullname" . }}-secret + key: sql_username - name: POSTGRES_PASSWORD - value: "{{ .Values.db.password }}" + valueFrom: + secretKeyRef: + name: {{ include "chp-api.fullname" . }}-secret + key: sql_password - name: DJANGO_SUPERUSER_PASSWORD - value: "{{ .Values.app.djangoSuperuserPassword }}" + valueFrom: + secretKeyRef: + name: {{ include "chp-api.fullname" . }}-secret + key: django_superuser_password - name: SQL_ENGINE value: "{{ .Values.db.engine }}" - name: POSTGRES_HOST From 23583ea8dd754eefba975ae3bfec8145c71a4e77 Mon Sep 17 00:00:00 2001 From: bettyli037 Date: Thu, 8 Jun 2023 10:16:17 -0400 Subject: [PATCH 073/132] feat: update secret.yaml --- deploy/chp-api/templates/secret.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 deploy/chp-api/templates/secret.yaml diff --git a/deploy/chp-api/templates/secret.yaml b/deploy/chp-api/templates/secret.yaml new file mode 100644 index 0000000..c435a39 --- /dev/null +++ b/deploy/chp-api/templates/secret.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "chp-api.fullname" . }}-secret + labels: + {{- include "chp-api.labels" . | nindent 4 }} +stringData: + sql_database: {{ .Values.db.database }} + sql_username: {{ .Values.db.username }} + sql_password: {{ .Values.db.password }} + secret_key: {{ .Values.app.secret_key }} + django_superuser_password: {{ .Values.app.djangoSuperuserPassword }} From 91163f70019d9fb9bafa7c9a30ba8c1fe676678e Mon Sep 17 00:00:00 2001 From: Chase Yakaboski Date: Mon, 12 Jun 2023 13:12:19 -0400 Subject: [PATCH 074/132] Updated deployment script and compose files for AWS deployment. --- dev-deployment-script | 2 +- docker-compose.yml | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/dev-deployment-script b/dev-deployment-script index a9f454c..f56ee4b 100755 --- a/dev-deployment-script +++ b/dev-deployment-script @@ -6,7 +6,7 @@ django_superuser_email='cat secrets/chp_api/django_superuser_email.txt' # Only to be run when building on dev machine # use --no-cache if need to rebuild submodules -docker compose build #--no-cache +docker compose build --no-cache docker compose up -d diff --git a/docker-compose.yml b/docker-compose.yml index 0253de9..79a39d7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -37,8 +37,8 @@ services: - django-superuser-email - django-superuser-password environment: - - POSTGRES_DB=chp_db - - POSTGRES_USER=postgres + - POSTGRES_DB=chpapi + - POSTGRES_USER=chpapi_user - POSTGRES_PASSWORD_FILE=/run/secrets/db-password - POSTGRES_HOST=db - POSTGRES_PORT=5432 @@ -70,18 +70,18 @@ services: db: image: postgres restart: always - user: postgres secrets: - db-password volumes: - db-data:/var/lib/postgresql/data environment: - - POSTGRES_DB=chp_db + - POSTGRES_DB=chpapi + - POSTGRES_USER=chpapi_user - POSTGRES_PASSWORD_FILE=/run/secrets/db-password expose: - 5432 healthcheck: - test: [ "CMD", "pg_isready" ] + test: [ "CMD", "pg_isready -d chpapi -U chpapi_user" ] interval: 10s timeout: 5s retries: 5 From e8af8a8b65e68fcd9b63a62759d7054605f325dd Mon Sep 17 00:00:00 2001 From: bettyli037 Date: Thu, 15 Jun 2023 09:45:37 -0400 Subject: [PATCH 075/132] feat: update deployment.yaml --- deploy/chp-api/templates/deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/chp-api/templates/deployment.yaml b/deploy/chp-api/templates/deployment.yaml index 06a4e8e..7c806bc 100644 --- a/deploy/chp-api/templates/deployment.yaml +++ b/deploy/chp-api/templates/deployment.yaml @@ -26,7 +26,7 @@ spec: image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" imagePullPolicy: {{ .Values.image.pullPolicy }} command: ["/bin/sh"] - args: ["-c", "gunicorn -c gunicorn.config.py --log-file=- --env DJANGO_SETTINGS_MODULE=chp_api.settings chp_api.wsgi:application --bind 0.0.0.0:8000"] + args: ["-c", "python3 manage.py collectstatic --noinput && gunicorn -c gunicorn.config.py --log-file=- --env DJANGO_SETTINGS_MODULE=chp_api.settings chp_api.wsgi:application --bind 0.0.0.0:8000"] ports: - name: http-app containerPort: 8000 From 4e931230ede94f6adc8439c024f7d8f2ee2750c8 Mon Sep 17 00:00:00 2001 From: Chase Yakaboski Date: Thu, 22 Jun 2023 13:52:18 -0400 Subject: [PATCH 076/132] Saving work. --- chp_api/chp_api/serializers.py | 12 ++ chp_api/chp_api/settings.py | 11 +- chp_api/chp_api/urls.py | 8 + chp_api/gennifer/admin.py | 3 +- .../migrations/0006_algorithm_directed.py | 19 ++ .../migrations/0007_useranalysissession.py | 27 +++ chp_api/gennifer/models.py | 21 ++ chp_api/gennifer/scripts/algorithm_loader.py | 1 + chp_api/gennifer/serializers.py | 40 +++- chp_api/gennifer/urls.py | 3 + chp_api/gennifer/views.py | 186 +++++++++++++++++- chp_api/requirements.txt | 1 + gennifer | 2 +- nginx/nginx.conf | 2 +- 14 files changed, 322 insertions(+), 14 deletions(-) create mode 100644 chp_api/chp_api/serializers.py create mode 100755 chp_api/gennifer/migrations/0006_algorithm_directed.py create mode 100755 chp_api/gennifer/migrations/0007_useranalysissession.py diff --git a/chp_api/chp_api/serializers.py b/chp_api/chp_api/serializers.py new file mode 100644 index 0000000..fb8942d --- /dev/null +++ b/chp_api/chp_api/serializers.py @@ -0,0 +1,12 @@ +from rest_framework_simplejwt.serializers import TokenObtainPairSerializer + +class ChpTokenObtainPairSerializer(TokenObtainPairSerializer): + @classmethod + def get_token(cls, user): + token = super().get_token(user) + + # Add custom claims + token['email'] = user.email + token['username'] = user.username + + return token diff --git a/chp_api/chp_api/settings.py b/chp_api/chp_api/settings.py index 99c3ed9..34cc2f7 100644 --- a/chp_api/chp_api/settings.py +++ b/chp_api/chp_api/settings.py @@ -29,7 +29,10 @@ REST_FRAMEWORK = { 'DEFAULT_PARSER_CLASSES': [ 'rest_framework.parsers.JSONParser', - ] + ], + 'DEFAULT_AUTHENTICATION_CLASSES': ( + 'rest_framework_simplejwt.authentication.JWTAuthentication', + ) } # Application definition @@ -41,6 +44,7 @@ 'django.contrib.messages', 'django.contrib.staticfiles', 'rest_framework', + 'rest_framework_simplejwt', 'django_filters', 'dispatcher.apps.DispatcherConfig', 'django_extensions', @@ -174,6 +178,11 @@ with open(env("DJANGO_SUPERUSER_PASSWORD_FILE"), 'r') as dsp_file: os.environ["DJANGO_SUPERUSER_PASSWORD"] = dsp_file.readline().strip() +# Simple JWT Settings +SIMPLE_JWT = { + "TOKEN_OBTAIN_SERIALIZER": "chp_api.serializers.ChpTokenObtainPairSerializer", + } + # Celery Settings CELERY_BROKER_URL = os.environ.get("CELERY_BROKER_URL", "redis://localhost:6379") CELERY_RESULT_BACKEND = os.environ.get("CELERY_RESULT_BACKEND", "redis://localhost:6379") diff --git a/chp_api/chp_api/urls.py b/chp_api/chp_api/urls.py index 4ba98dc..f42e50e 100644 --- a/chp_api/chp_api/urls.py +++ b/chp_api/chp_api/urls.py @@ -17,10 +17,18 @@ from django.urls import path, include from rest_framework.urlpatterns import format_suffix_patterns +from rest_framework_simplejwt.views import ( + TokenObtainPairView, + TokenRefreshView, +) + + urlpatterns = [ path('admin/', admin.site.urls), path('', include('dispatcher.urls')), path('gennifer/api/', include('gennifer.urls')), + path('api/token/', TokenObtainPairView.as_view(), name='token_obtain_pair'), + path('api/token/refresh/', TokenRefreshView.as_view(), name='token_refresh'), ] diff --git a/chp_api/gennifer/admin.py b/chp_api/gennifer/admin.py index d2ab304..abca8d6 100644 --- a/chp_api/gennifer/admin.py +++ b/chp_api/gennifer/admin.py @@ -1,9 +1,10 @@ from django.contrib import admin -from .models import Algorithm, Dataset, InferenceStudy, InferenceResult, Gene +from .models import Algorithm, Dataset, InferenceStudy, InferenceResult, Gene, UserAnalysisSession admin.site.register(Algorithm) admin.site.register(Dataset) admin.site.register(InferenceStudy) admin.site.register(InferenceResult) admin.site.register(Gene) +admin.site.register(UserAnalysisSession) diff --git a/chp_api/gennifer/migrations/0006_algorithm_directed.py b/chp_api/gennifer/migrations/0006_algorithm_directed.py new file mode 100755 index 0000000..a7382ec --- /dev/null +++ b/chp_api/gennifer/migrations/0006_algorithm_directed.py @@ -0,0 +1,19 @@ +# Generated by Django 4.2.1 on 2023-06-04 02:08 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('gennifer', '0005_gene_chp_preferred_curie'), + ] + + operations = [ + migrations.AddField( + model_name='algorithm', + name='directed', + field=models.BooleanField(default=False), + preserve_default=False, + ), + ] diff --git a/chp_api/gennifer/migrations/0007_useranalysissession.py b/chp_api/gennifer/migrations/0007_useranalysissession.py new file mode 100755 index 0000000..76be5da --- /dev/null +++ b/chp_api/gennifer/migrations/0007_useranalysissession.py @@ -0,0 +1,27 @@ +# Generated by Django 4.2.2 on 2023-06-18 22:33 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('gennifer', '0006_algorithm_directed'), + ] + + operations = [ + migrations.CreateModel( + name='UserAnalysisSession', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('name', models.CharField(max_length=128)), + ('session_data', models.JSONField()), + ('is_saved', models.BooleanField(default=False)), + ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + ), + ] diff --git a/chp_api/gennifer/models.py b/chp_api/gennifer/models.py index 8abde76..d5f2016 100644 --- a/chp_api/gennifer/models.py +++ b/chp_api/gennifer/models.py @@ -1,15 +1,31 @@ import requests +import uuid from django.db import models from django.contrib.auth.models import User +class UserAnalysisSession(models.Model): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + name = models.CharField(max_length=128) + user = models.ForeignKey(User, on_delete=models.CASCADE) + session_data = models.JSONField() + is_saved = models.BooleanField(default=False) + + def update_session_data(self, new_data): + self.session_data.update(new_data) + self.save() + + def __str__(self): + return self.name + class Algorithm(models.Model): name = models.CharField(max_length=128) url = models.CharField(max_length=128) edge_weight_description = models.TextField(null=True, blank=True) edge_weight_type = models.CharField(max_length=128, null=True, blank=True) description = models.TextField(null=True, blank=True) + directed = models.BooleanField() def __str__(self): return self.name @@ -40,6 +56,9 @@ def save(self, *args, **kwargs): CLEANR = re.compile('<.*?>') info = self.get_record() + if 'status' in info and 'message' in info and len(info) == 2: + # This means that retrieval failed + raise ValueError(f'Could not retrieve zenodo record {self.zenodo_id}. Failed with message: {info["message"]}') self.doi = info["doi"] self.description = re.sub(CLEANR, '', info["metadata"]["description"]) self.title = re.sub(CLEANR, '', info["metadata"]["title"]) @@ -49,6 +68,8 @@ def save(self, *args, **kwargs): def get_record(self): return requests.get(f"https://zenodo.org/api/records/{self.zenodo_id}").json() + def __str__(self): + return f'zenodo:{self.zenodo_id}' class Gene(models.Model): name = models.CharField(max_length=128) diff --git a/chp_api/gennifer/scripts/algorithm_loader.py b/chp_api/gennifer/scripts/algorithm_loader.py index af2893f..1236632 100644 --- a/chp_api/gennifer/scripts/algorithm_loader.py +++ b/chp_api/gennifer/scripts/algorithm_loader.py @@ -14,5 +14,6 @@ def run(): edge_weight_description=algo_info["edge_weight_description"], edge_weight_type=algo_info["edge_weight_type"], description=algo_info["description"], + directed=algo_info["directed"], ) algo.save() diff --git a/chp_api/gennifer/serializers.py b/chp_api/gennifer/serializers.py index 64b85da..818f215 100644 --- a/chp_api/gennifer/serializers.py +++ b/chp_api/gennifer/serializers.py @@ -1,45 +1,71 @@ from rest_framework import serializers -from .models import Dataset, InferenceStudy, InferenceResult, Algorithm +from .models import Dataset, InferenceStudy, InferenceResult, Algorithm, Gene, UserAnalysisSession + + +class UserAnalysisSessionSerializer(serializers.ModelSerializer): + class Meta: + model = UserAnalysisSession + fields = ['id', 'user', 'name', 'session_data', 'is_saved'] class DatasetSerializer(serializers.ModelSerializer): class Meta: model = Dataset - fields = ['upload_user', 'title', 'zenodo_id', 'doi', 'description'] + fields = ['title', 'zenodo_id', 'doi', 'description'] + read_only_fields = ['title', 'doi', 'description'] class InferenceStudySerializer(serializers.ModelSerializer): + name = serializers.SerializerMethodField('get_name') + + def get_name(self, study): + return f'{study.algorithm_instance.algorithm.name} on {study.dataset.title}' + class Meta: model = InferenceStudy fields = [ + 'pk', 'algorithm_instance', - 'user', 'dataset', 'timestamp', 'max_study_edge_weight', 'min_study_edge_weight', 'avg_study_edge_weight', - 'std_study_edge_weight', - 'is_public', + 'std_study_edge_weight', + 'name', + 'status', ] class InferenceResultSerializer(serializers.ModelSerializer): class Meta: model = InferenceResult fields = [ + 'pk', 'tf', 'target', 'edge_weight', 'study', - 'is_public', - 'user', ] class AlgorithmSerializer(serializers.ModelSerializer): class Meta: model = Algorithm fields = [ + 'pk', 'name', 'description', 'edge_weight_type', + 'directed', + ] + + +class GeneSerializer(serializers.ModelSerializer): + class Meta: + model = Gene + fields = [ + 'pk', + 'name', + 'curie', + 'variant', + 'chp_preferred_curie', ] diff --git a/chp_api/gennifer/urls.py b/chp_api/gennifer/urls.py index 621ea9a..29fae6a 100644 --- a/chp_api/gennifer/urls.py +++ b/chp_api/gennifer/urls.py @@ -9,8 +9,11 @@ router.register(r'inference_studies', views.InferenceStudyViewSet, basename='inference_study') router.register(r'inference_results', views.InferenceResultViewSet, basename='inference_result') router.register(r'algorithms', views.AlgorithmViewSet, basename='algorithm') +router.register(r'genes', views.GeneViewSet, basename='genes') +router.register(r'analyses', views.UserAnalysisSessionViewSet, basename='analyses') urlpatterns = [ path('', include(router.urls)), path('run', views.run.as_view()), + path('graph', views.CytoscapeView.as_view()), ] diff --git a/chp_api/gennifer/views.py b/chp_api/gennifer/views.py index 486b53e..68b3f76 100644 --- a/chp_api/gennifer/views.py +++ b/chp_api/gennifer/views.py @@ -3,18 +3,33 @@ from django.http import HttpResponse, JsonResponse from django.shortcuts import get_object_or_404 from django.core.exceptions import ObjectDoesNotExist +from django.db.models import Q from rest_framework import viewsets from rest_framework.views import APIView from rest_framework.response import Response from rest_framework.permissions import IsAuthenticated, IsAuthenticatedOrReadOnly +from rest_framework.exceptions import ValidationError from django_filters.rest_framework import DjangoFilterBackend -from .models import Dataset, InferenceStudy, InferenceResult, Algorithm -from .serializers import DatasetSerializer, InferenceStudySerializer, InferenceResultSerializer, AlgorithmSerializer +from .models import Dataset, InferenceStudy, InferenceResult, Algorithm, Gene, UserAnalysisSession +from .serializers import DatasetSerializer, InferenceStudySerializer, InferenceResultSerializer, AlgorithmSerializer, GeneSerializer, UserAnalysisSessionSerializer from .tasks import create_task from .permissions import IsOwnerOrReadOnly, IsAdminOrReadOnly + +class UserAnalysisSessionViewSet(viewsets.ModelViewSet): + serializer_class = UserAnalysisSessionSerializer + #filter_backends = [DjangoFilterBackend] + #filterset_fields = ['id', 'name', 'is_saved'] + permission_classes = [IsOwnerOrReadOnly, IsAuthenticated] + + def get_queryset(self): + user = self.request.user + print(f'User is {user}') + return UserAnalysisSession.objects.filter(user=user) + + class DatasetViewSet(viewsets.ModelViewSet): queryset = Dataset.objects.all() serializer_class = DatasetSerializer @@ -23,6 +38,13 @@ class DatasetViewSet(viewsets.ModelViewSet): permission_classes = [IsAuthenticatedOrReadOnly] + def perform_create(self, serializers): + try: + serializers.save(upload_user=self.request.user) + except ValueError as e: + raise ValidationError(str(e)) + + class InferenceStudyViewSet(viewsets.ModelViewSet): queryset = InferenceStudy.objects.all() serializer_class = InferenceStudySerializer @@ -51,9 +73,167 @@ class AlgorithmViewSet(viewsets.ModelViewSet): queryset = Algorithm.objects.all() permissions = [IsAdminOrReadOnly] +class GeneViewSet(viewsets.ModelViewSet): + serializer_class = GeneSerializer + queryset = Gene.objects.all() + permissions = [IsAdminOrReadOnly] -class run(APIView): + +class CytoscapeView(APIView): + + def construct_node(self, gene_obj): + if gene_obj.variant: + name = f'{gene_obj.name}({gene_obj.variant})' + curie = f'{gene_obj.curie}({gene_obj.variant})' + chp_preferred_curie = f'{gene_obj.chp_preferred_curie}({gene_obj.variant})' + else: + name = gene_obj.name + curie = gene_obj.curie + chp_preferred_curie = gene_obj.chp_preferred_curie + + node = { + "data": { + "id": str(gene_obj.pk), + "name": name, + "curie": curie, + "chp_preferred_curie": chp_preferred_curie + } + } + return node, str(gene_obj.pk) + def construct_edge(self, res, source_id, target_id): + # Normalize edge weight based on the study + normalized_weight = (res.edge_weight - res.study.min_study_edge_weight) / (res.study.max_study_edge_weight - res.study.min_study_edge_weight) + directed = res.study.algorithm_instance.algorithm.directed + edge_tuple = tuple(sorted([source_id, target_id])) + edge = { + "data": { + "id": str(res.pk), + "source": source_id, + "target": target_id, + "dataset": str(res.study.dataset), + "weight": normalized_weight, + "algorithm": str(res.study.algorithm_instance), + "directed": directed, + } + } + return edge, edge_tuple, directed + + def add(self, res, nodes, edges, processed_node_ids, processed_undirected_edges): + # Construct nodes + tf_node, tf_id = self.construct_node(res.tf) + target_node, target_id = self.construct_node(res.target) + # Add nodes if not already added by another result + if tf_id not in processed_node_ids: + nodes.append(tf_node) + processed_node_ids.add(tf_id) + if target_id not in processed_node_ids: + nodes.append(target_node) + processed_node_ids.add(target_id) + # Add and construct edge + edge, edge_tuple, edge_is_directed = self.construct_edge(res, tf_id, target_id) + if edge_is_directed: + edges.append(edge) + elif not edge_is_directed and edge_tuple not in processed_undirected_edges: + edges.append(edge) + processed_undirected_edges.add(edge_tuple) + else: + pass + return nodes, edges, processed_node_ids, processed_undirected_edges + + def construct_cytoscape_data(self, results): + nodes = [] + edges = [] + processed_node_ids = set() + processed_undirected_edges = set() + elements = [] + # Construct graph + for res in results: + nodes, edges, processed_node_ids, processed_undirected_edges = self.add( + res, + nodes, + edges, + processed_node_ids, + processed_undirected_edges, + ) + elements.extend(nodes) + elements.extend(edges) + return { + "elements": elements + } + + def get(self, request): + results = InferenceResult.objects.all() + cyto = self.construct_cytoscape_data(results) + return JsonResponse(cyto) + + def post(self, request): + elements = [] + gene_ids = request.data.get("gene_ids", None) + study_ids = request.data.get("study_ids", None) + algorithm_ids = request.data.get("algorithm_ids", None) + dataset_ids = request.data.get("dataset_ids", None) + cached_inference_result_ids = request.data.get("cached_results", None) + + if not (study_ids and gene_ids) and not (algorithm_ids and dataset_ids and gene_ids): + return JsonResponse({"elements": elements}) + + # Create Filter + filters = [] + if gene_ids: + filters.extend( + [ + {"field": 'tf__pk', "operator": 'in', "value": gene_ids}, + {"field": 'target__pk', "operator": 'in', "value": gene_ids}, + ] + ) + if study_ids: + filters.append({"field": 'study__pk', "operator": 'in', "value": study_ids}) + if algorithm_ids: + filters.append({"field": 'study__algorithm_instance__algorithm__pk', "operator": 'in', "value": study_ids}) + if dataset_ids: + filters.append({"field": 'study__dataset__zenodo_id', "operator": 'in', "value": dataset_ids}) + + # Construct Query + query = Q() + for filter_item in filters: + field = filter_item["field"] + operator = filter_item["operator"] + value = filter_item["value"] + query &= Q(**{f'{field}__{operator}': value}) + + # Get matching results + results = InferenceResult.objects.filter(query) + + if len(results) == 0: + return JsonResponse({"elements": elements}) + + # Exclude results that have already been sent to user + if cached_inference_result_ids: + logs.append('filtering') + results = results.exclude(pk__in=cached_inference_result_ids) + + nodes = [] + edges = [] + processed_node_ids = set() + processed_undirected_edges = set() + for res in results: + nodes, edges, processed_node_ids, processed_undirected_edges = self.add( + res, + nodes, + edges, + processed_node_ids, + processed_undirected_edges, + ) + elements.extend(nodes) + elements.extend(edges) + return JsonResponse({"elements": elements}) + + + +class run(APIView): + permission_classes = [IsAuthenticated] + def post(self, request): """ Request comes in as a list of algorithms to run. """ diff --git a/chp_api/requirements.txt b/chp_api/requirements.txt index e7d5df4..a544600 100644 --- a/chp_api/requirements.txt +++ b/chp_api/requirements.txt @@ -1,5 +1,6 @@ tqdm djangorestframework +djangorestframework-simplejwt psycopg2-binary bmt reasoner_pydantic diff --git a/gennifer b/gennifer index ea758cf..3b27cbd 160000 --- a/gennifer +++ b/gennifer @@ -1 +1 @@ -Subproject commit ea758cfec58c88b69fd8ff029b03b4a66012c472 +Subproject commit 3b27cbd61ab3561e02f9d5a6dfe15038804699f5 diff --git a/nginx/nginx.conf b/nginx/nginx.conf index e6700a3..dcd0bb8 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -34,7 +34,7 @@ http { # Size Limits client_body_buffer_size 10K; - client_header_buffer_size 1k; + client_header_buffer_size 128k; client_max_body_size 8m; large_client_header_buffers 2 1k; From 838632682502c94eeab27c4a9b42940211072eec Mon Sep 17 00:00:00 2001 From: Chase Yakaboski Date: Thu, 22 Jun 2023 13:57:33 -0400 Subject: [PATCH 077/132] Added static files mount volume and changed mount points. --- deploy/chp-api/templates/deployment.yaml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/deploy/chp-api/templates/deployment.yaml b/deploy/chp-api/templates/deployment.yaml index d97d8ed..4863b2d 100644 --- a/deploy/chp-api/templates/deployment.yaml +++ b/deploy/chp-api/templates/deployment.yaml @@ -26,14 +26,14 @@ spec: image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" imagePullPolicy: {{ .Values.image.pullPolicy }} command: ["/bin/sh"] - args: ["-c", "gunicorn -c gunicorn.config.py --log-file=- --env DJANGO_SETTINGS_MODULE=chp_api.settings chp_api.wsgi:application --bind 0.0.0.0:8000"] + args: ["-c", "python3 manage.py collectstatic --no-input && gunicorn -c gunicorn.config.py --log-file=- --env DJANGO_SETTINGS_MODULE=chp_api.settings chp_api.wsgi:application --bind 0.0.0.0:8000"] ports: - name: http-app containerPort: 8000 protocol: TCP volumeMounts: - - name: {{ include "chp-api.fullname" . }}-pvc - mountPath: /home/chp_api/web/chp_api/staticfiles + - name: {{ include "chp-api.fullname" . }}-static + mountPath: /home/chp_api/staticfiles env: - name: SECRET_KEY valueFrom: @@ -102,7 +102,7 @@ spec: containerPort: 8080 protocol: TCP volumeMounts: - - name: {{ include "chp-api.fullname" . }}-pvc + - name: {{ include "chp-api.fullname" . }}-static mountPath: /var/www/static env: - name: FOLDER @@ -113,6 +113,8 @@ spec: - name: config-vol configMap: name: {{ include "chp-api.fullname" . }}-configs + - name: static-vol + configMap: {{ include "chp-api.fullname" . }}-static {{- with .Values.affinity }} affinity: {{- toYaml . | nindent 8 }} From 6b4f081949553d6fe0eb8659bf52e9bd709468e5 Mon Sep 17 00:00:00 2001 From: yakaboskic <35247528+yakaboskic@users.noreply.github.com> Date: Fri, 23 Jun 2023 11:53:06 -0400 Subject: [PATCH 078/132] Update deployment.yaml --- deploy/chp-api/templates/deployment.yaml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/deploy/chp-api/templates/deployment.yaml b/deploy/chp-api/templates/deployment.yaml index 4863b2d..d5cfa8b 100644 --- a/deploy/chp-api/templates/deployment.yaml +++ b/deploy/chp-api/templates/deployment.yaml @@ -32,7 +32,7 @@ spec: containerPort: 8000 protocol: TCP volumeMounts: - - name: {{ include "chp-api.fullname" . }}-static + - name: static-vol mountPath: /home/chp_api/staticfiles env: - name: SECRET_KEY @@ -87,8 +87,6 @@ spec: containerPort: 80 protocol: TCP volumeMounts: - - name: {{ include "chp-api.fullname" . }}-pvc - mountPath: /home/chp_api/web/staticfiles - name: config-vol mountPath: /etc/nginx/conf.d/default.conf subPath: nginx.conf @@ -102,7 +100,7 @@ spec: containerPort: 8080 protocol: TCP volumeMounts: - - name: {{ include "chp-api.fullname" . }}-static + - name: static-vol mountPath: /var/www/static env: - name: FOLDER From 0fffcfea922ffd996c6ea1b5d2cdf54bd7c28212 Mon Sep 17 00:00:00 2001 From: yakaboskic <35247528+yakaboskic@users.noreply.github.com> Date: Fri, 23 Jun 2023 11:58:00 -0400 Subject: [PATCH 079/132] Update deployment.yaml --- deploy/chp-api/templates/deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/chp-api/templates/deployment.yaml b/deploy/chp-api/templates/deployment.yaml index d5cfa8b..98c543e 100644 --- a/deploy/chp-api/templates/deployment.yaml +++ b/deploy/chp-api/templates/deployment.yaml @@ -33,7 +33,7 @@ spec: protocol: TCP volumeMounts: - name: static-vol - mountPath: /home/chp_api/staticfiles + mountPath: /home/chp_api/web/staticfiles env: - name: SECRET_KEY valueFrom: From 02f5f44ca68682f5a09ef1959db28091646f9e64 Mon Sep 17 00:00:00 2001 From: bettyli037 Date: Fri, 23 Jun 2023 12:14:42 -0400 Subject: [PATCH 080/132] feat: update Jenkinsfile --- deploy/chp-api/Jenkinsfile | 7 ------- 1 file changed, 7 deletions(-) diff --git a/deploy/chp-api/Jenkinsfile b/deploy/chp-api/Jenkinsfile index c04e6eb..b0c343f 100644 --- a/deploy/chp-api/Jenkinsfile +++ b/deploy/chp-api/Jenkinsfile @@ -33,12 +33,6 @@ pipeline { } } } - stage('Checkout source code') { - steps { - cleanWs() - checkout scm - } - } stage('Build Docker') { when { expression { return env.BUILD == 'true' }} steps { @@ -60,7 +54,6 @@ pipeline { stage('Deploy to AWS EKS') { agent { label 'translator && ci && deploy'} steps { - checkout scm configFileProvider([ configFile(fileId: 'values-ci.yaml', targetLocation: 'deploy/chp-api/values-ncats.yaml'), configFile(fileId: 'prepare.sh', targetLocation: 'deploy/chp-api/prepare.sh') From 6df12e44d06d4a1b3670525d20e7cf7b8a40540c Mon Sep 17 00:00:00 2001 From: bettyli037 Date: Fri, 23 Jun 2023 13:13:39 -0400 Subject: [PATCH 081/132] feat: update deployment.yaml --- deploy/chp-api/templates/deployment.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deploy/chp-api/templates/deployment.yaml b/deploy/chp-api/templates/deployment.yaml index a43e35a..af13933 100644 --- a/deploy/chp-api/templates/deployment.yaml +++ b/deploy/chp-api/templates/deployment.yaml @@ -113,7 +113,8 @@ spec: configMap: name: {{ include "chp-api.fullname" . }}-configs - name: static-vol - configMap: {{ include "chp-api.fullname" . }}-static + configMap: + name: {{ include "chp-api.fullname" . }}-static {{- with .Values.affinity }} affinity: {{- toYaml . | nindent 8 }} From b7e7fcf8c4997932ef2ae12d3ee32789b031b0a6 Mon Sep 17 00:00:00 2001 From: yakaboskic <35247528+yakaboskic@users.noreply.github.com> Date: Fri, 23 Jun 2023 13:14:07 -0400 Subject: [PATCH 082/132] Update deployment.yaml --- deploy/chp-api/templates/deployment.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deploy/chp-api/templates/deployment.yaml b/deploy/chp-api/templates/deployment.yaml index a43e35a..af13933 100644 --- a/deploy/chp-api/templates/deployment.yaml +++ b/deploy/chp-api/templates/deployment.yaml @@ -113,7 +113,8 @@ spec: configMap: name: {{ include "chp-api.fullname" . }}-configs - name: static-vol - configMap: {{ include "chp-api.fullname" . }}-static + configMap: + name: {{ include "chp-api.fullname" . }}-static {{- with .Values.affinity }} affinity: {{- toYaml . | nindent 8 }} From 1c7a0b9df649143803d20e581e8a0de416284082 Mon Sep 17 00:00:00 2001 From: bettyli037 Date: Fri, 23 Jun 2023 13:17:34 -0400 Subject: [PATCH 083/132] feat: update Jenkinsfile --- deploy/chp-api/Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/deploy/chp-api/Jenkinsfile b/deploy/chp-api/Jenkinsfile index b0c343f..74e4c50 100644 --- a/deploy/chp-api/Jenkinsfile +++ b/deploy/chp-api/Jenkinsfile @@ -63,6 +63,7 @@ pipeline { aws --region ${AWS_REGION} eks update-kubeconfig --name ${KUBERNETES_CLUSTER_NAME} cd deploy/chp-api source prepare.sh + ls /bin/bash deploy.sh ''' } From a6e417de494f2092628cdfeda1d3c6270dc79dfe Mon Sep 17 00:00:00 2001 From: bettyli037 Date: Fri, 23 Jun 2023 13:21:39 -0400 Subject: [PATCH 084/132] feat: update Jenkinsfile --- deploy/chp-api/Jenkinsfile | 1 - 1 file changed, 1 deletion(-) diff --git a/deploy/chp-api/Jenkinsfile b/deploy/chp-api/Jenkinsfile index 74e4c50..43af563 100644 --- a/deploy/chp-api/Jenkinsfile +++ b/deploy/chp-api/Jenkinsfile @@ -1,7 +1,6 @@ pipeline { options { timestamps() - skipDefaultCheckout() disableConcurrentBuilds() } agent { From 89602b33a7e6885f8759de3ad79c6bfa6167dc5e Mon Sep 17 00:00:00 2001 From: bettyli037 Date: Mon, 26 Jun 2023 09:15:50 -0400 Subject: [PATCH 085/132] feat: update deployment.yaml --- deploy/chp-api/templates/deployment.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/deploy/chp-api/templates/deployment.yaml b/deploy/chp-api/templates/deployment.yaml index af13933..1acc4c7 100644 --- a/deploy/chp-api/templates/deployment.yaml +++ b/deploy/chp-api/templates/deployment.yaml @@ -113,8 +113,7 @@ spec: configMap: name: {{ include "chp-api.fullname" . }}-configs - name: static-vol - configMap: - name: {{ include "chp-api.fullname" . }}-static + emptyDir: {} {{- with .Values.affinity }} affinity: {{- toYaml . | nindent 8 }} From 011fee7511c7f071b9b3f939099b3f1fd41d3c35 Mon Sep 17 00:00:00 2001 From: bettyli037 Date: Mon, 26 Jun 2023 10:01:52 -0400 Subject: [PATCH 086/132] feat: update deployment.yaml --- deploy/chp-api/templates/deployment.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deploy/chp-api/templates/deployment.yaml b/deploy/chp-api/templates/deployment.yaml index 1acc4c7..738f0e7 100644 --- a/deploy/chp-api/templates/deployment.yaml +++ b/deploy/chp-api/templates/deployment.yaml @@ -113,7 +113,8 @@ spec: configMap: name: {{ include "chp-api.fullname" . }}-configs - name: static-vol - emptyDir: {} + persistentVolumeClaim: + claimName: {{ include "chp-api.fullname" . }}-pvc {{- with .Values.affinity }} affinity: {{- toYaml . | nindent 8 }} From d1d2eab287668f6764796d12949d922b9cbb6c92 Mon Sep 17 00:00:00 2001 From: bettyli037 Date: Mon, 26 Jun 2023 10:15:26 -0400 Subject: [PATCH 087/132] feat: update deployment.yaml --- deploy/chp-api/templates/deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/chp-api/templates/deployment.yaml b/deploy/chp-api/templates/deployment.yaml index 738f0e7..08d2f1d 100644 --- a/deploy/chp-api/templates/deployment.yaml +++ b/deploy/chp-api/templates/deployment.yaml @@ -114,7 +114,7 @@ spec: name: {{ include "chp-api.fullname" . }}-configs - name: static-vol persistentVolumeClaim: - claimName: {{ include "chp-api.fullname" . }}-pvc + claimName: chp-api-pvc-chp-api-0 {{- with .Values.affinity }} affinity: {{- toYaml . | nindent 8 }} From 98e6283c6240d2c0a2dce80d6cbe870b65f0a07a Mon Sep 17 00:00:00 2001 From: Chase Yakaboski Date: Mon, 26 Jun 2023 12:52:45 -0400 Subject: [PATCH 088/132] Added oauth2 and shifted gennifer models around. --- chp_api/Dockerfile | 14 ++-- chp_api/chp_api/settings.py | 23 +++++ chp_api/chp_api/urls.py | 1 + chp_api/gennifer/admin.py | 7 +- .../0008_result_study_task_and_more.py | 83 +++++++++++++++++++ ...user_result_target_result_task_and_more.py | 46 ++++++++++ chp_api/gennifer/models.py | 32 +++++-- chp_api/gennifer/serializers.py | 18 ++-- chp_api/gennifer/tasks.py | 81 +++++++++--------- chp_api/gennifer/trapi_interface.py | 10 +-- chp_api/gennifer/urls.py | 5 +- chp_api/gennifer/views.py | 79 ++++++++++++------ chp_api/requirements.txt | 3 +- chp_api/users/__init__.py | 0 chp_api/users/admin.py | 6 ++ chp_api/users/apps.py | 6 ++ chp_api/users/migrations/0001_initial.py | 44 ++++++++++ chp_api/users/migrations/__init__.py | 0 chp_api/users/models.py | 5 ++ chp_api/users/tests.py | 3 + chp_api/users/views.py | 3 + copy-migrations | 2 +- gennifer | 2 +- gennifer-sample.json | 2 + 24 files changed, 379 insertions(+), 96 deletions(-) create mode 100755 chp_api/gennifer/migrations/0008_result_study_task_and_more.py create mode 100755 chp_api/gennifer/migrations/0009_task_user_study_user_result_target_result_task_and_more.py create mode 100644 chp_api/users/__init__.py create mode 100644 chp_api/users/admin.py create mode 100644 chp_api/users/apps.py create mode 100755 chp_api/users/migrations/0001_initial.py create mode 100644 chp_api/users/migrations/__init__.py create mode 100644 chp_api/users/models.py create mode 100644 chp_api/users/tests.py create mode 100644 chp_api/users/views.py diff --git a/chp_api/Dockerfile b/chp_api/Dockerfile index 10c0b1f..b4008d2 100644 --- a/chp_api/Dockerfile +++ b/chp_api/Dockerfile @@ -10,6 +10,9 @@ WORKDIR /usr/src/chp_api RUN git clone --single-branch --branch gene_spec_pydantic-ghyde https://github.com/di2ag/gene-specificity.git +# Upgrade pip +RUN pip3 install --upgrade pip + # install dependencies COPY ./requirements.txt . RUN pip3 wheel --no-cache-dir --no-deps --wheel-dir /usr/src/chp_api/wheels -r requirements.txt @@ -48,12 +51,13 @@ COPY --from=intermediate /usr/src/chp_api/requirements.txt . RUN pip3 install --no-cache /wheels/* # copy project -COPY ./chp_api $APP_HOME/chp_api -COPY ./manage.py $APP_HOME -COPY ./dispatcher $APP_HOME/dispatcher -COPY ./gennifer $APP_HOME/gennifer +COPY . . +#COPY ./chp_api $APP_HOME/chp_api +#COPY ./manage.py $APP_HOME +#COPY ./dispatcher $APP_HOME/dispatcher +#COPY ./gennifer $APP_HOME/gennifer #COPY ./chp_db_fixture.json.gz $APP_HOME -COPY ./gunicorn.config.py $APP_HOME +#COPY ./gunicorn.config.py $APP_HOME # chown all the files to the app user RUN chown -R chp_api:chp_api $APP_HOME \ diff --git a/chp_api/chp_api/settings.py b/chp_api/chp_api/settings.py index 34cc2f7..d04df17 100644 --- a/chp_api/chp_api/settings.py +++ b/chp_api/chp_api/settings.py @@ -35,6 +35,19 @@ ) } +AUTHENTICATION_BACKENDS = [ + 'oauth2_provider.backends.OAuth2Backend', + # Uncomment following if you want to access the admin + #'django.contrib.auth.backends.ModelBackend', + '...', +] + +# Cors stuff (must go before installed apps) +CORS_ALLOWED_ORIGINS = [ + 'http://localhost', + 'http://localhost:3000', + ] + # Application definition INSTALLED_BASE_APPS = [ 'django.contrib.admin', @@ -43,11 +56,14 @@ 'django.contrib.sessions', 'django.contrib.messages', 'django.contrib.staticfiles', + 'corsheaders', 'rest_framework', 'rest_framework_simplejwt', 'django_filters', 'dispatcher.apps.DispatcherConfig', 'django_extensions', + 'users', + 'oauth2_provider', #'gennifer', # Need to make into CHP app ] @@ -63,6 +79,7 @@ INSTALLED_APPS = INSTALLED_BASE_APPS + INSTALLED_CHP_APPS MIDDLEWARE = [ + 'corsheaders.middleware.CorsMiddleware', 'django.middleware.security.SecurityMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', 'django.middleware.common.CommonMiddleware', @@ -70,6 +87,8 @@ 'django.contrib.auth.middleware.AuthenticationMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', 'django.middleware.clickjacking.XFrameOptionsMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'oauth2_provider.middleware.OAuth2TokenMiddleware', ] ROOT_URLCONF = 'chp_api.urls' @@ -125,6 +144,10 @@ }, ] +# Authorization +AUTH_USER_MODEL='users.User' +LOGIN_URL='/admin/login/' + # Internationalization # https://docs.djangoproject.com/en/3.0/topics/i18n/ LANGUAGE_CODE = 'en-us' diff --git a/chp_api/chp_api/urls.py b/chp_api/chp_api/urls.py index f42e50e..36727e8 100644 --- a/chp_api/chp_api/urls.py +++ b/chp_api/chp_api/urls.py @@ -29,6 +29,7 @@ path('gennifer/api/', include('gennifer.urls')), path('api/token/', TokenObtainPairView.as_view(), name='token_obtain_pair'), path('api/token/refresh/', TokenRefreshView.as_view(), name='token_refresh'), + path('o/', include('oauth2_provider.urls', namespace='oauth2_provider')) ] diff --git a/chp_api/gennifer/admin.py b/chp_api/gennifer/admin.py index abca8d6..51303f2 100644 --- a/chp_api/gennifer/admin.py +++ b/chp_api/gennifer/admin.py @@ -1,10 +1,11 @@ from django.contrib import admin -from .models import Algorithm, Dataset, InferenceStudy, InferenceResult, Gene, UserAnalysisSession +from .models import Algorithm, Dataset, Study, Task, Result, Gene, UserAnalysisSession admin.site.register(Algorithm) admin.site.register(Dataset) -admin.site.register(InferenceStudy) -admin.site.register(InferenceResult) +admin.site.register(Study) +admin.site.register(Task) +admin.site.register(Result) admin.site.register(Gene) admin.site.register(UserAnalysisSession) diff --git a/chp_api/gennifer/migrations/0008_result_study_task_and_more.py b/chp_api/gennifer/migrations/0008_result_study_task_and_more.py new file mode 100755 index 0000000..a50eeb5 --- /dev/null +++ b/chp_api/gennifer/migrations/0008_result_study_task_and_more.py @@ -0,0 +1,83 @@ +# Generated by Django 4.2.2 on 2023-06-25 23:19 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('gennifer', '0007_useranalysissession'), + ] + + operations = [ + migrations.CreateModel( + name='Result', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('edge_weight', models.FloatField()), + ('is_public', models.BooleanField(default=False)), + ], + ), + migrations.CreateModel( + name='Study', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=128)), + ('description', models.TextField(blank=True, null=True)), + ('status', models.CharField(max_length=10)), + ('timestamp', models.DateTimeField(auto_now_add=True)), + ], + options={ + 'verbose_name_plural': 'studies', + }, + ), + migrations.CreateModel( + name='Task', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('timestamp', models.DateTimeField(auto_now_add=True)), + ('max_study_edge_weight', models.FloatField(null=True)), + ('min_study_edge_weight', models.FloatField(null=True)), + ('avg_study_edge_weight', models.FloatField(null=True)), + ('std_study_edge_weight', models.FloatField(null=True)), + ('is_public', models.BooleanField(default=False)), + ('status', models.CharField(max_length=10)), + ('error_message', models.TextField(blank=True, null=True)), + ('algorithm_instance', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='tasks', to='gennifer.algorithminstance')), + ], + ), + migrations.RemoveField( + model_name='inferencestudy', + name='algorithm_instance', + ), + migrations.RemoveField( + model_name='inferencestudy', + name='dataset', + ), + migrations.RemoveField( + model_name='inferencestudy', + name='user', + ), + migrations.RenameField( + model_name='dataset', + old_name='upload_user', + new_name='user', + ), + migrations.DeleteModel( + name='InferenceResult', + ), + migrations.DeleteModel( + name='InferenceStudy', + ), + migrations.AddField( + model_name='task', + name='dataset', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='tasks', to='gennifer.dataset'), + ), + migrations.AddField( + model_name='task', + name='study', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='tasks', to='gennifer.study'), + ), + ] diff --git a/chp_api/gennifer/migrations/0009_task_user_study_user_result_target_result_task_and_more.py b/chp_api/gennifer/migrations/0009_task_user_study_user_result_target_result_task_and_more.py new file mode 100755 index 0000000..83374e4 --- /dev/null +++ b/chp_api/gennifer/migrations/0009_task_user_study_user_result_target_result_task_and_more.py @@ -0,0 +1,46 @@ +# Generated by Django 4.2.2 on 2023-06-25 23:19 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('gennifer', '0008_result_study_task_and_more'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.AddField( + model_name='task', + name='user', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='tasks', to=settings.AUTH_USER_MODEL), + ), + migrations.AddField( + model_name='study', + name='user', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='studies', to=settings.AUTH_USER_MODEL), + ), + migrations.AddField( + model_name='result', + name='target', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='inference_result_target', to='gennifer.gene'), + ), + migrations.AddField( + model_name='result', + name='task', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='results', to='gennifer.task'), + ), + migrations.AddField( + model_name='result', + name='tf', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='inference_result_tf', to='gennifer.gene'), + ), + migrations.AddField( + model_name='result', + name='user', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='results', to=settings.AUTH_USER_MODEL), + ), + ] diff --git a/chp_api/gennifer/models.py b/chp_api/gennifer/models.py index d5f2016..3e38f11 100644 --- a/chp_api/gennifer/models.py +++ b/chp_api/gennifer/models.py @@ -1,6 +1,7 @@ import requests import uuid +from django.conf import settings from django.db import models from django.contrib.auth.models import User @@ -8,7 +9,7 @@ class UserAnalysisSession(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) name = models.CharField(max_length=128) - user = models.ForeignKey(User, on_delete=models.CASCADE) + user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE) session_data = models.JSONField() is_saved = models.BooleanField(default=False) @@ -48,7 +49,7 @@ class Dataset(models.Model): zenodo_id = models.CharField(max_length=128, primary_key=True) doi = models.CharField(max_length=128) description = models.TextField(null=True, blank=True) - upload_user = models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True) + user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, null=True, blank=True) def save(self, *args, **kwargs): import re @@ -80,11 +81,23 @@ class Gene(models.Model): def __str__(self): return self.name +class Study(models.Model): + class Meta: + verbose_name_plural = "studies" -class InferenceStudy(models.Model): - algorithm_instance = models.ForeignKey(AlgorithmInstance, on_delete=models.CASCADE, related_name='studies') - user = models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True, related_name='studies') - dataset = models.ForeignKey(Dataset, on_delete=models.CASCADE, related_name='studies') + name = models.CharField(max_length=128) + description = models.TextField(null=True, blank=True) + status = models.CharField(max_length=10) + timestamp = models.DateTimeField(auto_now_add=True) + user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, null=True, blank=True, related_name='studies') + + def __str__(self): + return self.name + +class Task(models.Model): + algorithm_instance = models.ForeignKey(AlgorithmInstance, on_delete=models.CASCADE, related_name='tasks') + user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, null=True, blank=True, related_name='tasks') + dataset = models.ForeignKey(Dataset, on_delete=models.CASCADE, related_name='tasks') timestamp = models.DateTimeField(auto_now_add=True) # Study characteristics for all edge weights in a given study over a dataset max_study_edge_weight = models.FloatField(null=True) @@ -94,20 +107,21 @@ class InferenceStudy(models.Model): is_public = models.BooleanField(default=False) status = models.CharField(max_length=10) error_message = models.TextField(null=True, blank=True) + study = models.ForeignKey(Study, on_delete=models.CASCADE, related_name='tasks') def __str__(self): return f'{self.algorithm_instance} on {self.dataset.zenodo_id}' -class InferenceResult(models.Model): +class Result(models.Model): # Stands for transcription factor tf = models.ForeignKey(Gene, on_delete=models.CASCADE, related_name='inference_result_tf') # Target is the gene that is regulated by the transcription factor target = models.ForeignKey(Gene, on_delete=models.CASCADE, related_name='inference_result_target') edge_weight = models.FloatField() - study = models.ForeignKey(InferenceStudy, on_delete=models.CASCADE, related_name='results') + task = models.ForeignKey(Task, on_delete=models.CASCADE, related_name='results') is_public = models.BooleanField(default=False) - user = models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True, related_name='results') + user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, null=True, blank=True, related_name='results') def __str__(self): return f'{self.tf}:{self.tf.curie} -> regulates -> {self.target}:{self.target.curie}' diff --git a/chp_api/gennifer/serializers.py b/chp_api/gennifer/serializers.py index 818f215..a3932d4 100644 --- a/chp_api/gennifer/serializers.py +++ b/chp_api/gennifer/serializers.py @@ -1,6 +1,6 @@ from rest_framework import serializers -from .models import Dataset, InferenceStudy, InferenceResult, Algorithm, Gene, UserAnalysisSession +from .models import Dataset, Study, Task, Result, Algorithm, Gene, UserAnalysisSession class UserAnalysisSessionSerializer(serializers.ModelSerializer): @@ -14,15 +14,20 @@ class Meta: fields = ['title', 'zenodo_id', 'doi', 'description'] read_only_fields = ['title', 'doi', 'description'] +class StudySerializer(serializers.ModelSerializer): + class Meta: + model = Study + fields = ['name', 'status', 'description', 'timestamp', 'user'] + -class InferenceStudySerializer(serializers.ModelSerializer): +class TaskSerializer(serializers.ModelSerializer): name = serializers.SerializerMethodField('get_name') def get_name(self, study): return f'{study.algorithm_instance.algorithm.name} on {study.dataset.title}' class Meta: - model = InferenceStudy + model = Task fields = [ 'pk', 'algorithm_instance', @@ -33,18 +38,19 @@ class Meta: 'avg_study_edge_weight', 'std_study_edge_weight', 'name', + 'study', 'status', ] -class InferenceResultSerializer(serializers.ModelSerializer): +class ResultSerializer(serializers.ModelSerializer): class Meta: - model = InferenceResult + model = Result fields = [ 'pk', 'tf', 'target', 'edge_weight', - 'study', + 'task', ] class AlgorithmSerializer(serializers.ModelSerializer): diff --git a/chp_api/gennifer/tasks.py b/chp_api/gennifer/tasks.py index 70dd1dc..cf1a088 100644 --- a/chp_api/gennifer/tasks.py +++ b/chp_api/gennifer/tasks.py @@ -4,15 +4,16 @@ import requests from django.db import transaction -from django.contrib.auth.models import User +from django.contrib.auth import get_user_model from celery import shared_task from celery.utils.log import get_task_logger from copy import deepcopy -from .models import Dataset, Gene, InferenceStudy, InferenceResult, Algorithm, AlgorithmInstance +from .models import Dataset, Gene, Study, Task, Result, Algorithm, AlgorithmInstance from dispatcher.models import DispatcherSetting logger = get_task_logger(__name__) +User = get_user_model() def normalize_nodes(curies): dispatcher_settings = DispatcherSetting.load() @@ -35,20 +36,20 @@ def get_chp_preferred_curie(info): return _id['identifier'] return None -def save_inference_study(study, status, failed=False): - study.status = status["task_status"] +def save_inference_task(task, status, failed=False): + task.status = status["task_status"] if failed: - study.message = status["task_result"] + task.message = status["task_result"] else: # Construct Dataframe from result df = pd.DataFrame.from_records(status["task_result"]) - # Add study edge weight features + # Add task edge weight features stats = df["EdgeWeight"].astype(float).describe() - study.max_study_edge_weight = stats["max"] - study.min_study_edge_weight = stats["min"] - study.avg_study_edge_weight = stats["mean"] - study.std_study_edge_weight = stats["std"] + task.max_task_edge_weight = stats["max"] + task.min_task_edge_weight = stats["min"] + task.avg_task_edge_weight = stats["mean"] + task.std_task_edge_weight = stats["std"] # Collect all genes genes = set() @@ -95,41 +96,41 @@ def save_inference_study(study, status, failed=False): if created: gene2_obj.save() # Construct and save Result - result = InferenceResult.objects.create( + result = Result.objects.create( tf=gene1_obj, target=gene2_obj, edge_weight=row["EdgeWeight"], - study=study, - user=study.user, + task=task, + user=task.user, ) result.save() - study.save() + task.save() return True def get_status(algo, task_id): return requests.get(f'{algo.url}/status/{task_id}', headers={'Cache-Control': 'no-cache'}).json() -def return_saved_study(studies, user): - study = studies[0] - # Copy study results - results = deepcopy(study.results) - # Create a new study that is a duplicate but assign to this user. - study.pk = None - study.results = None - study.save() +def return_saved_task(tasks, user): + task = studies[0] + # Copy task results + results = deepcopy(task.results) + # Create a new task that is a duplicate but assign to this user. + task.pk = None + task.results = None + task.save() - # Now go through and assign all results to this study and user. + # Now go through and assign all results to this task and user. for result in results: result.pk = None - result.study = study + result.task = task result.user = user result.save() return True @shared_task(name="create_gennifer_task") -def create_task(algorithm_name, zenodo_id, hyperparameters, user_pk): +def create_task(algorithm_name, zenodo_id, hyperparameters, user_pk, study_pk): # Get algorithm obj algo = Algorithm.objects.get(name=algorithm_name) @@ -147,26 +148,29 @@ def create_task(algorithm_name, zenodo_id, hyperparameters, user_pk): # Get User obj user = User.objects.get(pk=user_pk) + + # Get Study obj + study = Study.objects.get(pk=study_pk) # Initialize dataset instance dataset, dataset_created = Dataset.objects.get_or_create( zenodo_id=zenodo_id, - upload_user=user, + user=user, ) if dataset_created: dataset.save() if not algo_instance_created and not dataset_created: - # This means we've already run the study. So let's just return that and not bother our workers. - studies = InferenceStudy.objects.filter( + # This means we've already run the task. So let's just return that and not bother our workers. + tasks = Task.objects.filter( algorithm_instance=algo_instance, dataset=dataset, status='SUCCESS', ) #TODO: Probably should add some timestamp handling here if len(studies) > 0: - return_saved_study(studies, user) + return_saved_task(tasks, user) # Send to gennifer app gennifer_request = { @@ -181,25 +185,26 @@ def create_task(algorithm_name, zenodo_id, hyperparameters, user_pk): status = get_status(algo, task_id) # Create Inference Study - study = InferenceStudy.objects.create( + task = Task.objects.create( algorithm_instance=algo_instance, user=user, dataset=dataset, status=status["task_status"], + study=study, ) - # Save initial study - study.save() + # Save initial task + task.save() - # Enter a loop to keep checking back in and populate the study once it has completed. + # Enter a loop to keep checking back in and populate the task once it has completed. #TODO: Not sure if this is best practice while True: # Check in every 2 seconds time.sleep(5) status = get_status(algo, task_id) if status["task_status"] == 'SUCCESS': - return save_inference_study(study, status) + return save_inference_task(task, status) if status["task_status"] == "FAILURE": - return save_inference_study(study, status, failed=True) - if status["task_status"] != study.status: - study.status = status["task_status"] - study.save() + return save_inference_task(task, status, failed=True) + if status["task_status"] != task.status: + task.status = status["task_status"] + task.save() diff --git a/chp_api/gennifer/trapi_interface.py b/chp_api/gennifer/trapi_interface.py index cf83e75..522b75a 100644 --- a/chp_api/gennifer/trapi_interface.py +++ b/chp_api/gennifer/trapi_interface.py @@ -12,7 +12,7 @@ from reasoner_pydantic.kgraph import RetrievalSource, Attribute from reasoner_pydantic.results import NodeBinding, EdgeBinding, Result, Results, Analysis -from .models import InferenceResult, Gene +from .models import Result, Gene # Setup logging logging.addLevelName(25, "NOTE") @@ -144,12 +144,12 @@ def get_response(self, message: Message, logger): if predicate == 'biolink:regulates': results = [] for obj_gene in obj_genes: - results.extend(InferenceResult.objects.filter(target=obj_gene, is_public=True)) + results.extend(Result.objects.filter(target=obj_gene, is_public=True)) subject_curies = [r.tf.chp_preferred_curie for r in results] elif predicate == 'biolink:regulated_by': results = [] for obj_gene in obj_genes: - results.extend(InferenceResult.objects.filter(tf=obj_gene, is_public=True)) + results.extend(Result.objects.filter(tf=obj_gene, is_public=True)) subject_curies = [r.target.chp_preferred_curie for r in results] else: raise ValueError(f'Unknown predicate: {predicate}.') @@ -182,12 +182,12 @@ def get_response(self, message: Message, logger): if predicate == 'biolink:regulates': results = [] for sub_gene in sub_genes: - results.extend(InferenceResult.objects.filter(tf=sub_gene, is_public=True)) + results.extend(Result.objects.filter(tf=sub_gene, is_public=True)) object_curies = [r.target.chp_preferred_curie for r in results] elif predicate == 'biolink:regulated_by': results = [] for sub_gene in sub_genes: - results.extend(InferenceResult.objects.filter(target=sub_gene, is_public=True)) + results.extend(Result.objects.filter(target=sub_gene, is_public=True)) object_curies = [r.tf.chp_preferred_curie for r in results] else: raise ValueError(f'Unknown predicate: {predicate}.') diff --git a/chp_api/gennifer/urls.py b/chp_api/gennifer/urls.py index 29fae6a..fee431c 100644 --- a/chp_api/gennifer/urls.py +++ b/chp_api/gennifer/urls.py @@ -6,8 +6,9 @@ # Create router and register viewsets router = DefaultRouter() router.register(r'datasets', views.DatasetViewSet, basename='dataset') -router.register(r'inference_studies', views.InferenceStudyViewSet, basename='inference_study') -router.register(r'inference_results', views.InferenceResultViewSet, basename='inference_result') +router.register(r'studies', views.StudyViewSet, basename='study') +router.register(r'tasks', views.TaskViewSet, basename='task') +router.register(r'results', views.ResultViewSet, basename='result') router.register(r'algorithms', views.AlgorithmViewSet, basename='algorithm') router.register(r'genes', views.GeneViewSet, basename='genes') router.register(r'analyses', views.UserAnalysisSessionViewSet, basename='analyses') diff --git a/chp_api/gennifer/views.py b/chp_api/gennifer/views.py index 68b3f76..5dd8dfb 100644 --- a/chp_api/gennifer/views.py +++ b/chp_api/gennifer/views.py @@ -11,9 +11,26 @@ from rest_framework.permissions import IsAuthenticated, IsAuthenticatedOrReadOnly from rest_framework.exceptions import ValidationError from django_filters.rest_framework import DjangoFilterBackend - -from .models import Dataset, InferenceStudy, InferenceResult, Algorithm, Gene, UserAnalysisSession -from .serializers import DatasetSerializer, InferenceStudySerializer, InferenceResultSerializer, AlgorithmSerializer, GeneSerializer, UserAnalysisSessionSerializer +from oauth2_provider.contrib.rest_framework import TokenHasReadWriteScope + +from .models import ( + Dataset, + Study, + Task, + Result, + Algorithm, + Gene, + UserAnalysisSession + ) +from .serializers import ( + DatasetSerializer, + StudySerializer, + TaskSerializer, + ResultSerializer, + AlgorithmSerializer, + GeneSerializer, + UserAnalysisSessionSerializer + ) from .tasks import create_task from .permissions import IsOwnerOrReadOnly, IsAdminOrReadOnly @@ -34,20 +51,24 @@ class DatasetViewSet(viewsets.ModelViewSet): queryset = Dataset.objects.all() serializer_class = DatasetSerializer filter_backends = [DjangoFilterBackend] - filterset_fields = ['upload_user', 'zenodo_id'] - permission_classes = [IsAuthenticatedOrReadOnly] + filterset_fields = ['user', 'zenodo_id'] + permission_classes = [IsOwnerOrReadOnly] def perform_create(self, serializers): try: - serializers.save(upload_user=self.request.user) + serializers.save(user=self.request.user) except ValueError as e: raise ValidationError(str(e)) +class StudyViewSet(viewsets.ModelViewSet): + queryset = Study.objects.all() + serializer_class = StudySerializer + permission_classes = [IsOwnerOrReadOnly] -class InferenceStudyViewSet(viewsets.ModelViewSet): - queryset = InferenceStudy.objects.all() - serializer_class = InferenceStudySerializer +class TaskViewSet(viewsets.ModelViewSet): + queryset = Task.objects.all() + serializer_class = TaskSerializer filter_backends = [DjangoFilterBackend] filterset_fields = ['is_public', 'dataset', 'algorithm_instance'] permission_classes = [IsOwnerOrReadOnly] @@ -57,11 +78,11 @@ class InferenceStudyViewSet(viewsets.ModelViewSet): # return InferenceStudy.objects.filter(user=user) -class InferenceResultViewSet(viewsets.ModelViewSet): - queryset = InferenceResult.objects.all() - serializer_class = InferenceResultSerializer +class ResultViewSet(viewsets.ModelViewSet): + queryset = Result.objects.all() + serializer_class = ResultSerializer filter_backends = [DjangoFilterBackend] - filterset_fields = ['is_public', 'study', 'tf', 'target'] + filterset_fields = ['is_public', 'task', 'tf', 'target'] permission_classes = [IsOwnerOrReadOnly] #def get_queryset(self): @@ -103,17 +124,17 @@ def construct_node(self, gene_obj): def construct_edge(self, res, source_id, target_id): # Normalize edge weight based on the study - normalized_weight = (res.edge_weight - res.study.min_study_edge_weight) / (res.study.max_study_edge_weight - res.study.min_study_edge_weight) - directed = res.study.algorithm_instance.algorithm.directed + normalized_weight = (res.edge_weight - res.task.min_study_edge_weight) / (res.task.max_study_edge_weight - res.task.min_study_edge_weight) + directed = res.task.algorithm_instance.algorithm.directed edge_tuple = tuple(sorted([source_id, target_id])) edge = { "data": { "id": str(res.pk), "source": source_id, "target": target_id, - "dataset": str(res.study.dataset), + "dataset": str(res.task.dataset), "weight": normalized_weight, - "algorithm": str(res.study.algorithm_instance), + "algorithm": str(res.task.algorithm_instance), "directed": directed, } } @@ -163,14 +184,14 @@ def construct_cytoscape_data(self, results): } def get(self, request): - results = InferenceResult.objects.all() + results = Result.objects.all() cyto = self.construct_cytoscape_data(results) return JsonResponse(cyto) def post(self, request): elements = [] gene_ids = request.data.get("gene_ids", None) - study_ids = request.data.get("study_ids", None) + task_ids = request.data.get("task_ids", None) algorithm_ids = request.data.get("algorithm_ids", None) dataset_ids = request.data.get("dataset_ids", None) cached_inference_result_ids = request.data.get("cached_results", None) @@ -188,11 +209,11 @@ def post(self, request): ] ) if study_ids: - filters.append({"field": 'study__pk', "operator": 'in', "value": study_ids}) + filters.append({"field": 'task__pk', "operator": 'in', "value": study_ids}) if algorithm_ids: - filters.append({"field": 'study__algorithm_instance__algorithm__pk', "operator": 'in', "value": study_ids}) + filters.append({"field": 'task__algorithm_instance__algorithm__pk', "operator": 'in', "value": study_ids}) if dataset_ids: - filters.append({"field": 'study__dataset__zenodo_id', "operator": 'in', "value": dataset_ids}) + filters.append({"field": 'task__dataset__zenodo_id', "operator": 'in', "value": dataset_ids}) # Construct Query query = Q() @@ -203,7 +224,7 @@ def post(self, request): query &= Q(**{f'{field}__{operator}': value}) # Get matching results - results = InferenceResult.objects.filter(query) + results = Result.objects.filter(query) if len(results) == 0: return JsonResponse({"elements": elements}) @@ -237,9 +258,17 @@ class run(APIView): def post(self, request): """ Request comes in as a list of algorithms to run. """ + # Create study + study = Study.objects.create( + name = request.data['name'], + description = request.data.get('description', None), + status = 'RECIEVED', + user = request.user, + ) + study.save() # Build gennifer requests tasks = request.data['tasks'] - response = {"tasks": []} + response = {"study_id": study.pk, "tasks": []} for task in tasks: algorithm_name = task.get("algorithm_name", None) zenodo_id = task.get("zenodo_id", None) @@ -262,6 +291,6 @@ def post(self, request): response["tasks"].append(task) continue # If all pass, now send to gennifer services - task["task_id"] = create_task.delay(algo.name, zenodo_id, hyperparameters, request.user.pk).id + task["task_id"] = create_task.delay(algo.name, zenodo_id, hyperparameters, request.user.pk, study.pk).id response["tasks"].append(task) return JsonResponse(response) diff --git a/chp_api/requirements.txt b/chp_api/requirements.txt index a544600..75b8cd4 100644 --- a/chp_api/requirements.txt +++ b/chp_api/requirements.txt @@ -4,7 +4,6 @@ djangorestframework-simplejwt psycopg2-binary bmt reasoner_pydantic -#reasoner-validator django-environ django-hosts gunicorn @@ -16,3 +15,5 @@ celery flower redis pandas +django-cors-headers +django-oauth-toolkit diff --git a/chp_api/users/__init__.py b/chp_api/users/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/chp_api/users/admin.py b/chp_api/users/admin.py new file mode 100644 index 0000000..88e1740 --- /dev/null +++ b/chp_api/users/admin.py @@ -0,0 +1,6 @@ +from django.contrib import admin +from django.contrib.auth.admin import UserAdmin +from .models import User + +admin.site.register(User, UserAdmin) + diff --git a/chp_api/users/apps.py b/chp_api/users/apps.py new file mode 100644 index 0000000..72b1401 --- /dev/null +++ b/chp_api/users/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class UsersConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'users' diff --git a/chp_api/users/migrations/0001_initial.py b/chp_api/users/migrations/0001_initial.py new file mode 100755 index 0000000..b754ae9 --- /dev/null +++ b/chp_api/users/migrations/0001_initial.py @@ -0,0 +1,44 @@ +# Generated by Django 4.2.2 on 2023-06-25 23:23 + +import django.contrib.auth.models +import django.contrib.auth.validators +from django.db import migrations, models +import django.utils.timezone + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('auth', '0012_alter_user_first_name_max_length'), + ] + + operations = [ + migrations.CreateModel( + name='User', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('password', models.CharField(max_length=128, verbose_name='password')), + ('last_login', models.DateTimeField(blank=True, null=True, verbose_name='last login')), + ('is_superuser', models.BooleanField(default=False, help_text='Designates that this user has all permissions without explicitly assigning them.', verbose_name='superuser status')), + ('username', models.CharField(error_messages={'unique': 'A user with that username already exists.'}, help_text='Required. 150 characters or fewer. Letters, digits and @/./+/-/_ only.', max_length=150, unique=True, validators=[django.contrib.auth.validators.UnicodeUsernameValidator()], verbose_name='username')), + ('first_name', models.CharField(blank=True, max_length=150, verbose_name='first name')), + ('last_name', models.CharField(blank=True, max_length=150, verbose_name='last name')), + ('email', models.EmailField(blank=True, max_length=254, verbose_name='email address')), + ('is_staff', models.BooleanField(default=False, help_text='Designates whether the user can log into this admin site.', verbose_name='staff status')), + ('is_active', models.BooleanField(default=True, help_text='Designates whether this user should be treated as active. Unselect this instead of deleting accounts.', verbose_name='active')), + ('date_joined', models.DateTimeField(default=django.utils.timezone.now, verbose_name='date joined')), + ('groups', models.ManyToManyField(blank=True, help_text='The groups this user belongs to. A user will get all permissions granted to each of their groups.', related_name='user_set', related_query_name='user', to='auth.group', verbose_name='groups')), + ('user_permissions', models.ManyToManyField(blank=True, help_text='Specific permissions for this user.', related_name='user_set', related_query_name='user', to='auth.permission', verbose_name='user permissions')), + ], + options={ + 'verbose_name': 'user', + 'verbose_name_plural': 'users', + 'abstract': False, + }, + managers=[ + ('objects', django.contrib.auth.models.UserManager()), + ], + ), + ] diff --git a/chp_api/users/migrations/__init__.py b/chp_api/users/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/chp_api/users/models.py b/chp_api/users/models.py new file mode 100644 index 0000000..da5acd7 --- /dev/null +++ b/chp_api/users/models.py @@ -0,0 +1,5 @@ +from django.db import models +from django.contrib.auth.models import AbstractUser + +class User(AbstractUser): + pass diff --git a/chp_api/users/tests.py b/chp_api/users/tests.py new file mode 100644 index 0000000..7ce503c --- /dev/null +++ b/chp_api/users/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/chp_api/users/views.py b/chp_api/users/views.py new file mode 100644 index 0000000..91ea44a --- /dev/null +++ b/chp_api/users/views.py @@ -0,0 +1,3 @@ +from django.shortcuts import render + +# Create your views here. diff --git a/copy-migrations b/copy-migrations index c362e86..7f9244c 100755 --- a/copy-migrations +++ b/copy-migrations @@ -2,4 +2,4 @@ docker compose -f compose.chp-api.yaml run -v migrations:/home/migrations \ --user root api \ - bash -c "python3 manage.py makemigrations && cp -r /home/chp_api/web/dispatcher/migrations /home/migrations/dispatcher && cp -r /home/chp_api/web/gennifer/migrations /home/migrations/gennifer" + bash -c "python3 manage.py makemigrations && cp -r /home/chp_api/web/dispatcher/migrations /home/migrations/dispatcher && cp -r /home/chp_api/web/gennifer/migrations /home/migrations/gennifer && cp -r /home/chp_api/web/users/migrations /home/migrations/users" diff --git a/gennifer b/gennifer index 3b27cbd..71038d5 160000 --- a/gennifer +++ b/gennifer @@ -1 +1 @@ -Subproject commit 3b27cbd61ab3561e02f9d5a6dfe15038804699f5 +Subproject commit 71038d5b152b4fa223db9563736ada1691a01ac4 diff --git a/gennifer-sample.json b/gennifer-sample.json index 803f0f0..c0e65c6 100644 --- a/gennifer-sample.json +++ b/gennifer-sample.json @@ -1,4 +1,6 @@ { + "name": "GSD Study", + "description": "This is a test study with one task.", "tasks": [ { "algorithm_name": "pidc", From 69094c276d2ff7db7c8bb9dd118f6d7afa48680d Mon Sep 17 00:00:00 2001 From: bettyli037 Date: Mon, 26 Jun 2023 15:32:41 -0400 Subject: [PATCH 089/132] feat: update deployment.yaml --- deploy/chp-api/templates/deployment.yaml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/deploy/chp-api/templates/deployment.yaml b/deploy/chp-api/templates/deployment.yaml index 08d2f1d..93c3dcf 100644 --- a/deploy/chp-api/templates/deployment.yaml +++ b/deploy/chp-api/templates/deployment.yaml @@ -32,7 +32,7 @@ spec: containerPort: 8000 protocol: TCP volumeMounts: - - name: static-vol + - name: {{ include "chp-api.fullname" . }}-pvc mountPath: /home/chp_api/web/staticfiles env: - name: SECRET_KEY @@ -101,7 +101,7 @@ spec: containerPort: 8080 protocol: TCP volumeMounts: - - name: static-vol + - name: {{ include "chp-api.fullname" . }}-pvc mountPath: /var/www/static env: - name: FOLDER @@ -112,9 +112,6 @@ spec: - name: config-vol configMap: name: {{ include "chp-api.fullname" . }}-configs - - name: static-vol - persistentVolumeClaim: - claimName: chp-api-pvc-chp-api-0 {{- with .Values.affinity }} affinity: {{- toYaml . | nindent 8 }} From a2bd97163f1ac26c510d11276977f745b448e50b Mon Sep 17 00:00:00 2001 From: Chase Yakaboski Date: Mon, 26 Jun 2023 17:38:29 -0400 Subject: [PATCH 090/132] Fixed bug in settings. --- chp_api/chp_api/settings.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/chp_api/chp_api/settings.py b/chp_api/chp_api/settings.py index 72d00d5..269d0d3 100644 --- a/chp_api/chp_api/settings.py +++ b/chp_api/chp_api/settings.py @@ -39,7 +39,6 @@ 'oauth2_provider.backends.OAuth2Backend', # Uncomment following if you want to access the admin #'django.contrib.auth.backends.ModelBackend', - '...', ] # Cors stuff (must go before installed apps) @@ -230,5 +229,6 @@ # Gennifer settings GENNIFER_ALGORITHM_URLS = [ - "http://pidc:5000" + "http://pidc:5000", + "http://grisli:5000", ] From 7c093f95ae7d0b738d15eee246bcc8969e80f0c6 Mon Sep 17 00:00:00 2001 From: bettyli037 Date: Tue, 27 Jun 2023 10:31:08 -0400 Subject: [PATCH 091/132] feat: update the mountPath for chp-api container --- deploy/chp-api/templates/deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/chp-api/templates/deployment.yaml b/deploy/chp-api/templates/deployment.yaml index 93c3dcf..28ded33 100644 --- a/deploy/chp-api/templates/deployment.yaml +++ b/deploy/chp-api/templates/deployment.yaml @@ -33,7 +33,7 @@ spec: protocol: TCP volumeMounts: - name: {{ include "chp-api.fullname" . }}-pvc - mountPath: /home/chp_api/web/staticfiles + mountPath: /home/chp_api/staticfiles env: - name: SECRET_KEY valueFrom: From ef3f140aad70e11cc27e9675ffcc1824a6106197 Mon Sep 17 00:00:00 2001 From: Chase Yakaboski Date: Tue, 27 Jun 2023 17:03:03 -0400 Subject: [PATCH 092/132] Added CSRF trusted origins to deployment and settings. --- chp_api/chp_api/settings.py | 7 +++++++ deploy/chp-api/templates/deployment.yaml | 2 ++ deploy/chp-api/values.yaml | 1 + docker-compose.yml | 6 +++++- 4 files changed, 15 insertions(+), 1 deletion(-) diff --git a/chp_api/chp_api/settings.py b/chp_api/chp_api/settings.py index c8e3018..461d3b3 100644 --- a/chp_api/chp_api/settings.py +++ b/chp_api/chp_api/settings.py @@ -171,6 +171,13 @@ with open(env("SECRET_KEY_FILE"), 'r') as sk_file: SECRET_KEY = sk_file.readline().strip() +CSRF_TRUSTED_ORIGINS = env("CSRF_TRUSTED_ORIGINS", default=None) +if not CSRF_TRUSTED_ORIGINS: + with open(env("CSRF_TRUSTED_ORIGINS_FILE"), 'r') as csrf_file: + CSRF_TRUSTED_ORIGINS = csrf_file.readline().strip().split(" ") +else: + CSRF_TRUSTED_ORIGINS = CSRF_TRUSTED_ORIGINS.split(',') + # Set UN, Email and Password for superuser DJANGO_SUPERUSER_USERNAME = env("DJANGO_SUPERUSER_USERNAME", default=None) if not DJANGO_SUPERUSER_USERNAME: diff --git a/deploy/chp-api/templates/deployment.yaml b/deploy/chp-api/templates/deployment.yaml index af13933..b7bee5d 100644 --- a/deploy/chp-api/templates/deployment.yaml +++ b/deploy/chp-api/templates/deployment.yaml @@ -72,6 +72,8 @@ spec: value: "{{ .Values.app.debug }}" - name: DJANGO_ALLOWED_HOSTS value: "{{ .Values.app.djangoAllowedHosts }}" + - name: CSRF_TRUSTED_ORIGINS + value: "{{ .Values.app.djangoCSRFTrustedOrigins }}" - name: DJANGO_SETTINGS_MODULE value: "{{ .Values.app.djangoSettingsModule }}" - name: DJANGO_SUPERUSER_USERNAME diff --git a/deploy/chp-api/values.yaml b/deploy/chp-api/values.yaml index 0d415a0..c34c476 100644 --- a/deploy/chp-api/values.yaml +++ b/deploy/chp-api/values.yaml @@ -20,6 +20,7 @@ fullnameOverride: "" app: debug: "0" secret_key: "" + djangoCSRFTrustedOrigins: "" djangoAllowedHosts: "" djangoSettingsModule: "chp_api.settings" djangoSuperuserUsername: "chp_admin" diff --git a/docker-compose.yml b/docker-compose.yml index 211128f..79e4e17 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -33,6 +33,7 @@ services: - db-password - django-key - allowed-hosts + - csrf-trusted-origins - django-superuser-username - django-superuser-email - django-superuser-password @@ -47,11 +48,12 @@ services: #- POSTGRES_PASSWORD=31173e51d8f78b56606d06dfb66a1b126630cdf4711bed9427025d8979976f31 #- SECRET_KEY=e1743ca40af220389cd1165d213e3d677f2d59c00d7b0f94e7a302c91f95f029 #- DJANGO_ALLOWED_HOSTS=localhost,chp.thayer.dartmouth.edu + #- CSRF_TRUSTED_ORIGINS=localhost,chp.thayer.dartmouth.edu #- DJANGO_SUPERUSER_USERNAME=chp_admin #- DJANGO_SUPERUSER_EMAIL=chp_admin@chp.com #- DJANGO_SUPERUSER_PASSWORD=e12ff26f070819d9a72e317898148679680e6b3976e464b4102bd6eb18357919 - - POSTGRES_PASSWORD_FILE=/run/secrets/db-password - SECRET_KEY_FILE=/run/secrets/django-key + - CSRF_TRUSTED_ORIGINS_FILE=/run/secrets/csrf-trusted-origins - DJANGO_ALLOWED_HOSTS_FILE=/run/secrets/allowed-hosts - DJANGO_SUPERUSER_USERNAME_FILE=/run/secrets/django-superuser-username - DJANGO_SUPERUSER_EMAIL_FILE=/run/secrets/django-superuser-email @@ -109,6 +111,8 @@ volumes: secrets: allowed-hosts: file: secrets/chp_api/allowed_hosts.txt + csrf-trusted-origins: + file: secrets/chp_api/csrf_trusted_origins.txt db-password: file: secrets/db/password.txt django-key: From 003c05b707fe5ee000420279bfc787a4cb962036 Mon Sep 17 00:00:00 2001 From: Chase Yakaboski Date: Tue, 27 Jun 2023 17:06:39 -0400 Subject: [PATCH 093/132] Updated docker compose. --- docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 79e4e17..f2d497b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -48,12 +48,12 @@ services: #- POSTGRES_PASSWORD=31173e51d8f78b56606d06dfb66a1b126630cdf4711bed9427025d8979976f31 #- SECRET_KEY=e1743ca40af220389cd1165d213e3d677f2d59c00d7b0f94e7a302c91f95f029 #- DJANGO_ALLOWED_HOSTS=localhost,chp.thayer.dartmouth.edu - #- CSRF_TRUSTED_ORIGINS=localhost,chp.thayer.dartmouth.edu + - CSRF_TRUSTED_ORIGINS=http://localhost,https://chp.thayer.dartmouth.edu #- DJANGO_SUPERUSER_USERNAME=chp_admin #- DJANGO_SUPERUSER_EMAIL=chp_admin@chp.com #- DJANGO_SUPERUSER_PASSWORD=e12ff26f070819d9a72e317898148679680e6b3976e464b4102bd6eb18357919 - SECRET_KEY_FILE=/run/secrets/django-key - - CSRF_TRUSTED_ORIGINS_FILE=/run/secrets/csrf-trusted-origins + #- CSRF_TRUSTED_ORIGINS_FILE=/run/secrets/csrf-trusted-origins - DJANGO_ALLOWED_HOSTS_FILE=/run/secrets/allowed-hosts - DJANGO_SUPERUSER_USERNAME_FILE=/run/secrets/django-superuser-username - DJANGO_SUPERUSER_EMAIL_FILE=/run/secrets/django-superuser-email From 61be809af250f7bd836f5b351c04628636ad4261 Mon Sep 17 00:00:00 2001 From: Chase Yakaboski Date: Tue, 27 Jun 2023 17:09:03 -0400 Subject: [PATCH 094/132] Update from dev machine. --- compose.chp-api.yaml | 3 +- compose.gennifer.yaml | 97 ++++++++++++++++++++++++++++++++++++++++++- gennifer-sample.json | 7 +++- 3 files changed, 103 insertions(+), 4 deletions(-) diff --git a/compose.chp-api.yaml b/compose.chp-api.yaml index f4aa18d..c4476af 100644 --- a/compose.chp-api.yaml +++ b/compose.chp-api.yaml @@ -43,7 +43,7 @@ services: - POSTGRES_PASSWORD_FILE=/run/secrets/db-password - POSTGRES_HOST=db - POSTGRES_PORT=5432 - - DEBUG=1 + - DEBUG=0 # For Helm testing purposes #- POSTGRES_PASSWORD=31173e51d8f78b56606d06dfb66a1b126630cdf4711bed9427025d8979976f31 #- SECRET_KEY=e1743ca40af220389cd1165d213e3d677f2d59c00d7b0f94e7a302c91f95f029 @@ -51,7 +51,6 @@ services: #- DJANGO_SUPERUSER_USERNAME=chp_admin #- DJANGO_SUPERUSER_EMAIL=chp_admin@chp.com #- DJANGO_SUPERUSER_PASSWORD=e12ff26f070819d9a72e317898148679680e6b3976e464b4102bd6eb18357919 - - POSTGRES_PASSWORD_FILE=/run/secrets/db-password - SECRET_KEY_FILE=/run/secrets/django-key - DJANGO_ALLOWED_HOSTS_FILE=/run/secrets/allowed-hosts - DJANGO_SUPERUSER_USERNAME_FILE=/run/secrets/django-superuser-username diff --git a/compose.gennifer.yaml b/compose.gennifer.yaml index fac4a43..ffa7a9c 100644 --- a/compose.gennifer.yaml +++ b/compose.gennifer.yaml @@ -20,7 +20,7 @@ services: command: flask --app pidc run --debug --host 0.0.0.0 depends_on: - redis - + worker-pidc: build: context: ./gennifer/pidc @@ -32,6 +32,101 @@ services: depends_on: - pidc - redis + + grisli: + build: + context: ./gennifer/grisli + dockerfile: Dockerfile + restart: always + user: gennifer_user + ports: + - 5005:5000 + secrets: + - gennifer_key + environment: + - SECRET_KEY_FILE=/run/secrets/gennifer_key + - CELERY_BROKER_URL=redis://redis:6379/0 + - CELERY_RESULT_BACKEND=redis://redis:6379/0 + - PYTHONUNBUFFERED=1 + #command: gunicorn -c gunicorn.config.py -b 0.0.0.0:5000 'pidc:create_app()' + command: flask --app grisli run --debug --host 0.0.0.0 + depends_on: + - redis + + worker-grisli: + build: + context: ./gennifer/grisli + dockerfile: Dockerfile + command: celery --app grisli.tasks.celery worker -Q grisli --loglevel=info + environment: + - CELERY_BROKER_URL=redis://redis:6379/0 + - CELERY_RESULT_BACKEND=redis://redis:6379/0 + - PYTHONUNBUFFERED=1 + depends_on: + - grisli + - redis + + genie3: + build: + context: ./gennifer/genie3 + dockerfile: Dockerfile + restart: always + user: gennifer_user + ports: + - 5006:5000 + secrets: + - gennifer_key + environment: + - SECRET_KEY_FILE=/run/secrets/gennifer_key + - CELERY_BROKER_URL=redis://redis:6379/0 + - CELERY_RESULT_BACKEND=redis://redis:6379/0 + #command: gunicorn -c gunicorn.config.py -b 0.0.0.0:5000 'genie3:create_app()' + command: flask --app genie3 run --debug --host 0.0.0.0 + depends_on: + - redis + + worker-genie3: + build: + context: ./gennifer/genie3 + dockerfile: Dockerfile + command: celery --app genie3.tasks.celery worker -Q genie3 --loglevel=info + environment: + - CELERY_BROKER_URL=redis://redis:6379/0 + - CELERY_RESULT_BACKEND=redis://redis:6379/0 + depends_on: + - genie3 + - redis + + grnboost2: + build: + context: ./gennifer/grnboost2 + dockerfile: Dockerfile + restart: always + user: gennifer_user + ports: + - 5007:5000 + secrets: + - gennifer_key + environment: + - SECRET_KEY_FILE=/run/secrets/gennifer_key + - CELERY_BROKER_URL=redis://redis:6379/0 + - CELERY_RESULT_BACKEND=redis://redis:6379/0 + #command: gunicorn -c gunicorn.config.py -b 0.0.0.0:5000 'grnboost2:create_app()' + command: flask --app grnboost2 run --debug --host 0.0.0.0 + depends_on: + - redis + + worker-grnboost2: + build: + context: ./gennifer/grnboost2 + dockerfile: Dockerfile + command: celery --app grnboost2.tasks.celery worker -Q grnboost2 --loglevel=info + environment: + - CELERY_BROKER_URL=redis://redis:6379/0 + - CELERY_RESULT_BACKEND=redis://redis:6379/0 + depends_on: + - grnboost2 + - redis secrets: gennifer_key: diff --git a/gennifer-sample.json b/gennifer-sample.json index c0e65c6..e91e778 100644 --- a/gennifer-sample.json +++ b/gennifer-sample.json @@ -6,6 +6,11 @@ "algorithm_name": "pidc", "zenodo_id":"7988181", "hyperparameters": null - } + }, + { + "algorithm_name": "grisli", + "zenodo_id":"8057216", + "hyperparameters": null + } ] } From 747830c865bdead400dcad9152cbc80a9a30ce5a Mon Sep 17 00:00:00 2001 From: bettyli037 Date: Tue, 27 Jun 2023 17:09:35 -0400 Subject: [PATCH 095/132] feat: update Jenkinsfile --- deploy/chp-api/Jenkinsfile | 1 - 1 file changed, 1 deletion(-) diff --git a/deploy/chp-api/Jenkinsfile b/deploy/chp-api/Jenkinsfile index 43af563..f432639 100644 --- a/deploy/chp-api/Jenkinsfile +++ b/deploy/chp-api/Jenkinsfile @@ -62,7 +62,6 @@ pipeline { aws --region ${AWS_REGION} eks update-kubeconfig --name ${KUBERNETES_CLUSTER_NAME} cd deploy/chp-api source prepare.sh - ls /bin/bash deploy.sh ''' } From df20564de356b097843eb0ece8d7dcaafbc286db Mon Sep 17 00:00:00 2001 From: bettyli037 Date: Wed, 28 Jun 2023 14:39:41 -0400 Subject: [PATCH 096/132] feat: update values.yaml --- deploy/chp-api/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/chp-api/values.yaml b/deploy/chp-api/values.yaml index c34c476..a4baa1d 100644 --- a/deploy/chp-api/values.yaml +++ b/deploy/chp-api/values.yaml @@ -18,7 +18,7 @@ fullnameOverride: "" # django applicaiton configuration app: - debug: "0" + debug: "1" secret_key: "" djangoCSRFTrustedOrigins: "" djangoAllowedHosts: "" From e67d5dbe0cea59a7fcf0a53d402125c4f31ff4dd Mon Sep 17 00:00:00 2001 From: Chase Yakaboski Date: Tue, 4 Jul 2023 16:16:46 -0400 Subject: [PATCH 097/132] Nextjs application support. --- compose.chp-api.yaml | 2 ++ nginx/default.conf | 16 ++++++++++++++++ nginx/nginx.conf | 10 ++++++++++ nginx/start.sh | 2 +- 4 files changed, 29 insertions(+), 1 deletion(-) diff --git a/compose.chp-api.yaml b/compose.chp-api.yaml index c4476af..c5c124e 100644 --- a/compose.chp-api.yaml +++ b/compose.chp-api.yaml @@ -11,6 +11,8 @@ services: - DJANGO_SERVER_ADDR=api:8000 - STATIC_SERVER_ADDR=static-fs:8080 - FLOWER_DASHBOARD_ADDR=dashboard:5556 + #- NEXTJS_SERVER_ADDR=chatgpt:3000 + - NEXTJS_SERVER_ADDR=api:8000 ports: - "80:80" depends_on: diff --git a/nginx/default.conf b/nginx/default.conf index 8d113d4..03c4dbc 100644 --- a/nginx/default.conf +++ b/nginx/default.conf @@ -31,6 +31,9 @@ server { proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_cache cache; + proxy_ignore_headers Cache-Control; + proxy_cache_valid 60m; } location /flower-dashboard { @@ -39,5 +42,18 @@ server { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; } + + location /_next/static { + proxy_cache cache; + proxy_pass http://$NEXTJS_SERVER_ADDR; + } + + location /api { + proxy_pass http://$NEXTJS_SERVER_ADDR; + } + + location /chat { + proxy_pass http://$NEXTJS_SERVER_ADDR/; + } } diff --git a/nginx/nginx.conf b/nginx/nginx.conf index dcd0bb8..ca4394f 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -28,10 +28,20 @@ http { # Enable Compression gzip on; + gzip_proxied any; + gzip_comp_level 4; + gzip_types text/css application/javascript image/svg+xml; # Disable Display of NGINX Version server_tokens off; + # Headers + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection 'upgrade'; + proxy_set_header Host $host; + proxy_cache_bypass $http_upgrade; + # Size Limits client_body_buffer_size 10K; client_header_buffer_size 128k; diff --git a/nginx/start.sh b/nginx/start.sh index eeb5d3a..4fea892 100644 --- a/nginx/start.sh +++ b/nginx/start.sh @@ -1,2 +1,2 @@ #!/bin/bash -envsubst '$DJANGO_SERVER_ADDR,$STATIC_SERVER_ADDR,$FLOWER_DASHBOARD_ADDR' < /tmp/default.conf > /etc/nginx/conf.d/default.conf && nginx -g 'daemon off;' +envsubst '$DJANGO_SERVER_ADDR,$STATIC_SERVER_ADDR,$FLOWER_DASHBOARD_ADDR,$NEXTJS_SERVER_ADDR' < /tmp/default.conf > /etc/nginx/conf.d/default.conf && nginx -g 'daemon off;' From b71ccd408295942b35714cf1d5e8fd35e878d17c Mon Sep 17 00:00:00 2001 From: Chase Yakaboski Date: Wed, 5 Jul 2023 23:00:35 -0400 Subject: [PATCH 098/132] Updated settings and user models. --- chp_api/chp_api/settings.py | 13 ++++++++++-- chp_api/chp_api/urls.py | 3 ++- chp_api/dispatcher/templates.py.save | 2 ++ chp_api/gennifer/views.py | 2 +- chp_api/users/serializers.py | 9 +++++++++ chp_api/users/urls.py | 7 +++++++ chp_api/users/views.py | 15 ++++++++++++-- compose.chp-api.yaml | 30 ++++++++++++++++++++++------ 8 files changed, 69 insertions(+), 12 deletions(-) create mode 100644 chp_api/dispatcher/templates.py.save create mode 100644 chp_api/users/serializers.py create mode 100644 chp_api/users/urls.py diff --git a/chp_api/chp_api/settings.py b/chp_api/chp_api/settings.py index 665beab..38952c9 100644 --- a/chp_api/chp_api/settings.py +++ b/chp_api/chp_api/settings.py @@ -31,16 +31,25 @@ 'rest_framework.parsers.JSONParser', ], 'DEFAULT_AUTHENTICATION_CLASSES': ( - 'rest_framework_simplejwt.authentication.JWTAuthentication', + #'rest_framework_simplejwt.authentication.JWTAuthentication', + 'oauth2_provider.contrib.rest_framework.OAuth2Authentication', + ), + 'DEFAULT_PERMISSION_CLASSES': ( + 'rest_framework.permissions.IsAuthenticated', ) } AUTHENTICATION_BACKENDS = [ 'oauth2_provider.backends.OAuth2Backend', # Uncomment following if you want to access the admin - #'django.contrib.auth.backends.ModelBackend', + 'django.contrib.auth.backends.ModelBackend', ] +OAUTH2_PROVIDER = { + # this is the list of available scopes + 'SCOPES': {'read': 'Read scope', 'write': 'Write scope', 'groups': 'Access to your groups'} +} + # Cors stuff (must go before installed apps) CORS_ALLOWED_ORIGINS = [ 'http://localhost', diff --git a/chp_api/chp_api/urls.py b/chp_api/chp_api/urls.py index 36727e8..5a71ddc 100644 --- a/chp_api/chp_api/urls.py +++ b/chp_api/chp_api/urls.py @@ -29,7 +29,8 @@ path('gennifer/api/', include('gennifer.urls')), path('api/token/', TokenObtainPairView.as_view(), name='token_obtain_pair'), path('api/token/refresh/', TokenRefreshView.as_view(), name='token_refresh'), - path('o/', include('oauth2_provider.urls', namespace='oauth2_provider')) + path('o/', include('oauth2_provider.urls', namespace='oauth2_provider')), + path('users/', include('users.urls')), ] diff --git a/chp_api/dispatcher/templates.py.save b/chp_api/dispatcher/templates.py.save new file mode 100644 index 0000000..f4de3a8 --- /dev/null +++ b/chp_api/dispatcher/templates.py.save @@ -0,0 +1,2 @@ +8 85.4% - 396072s +H 0 0 83.0000000 12.25038 diff --git a/chp_api/gennifer/views.py b/chp_api/gennifer/views.py index 5dd8dfb..8b4c5fb 100644 --- a/chp_api/gennifer/views.py +++ b/chp_api/gennifer/views.py @@ -52,7 +52,7 @@ class DatasetViewSet(viewsets.ModelViewSet): serializer_class = DatasetSerializer filter_backends = [DjangoFilterBackend] filterset_fields = ['user', 'zenodo_id'] - permission_classes = [IsOwnerOrReadOnly] + permission_classes = [IsAuthenticated, TokenHasReadWriteScope] def perform_create(self, serializers): diff --git a/chp_api/users/serializers.py b/chp_api/users/serializers.py new file mode 100644 index 0000000..53309e2 --- /dev/null +++ b/chp_api/users/serializers.py @@ -0,0 +1,9 @@ +from django.contrib.auth import get_user_model + +from rest_framework import serializers + + +class UserSerializer(serializers.ModelSerializer): + class Meta: + model = get_user_model() + fields = ['username', 'email', 'first_name', 'last_name'] diff --git a/chp_api/users/urls.py b/chp_api/users/urls.py new file mode 100644 index 0000000..a1c2f77 --- /dev/null +++ b/chp_api/users/urls.py @@ -0,0 +1,7 @@ +from django.urls import path, include + +from .views import UserDetails + +urlpatterns = [ + path('me/', UserDetails.as_view()) + ] diff --git a/chp_api/users/views.py b/chp_api/users/views.py index 91ea44a..8aaaf81 100644 --- a/chp_api/users/views.py +++ b/chp_api/users/views.py @@ -1,3 +1,14 @@ -from django.shortcuts import render +from rest_framework import permissions +from rest_framework.views import APIView +from rest_framework.response import Response +from oauth2_provider.contrib.rest_framework import TokenHasReadWriteScope -# Create your views here. +from .serializers import UserSerializer + + +class UserDetails(APIView): + permission_classes = [permissions.IsAuthenticated, TokenHasReadWriteScope] + + def get(self, request): + user = request.user + return Response(UserSerializer(user).data) diff --git a/compose.chp-api.yaml b/compose.chp-api.yaml index af35a16..2e9a63a 100644 --- a/compose.chp-api.yaml +++ b/compose.chp-api.yaml @@ -94,13 +94,22 @@ services: - django-superuser-email - django-superuser-password environment: - - POSTGRES_DB=chp_db - - POSTGRES_USER=postgres + - POSTGRES_DB=chpapi + - POSTGRES_USER=chpapi_user - POSTGRES_PASSWORD_FILE=/run/secrets/db-password - POSTGRES_HOST=db - POSTGRES_PORT=5432 + - DEBUG=0 + # For Helm testing purposes + #- POSTGRES_PASSWORD=31173e51d8f78b56606d06dfb66a1b126630cdf4711bed9427025d8979976f31 + #- SECRET_KEY=e1743ca40af220389cd1165d213e3d677f2d59c00d7b0f94e7a302c91f95f029 + #- DJANGO_ALLOWED_HOSTS=localhost,chp.thayer.dartmouth.edu + - CSRF_TRUSTED_ORIGINS=http://localhost,https://chp.thayer.dartmouth.edu + #- DJANGO_SUPERUSER_USERNAME=chp_admin + #- DJANGO_SUPERUSER_EMAIL=chp_admin@chp.com + #- DJANGO_SUPERUSER_PASSWORD=e12ff26f070819d9a72e317898148679680e6b3976e464b4102bd6eb18357919 - SECRET_KEY_FILE=/run/secrets/django-key - - DEBUG=1 + #- CSRF_TRUSTED_ORIGINS_FILE=/run/secrets/csrf-trusted-origins - DJANGO_ALLOWED_HOSTS_FILE=/run/secrets/allowed-hosts - DJANGO_SUPERUSER_USERNAME_FILE=/run/secrets/django-superuser-username - DJANGO_SUPERUSER_EMAIL_FILE=/run/secrets/django-superuser-email @@ -124,13 +133,22 @@ services: - django-superuser-email - django-superuser-password environment: - - POSTGRES_DB=chp_db - - POSTGRES_USER=postgres + - POSTGRES_DB=chpapi + - POSTGRES_USER=chpapi_user - POSTGRES_PASSWORD_FILE=/run/secrets/db-password - POSTGRES_HOST=db - POSTGRES_PORT=5432 + - DEBUG=0 + # For Helm testing purposes + #- POSTGRES_PASSWORD=31173e51d8f78b56606d06dfb66a1b126630cdf4711bed9427025d8979976f31 + #- SECRET_KEY=e1743ca40af220389cd1165d213e3d677f2d59c00d7b0f94e7a302c91f95f029 + #- DJANGO_ALLOWED_HOSTS=localhost,chp.thayer.dartmouth.edu + - CSRF_TRUSTED_ORIGINS=http://localhost,https://chp.thayer.dartmouth.edu + #- DJANGO_SUPERUSER_USERNAME=chp_admin + #- DJANGO_SUPERUSER_EMAIL=chp_admin@chp.com + #- DJANGO_SUPERUSER_PASSWORD=e12ff26f070819d9a72e317898148679680e6b3976e464b4102bd6eb18357919 - SECRET_KEY_FILE=/run/secrets/django-key - - DEBUG=1 + #- CSRF_TRUSTED_ORIGINS_FILE=/run/secrets/csrf-trusted-origins - DJANGO_ALLOWED_HOSTS_FILE=/run/secrets/allowed-hosts - DJANGO_SUPERUSER_USERNAME_FILE=/run/secrets/django-superuser-username - DJANGO_SUPERUSER_EMAIL_FILE=/run/secrets/django-superuser-email From f29acc9ccc5d8a2aaf2a60052fd02a892d2ce611 Mon Sep 17 00:00:00 2001 From: Chase Yakaboski Date: Mon, 10 Jul 2023 17:53:30 -0400 Subject: [PATCH 099/132] Added Hyperparameters and updated Task. --- chp_api/chp_api/settings.py | 2 +- chp_api/gennifer/admin.py | 4 +- .../0010_hyperparameter_and_more.py | 57 ++++++++ chp_api/gennifer/models.py | 51 ++++++- chp_api/gennifer/scripts/algorithm_loader.py | 12 +- chp_api/gennifer/serializers.py | 75 ++++++++-- chp_api/gennifer/tasks.py | 102 +++++++------- chp_api/gennifer/urls.py | 7 +- chp_api/gennifer/views.py | 129 +++++++++++------- 9 files changed, 322 insertions(+), 117 deletions(-) create mode 100755 chp_api/gennifer/migrations/0010_hyperparameter_and_more.py diff --git a/chp_api/chp_api/settings.py b/chp_api/chp_api/settings.py index 38952c9..da9d796 100644 --- a/chp_api/chp_api/settings.py +++ b/chp_api/chp_api/settings.py @@ -35,7 +35,7 @@ 'oauth2_provider.contrib.rest_framework.OAuth2Authentication', ), 'DEFAULT_PERMISSION_CLASSES': ( - 'rest_framework.permissions.IsAuthenticated', + 'rest_framework.permissions.IsAuthenticatedOrReadOnly', ) } diff --git a/chp_api/gennifer/admin.py b/chp_api/gennifer/admin.py index 51303f2..088d222 100644 --- a/chp_api/gennifer/admin.py +++ b/chp_api/gennifer/admin.py @@ -1,8 +1,10 @@ from django.contrib import admin -from .models import Algorithm, Dataset, Study, Task, Result, Gene, UserAnalysisSession +from .models import Algorithm, Dataset, Study, Task, Result, Gene, UserAnalysisSession, AlgorithmInstance, Hyperparameter admin.site.register(Algorithm) +admin.site.register(AlgorithmInstance) +admin.site.register(Hyperparameter) admin.site.register(Dataset) admin.site.register(Study) admin.site.register(Task) diff --git a/chp_api/gennifer/migrations/0010_hyperparameter_and_more.py b/chp_api/gennifer/migrations/0010_hyperparameter_and_more.py new file mode 100755 index 0000000..1f0d671 --- /dev/null +++ b/chp_api/gennifer/migrations/0010_hyperparameter_and_more.py @@ -0,0 +1,57 @@ +# Generated by Django 4.2.3 on 2023-07-10 19:16 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('gennifer', '0009_task_user_study_user_result_target_result_task_and_more'), + ] + + operations = [ + migrations.CreateModel( + name='Hyperparameter', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=128)), + ('type', models.CharField(choices=[('int', 'Integer'), ('bool', 'Boolean'), ('str', 'String'), ('float', 'Float')], default='float', max_length=5)), + ('info', models.TextField(blank=True, null=True)), + ('algorithm', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='hyperparameters', to='gennifer.algorithm')), + ], + ), + migrations.RenameField( + model_name='task', + old_name='avg_study_edge_weight', + new_name='avg_task_edge_weight', + ), + migrations.RenameField( + model_name='task', + old_name='max_study_edge_weight', + new_name='max_task_edge_weight', + ), + migrations.RenameField( + model_name='task', + old_name='min_study_edge_weight', + new_name='min_task_edge_weight', + ), + migrations.RenameField( + model_name='task', + old_name='std_study_edge_weight', + new_name='std_task_edge_weight', + ), + migrations.RemoveField( + model_name='algorithminstance', + name='hyperparameters', + ), + migrations.CreateModel( + name='HyperparameterInstance', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('value_str', models.CharField(max_length=128)), + ('algorithm_instance', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='hyperparameters', to='gennifer.algorithminstance')), + ('hyperparameter', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='instances', to='gennifer.hyperparameter')), + ], + ), + ] diff --git a/chp_api/gennifer/models.py b/chp_api/gennifer/models.py index 3e38f11..a025099 100644 --- a/chp_api/gennifer/models.py +++ b/chp_api/gennifer/models.py @@ -34,16 +34,55 @@ def __str__(self): class AlgorithmInstance(models.Model): algorithm = models.ForeignKey(Algorithm, on_delete=models.CASCADE, related_name='instances') - hyperparameters = models.JSONField(null=True) def __str__(self): if self.hyperparameters: - hypers = tuple([f'{k}={v}' for k, v in self.hyperparameters.items()]) + hypers = tuple([f'{h}' for h in self.hyperparameters]) else: hypers = '()' return f'{self.algorithm.name}{hypers}' +class Hyperparameter(models.Model): + INT = "int" + BOOL = "bool" + STR = "str" + FLOAT = "float" + TYPE_CHOICES = ( + (INT, "Integer"), + (BOOL, "Boolean"), + (STR, "String"), + (FLOAT, "Float"), + ) + name = models.CharField(max_length=128) + type = models.CharField(max_length=5, choices=TYPE_CHOICES, default=FLOAT) + algorithm = models.ForeignKey(Algorithm, on_delete=models.CASCADE, related_name='hyperparameters') + info = models.TextField(null=True, blank=True) + + def get_type(self): + known_types = { + "int": int, + "bool": bool, + "str": str, + "float": float, + } + return known_types[self.type] + + def __str__(self): + return self.name + + +class HyperparameterInstance(models.Model): + hyperparameter = models.ForeignKey(Hyperparameter, on_delete=models.CASCADE, related_name='instances') + value_str = models.CharField(max_length=128) + algorithm_instance = models.ForeignKey(AlgorithmInstance, on_delete=models.CASCADE, related_name='hyperparameters') + + def get_value(self): + return self.hyperparameter.get_type()(self.value_str) + + def __str__(self): + return f'{self.hyperparameter.name}={self.value}' + class Dataset(models.Model): title = models.CharField(max_length=128) zenodo_id = models.CharField(max_length=128, primary_key=True) @@ -100,10 +139,10 @@ class Task(models.Model): dataset = models.ForeignKey(Dataset, on_delete=models.CASCADE, related_name='tasks') timestamp = models.DateTimeField(auto_now_add=True) # Study characteristics for all edge weights in a given study over a dataset - max_study_edge_weight = models.FloatField(null=True) - min_study_edge_weight = models.FloatField(null=True) - avg_study_edge_weight = models.FloatField(null=True) - std_study_edge_weight = models.FloatField(null=True) + max_task_edge_weight = models.FloatField(null=True) + min_task_edge_weight = models.FloatField(null=True) + avg_task_edge_weight = models.FloatField(null=True) + std_task_edge_weight = models.FloatField(null=True) is_public = models.BooleanField(default=False) status = models.CharField(max_length=10) error_message = models.TextField(null=True, blank=True) diff --git a/chp_api/gennifer/scripts/algorithm_loader.py b/chp_api/gennifer/scripts/algorithm_loader.py index 1236632..9a9dab8 100644 --- a/chp_api/gennifer/scripts/algorithm_loader.py +++ b/chp_api/gennifer/scripts/algorithm_loader.py @@ -1,7 +1,7 @@ import requests from django.conf import settings -from ..models import Algorithm +from ..models import Algorithm, Hyperparameter def run(): @@ -17,3 +17,13 @@ def run(): directed=algo_info["directed"], ) algo.save() + # Load Hyperparameters + if algo_info["hyperparameters"]: + for hp_name, hp_info in algo_info["hyperparameters"].items(): + hp = Hyperparameter.objects.create( + name=hp_name, + type=getattr(Hyperparameter, hp_info["type"]), + algorithm=algo, + info=hp_info["info"], + ) + hp.save() diff --git a/chp_api/gennifer/serializers.py b/chp_api/gennifer/serializers.py index a3932d4..d310202 100644 --- a/chp_api/gennifer/serializers.py +++ b/chp_api/gennifer/serializers.py @@ -1,6 +1,17 @@ from rest_framework import serializers -from .models import Dataset, Study, Task, Result, Algorithm, Gene, UserAnalysisSession +from .models import ( + Dataset, + Study, + Task, + Result, + Algorithm, + Gene, + UserAnalysisSession, + AlgorithmInstance, + Hyperparameter, + HyperparameterInstance, + ) class UserAnalysisSessionSerializer(serializers.ModelSerializer): @@ -11,13 +22,14 @@ class Meta: class DatasetSerializer(serializers.ModelSerializer): class Meta: model = Dataset - fields = ['title', 'zenodo_id', 'doi', 'description'] - read_only_fields = ['title', 'doi', 'description'] + fields = ['pk', 'title', 'zenodo_id', 'doi', 'description'] + read_only_fields = ['pk', 'title', 'doi', 'description'] class StudySerializer(serializers.ModelSerializer): class Meta: model = Study - fields = ['name', 'status', 'description', 'timestamp', 'user'] + fields = ['pk', 'name', 'status', 'description', 'timestamp', 'user', 'tasks'] + read_only_fields = ['pk', 'status'] class TaskSerializer(serializers.ModelSerializer): @@ -33,14 +45,23 @@ class Meta: 'algorithm_instance', 'dataset', 'timestamp', - 'max_study_edge_weight', - 'min_study_edge_weight', - 'avg_study_edge_weight', - 'std_study_edge_weight', + 'max_task_edge_weight', + 'min_task_edge_weight', + 'avg_task_edge_weight', + 'std_task_edge_weight', 'name', 'study', 'status', ] + read_only_fields = [ + 'pk', + 'max_task_edge_weight', + 'min_task_edge_weight', + 'avg_task_edge_weight', + 'std_task_edge_weight', + 'name', + 'status', + ] class ResultSerializer(serializers.ModelSerializer): class Meta: @@ -64,6 +85,44 @@ class Meta: 'directed', ] +class AlgorithmInstanceSerializer(serializers.ModelSerializer): + class Meta: + model = AlgorithmInstance + fields = [ + 'pk', + 'algorithm', + ] + read_only_fields = ['pk'] + + def create(self, validated_data): + instance, _ = AlgorithmInstance.objects.get_or_create(**validated_data) + return instance + +class HyperparameterSerializer(serializers.ModelSerializer): + class Meta: + model = Hyperparameter + fields = [ + 'pk', + 'name', + 'algorithm', + 'type', + ] + read_only_fields = ['pk', 'name', 'algorithm', 'type'] + +class HyperparameterInstanceSerializer(serializers.ModelSerializer): + class Meta: + model = HyperparameterInstance + fields = [ + 'pk', + 'algorithm_instance', + 'value_str', + 'hyperparameter', + ] + read_only_fields = ['pk'] + + def create(self, validated_data): + instance, _ = HyperparameterInstance.objects.get_or_create(**validated_data) + return instance class GeneSerializer(serializers.ModelSerializer): class Meta: diff --git a/chp_api/gennifer/tasks.py b/chp_api/gennifer/tasks.py index cf1a088..560b143 100644 --- a/chp_api/gennifer/tasks.py +++ b/chp_api/gennifer/tasks.py @@ -128,53 +128,61 @@ def return_saved_task(tasks, user): result.save() return True - @shared_task(name="create_gennifer_task") -def create_task(algorithm_name, zenodo_id, hyperparameters, user_pk, study_pk): - # Get algorithm obj - algo = Algorithm.objects.get(name=algorithm_name) - - # Get or create a new algorithm instance based on the hyperparameters - if not hyperparameters: - algo_instance, algo_instance_created = AlgorithmInstance.objects.get_or_create( - algorithm=algo, - hyperparameters__isnull=True, - ) - else: - algo_instance, algo_instance_created = AlgorithmInstance.objects.get_or_create( - algorithm=algo, - hyperparameters=hyperparameters, - ) +def create_task(task_pk): + # Get task + task = Task.objects.get(pk=task_pk) + algo = task.algorithm_instance.algorithm + user = task.user + ## Get algorithm obj + #algo = Algorithm.objects.get(name=algorithm_name) + + ## Get or create a new algorithm instance based on the hyperparameters + #if not hyperparameters: + # algo_instance, algo_instance_created = AlgorithmInstance.objects.get_or_create( + # algorithm=algo, + # hyperparameters__isnull=True, + # ) + #else: + # algo_instance, algo_instance_created = AlgorithmInstance.objects.get_or_create( + # algorithm=algo, + # hyperparameters=hyperparameters, + # ) # Get User obj - user = User.objects.get(pk=user_pk) + #user = User.objects.get(pk=user_pk) # Get Study obj - study = Study.objects.get(pk=study_pk) + #study = Study.objects.get(pk=study_pk) # Initialize dataset instance - dataset, dataset_created = Dataset.objects.get_or_create( - zenodo_id=zenodo_id, - user=user, - ) - - if dataset_created: - dataset.save() - - if not algo_instance_created and not dataset_created: - # This means we've already run the task. So let's just return that and not bother our workers. - tasks = Task.objects.filter( - algorithm_instance=algo_instance, - dataset=dataset, - status='SUCCESS', - ) - #TODO: Probably should add some timestamp handling here - if len(studies) > 0: - return_saved_task(tasks, user) - + #dataset, dataset_created = Dataset.objects.get_or_create( + # zenodo_id=zenodo_id, + # user=user, + # ) + + #if dataset_created: + # dataset.save() + + #if not algo_instance_created and not dataset_created: + # # This means we've already run the task. So let's just return that and not bother our workers. + # tasks = Task.objects.filter( + # algorithm_instance=algo_instance, + # dataset=dataset, + # status='SUCCESS', + # ) + # #TODO: Probably should add some timestamp handling here + # if len(studies) > 0: + # return_saved_task(tasks, user) + + # Create Hyperparameter serialization + hyperparameters = {} + for h in task.algorithm_instance.hyperparameters.all(): + hyperparameters[h.hyperparameter.name] = h.get_value() + # Send to gennifer app gennifer_request = { - "zenodo_id": zenodo_id, + "zenodo_id": task.dataset.zenodo_id, "hyperparameters": hyperparameters, } task_id = requests.post(f'{algo.url}/run', json=gennifer_request).json()["task_id"] @@ -184,16 +192,16 @@ def create_task(algorithm_name, zenodo_id, hyperparameters, user_pk, study_pk): # Get initial status status = get_status(algo, task_id) - # Create Inference Study - task = Task.objects.create( - algorithm_instance=algo_instance, - user=user, - dataset=dataset, - status=status["task_status"], - study=study, - ) + + #task = Task.objects.create( + # algorithm_instance=algo_instance, + # user=user, + # dataset=dataset, + # status=status["task_status"], + # study=study, + # ) # Save initial task - task.save() + #task.save() # Enter a loop to keep checking back in and populate the task once it has completed. #TODO: Not sure if this is best practice diff --git a/chp_api/gennifer/urls.py b/chp_api/gennifer/urls.py index fee431c..4d9671e 100644 --- a/chp_api/gennifer/urls.py +++ b/chp_api/gennifer/urls.py @@ -10,11 +10,14 @@ router.register(r'tasks', views.TaskViewSet, basename='task') router.register(r'results', views.ResultViewSet, basename='result') router.register(r'algorithms', views.AlgorithmViewSet, basename='algorithm') +router.register(r'algorithm_instances', views.AlgorithmInstanceViewSet, basename='algorithm_instance') +router.register(r'hyperparameters', views.HyperparameterViewSet, basename='hyperparameter') +router.register(r'hyperparameter_instances', views.HyperparameterInstanceViewSet, basename='hyperparameter_instance') router.register(r'genes', views.GeneViewSet, basename='genes') router.register(r'analyses', views.UserAnalysisSessionViewSet, basename='analyses') urlpatterns = [ path('', include(router.urls)), - path('run', views.run.as_view()), - path('graph', views.CytoscapeView.as_view()), + path('run/', views.run.as_view()), + path('graph/', views.CytoscapeView.as_view()), ] diff --git a/chp_api/gennifer/views.py b/chp_api/gennifer/views.py index 8b4c5fb..a795507 100644 --- a/chp_api/gennifer/views.py +++ b/chp_api/gennifer/views.py @@ -11,7 +11,7 @@ from rest_framework.permissions import IsAuthenticated, IsAuthenticatedOrReadOnly from rest_framework.exceptions import ValidationError from django_filters.rest_framework import DjangoFilterBackend -from oauth2_provider.contrib.rest_framework import TokenHasReadWriteScope +#from oauth2_provider.contrib.rest_framework import TokenHasReadWriteScope, TokenHasScope from .models import ( Dataset, @@ -20,7 +20,10 @@ Result, Algorithm, Gene, - UserAnalysisSession + UserAnalysisSession, + AlgorithmInstance, + Hyperparameter, + HyperparameterInstance ) from .serializers import ( DatasetSerializer, @@ -29,7 +32,10 @@ ResultSerializer, AlgorithmSerializer, GeneSerializer, - UserAnalysisSessionSerializer + UserAnalysisSessionSerializer, + AlgorithmInstanceSerializer, + HyperparameterSerializer, + HyperparameterInstanceSerializer, ) from .tasks import create_task from .permissions import IsOwnerOrReadOnly, IsAdminOrReadOnly @@ -39,7 +45,7 @@ class UserAnalysisSessionViewSet(viewsets.ModelViewSet): serializer_class = UserAnalysisSessionSerializer #filter_backends = [DjangoFilterBackend] #filterset_fields = ['id', 'name', 'is_saved'] - permission_classes = [IsOwnerOrReadOnly, IsAuthenticated] + permission_classes = [IsAuthenticated, IsOwnerOrReadOnly]#, TokenHasReadWriteScope] def get_queryset(self): user = self.request.user @@ -52,8 +58,7 @@ class DatasetViewSet(viewsets.ModelViewSet): serializer_class = DatasetSerializer filter_backends = [DjangoFilterBackend] filterset_fields = ['user', 'zenodo_id'] - permission_classes = [IsAuthenticated, TokenHasReadWriteScope] - + permission_classes = [IsAuthenticated, IsOwnerOrReadOnly]#, TokenHasReadWriteScope] def perform_create(self, serializers): try: @@ -62,42 +67,77 @@ def perform_create(self, serializers): raise ValidationError(str(e)) class StudyViewSet(viewsets.ModelViewSet): - queryset = Study.objects.all() + #queryset = Study.objects.all() serializer_class = StudySerializer - permission_classes = [IsOwnerOrReadOnly] + permission_classes = [IsAuthenticated, IsOwnerOrReadOnly]#, TokenHasReadWriteScope] + + def get_queryset(self): + user = self.request.user + return Study.objects.filter(user=user) + + def perform_create(self, serializers): + try: + serializers.save(user=self.request.user, status='RECEIVED') + except ValueError as e: + raise ValidationError(str(e)) class TaskViewSet(viewsets.ModelViewSet): - queryset = Task.objects.all() + #queryset = Task.objects.all() serializer_class = TaskSerializer filter_backends = [DjangoFilterBackend] filterset_fields = ['is_public', 'dataset', 'algorithm_instance'] - permission_classes = [IsOwnerOrReadOnly] + permission_classes = [IsAuthenticated, IsOwnerOrReadOnly]#, TokenHasReadWriteScope] - #def get_queryset(self): - # user = self.request.user - # return InferenceStudy.objects.filter(user=user) + def get_queryset(self): + user = self.request.user + return Task.objects.filter(user=user) + + def perform_create(self, serializers): + try: + serializers.save(user=self.request.user, status='RECEIVED') + except ValueError as e: + raise ValidationError(str(e)) class ResultViewSet(viewsets.ModelViewSet): - queryset = Result.objects.all() + #queryset = Result.objects.all() serializer_class = ResultSerializer filter_backends = [DjangoFilterBackend] filterset_fields = ['is_public', 'task', 'tf', 'target'] - permission_classes = [IsOwnerOrReadOnly] + permission_classes = [IsAuthenticated, IsOwnerOrReadOnly]#, TokenHasReadWriteScope] - #def get_queryset(self): - # user = self.request.user - # return InferenceResult.objects.filter(user=user) + def get_queryset(self): + user = self.request.user + return Result.objects.filter(user=user) class AlgorithmViewSet(viewsets.ModelViewSet): serializer_class = AlgorithmSerializer queryset = Algorithm.objects.all() - permissions = [IsAdminOrReadOnly] + permission_classes = [IsAuthenticated, IsAdminOrReadOnly]#, TokenHasReadWriteScope] + #required_scopes = ['read'] + +class AlgorithmInstanceViewSet(viewsets.ModelViewSet): + queryset = AlgorithmInstance.objects.all() + serializer_class = AlgorithmInstanceSerializer + permission_classes = [IsAuthenticated, IsOwnerOrReadOnly]#, TokenHasReadWriteScope] + +class HyperparameterViewSet(viewsets.ModelViewSet): + serializer_class = HyperparameterSerializer + queryset = Hyperparameter.objects.all() + filter_backends = [DjangoFilterBackend] + filterset_fields = ['algorithm'] + permission_classes = [IsAuthenticated, IsAdminOrReadOnly]#, TokenHasReadWriteScope] + #required_scopes = ['read'] + +class HyperparameterInstanceViewSet(viewsets.ModelViewSet): + queryset = HyperparameterInstance.objects.all() + serializer_class = HyperparameterInstanceSerializer + permission_classes = [IsAuthenticated, IsOwnerOrReadOnly]#, TokenHasReadWriteScope] class GeneViewSet(viewsets.ModelViewSet): serializer_class = GeneSerializer queryset = Gene.objects.all() - permissions = [IsAdminOrReadOnly] + permission_classes = [IsAuthenticated, IsAdminOrReadOnly]#, TokenHasReadWriteScope] class CytoscapeView(APIView): @@ -258,39 +298,26 @@ class run(APIView): def post(self, request): """ Request comes in as a list of algorithms to run. """ - # Create study - study = Study.objects.create( - name = request.data['name'], - description = request.data.get('description', None), - status = 'RECIEVED', - user = request.user, - ) + study_id = request.data.get("study_id", None) + if not study_id: + return JsonResponse({"error": 'Must pass a study_id.'}) + response = { + "study_id": study_id, + "task_status": [], + } + # Get study + try: + study = Study.objects.get(pk=study_id, user=request.user) + except ObjectDoesNotExist: + response["error"] = 'The study does not exist for request user.' + return JsonResponse(response) + # Set Study Status to Started. + study.status = 'STARTED' study.save() # Build gennifer requests - tasks = request.data['tasks'] - response = {"study_id": study.pk, "tasks": []} + tasks = Task.objects.filter(study=study) for task in tasks: - algorithm_name = task.get("algorithm_name", None) - zenodo_id = task.get("zenodo_id", None) - hyperparameters = task.get("hyperparameters", None) - if hyperparameters: - if len(hyperparameters) == 0: - hyperparameters = None - if not algorithm_name: - task["error"] = "No algorithm name provided." - response["tasks"].append(task) - continue - if not zenodo_id: - task["error"] = "No dataset Zenodo identifer provided." - response["tasks"].append(task) - continue - try: - algo = Algorithm.objects.get(name=algorithm_name) - except ObjectDoesNotExist: - task["error"] = f"The algorithm: {algorithm_name} is not supported in Gennifer." - response["tasks"].append(task) - continue # If all pass, now send to gennifer services - task["task_id"] = create_task.delay(algo.name, zenodo_id, hyperparameters, request.user.pk, study.pk).id - response["tasks"].append(task) + task_id = create_task.delay(task.pk).id + response["task_status"].append(task_id) return JsonResponse(response) From 934aabdd6745b3e8484622373c6927ae2214704e Mon Sep 17 00:00:00 2001 From: Chase Yakaboski Date: Fri, 28 Jul 2023 13:58:35 -0400 Subject: [PATCH 100/132] All five algorithms work as expected. --- chp_api/chp_api/settings.py | 3 + chp_api/gennifer/models.py | 5 +- chp_api/gennifer/serializers.py | 19 ++++- chp_api/gennifer/tasks.py | 6 ++ chp_api/gennifer/urls.py | 1 + chp_api/gennifer/views.py | 137 +++++++++++++++++++++++++------- compose.chp-api.yaml | 4 +- compose.gennifer.yaml | 54 +++++++++++++ deployment-script | 2 +- nginx/default.conf | 18 ++++- 10 files changed, 211 insertions(+), 38 deletions(-) diff --git a/chp_api/chp_api/settings.py b/chp_api/chp_api/settings.py index da9d796..3ae79b7 100644 --- a/chp_api/chp_api/settings.py +++ b/chp_api/chp_api/settings.py @@ -247,4 +247,7 @@ GENNIFER_ALGORITHM_URLS = [ "http://pidc:5000", "http://grisli:5000", + "http://genie3:5000", + "http://grnboost2:5000", + "http://bkb-grn:5000", ] diff --git a/chp_api/gennifer/models.py b/chp_api/gennifer/models.py index a025099..d506c26 100644 --- a/chp_api/gennifer/models.py +++ b/chp_api/gennifer/models.py @@ -37,7 +37,7 @@ class AlgorithmInstance(models.Model): def __str__(self): if self.hyperparameters: - hypers = tuple([f'{h}' for h in self.hyperparameters]) + hypers = tuple([f'{h}' for h in self.hyperparameters.all()]) else: hypers = '()' return f'{self.algorithm.name}{hypers}' @@ -81,7 +81,7 @@ def get_value(self): return self.hyperparameter.get_type()(self.value_str) def __str__(self): - return f'{self.hyperparameter.name}={self.value}' + return f'{self.hyperparameter.name}={self.value_str}' class Dataset(models.Model): title = models.CharField(max_length=128) @@ -133,6 +133,7 @@ class Meta: def __str__(self): return self.name + class Task(models.Model): algorithm_instance = models.ForeignKey(AlgorithmInstance, on_delete=models.CASCADE, related_name='tasks') user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, null=True, blank=True, related_name='tasks') diff --git a/chp_api/gennifer/serializers.py b/chp_api/gennifer/serializers.py index d310202..1e5e745 100644 --- a/chp_api/gennifer/serializers.py +++ b/chp_api/gennifer/serializers.py @@ -1,3 +1,4 @@ +from collections import defaultdict from rest_framework import serializers from .models import ( @@ -26,10 +27,24 @@ class Meta: read_only_fields = ['pk', 'title', 'doi', 'description'] class StudySerializer(serializers.ModelSerializer): + task_status = serializers.SerializerMethodField('get_task_status') + + def get_task_status(self, study): + status = defaultdict(int) + for task in study.tasks.all(): + status[task.status] += 1 + status = dict(status) + status = sorted([f'{count} {state}'.title() for state, count in status.items()]) + if len(status) == 0: + return '' + elif len(status) == 1: + return status[0] + return ' and '.join([', '.join(status[:-1]), status[-1]]) + class Meta: model = Study - fields = ['pk', 'name', 'status', 'description', 'timestamp', 'user', 'tasks'] - read_only_fields = ['pk', 'status'] + fields = ['pk', 'name', 'status', 'task_status', 'description', 'timestamp', 'user', 'tasks'] + read_only_fields = ['pk', 'status', 'task_status'] class TaskSerializer(serializers.ModelSerializer): diff --git a/chp_api/gennifer/tasks.py b/chp_api/gennifer/tasks.py index 560b143..25f3b4d 100644 --- a/chp_api/gennifer/tasks.py +++ b/chp_api/gennifer/tasks.py @@ -73,12 +73,18 @@ def save_inference_task(task, status, failed=False): except TypeError: gene1_name = 'Not found in SRI Node Normalizer.' gene1_chp_preferred_curie = None + except KeyError: + _, gene1_name = res[gene1]["id"]["identifier"].split(':') + gene1_chp_preferred_curie = get_chp_preferred_curie(res[gene1]) try: gene2_name = res[gene2]["id"]["label"] gene2_chp_preferred_curie = get_chp_preferred_curie(res[gene2]) except TypeError: gene2_name = 'Not found in SRI Node Normalizer.' gene2_chp_preferred_curie = None + except KeyError: + _, gene2_name = res[gene2]["id"]["identifier"].split(':') + gene2_chp_preferred_curie = get_chp_preferred_curie(res[gene2]) gene1_obj, created = Gene.objects.get_or_create( name=gene1_name, curie=gene1, diff --git a/chp_api/gennifer/urls.py b/chp_api/gennifer/urls.py index 4d9671e..3e380c0 100644 --- a/chp_api/gennifer/urls.py +++ b/chp_api/gennifer/urls.py @@ -20,4 +20,5 @@ path('', include(router.urls)), path('run/', views.run.as_view()), path('graph/', views.CytoscapeView.as_view()), + path('download_study/', views.StudyDownloadView.as_view()) ] diff --git a/chp_api/gennifer/views.py b/chp_api/gennifer/views.py index a795507..64f7fd9 100644 --- a/chp_api/gennifer/views.py +++ b/chp_api/gennifer/views.py @@ -136,12 +136,23 @@ class HyperparameterInstanceViewSet(viewsets.ModelViewSet): class GeneViewSet(viewsets.ModelViewSet): serializer_class = GeneSerializer - queryset = Gene.objects.all() + #queryset = Gene.objects.all() permission_classes = [IsAuthenticated, IsAdminOrReadOnly]#, TokenHasReadWriteScope] - - -class CytoscapeView(APIView): - + + def get_queryset(self): + user = self.request.user + # Get user results + results = Result.objects.filter(user=user) + tf_genes = Gene.objects.filter(inference_result_tf__pk__in=results) + target_genes = Gene.objects.filter(inference_result_target__pk__in=results) + genes_union = tf_genes.union(target_genes) + print(len(genes_union)) + return genes_union + +class CytoscapeHandler: + def __init__(self, results): + self.results = results + def construct_node(self, gene_obj): if gene_obj.variant: name = f'{gene_obj.name}({gene_obj.variant})' @@ -164,7 +175,7 @@ def construct_node(self, gene_obj): def construct_edge(self, res, source_id, target_id): # Normalize edge weight based on the study - normalized_weight = (res.edge_weight - res.task.min_study_edge_weight) / (res.task.max_study_edge_weight - res.task.min_study_edge_weight) + normalized_weight = (res.edge_weight - res.task.min_task_edge_weight) / (res.task.max_task_edge_weight - res.task.min_task_edge_weight) directed = res.task.algorithm_instance.algorithm.directed edge_tuple = tuple(sorted([source_id, target_id])) edge = { @@ -202,14 +213,14 @@ def add(self, res, nodes, edges, processed_node_ids, processed_undirected_edges) pass return nodes, edges, processed_node_ids, processed_undirected_edges - def construct_cytoscape_data(self, results): + def construct_cytoscape_data(self): nodes = [] edges = [] processed_node_ids = set() processed_undirected_edges = set() elements = [] # Construct graph - for res in results: + for res in self.results: nodes, edges, processed_node_ids, processed_undirected_edges = self.add( res, nodes, @@ -223,21 +234,26 @@ def construct_cytoscape_data(self, results): "elements": elements } + +class CytoscapeView(APIView): + + def get(self, request): results = Result.objects.all() - cyto = self.construct_cytoscape_data(results) + cyto_handler = CytoscapeHandler(results) + cyto = cyto_handler.construct_cytoscape_data() return JsonResponse(cyto) def post(self, request): elements = [] gene_ids = request.data.get("gene_ids", None) - task_ids = request.data.get("task_ids", None) + study_ids = request.data.get("study_ids", None) algorithm_ids = request.data.get("algorithm_ids", None) dataset_ids = request.data.get("dataset_ids", None) cached_inference_result_ids = request.data.get("cached_results", None) if not (study_ids and gene_ids) and not (algorithm_ids and dataset_ids and gene_ids): - return JsonResponse({"elements": elements}) + return JsonResponse({"elements": elements, "result_ids": []}) # Create Filter filters = [] @@ -249,9 +265,11 @@ def post(self, request): ] ) if study_ids: - filters.append({"field": 'task__pk', "operator": 'in', "value": study_ids}) + tasks = Task.objects.filter(study__pk__in=study_ids) + task_ids = [task.pk for task in tasks] + filters.append({"field": 'task__pk', "operator": 'in', "value": task_ids}) if algorithm_ids: - filters.append({"field": 'task__algorithm_instance__algorithm__pk', "operator": 'in', "value": study_ids}) + filters.append({"field": 'task__algorithm_instance__algorithm__pk', "operator": 'in', "value": algorithm_ids}) if dataset_ids: filters.append({"field": 'task__dataset__zenodo_id', "operator": 'in', "value": dataset_ids}) @@ -267,30 +285,89 @@ def post(self, request): results = Result.objects.filter(query) if len(results) == 0: - return JsonResponse({"elements": elements}) + return JsonResponse({"elements": elements, "result_ids": []}) # Exclude results that have already been sent to user if cached_inference_result_ids: - logs.append('filtering') results = results.exclude(pk__in=cached_inference_result_ids) - nodes = [] - edges = [] - processed_node_ids = set() - processed_undirected_edges = set() - for res in results: - nodes, edges, processed_node_ids, processed_undirected_edges = self.add( - res, - nodes, - edges, - processed_node_ids, - processed_undirected_edges, - ) - elements.extend(nodes) - elements.extend(edges) - return JsonResponse({"elements": elements}) + # Capture result_ids + result_ids = [res.pk for res in results] + + # Initialize Cytoscape Handler + cyto_handler = CytoscapeHandler(results) + elements_dict = cyto_handler.construct_cytoscape_data() + + elements_dict["result_ids"] = result_ids + return JsonResponse(elements_dict) +class StudyDownloadView(APIView): + permission_classes = [IsAuthenticated] + def get(self, request, study_id=None): + if not study_id: + return JsonResponse({"detail": 'No study ID was passed.'}) + # Set response + response = { + "study_id": study_id, + } + # Get study + try: + study = Study.objects.get(pk=study_id, user=request.user) + except ObjectDoesNotExist: + response["error"] = 'The study does not exist for request user.' + return JsonResponse(response) + + response["description"] = study.description + response["status"] = study.status + response["tasks"] = [] + + # Get tasks assocaited with study + tasks = Task.objects.filter(study=study) + + # Collect task information + for task in tasks: + task_json = {} + # Collect task information + task_json["max_task_edge_weight"] = task.max_task_edge_weight + task_json["min_task_edge_weight"] = task.min_task_edge_weight + task_json["avg_task_edge_weight"] = task.avg_task_edge_weight + task_json["std_task_edge_weight"] = task.std_task_edge_weight + task_json["status"] = task.status + # Collect algo hyperparameters + hyper_instance_objs = task.algorithm_instance.hyperparameters.all() + hypers = {} + for hyper in hyper_instance_objs: + hypers[hyper.hyperparameter.name] = { + "value": hyper.value_str, + "info": hyper.hyperparameter.info + } + # Collect Algorithm instance information + task_json["algorithm"] = { + "name": task.algorithm_instance.algorithm.name, + "description": task.algorithm_instance.algorithm.description, + "edge_weight_description": task.algorithm_instance.algorithm.edge_weight_description, + "edge_weight_type": task.algorithm_instance.algorithm.edge_weight_type, + "directed": task.algorithm_instance.algorithm.directed, + "hyperparameters": hypers + } + # Collect Dataset information + task_json["dataset"] = { + "title": task.dataset.title, + "zenodo_id": task.dataset.zenodo_id, + "description": task.dataset.description, + } + # Build cytoscape graph + if task.status == 'SUCCESS': + results = Result.objects.filter(task=task) + cyto_handler = CytoscapeHandler(results) + task_json["graph"] = cyto_handler.construct_cytoscape_data() + else: + task_json["graph"] = None + response["tasks"].append(task_json) + + return JsonResponse(response) + class run(APIView): permission_classes = [IsAuthenticated] diff --git a/compose.chp-api.yaml b/compose.chp-api.yaml index 2e9a63a..be95e4d 100644 --- a/compose.chp-api.yaml +++ b/compose.chp-api.yaml @@ -11,8 +11,8 @@ services: - DJANGO_SERVER_ADDR=api:8000 - STATIC_SERVER_ADDR=static-fs:8080 - FLOWER_DASHBOARD_ADDR=dashboard:5556 - #- NEXTJS_SERVER_ADDR=chatgpt:3000 - - NEXTJS_SERVER_ADDR=api:8000 + - NEXTJS_SERVER_ADDR=frontend:3000 + #- NEXTJS_SERVER_ADDR=api:8000 ports: - "80:80" depends_on: diff --git a/compose.gennifer.yaml b/compose.gennifer.yaml index ffa7a9c..6dc86c8 100644 --- a/compose.gennifer.yaml +++ b/compose.gennifer.yaml @@ -1,6 +1,22 @@ version: '3.8' services: + frontend: + build: + context: ./gennifer/frontend + dockerfile: Dockerfile + ports: + - 3000:3000 + environment: + - NEXTAUTH_SECRET=X/NTPIqf088gXiYFi7WF0iH3NRJRPE3nZ0oOkRXf5es= + - NEXTAUTH_URL=https://chp.thayer.dartmouth.edu + - NEXTAUTH_URL_INTERNAL=http://127.0.0.1:3000 + - CREDENTIALS_URL=https://chp.thayer.dartmouth.edu/o/token/ + - GENNIFER_CLIENT_ID=jHM4ETk5wi2WUVPElpMFJtZqwY2oBKHVMmTsY9ry + - GENNIFER_CLIENT_SECRET=hY0XfS8YLGMojWuvUOPga4sJpEO9isltF7Xk7wXjyFHwWmxkifRXcPbnmhUM0oVO4Zlz349jbtBePIlaafkWubReqEBCoIcCzaZLa2a9pIlq55yow2TBvMDHnImrXvig + - GENNIFER_USER_DETAILS_URL=https://chp.thayer.dartmouth.edu/users/me/ + - GENNIFER_BASE_URL=https://chp.thayer.dartmouth.edu/gennifer/api/ + - NEXT_PUBLIC_GENNIFER_BASE_URL=https://chp.thayer.dartmouth.edu/gennifer/api/ pidc: build: @@ -128,6 +144,44 @@ services: - grnboost2 - redis + bkb-grn: + build: + context: ./gennifer/bkb-grn + dockerfile: Dockerfile + restart: always + user: gennifer_user + ports: + - 5008:5000 + secrets: + - gennifer_key + environment: + - PYTHONUNBUFFERED=1 + - SECRET_KEY_FILE=/run/secrets/gennifer_key + - CELERY_BROKER_URL=redis://redis:6379/0 + - CELERY_RESULT_BACKEND=redis://redis:6379/0 + #command: gunicorn -c gunicorn.config.py -b 0.0.0.0:5000 'grnboost2:create_app()' + command: flask --app bkb_grn run --debug --host 0.0.0.0 + depends_on: + - redis + + worker-bkb-grn: + build: + context: ./gennifer/bkb-grn + dockerfile: Dockerfile + secrets: + - gurobi_lic + command: celery --app bkb_grn.tasks.celery worker -Q bkb_grn --loglevel=info + environment: + - PYTHONUNBUFFERED=1 + - CELERY_BROKER_URL=redis://redis:6379/0 + - CELERY_RESULT_BACKEND=redis://redis:6379/0 + - GRB_LICENSE_FILE=/run/secrets/gurobi_lic + depends_on: + - bkb-grn + - redis + secrets: gennifer_key: file: secrets/gennifer/secret_key.txt + gurobi_lic: + file: secrets/gennifer/gurobi.lic diff --git a/deployment-script b/deployment-script index c077fa1..ab3de33 100755 --- a/deployment-script +++ b/deployment-script @@ -20,4 +20,4 @@ docker compose -f compose.chp-api.yaml run api python3 manage.py runscript algor docker compose -f compose.chp-api.yaml run --user root api python3 manage.py collectstatic --noinput -echo "Check logs with: docker compose logs -f" +echo "Check logs with: docker compose -f compose.chp-api.yaml -f compose.gennifer.yaml logs -f" diff --git a/nginx/default.conf b/nginx/default.conf index 03c4dbc..ee86726 100644 --- a/nginx/default.conf +++ b/nginx/default.conf @@ -52,8 +52,24 @@ server { proxy_pass http://$NEXTJS_SERVER_ADDR; } - location /chat { + location /ui { proxy_pass http://$NEXTJS_SERVER_ADDR/; } + location /dashboard { + proxy_pass http://$NEXTJS_SERVER_ADDR; + } + + location /studies { + proxy_pass http://$NEXTJS_SERVER_ADDR; + } + + location /explore { + proxy_pass http://$NEXTJS_SERVER_ADDR; + } + + location /documentation { + proxy_pass http://$NEXTJS_SERVER_ADDR; + } + } From 9080aa77c173457487841b0890b1eeeaeb44af24 Mon Sep 17 00:00:00 2001 From: "CHP Dev Machine User (Chase, Greg, or Anthony)" Date: Fri, 28 Jul 2023 16:29:47 -0400 Subject: [PATCH 101/132] Updating submodule commit. --- gennifer | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gennifer b/gennifer index 71038d5..46cd4bb 160000 --- a/gennifer +++ b/gennifer @@ -1 +1 @@ -Subproject commit 71038d5b152b4fa223db9563736ada1691a01ac4 +Subproject commit 46cd4bb8d7494f96b2c7e9b0295007f844e98466 From e810439dbc88288574d5979d160428e78e1fbdf6 Mon Sep 17 00:00:00 2001 From: "CHP Dev Machine User (Chase, Greg, or Anthony)" Date: Thu, 7 Sep 2023 15:23:15 -0400 Subject: [PATCH 102/132] push local changes for transfer --- chp_api/Dockerfile | 2 ++ .../scripts/gene_spec_curie_templater.py | 10 +++---- chp_api/dispatcher/views.py | 2 ++ compose.chp-api.yaml | 8 ++++-- compose.gennifer.yaml | 26 ++++++++++++------- 5 files changed, 31 insertions(+), 17 deletions(-) diff --git a/chp_api/Dockerfile b/chp_api/Dockerfile index b4008d2..dae1f58 100644 --- a/chp_api/Dockerfile +++ b/chp_api/Dockerfile @@ -48,6 +48,8 @@ ARG DEBIAN_FRONTEND=noninterative # copy repo to new image COPY --from=intermediate /usr/src/chp_api/wheels /wheels COPY --from=intermediate /usr/src/chp_api/requirements.txt . +# Running this command to ensure version 1 of pydantic is being used until reasoner-pydantic is updated. +RUN pip3 install pydantic==1.10.12 RUN pip3 install --no-cache /wheels/* # copy project diff --git a/chp_api/dispatcher/scripts/gene_spec_curie_templater.py b/chp_api/dispatcher/scripts/gene_spec_curie_templater.py index 2e6e78b..63a6ead 100644 --- a/chp_api/dispatcher/scripts/gene_spec_curie_templater.py +++ b/chp_api/dispatcher/scripts/gene_spec_curie_templater.py @@ -37,13 +37,13 @@ def _get_ascendants(curies, category): "query_graph": query_graph, } } - url = 'https://ontology-kp.apps.renci.org/query' + url = 'https://automat.renci.org/ubergraph/1.4/query/' r = requests.post(url, json=query, timeout=1000) answer = json.loads(r.content) - for edge_id, edge in answer['message']['knowledge_graph']['edges'].items(): - subject = edge['subject'] - object = edge['object'] - mapping[object].add(subject) + for result in answer['message']['results']: + child = result['node_bindings']['n1'][0]['id'] + parent = result['node_bindings']['n0'][0]['id'] + mapping[parent].add(child) return dict(mapping) diff --git a/chp_api/dispatcher/views.py b/chp_api/dispatcher/views.py index afddf72..413f6c2 100644 --- a/chp_api/dispatcher/views.py +++ b/chp_api/dispatcher/views.py @@ -7,6 +7,7 @@ from .base import Dispatcher from .models import Transaction, DispatcherSetting from .serializers import TransactionListSerializer, TransactionDetailSerializer +from .permissions import CustomQueryPostPermission from django.http import HttpResponse, JsonResponse from django.shortcuts import get_object_or_404 @@ -19,6 +20,7 @@ TOOLKIT = Toolkit() class query(APIView): + permission_classes = [CustomQueryPostPermission] def post(self, request): # Get current trapi and biolink versions diff --git a/compose.chp-api.yaml b/compose.chp-api.yaml index be95e4d..b7d117d 100644 --- a/compose.chp-api.yaml +++ b/compose.chp-api.yaml @@ -79,13 +79,14 @@ services: retries: 3 volumes: - static-files:/home/chp_api/staticfiles - #command: gunicorn -c gunicorn.config.py -b 0.0.0.0:8000 chp_api.wsgi:application - command: python3 manage.py runserver 0.0.0.0:8000 + command: gunicorn -c gunicorn.config.py -b 0.0.0.0:8000 chp_api.wsgi:application + #command: python3 manage.py runserver 0.0.0.0:8000 worker-api: build: context: ./chp_api dockerfile: Dockerfile + restart: always secrets: - db-password - django-key @@ -125,6 +126,7 @@ services: build: context: ./chp_api dockerfile: Dockerfile + restart: always secrets: - db-password - django-key @@ -164,6 +166,7 @@ services: - worker-api redis: + restart: always image: redis:6-alpine db: @@ -189,6 +192,7 @@ services: static-fs: image: halverneus/static-file-server:latest + restart: always environment: - FOLDER=/var/www - DEBUG=true diff --git a/compose.gennifer.yaml b/compose.gennifer.yaml index 6dc86c8..4ccd4a7 100644 --- a/compose.gennifer.yaml +++ b/compose.gennifer.yaml @@ -5,6 +5,7 @@ services: build: context: ./gennifer/frontend dockerfile: Dockerfile + restart: always ports: - 3000:3000 environment: @@ -32,8 +33,8 @@ services: - SECRET_KEY_FILE=/run/secrets/gennifer_key - CELERY_BROKER_URL=redis://redis:6379/0 - CELERY_RESULT_BACKEND=redis://redis:6379/0 - #command: gunicorn -c gunicorn.config.py -b 0.0.0.0:5000 'pidc:create_app()' - command: flask --app pidc run --debug --host 0.0.0.0 + command: gunicorn -c gunicorn.config.py -b 0.0.0.0:5000 'pidc:create_app()' + #command: flask --app pidc run --debug --host 0.0.0.0 depends_on: - redis @@ -41,6 +42,7 @@ services: build: context: ./gennifer/pidc dockerfile: Dockerfile + restart: always command: celery --app pidc.tasks.celery worker -Q pidc --loglevel=info environment: - CELERY_BROKER_URL=redis://redis:6379/0 @@ -64,8 +66,8 @@ services: - CELERY_BROKER_URL=redis://redis:6379/0 - CELERY_RESULT_BACKEND=redis://redis:6379/0 - PYTHONUNBUFFERED=1 - #command: gunicorn -c gunicorn.config.py -b 0.0.0.0:5000 'pidc:create_app()' - command: flask --app grisli run --debug --host 0.0.0.0 + command: gunicorn -c gunicorn.config.py -b 0.0.0.0:5000 'grisli:create_app()' + #command: flask --app grisli run --debug --host 0.0.0.0 depends_on: - redis @@ -73,6 +75,7 @@ services: build: context: ./gennifer/grisli dockerfile: Dockerfile + restart: always command: celery --app grisli.tasks.celery worker -Q grisli --loglevel=info environment: - CELERY_BROKER_URL=redis://redis:6379/0 @@ -96,8 +99,8 @@ services: - SECRET_KEY_FILE=/run/secrets/gennifer_key - CELERY_BROKER_URL=redis://redis:6379/0 - CELERY_RESULT_BACKEND=redis://redis:6379/0 - #command: gunicorn -c gunicorn.config.py -b 0.0.0.0:5000 'genie3:create_app()' - command: flask --app genie3 run --debug --host 0.0.0.0 + command: gunicorn -c gunicorn.config.py -b 0.0.0.0:5000 'genie3:create_app()' + #command: flask --app genie3 run --debug --host 0.0.0.0 depends_on: - redis @@ -105,6 +108,7 @@ services: build: context: ./gennifer/genie3 dockerfile: Dockerfile + restart: always command: celery --app genie3.tasks.celery worker -Q genie3 --loglevel=info environment: - CELERY_BROKER_URL=redis://redis:6379/0 @@ -127,8 +131,8 @@ services: - SECRET_KEY_FILE=/run/secrets/gennifer_key - CELERY_BROKER_URL=redis://redis:6379/0 - CELERY_RESULT_BACKEND=redis://redis:6379/0 - #command: gunicorn -c gunicorn.config.py -b 0.0.0.0:5000 'grnboost2:create_app()' - command: flask --app grnboost2 run --debug --host 0.0.0.0 + command: gunicorn -c gunicorn.config.py -b 0.0.0.0:5000 'grnboost2:create_app()' + #command: flask --app grnboost2 run --debug --host 0.0.0.0 depends_on: - redis @@ -136,6 +140,7 @@ services: build: context: ./gennifer/grnboost2 dockerfile: Dockerfile + restart: always command: celery --app grnboost2.tasks.celery worker -Q grnboost2 --loglevel=info environment: - CELERY_BROKER_URL=redis://redis:6379/0 @@ -159,8 +164,8 @@ services: - SECRET_KEY_FILE=/run/secrets/gennifer_key - CELERY_BROKER_URL=redis://redis:6379/0 - CELERY_RESULT_BACKEND=redis://redis:6379/0 - #command: gunicorn -c gunicorn.config.py -b 0.0.0.0:5000 'grnboost2:create_app()' - command: flask --app bkb_grn run --debug --host 0.0.0.0 + command: gunicorn -c gunicorn.config.py -b 0.0.0.0:5000 'bkb_grn:create_app()' + #command: flask --app bkb_grn run --debug --host 0.0.0.0 depends_on: - redis @@ -168,6 +173,7 @@ services: build: context: ./gennifer/bkb-grn dockerfile: Dockerfile + restart: always secrets: - gurobi_lic command: celery --app bkb_grn.tasks.celery worker -Q bkb_grn --loglevel=info From a5a4f716a8ecd3dff68c23e9bf8e30f5c9bbb42e Mon Sep 17 00:00:00 2001 From: "CHP Dev Machine User (Chase, Greg, or Anthony)" Date: Thu, 7 Sep 2023 15:54:13 -0400 Subject: [PATCH 103/132] permisions for push to transfer --- chp_api/dispatcher/permissions.py | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 chp_api/dispatcher/permissions.py diff --git a/chp_api/dispatcher/permissions.py b/chp_api/dispatcher/permissions.py new file mode 100644 index 0000000..7ffebf9 --- /dev/null +++ b/chp_api/dispatcher/permissions.py @@ -0,0 +1,10 @@ +from rest_framework import permissions + + +class CustomQueryPostPermission(permissions.BasePermission): + """ + Allows the query POST endpoint to work without any permissions. + """ + + def has_permission(self, request, view): + return True From 7fc34a55a53998aa7c8e10f9c768782f92c8c3fe Mon Sep 17 00:00:00 2001 From: bettyli037 Date: Wed, 20 Sep 2023 10:46:57 -0400 Subject: [PATCH 104/132] feat: update deployment.yaml with container resource definition --- deploy/chp-api/templates/deployment.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/deploy/chp-api/templates/deployment.yaml b/deploy/chp-api/templates/deployment.yaml index 1997bbd..4aa8fa0 100644 --- a/deploy/chp-api/templates/deployment.yaml +++ b/deploy/chp-api/templates/deployment.yaml @@ -21,6 +21,10 @@ spec: {{- toYaml .Values.podSecurityContext | nindent 8 }} containers: - name: {{ .Chart.Name }} + {{- with .Values.chp-api.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} securityContext: {{- toYaml .Values.securityContext | nindent 12 }} image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" @@ -81,6 +85,10 @@ spec: - name: DJANGO_SUPERUSER_EMAIL value: "{{ .Values.app.djangoSuperuserEmail }}" - name: {{ .Chart.Name }}-nginx + {{- with .Values.chp-api-nginx.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} securityContext: {{- toYaml .Values.securityContextNginx | nindent 12 }} image: "{{ .Values.image.repository }}:{{ .Values.image.nginxTag | default .Chart.AppVersion }}" @@ -94,6 +102,10 @@ spec: mountPath: /etc/nginx/conf.d/default.conf subPath: nginx.conf - name: {{ .Chart.Name }}-staticfs + {{- with .Values.chp-api-staticfs.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} securityContext: {{- toYaml .Values.securityContextStaticfs | nindent 12 }} image: "{{ .Values.image.repository }}:{{ .Values.image.staticfsTag | default .Chart.AppVersion }}" From 1a79069236c8f5dba4c6906ef847395f8d07904b Mon Sep 17 00:00:00 2001 From: bettyli037 Date: Wed, 20 Sep 2023 10:52:39 -0400 Subject: [PATCH 105/132] feat: update chp_api resources --- deploy/chp-api/templates/deployment.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/deploy/chp-api/templates/deployment.yaml b/deploy/chp-api/templates/deployment.yaml index 4aa8fa0..87294b6 100644 --- a/deploy/chp-api/templates/deployment.yaml +++ b/deploy/chp-api/templates/deployment.yaml @@ -21,7 +21,7 @@ spec: {{- toYaml .Values.podSecurityContext | nindent 8 }} containers: - name: {{ .Chart.Name }} - {{- with .Values.chp-api.resources }} + {{- with .Values.chp_api.resources }} resources: {{- toYaml . | nindent 12 }} {{- end }} @@ -85,7 +85,7 @@ spec: - name: DJANGO_SUPERUSER_EMAIL value: "{{ .Values.app.djangoSuperuserEmail }}" - name: {{ .Chart.Name }}-nginx - {{- with .Values.chp-api-nginx.resources }} + {{- with .Values.chp_api_nginx.resources }} resources: {{- toYaml . | nindent 12 }} {{- end }} @@ -102,7 +102,7 @@ spec: mountPath: /etc/nginx/conf.d/default.conf subPath: nginx.conf - name: {{ .Chart.Name }}-staticfs - {{- with .Values.chp-api-staticfs.resources }} + {{- with .Values.chp_api_staticfs.resources }} resources: {{- toYaml . | nindent 12 }} {{- end }} From 42ad68ed22028595cc31c9a17415ed1f783d78f9 Mon Sep 17 00:00:00 2001 From: bettyli037 Date: Wed, 20 Sep 2023 10:56:16 -0400 Subject: [PATCH 106/132] feat: helm chart version is updated to 0.1.1 --- deploy/chp-api/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/chp-api/Chart.yaml b/deploy/chp-api/Chart.yaml index 61b8921..6066d70 100644 --- a/deploy/chp-api/Chart.yaml +++ b/deploy/chp-api/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.0 +version: 0.1.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to From 20ac07726d75494b89901ac09c7c0ac8f5271409 Mon Sep 17 00:00:00 2001 From: bettyli037 Date: Wed, 20 Sep 2023 11:22:05 -0400 Subject: [PATCH 107/132] feat: restore debug value --- deploy/chp-api/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/chp-api/values.yaml b/deploy/chp-api/values.yaml index a4baa1d..c34c476 100644 --- a/deploy/chp-api/values.yaml +++ b/deploy/chp-api/values.yaml @@ -18,7 +18,7 @@ fullnameOverride: "" # django applicaiton configuration app: - debug: "1" + debug: "0" secret_key: "" djangoCSRFTrustedOrigins: "" djangoAllowedHosts: "" From 5ea10e4e4628018e037cfeb40efb0c64b70c5c65 Mon Sep 17 00:00:00 2001 From: di2ag-org Date: Wed, 20 Sep 2023 21:42:53 -0500 Subject: [PATCH 108/132] Saving work. --- chp_api/gennifer/admin.py | 4 +- ...ion_dataset_public_publication_and_more.py | 65 ++++++++ chp_api/gennifer/models.py | 32 ++++ chp_api/gennifer/serializers.py | 17 ++ chp_api/gennifer/tasks.py | 145 +++++++++++++++++- chp_api/gennifer/views.py | 38 ++++- chp_api/requirements.txt | 2 + compose.chp-api.yaml | 7 +- compose.gennifer.yaml | 38 +++++ 9 files changed, 334 insertions(+), 14 deletions(-) create mode 100755 chp_api/gennifer/migrations/0011_annotated_annotation_dataset_public_publication_and_more.py diff --git a/chp_api/gennifer/admin.py b/chp_api/gennifer/admin.py index 088d222..c50b8c4 100644 --- a/chp_api/gennifer/admin.py +++ b/chp_api/gennifer/admin.py @@ -1,6 +1,6 @@ from django.contrib import admin -from .models import Algorithm, Dataset, Study, Task, Result, Gene, UserAnalysisSession, AlgorithmInstance, Hyperparameter +from .models import Algorithm, Dataset, Study, Task, Result, Gene, UserAnalysisSession, AlgorithmInstance, Hyperparameter, Annotation, Annotated admin.site.register(Algorithm) admin.site.register(AlgorithmInstance) @@ -11,3 +11,5 @@ admin.site.register(Result) admin.site.register(Gene) admin.site.register(UserAnalysisSession) +admin.site.register(Annotation) +admin.site.register(Annotated) \ No newline at end of file diff --git a/chp_api/gennifer/migrations/0011_annotated_annotation_dataset_public_publication_and_more.py b/chp_api/gennifer/migrations/0011_annotated_annotation_dataset_public_publication_and_more.py new file mode 100755 index 0000000..3a33b2b --- /dev/null +++ b/chp_api/gennifer/migrations/0011_annotated_annotation_dataset_public_publication_and_more.py @@ -0,0 +1,65 @@ +# Generated by Django 4.2.5 on 2023-09-11 21:24 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('gennifer', '0010_hyperparameter_and_more'), + ] + + operations = [ + migrations.CreateModel( + name='Annotated', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ], + ), + migrations.CreateModel( + name='Annotation', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('type', models.CharField(choices=[('openai', 'OpenAI'), ('translator', 'Translator')], default='translator', max_length=32)), + ('timestamp', models.DateTimeField(auto_now_add=True)), + ('tr_formatted_relation_string', models.CharField(blank=True, max_length=256, null=True)), + ('tr_predicate', models.CharField(blank=True, max_length=128, null=True)), + ('tr_qualified_predicate', models.CharField(blank=True, max_length=128, null=True)), + ('tr_object_modifier', models.CharField(blank=True, max_length=128, null=True)), + ('tr_object_aspect', models.CharField(blank=True, max_length=128, null=True)), + ('tr_resource_id', models.CharField(blank=True, max_length=128, null=True)), + ('tr_primary_source', models.CharField(blank=True, max_length=128, null=True)), + ('oai_justification', models.TextField(blank=True, null=True)), + ('results', models.ManyToManyField(through='gennifer.Annotated', to='gennifer.result')), + ], + ), + migrations.AddField( + model_name='dataset', + name='public', + field=models.BooleanField(default=False), + ), + migrations.CreateModel( + name='Publication', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('curie', models.CharField(max_length=128)), + ('annotation', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='publications', to='gennifer.annotation')), + ], + ), + migrations.AddField( + model_name='annotated', + name='annotation', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='gennifer.annotation'), + ), + migrations.AddField( + model_name='annotated', + name='result', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='gennifer.result'), + ), + migrations.AddField( + model_name='result', + name='annotations', + field=models.ManyToManyField(through='gennifer.Annotated', to='gennifer.annotation'), + ), + ] diff --git a/chp_api/gennifer/models.py b/chp_api/gennifer/models.py index d506c26..3c09378 100644 --- a/chp_api/gennifer/models.py +++ b/chp_api/gennifer/models.py @@ -89,6 +89,7 @@ class Dataset(models.Model): doi = models.CharField(max_length=128) description = models.TextField(null=True, blank=True) user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, null=True, blank=True) + public = models.BooleanField(default=False) def save(self, *args, **kwargs): import re @@ -162,6 +163,37 @@ class Result(models.Model): task = models.ForeignKey(Task, on_delete=models.CASCADE, related_name='results') is_public = models.BooleanField(default=False) user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, null=True, blank=True, related_name='results') + annotations = models.ManyToManyField('Annotation', through='Annotated') def __str__(self): return f'{self.tf}:{self.tf.curie} -> regulates -> {self.target}:{self.target.curie}' + +class Annotation(models.Model): + # Type of annotation + TYPE_CHOICES = ( + ('openai', "OpenAI"), + ('translator', "Translator"), + ) + type = models.CharField(max_length=32, choices=TYPE_CHOICES, default='translator') + timestamp = models.DateTimeField(auto_now_add=True) + results = models.ManyToManyField('Result', through='Annotated') + # Translator fields + tr_formatted_relation_string = models.CharField(max_length=256, null=True, blank=True) + tr_predicate = models.CharField(max_length=128, null=True, blank=True) + tr_qualified_predicate = models.CharField(max_length=128, null=True, blank=True) + tr_object_modifier = models.CharField(max_length=128, null=True, blank=True) + tr_object_aspect = models.CharField(max_length=128, null=True, blank=True) + tr_resource_id = models.CharField(max_length=128, null=True, blank=True) + tr_primary_source = models.CharField(max_length=128, null=True, blank=True) + # OpenAI fields + oai_justification = models.TextField(null=True, blank=True) + + +class Publication(models.Model): + curie = models.CharField(max_length=128) + annotation = models.ForeignKey(Annotation, on_delete=models.CASCADE, related_name='publications') + +class Annotated(models.Model): + result = models.ForeignKey(Result, on_delete=models.CASCADE) + annotation = models.ForeignKey(Annotation, on_delete=models.CASCADE) + diff --git a/chp_api/gennifer/serializers.py b/chp_api/gennifer/serializers.py index 1e5e745..155cacd 100644 --- a/chp_api/gennifer/serializers.py +++ b/chp_api/gennifer/serializers.py @@ -12,6 +12,7 @@ AlgorithmInstance, Hyperparameter, HyperparameterInstance, + Annotation, ) @@ -149,3 +150,19 @@ class Meta: 'variant', 'chp_preferred_curie', ] + +class AnnotationSerializer(serializers.ModelSerializer): + class Meta: + model = Annotation + fields = [ + 'type', + 'timestamp', + 'tr_formatted_relation_string', + 'tr_predicate', + 'tr_qualified_predicate', + 'tr_object_modifier', + 'tr_object_aspect', + 'tr_resource_id', + 'tr_primary_source', + 'oai_justification', + ] \ No newline at end of file diff --git a/chp_api/gennifer/tasks.py b/chp_api/gennifer/tasks.py index 25f3b4d..9dd6330 100644 --- a/chp_api/gennifer/tasks.py +++ b/chp_api/gennifer/tasks.py @@ -5,15 +5,19 @@ from django.db import transaction from django.contrib.auth import get_user_model +from django.db.models import Q from celery import shared_task from celery.utils.log import get_task_logger from copy import deepcopy +from nltk.stem import WordNetLemmatizer +from pattern.en import conjugate -from .models import Dataset, Gene, Study, Task, Result, Algorithm, AlgorithmInstance +from .models import Dataset, Gene, Study, Task, Result, Algorithm, AlgorithmInstance, Annotated, Annotation, Publication from dispatcher.models import DispatcherSetting logger = get_task_logger(__name__) User = get_user_model() +wnl = WordNetLemmatizer() def normalize_nodes(curies): dispatcher_settings = DispatcherSetting.load() @@ -102,23 +106,29 @@ def save_inference_task(task, status, failed=False): if created: gene2_obj.save() # Construct and save Result - result = Result.objects.create( + result, created = Result.objects.get_or_create( tf=gene1_obj, target=gene2_obj, edge_weight=row["EdgeWeight"], task=task, user=task.user, ) - result.save() + if created: + result.save() task.save() + # Collect all result PKs for this task + result_pks = [res.pk for res in task.results.all()] + # Send to annotation worker + create_annotations_task(result_pks, task.algorithm_instance.algorithm.directed) return True -def get_status(algo, task_id): +def get_status(algo, task_id, url=None): + if url: + return requests.get(f'{url}/status/{task_id}', headers={'Cache-Control': 'no-cache'}).json() return requests.get(f'{algo.url}/status/{task_id}', headers={'Cache-Control': 'no-cache'}).json() - def return_saved_task(tasks, user): - task = studies[0] + task = tasks[0] # Copy task results results = deepcopy(task.results) # Create a new task that is a duplicate but assign to this user. @@ -134,6 +144,129 @@ def return_saved_task(tasks, user): result.save() return True +def construct_annotation_request(results, directed): + data = [] + for result in results: + data.append({ + "source": { + "id": result.tf.curie, + "name": result.tf.name, + }, + "target": { + "id": result.target.curie, + "name": result.target.name, + }, + "result_pk": result.pk, + }) + return {"data": data, "directed": directed} + +def make_tr_formatted_relation( + predicate, + qualified_predicate, + object_modifier, + object_aspect, + ): + formatted_str = predicate.replace('biolink:', '').replace('_', ' ') + if qualified_predicate: + qp = wnl.lemmatize(qualified_predicate.replace('biolink:', ''), 'v') + try: + qp = conjugate(qp, 'part') + except RuntimeError: + # This function fails the first time its run so just run again, see: https://github.com/clips/pattern/issues/295 + qp = conjugate(qp, 'part') + pass + formatted_str += f' By {qp}' + if object_modifier: + om = object_modifier.replace('_', ' ') + formatted_str += f' {om}' + if object_aspect: + oa = object_aspect.replace('_', ' ') + formatted_str += f' {oa}' + return formatted_str.title() + +def save_annotation_task(status, failed=False): + if failed: + print('Annotation Failed') + return + annotations = status["task_result"] + for annotation in annotations: + result = Result.objects.get(pk=annotation["result_pk"]) + if annotation["justification"]: + # Make OpenAI Annotation + oai_justification = Annotation.objects.create( + type='openai', + oai_justification=annotation["justification"] + ) + oai_annotated = Annotated.objects.create( + result=result, + annotation=oai_justification, + ) + oai_justification.save() + oai_annotated.save() + # Make translator annotations + for tr_result in annotation["results"]: + tr_annotation = Annotation.objects.create( + type='translator', + tr_formatted_relation_string=make_tr_formatted_relation( + tr_result["predicate"], + tr_result["qualified_predicate"], + tr_result["object_modifier"], + tr_result["object_aspect"], + ), + tr_predicate= tr_result["predicate"], + tr_qualified_predicate=tr_result["qualified_predicate"], + tr_object_modifier=tr_result["object_modifier"], + tr_object_aspect=tr_result["object_aspect"], + tr_resource_id=tr_result["resource_id"], + tr_primary_source=tr_result["primary_source"], + ) + tr_annotation.save() + tr_annotated = Annotated.objects.create( + result=result, + annotation=tr_annotation, + ) + tr_annotated.save() + print('Saved annotations.') + return + +@shared_task(name="create_annotations_task") +def create_annotations_task(result_pks, directed): + results = Result.objects.filter(pk__in = result_pks) + results_to_be_annotated = [] + # First go through results and ensure we haven't already made an annotation request + for result in results: + matched_annotations = [a.annotation for a in Annotated.objects.filter( + result__tf__curie=result.tf.curie, + result__target__curie=result.target.curie, + result__task__algorithm_instance__algorithm__directed=result.task.algorithm_instance.algorithm.directed + )] + if len(matched_annotations) == 0: + results_to_be_annotated.append(result) + continue + for ma in matched_annotations: + annotated = Annotated.objects.create( + annotation = ma, + result=result, + ) + annotated.save() + # Construct annotation service request + r = construct_annotation_request(results_to_be_annotated, directed) + # Send to annotation service and wait + annotate_id = requests.post('http://annotator:5000/run', json=r).json()["task_id"] + # Get initial status + status = get_status(None, annotate_id, url='http://annotator:5000') + + # Enter a loop to keep checking back in and populate the task once it has completed. + #TODO: Not sure if this is best practice + while True: + # Check in every 10 seconds + time.sleep(10) + status = get_status(None, annotate_id, url='http://annotator:5000') + if status["task_status"] == 'SUCCESS': + return save_annotation_task(status) + if status["task_status"] == "FAILURE": + return save_annotation_task(status, failed=True) + @shared_task(name="create_gennifer_task") def create_task(task_pk): # Get task diff --git a/chp_api/gennifer/views.py b/chp_api/gennifer/views.py index 64f7fd9..a5ad669 100644 --- a/chp_api/gennifer/views.py +++ b/chp_api/gennifer/views.py @@ -1,5 +1,7 @@ import requests +from collections import defaultdict + from django.http import HttpResponse, JsonResponse from django.shortcuts import get_object_or_404 from django.core.exceptions import ObjectDoesNotExist @@ -23,7 +25,7 @@ UserAnalysisSession, AlgorithmInstance, Hyperparameter, - HyperparameterInstance + HyperparameterInstance, ) from .serializers import ( DatasetSerializer, @@ -173,11 +175,40 @@ def construct_node(self, gene_obj): } return node, str(gene_obj.pk) + def construct_edge_annotations(self, annotations): + obj = {"openai": {"justification": None}, "translator": []} + tr_dict = defaultdict(list) + for a in annotations: + if a.type == 'openai': + obj["openai"]["justification"] = a.oai_justification + continue + if a.type == 'translator': + tr_dict[a.tr_formatted_relation_string].append( + { + "predicate": a.tr_predicate, + "qualified_predicate": a.tr_qualified_predicate, + "object_modifier": a.tr_object_modifier, + "object_aspect": a.tr_object_aspect, + "resource_id": a.tr_resource_id, + "primary_source": a.tr_primary_source, + } + ) + # Reformat to list + for relation, tr_results in tr_dict.items(): + obj["translator"].append( + { + "formatted_relation": relation, + "results": tr_results + } + ) + return obj + def construct_edge(self, res, source_id, target_id): # Normalize edge weight based on the study normalized_weight = (res.edge_weight - res.task.min_task_edge_weight) / (res.task.max_task_edge_weight - res.task.min_task_edge_weight) directed = res.task.algorithm_instance.algorithm.directed edge_tuple = tuple(sorted([source_id, target_id])) + annotations = res.annotations.all() edge = { "data": { "id": str(res.pk), @@ -187,6 +218,7 @@ def construct_edge(self, res, source_id, target_id): "weight": normalized_weight, "algorithm": str(res.task.algorithm_instance), "directed": directed, + "annotations": self.construct_edge_annotations(annotations), } } return edge, edge_tuple, directed @@ -228,8 +260,8 @@ def construct_cytoscape_data(self): processed_node_ids, processed_undirected_edges, ) - elements.extend(nodes) - elements.extend(edges) + elements.extend(nodes) + elements.extend(edges) return { "elements": elements } diff --git a/chp_api/requirements.txt b/chp_api/requirements.txt index 75b8cd4..98f5258 100644 --- a/chp_api/requirements.txt +++ b/chp_api/requirements.txt @@ -17,3 +17,5 @@ redis pandas django-cors-headers django-oauth-toolkit +nltk +pattern diff --git a/compose.chp-api.yaml b/compose.chp-api.yaml index b7d117d..fd5e423 100644 --- a/compose.chp-api.yaml +++ b/compose.chp-api.yaml @@ -68,10 +68,9 @@ services: # Comment this for development #- DJANGO_SETTINGS_MODULE=mysite.settings.base depends_on: - db: - condition: service_healthy - depends_on: - - static-fs + - static-fs + - db + #condition: service_healthy healthcheck: #test: ["CMD-SHELL", "curl --silent --fail localhost:8000/flask-health-check || exit 1"] interval: 10s diff --git a/compose.gennifer.yaml b/compose.gennifer.yaml index 4ccd4a7..28230bd 100644 --- a/compose.gennifer.yaml +++ b/compose.gennifer.yaml @@ -185,9 +185,47 @@ services: depends_on: - bkb-grn - redis + + annotator: + build: + context: ./gennifer/annotator + dockerfile: Dockerfile + restart: always + user: gennifer_user + ports: + - 5009:5000 + secrets: + - gennifer_key + - openai_api_key + environment: + - SECRET_KEY_FILE=/run/secrets/gennifer_key + - CELERY_BROKER_URL=redis://redis:6379/0 + - CELERY_RESULT_BACKEND=redis://redis:6379/0 + - OPENAI_API_KEY_FILE=/run/secrets/openai_api_key + command: gunicorn -c gunicorn.config.py -b 0.0.0.0:5000 'annotator:create_app()' + #command: flask --app annotator run --debug --host 0.0.0.0 + depends_on: + - redis + + worker-annotator: + build: + context: ./gennifer/annotator + dockerfile: Dockerfile + command: celery --app annotator.tasks.celery worker -Q annotation --loglevel=info + secrets: + - openai_api_key + environment: + - CELERY_BROKER_URL=redis://redis:6379/0 + - CELERY_RESULT_BACKEND=redis://redis:6379/0 + - OPENAI_API_KEY_FILE=/run/secrets/openai_api_key + depends_on: + - annotator + - redis secrets: gennifer_key: file: secrets/gennifer/secret_key.txt gurobi_lic: file: secrets/gennifer/gurobi.lic + openai_api_key: + file: secrets/gennifer/openai_api_key.txt From c2e44b8eba60d45898d58276afd0e0db7864e080 Mon Sep 17 00:00:00 2001 From: di2ag-org Date: Wed, 18 Oct 2023 15:22:52 -0500 Subject: [PATCH 109/132] Updated gennifer commit. --- gennifer | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gennifer b/gennifer index 46cd4bb..01bbb48 160000 --- a/gennifer +++ b/gennifer @@ -1 +1 @@ -Subproject commit 46cd4bb8d7494f96b2c7e9b0295007f844e98466 +Subproject commit 01bbb4869d09ef1df65cbde161558b6208debfdf From dd0159c6016c691f50540a0e412cc62e48f797ee Mon Sep 17 00:00:00 2001 From: Chase Yakaboski Date: Thu, 19 Oct 2023 13:11:29 -0400 Subject: [PATCH 110/132] Updated pydantic version to install until reasoner-pydantic is fixed. --- Dockerfile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Dockerfile b/Dockerfile index 38965b0..1d3d682 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,6 +10,9 @@ WORKDIR /usr/src/chp_api RUN git clone --single-branch --branch gene_spec_pydantic-ghyde https://github.com/di2ag/gene-specificity.git +# Upgrade pip +RUN pip3 install --upgrade pip + # install dependencies COPY ./requirements.txt . RUN pip3 wheel --no-cache-dir --no-deps --wheel-dir /usr/src/chp_api/wheels -r requirements.txt @@ -45,6 +48,8 @@ ARG DEBIAN_FRONTEND=noninterative # copy repo to new image COPY --from=intermediate /usr/src/chp_api/wheels /wheels COPY --from=intermediate /usr/src/chp_api/requirements.txt . +# Running this command to ensure version 1 of pydantic is being used until reasoner-pydantic is updated. +RUN pip3 install pydantic==1.10.12 RUN pip3 install --no-cache /wheels/* # copy project From af9f342660f376258e3c480922d87c1648c415a1 Mon Sep 17 00:00:00 2001 From: akadapa Date: Wed, 15 Nov 2023 14:59:22 -0600 Subject: [PATCH 111/132] Added Tolerations to managed-app --- deploy/chp-api/values.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/deploy/chp-api/values.yaml b/deploy/chp-api/values.yaml index c34c476..bee66c9 100644 --- a/deploy/chp-api/values.yaml +++ b/deploy/chp-api/values.yaml @@ -76,7 +76,7 @@ ingress: tolerations: - key: "transltr" - value: "chp-api" + value: "managed-app" operator: "Equal" effect: "NoSchedule" @@ -89,9 +89,9 @@ affinity: - key: app operator: In values: - - chp-api + - managed-app topologyKey: "kubernetes.io/hostname" - # this ensures pod only runs on node with label application=chp-api + # this ensures pod only runs on node with label application=managed-app nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: @@ -99,4 +99,4 @@ affinity: - key: application operator: In values: - - chp-api + - managed-app From 9f02fbc6aab79706d3c910eb8889f7fa707e7ab1 Mon Sep 17 00:00:00 2001 From: "CHP Dev Machine User (Chase, Greg, or Anthony)" Date: Wed, 6 Dec 2023 16:03:35 -0500 Subject: [PATCH 112/132] updates to gennifer and reverting gene_spec branch back to master --- chp_api/Dockerfile | 2 +- gennifer | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/chp_api/Dockerfile b/chp_api/Dockerfile index dae1f58..849bb32 100644 --- a/chp_api/Dockerfile +++ b/chp_api/Dockerfile @@ -8,7 +8,7 @@ FROM python:3.8 as intermediate # set work directory WORKDIR /usr/src/chp_api -RUN git clone --single-branch --branch gene_spec_pydantic-ghyde https://github.com/di2ag/gene-specificity.git +RUN git clone --single-branch --branch master https://github.com/di2ag/gene-specificity.git # Upgrade pip RUN pip3 install --upgrade pip diff --git a/gennifer b/gennifer index 01bbb48..2077a30 160000 --- a/gennifer +++ b/gennifer @@ -1 +1 @@ -Subproject commit 01bbb4869d09ef1df65cbde161558b6208debfdf +Subproject commit 2077a3047e7260abaad32e1d242c53ecd40073bb From e31ad1b11590346f3aeeb178fbf7df019f2c4dd3 Mon Sep 17 00:00:00 2001 From: "CHP Dev Machine User (Chase, Greg, or Anthony)" Date: Thu, 7 Dec 2023 14:27:09 -0500 Subject: [PATCH 113/132] changes for new gene_spec data and models --- .../scripts/gene_spec_curie_templater.py | 22 +++++----- .../dispatcher/scripts/populate_gene_spec.py | 41 +++++++++++++++++++ 2 files changed, 53 insertions(+), 10 deletions(-) create mode 100644 chp_api/dispatcher/scripts/populate_gene_spec.py diff --git a/chp_api/dispatcher/scripts/gene_spec_curie_templater.py b/chp_api/dispatcher/scripts/gene_spec_curie_templater.py index 63a6ead..ddeea57 100644 --- a/chp_api/dispatcher/scripts/gene_spec_curie_templater.py +++ b/chp_api/dispatcher/scripts/gene_spec_curie_templater.py @@ -2,7 +2,7 @@ import json import requests from collections import defaultdict -from gene_specificity.models import CurieTemplate, CurieTemplateMatch, SpecificityMeanGene, SpecificityMeanTissue +from gene_specificity.models import CurieTemplate, CurieTemplateMatch, GeneToTissue, TissueToGene CHUNK_SIZE = 500 @@ -37,23 +37,25 @@ def _get_ascendants(curies, category): "query_graph": query_graph, } } - url = 'https://automat.renci.org/ubergraph/1.4/query/' + url = 'https://ontology-kp.apps.renci.org/query' r = requests.post(url, json=query, timeout=1000) answer = json.loads(r.content) - for result in answer['message']['results']: - child = result['node_bindings']['n1'][0]['id'] - parent = result['node_bindings']['n0'][0]['id'] - mapping[parent].add(child) + for edge_id, edge in answer['message']['knowledge_graph']['edges'].items(): + subject = edge['subject'] + object = edge['object'] + mapping[object].add(subject) return dict(mapping) def run(): - objects = SpecificityMeanGene.objects.all() + gene_objects = GeneToTissue.objects.all() + tissue_objects = TissueToGene.objects.all() gene_curies = set() + for gene_object in gene_objects: + gene_curies.add(gene_object.gene_id) tissue_curies = set() - for object in objects: - gene_curies.add(object.gene_curie) - tissue_curies.add(object.tissue_curie) + for tissue_object in tissue_objects: + tissue_curies.add(tissue_object.tissue_id) gene_ascendants = _get_ascendants(list(gene_curies), 'biolink:Gene') tissue_ascendants = _get_ascendants(list(tissue_curies), 'biolink:GrossAnatomicalStructure') diff --git a/chp_api/dispatcher/scripts/populate_gene_spec.py b/chp_api/dispatcher/scripts/populate_gene_spec.py new file mode 100644 index 0000000..dfe8a48 --- /dev/null +++ b/chp_api/dispatcher/scripts/populate_gene_spec.py @@ -0,0 +1,41 @@ +import json +from gene_specificity.models import GeneToTissue, TissueToGene + + +def run(): + max_count = 20 + p_val_thresh = 0.05 + + with open('gene_to_tissue.json', 'r') as f: + gene_to_tissue = json.load(f) + + for gene_id, tissue_dict in gene_to_tissue.items(): + i = 0 + for tissue_id, data_obj in tissue_dict.items(): + if i == max_count: + break + spec_val = data_obj['spec'] + norm_spec_val = data_obj['norm_spec'] + p_val = data_obj['p_val'] + if p_val > p_val_thresh: + break + gtt = GeneToTissue(gene_id = gene_id, tissue_id = tissue_id, spec = spec_val, norm_spec = norm_spec_val, p_val = p_val) + gtt.save() + i += 1 + + with open('tissue_to_gene.json', 'r') as f: + tissue_to_gene = json.load(f) + + for tissue_id, gene_dict in tissue_to_gene.items(): + i = 0 + for gene_id, data_obj in gene_dict.items(): + if i == max_count: + break + spec_val = data_obj['spec'] + norm_spec_val = data_obj['norm_spec'] + p_val = data_obj['p_val'] + if p_val > p_val_thresh: + break + ttg = TissueToGene(tissue_id = tissue_id, gene_id = gene_id, spec = spec_val, norm_spec = norm_spec_val, p_val = p_val) + ttg.save() + i += 1 From eb4da3f61d0236f2184d72b126cccc7125bb3361 Mon Sep 17 00:00:00 2001 From: "CHP Dev Machine User (Chase, Greg, or Anthony)" Date: Tue, 23 Apr 2024 16:40:37 -0400 Subject: [PATCH 114/132] updated for open telemetry, trapi 1.5 and biolink 4.2.0 --- chp_api/Dockerfile | 4 +- chp_api/gennifer/tasks.py | 1 + chp_api/manage.py | 8 +- chp_api/requirements.txt | 144 +++++++++++++++++++---- chp_api/requirements.txt.base | 23 ++++ compose.chp-api.yaml | 12 +- deploy/chp-api/templates/deployment.yaml | 10 +- deploy/chp-api/values.yaml | 4 + 8 files changed, 179 insertions(+), 27 deletions(-) create mode 100644 chp_api/requirements.txt.base diff --git a/chp_api/Dockerfile b/chp_api/Dockerfile index 849bb32..d342cfa 100644 --- a/chp_api/Dockerfile +++ b/chp_api/Dockerfile @@ -3,7 +3,7 @@ ########### # first stage of build to pull repos -FROM python:3.8 as intermediate +FROM python:3.9 as intermediate # set work directory WORKDIR /usr/src/chp_api @@ -25,7 +25,7 @@ RUN cd gene-specificity && python3 setup.py bdist_wheel && cd dist && cp gene_sp ######### #pull official base image -FROM python:3.8 +FROM python:3.9 # add app user RUN groupadd chp_api && useradd -ms /bin/bash -g chp_api chp_api diff --git a/chp_api/gennifer/tasks.py b/chp_api/gennifer/tasks.py index 9dd6330..af28b2e 100644 --- a/chp_api/gennifer/tasks.py +++ b/chp_api/gennifer/tasks.py @@ -348,6 +348,7 @@ def create_task(task_pk): # Check in every 2 seconds time.sleep(5) status = get_status(algo, task_id) + print(status) if status["task_status"] == 'SUCCESS': return save_inference_task(task, status) if status["task_status"] == "FAILURE": diff --git a/chp_api/manage.py b/chp_api/manage.py index 45e6ffe..59872a5 100644 --- a/chp_api/manage.py +++ b/chp_api/manage.py @@ -3,8 +3,14 @@ import os import sys +from opentelemetry.instrumentation.django import DjangoInstrumentor + def main(): os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'chp_api.settings') + + # This call is what makes the Django application be instrumented + DjangoInstrumentor().instrument() + try: from django.core.management import execute_from_command_line except ImportError as exc: @@ -16,4 +22,4 @@ def main(): execute_from_command_line(sys.argv) if __name__ == '__main__': - main() \ No newline at end of file + main() diff --git a/chp_api/requirements.txt b/chp_api/requirements.txt index 98f5258..2e5c27f 100644 --- a/chp_api/requirements.txt +++ b/chp_api/requirements.txt @@ -1,21 +1,123 @@ -tqdm -djangorestframework -djangorestframework-simplejwt -psycopg2-binary -bmt -reasoner_pydantic -django-environ -django-hosts -gunicorn -django -requests -requests-cache -django-filter -celery -flower -redis -pandas -django-cors-headers -django-oauth-toolkit -nltk -pattern +amqp==5.2.0 +asgiref==3.8.1 +async-timeout==4.0.3 +attrs==23.2.0 +autocommand==2.2.2 +backports.csv==1.0.7 +backports.tarfile==1.1.1 +beautifulsoup4==4.12.3 +billiard==4.2.0 +bmt==1.4.0 +cattrs==23.2.3 +celery==5.4.0 +certifi==2024.2.2 +cffi==1.16.0 +charset-normalizer==3.3.2 +cheroot==10.0.1 +CherryPy==18.9.0 +click==8.1.7 +click-didyoumean==0.3.1 +click-plugins==1.1.1 +click-repl==0.3.0 +cryptography==42.0.5 +curies==0.7.9 +Deprecated==1.2.14 +deprecation==2.1.0 +Django==4.2.11 +django-cors-headers==4.3.1 +django-environ==0.11.2 +django-extensions==3.2.3 +django-filter==24.2 +django-hosts==6.0 +django-oauth-toolkit==2.3.0 +djangorestframework==3.15.1 +djangorestframework-simplejwt==5.3.1 +exceptiongroup==1.2.1 +feedparser==6.0.11 +flower==2.0.1 +future==1.0.0 +hbreader==0.9.1 +humanize==4.9.0 +idna==3.7 +importlib-metadata==7.0.0 +inflect==7.2.0 +iniconfig==2.0.0 +isodate==0.6.1 +jaraco.collections==5.0.1 +jaraco.context==5.3.0 +jaraco.functools==4.0.1 +jaraco.text==3.12.0 +joblib==1.4.0 +json-flattener==0.1.9 +jsonasobj2==1.0.4 +jsonschema==4.21.1 +jsonschema-specifications==2023.12.1 +jwcrypto==1.5.6 +kombu==5.3.7 +linkml-runtime==1.7.5 +lxml==5.2.1 +more-itertools==10.2.0 +mysqlclient==2.2.4 +nltk==3.8.1 +numpy==1.26.4 +oauthlib==3.2.2 +opentelemetry-api==1.24.0 +opentelemetry-instrumentation==0.45b0 +opentelemetry-instrumentation-django==0.45b0 +opentelemetry-instrumentation-wsgi==0.45b0 +opentelemetry-sdk==1.24.0 +opentelemetry-semantic-conventions==0.45b0 +opentelemetry-util-http==0.45b0 +packaging==24.0 +pandas==2.2.2 +Pattern==3.6 +pdfminer.six==20231228 +platformdirs==4.2.1 +pluggy==1.5.0 +portend==3.2.0 +prefixcommons==0.1.12 +prefixmaps==0.2.4 +prometheus_client==0.20.0 +prompt-toolkit==3.0.43 +psycopg2-binary==2.9.9 +pycparser==2.22 +pydantic==1.10.12 +PyJWT==2.8.0 +pyparsing==3.1.2 +pytest==8.1.1 +pytest-logging==2015.11.4 +python-dateutil==2.9.0.post0 +python-docx==1.1.0 +PyTrie==0.4.0 +pytz==2024.1 +PyYAML==6.0.1 +rdflib==7.0.0 +reasoner-pydantic==5.0.2 +redis==5.0.4 +referencing==0.34.0 +regex==2024.4.16 +requests==2.31.0 +requests-cache==1.2.0 +rpds-py==0.18.0 +scipy==1.13.0 +sgmllib3k==1.0.0 +six==1.16.0 +sortedcontainers==2.4.0 +soupsieve==2.5 +sqlparse==0.5.0 +stringcase==1.2.0 +tempora==5.5.1 +tomli==2.0.1 +tornado==6.4 +tqdm==4.66.2 +typeguard==4.2.1 +typing_extensions==4.11.0 +tzdata==2024.1 +url-normalize==1.4.3 +urllib3==2.2.1 +uWSGI==2.0.25.1 +vine==5.1.0 +wcwidth==0.2.13 +wrapt==1.16.0 +zc.lockfile==3.0.post1 +zipp==3.18.1 diff --git a/chp_api/requirements.txt.base b/chp_api/requirements.txt.base new file mode 100644 index 0000000..4afc956 --- /dev/null +++ b/chp_api/requirements.txt.base @@ -0,0 +1,23 @@ +tqdm +djangorestframework +djangorestframework-simplejwt +psycopg2-binary +bmt +reasoner_pydantic +django-environ +django-hosts +django +requests +requests-cache +django-filter +celery +flower +redis +pandas +django-cors-headers +django-oauth-toolkit +nltk +pattern +opentelemetry-sdk +opentelemetry-instrumentation-django +uwsgi diff --git a/compose.chp-api.yaml b/compose.chp-api.yaml index fd5e423..b768bba 100644 --- a/compose.chp-api.yaml +++ b/compose.chp-api.yaml @@ -66,7 +66,12 @@ services: # Uncomment this for production #- DJANGO_SETTINGS_MODULE=mysite.settings.production # Comment this for development - #- DJANGO_SETTINGS_MODULE=mysite.settings.base + - DJANGO_SETTINGS_MODULE=chp_api.settings + # For Open Telemetry + - OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED=true + - OTEL_TRACES_EXPORTER=jaeger + - OTEL_EXPORTER_JAEGER_AGENT_HOST=jaeger-otel-agent.sri + - OTEL_EXPORTER_JAEGER_AGENT_PORT=6831 depends_on: - static-fs - db @@ -78,7 +83,10 @@ services: retries: 3 volumes: - static-files:/home/chp_api/staticfiles - command: gunicorn -c gunicorn.config.py -b 0.0.0.0:8000 chp_api.wsgi:application + #command: uwsgi --http :8000 --max-requests=200 --master --pidfile=/tmp/project-master.pid --logto /tmp/mylog.log --module chp_api.wsgi:application + #command: opentelemetry-instrument --traces_exporter console --metrics_exporter console uwsgi --http :8000 --max-requests=200 --master --pidfile=/tmp/project-master.pid --module chp_api.wsgi:application + command: opentelemetry-instrument --traces_exporter jaeger --metrics_exporter console uwsgi --http :8000 --max-requests=200 --master --pidfile=/tmp/project-master.pid --module chp_api.wsgi:application + #command: gunicorn -c gunicorn.config.py -b 0.0.0.0:8000 chp_api.wsgi:application #command: python3 manage.py runserver 0.0.0.0:8000 worker-api: diff --git a/deploy/chp-api/templates/deployment.yaml b/deploy/chp-api/templates/deployment.yaml index 1997bbd..1863ac0 100644 --- a/deploy/chp-api/templates/deployment.yaml +++ b/deploy/chp-api/templates/deployment.yaml @@ -26,7 +26,7 @@ spec: image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" imagePullPolicy: {{ .Values.image.pullPolicy }} command: ["/bin/sh"] - args: ["-c", "python3 manage.py collectstatic --noinput && gunicorn -c gunicorn.config.py --log-file=- --env DJANGO_SETTINGS_MODULE=chp_api.settings chp_api.wsgi:application --bind 0.0.0.0:8000"] + args: ["-c", "opentelemetry-instrument --traces_exporter jaeger --metrics_exporter console uwsgi --http :8000 --max-requests=200 --master --pidfile=/tmp/project-master.pid --module chp_api.wsgi:application"] ports: - name: http-app containerPort: 8000 @@ -80,6 +80,14 @@ spec: value: "{{ .Values.app.djangoSuperuserUsername }}" - name: DJANGO_SUPERUSER_EMAIL value: "{{ .Values.app.djangoSuperuserEmail }}" + - name: OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED + value: "{{ .Values.app.OtelPythonLoggingAutoInstrumentationEnabled }}" + - name: OTEL_TRACES_EXPORTER + value: "{{ .Values.app.OtelTracesExporter }}" + - name: OTEL_EXPORTER_JAEGER_AGENT_HOST + value: "{{ .Values.app.OtelExporterJaegerAgentHost }}" + - name: OTEL_EXPORTER_JAEGER_AGENT_PORT + value: "{{ .Values.app.OtelExporterJaegerAgentPort }}" - name: {{ .Chart.Name }}-nginx securityContext: {{- toYaml .Values.securityContextNginx | nindent 12 }} diff --git a/deploy/chp-api/values.yaml b/deploy/chp-api/values.yaml index c34c476..8921e43 100644 --- a/deploy/chp-api/values.yaml +++ b/deploy/chp-api/values.yaml @@ -28,6 +28,10 @@ app: djangoSuperuserEmail: "chp_admin@chp.com" staticfsFolder: "/var/www" staticfsDebug: "0" + OtelPythonLoggingAutoInstrumentationEnabled: "true" + OtelTracesExporter: "jaeger" + OtelExporterJaegerAgentHost: "jaeger-otel-agent.sri" + OtelExporterJaegerAgentPort: "6831" # database connection information db: From 727f2a0d4b8d9ee937e6380a1175ddab85522235 Mon Sep 17 00:00:00 2001 From: "CHP Dev Machine User (Chase, Greg, or Anthony)" Date: Tue, 30 Apr 2024 12:11:44 -0400 Subject: [PATCH 115/132] temp solution for CI build to find requirements --- requirements.txt | 123 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..2e5c27f --- /dev/null +++ b/requirements.txt @@ -0,0 +1,123 @@ +amqp==5.2.0 +asgiref==3.8.1 +async-timeout==4.0.3 +attrs==23.2.0 +autocommand==2.2.2 +backports.csv==1.0.7 +backports.tarfile==1.1.1 +beautifulsoup4==4.12.3 +billiard==4.2.0 +bmt==1.4.0 +cattrs==23.2.3 +celery==5.4.0 +certifi==2024.2.2 +cffi==1.16.0 +charset-normalizer==3.3.2 +cheroot==10.0.1 +CherryPy==18.9.0 +click==8.1.7 +click-didyoumean==0.3.1 +click-plugins==1.1.1 +click-repl==0.3.0 +cryptography==42.0.5 +curies==0.7.9 +Deprecated==1.2.14 +deprecation==2.1.0 +Django==4.2.11 +django-cors-headers==4.3.1 +django-environ==0.11.2 +django-extensions==3.2.3 +django-filter==24.2 +django-hosts==6.0 +django-oauth-toolkit==2.3.0 +djangorestframework==3.15.1 +djangorestframework-simplejwt==5.3.1 +exceptiongroup==1.2.1 +feedparser==6.0.11 +flower==2.0.1 +future==1.0.0 +hbreader==0.9.1 +humanize==4.9.0 +idna==3.7 +importlib-metadata==7.0.0 +inflect==7.2.0 +iniconfig==2.0.0 +isodate==0.6.1 +jaraco.collections==5.0.1 +jaraco.context==5.3.0 +jaraco.functools==4.0.1 +jaraco.text==3.12.0 +joblib==1.4.0 +json-flattener==0.1.9 +jsonasobj2==1.0.4 +jsonschema==4.21.1 +jsonschema-specifications==2023.12.1 +jwcrypto==1.5.6 +kombu==5.3.7 +linkml-runtime==1.7.5 +lxml==5.2.1 +more-itertools==10.2.0 +mysqlclient==2.2.4 +nltk==3.8.1 +numpy==1.26.4 +oauthlib==3.2.2 +opentelemetry-api==1.24.0 +opentelemetry-instrumentation==0.45b0 +opentelemetry-instrumentation-django==0.45b0 +opentelemetry-instrumentation-wsgi==0.45b0 +opentelemetry-sdk==1.24.0 +opentelemetry-semantic-conventions==0.45b0 +opentelemetry-util-http==0.45b0 +packaging==24.0 +pandas==2.2.2 +Pattern==3.6 +pdfminer.six==20231228 +platformdirs==4.2.1 +pluggy==1.5.0 +portend==3.2.0 +prefixcommons==0.1.12 +prefixmaps==0.2.4 +prometheus_client==0.20.0 +prompt-toolkit==3.0.43 +psycopg2-binary==2.9.9 +pycparser==2.22 +pydantic==1.10.12 +PyJWT==2.8.0 +pyparsing==3.1.2 +pytest==8.1.1 +pytest-logging==2015.11.4 +python-dateutil==2.9.0.post0 +python-docx==1.1.0 +PyTrie==0.4.0 +pytz==2024.1 +PyYAML==6.0.1 +rdflib==7.0.0 +reasoner-pydantic==5.0.2 +redis==5.0.4 +referencing==0.34.0 +regex==2024.4.16 +requests==2.31.0 +requests-cache==1.2.0 +rpds-py==0.18.0 +scipy==1.13.0 +sgmllib3k==1.0.0 +six==1.16.0 +sortedcontainers==2.4.0 +soupsieve==2.5 +sqlparse==0.5.0 +stringcase==1.2.0 +tempora==5.5.1 +tomli==2.0.1 +tornado==6.4 +tqdm==4.66.2 +typeguard==4.2.1 +typing_extensions==4.11.0 +tzdata==2024.1 +url-normalize==1.4.3 +urllib3==2.2.1 +uWSGI==2.0.25.1 +vine==5.1.0 +wcwidth==0.2.13 +wrapt==1.16.0 +zc.lockfile==3.0.post1 +zipp==3.18.1 From cd49eaf29dea284f123f84457c547c8463a965c7 Mon Sep 17 00:00:00 2001 From: Pouyan Ahmadi <70242801+pahmadi8740@users.noreply.github.com> Date: Tue, 30 Apr 2024 15:09:50 -0400 Subject: [PATCH 116/132] Update Jenkinsfile with the Docker build path --- deploy/chp-api/Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/chp-api/Jenkinsfile b/deploy/chp-api/Jenkinsfile index f432639..65360eb 100644 --- a/deploy/chp-api/Jenkinsfile +++ b/deploy/chp-api/Jenkinsfile @@ -36,7 +36,7 @@ pipeline { when { expression { return env.BUILD == 'true' }} steps { script { - docker.build(env.DOCKER_REPO_NAME, "--no-cache -f ./Dockerfile ./") + docker.build(env.DOCKER_REPO_NAME, "--no-cache -f ./chp_api/Dockerfile ./") sh 'docker login -u AWS -p $(aws ecr get-login-password --region us-east-1) 853771734544.dkr.ecr.us-east-1.amazonaws.com' docker.image(env.DOCKER_REPO_NAME).push("${BUILD_VERSION}") sh 'cp deploy/chp-api/configs/nginx.conf deploy/chp-api/nginx/' From 6458ae9f83dd528a06b1292f09a99f92393aed68 Mon Sep 17 00:00:00 2001 From: Pouyan Ahmadi <70242801+pahmadi8740@users.noreply.github.com> Date: Fri, 3 May 2024 11:03:36 -0400 Subject: [PATCH 117/132] Update values.yaml with new affinity --- deploy/chp-api/values.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/deploy/chp-api/values.yaml b/deploy/chp-api/values.yaml index 2ca91e2..3d75b9d 100644 --- a/deploy/chp-api/values.yaml +++ b/deploy/chp-api/values.yaml @@ -80,7 +80,7 @@ ingress: tolerations: - key: "transltr" - value: "managed-app" + value: "chp" operator: "Equal" effect: "NoSchedule" @@ -90,10 +90,10 @@ affinity: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: matchExpressions: - - key: app + - key: application operator: In values: - - managed-app + - chp topologyKey: "kubernetes.io/hostname" # this ensures pod only runs on node with label application=managed-app nodeAffinity: @@ -103,4 +103,4 @@ affinity: - key: application operator: In values: - - managed-app + - chp From 71e5d6c616692de0017bdfeb987b0491aefa523d Mon Sep 17 00:00:00 2001 From: akadapa <134102641+akadapa@users.noreply.github.com> Date: Tue, 7 May 2024 15:38:31 -0500 Subject: [PATCH 118/132] Update Jenkinsfile --- deploy/chp-api/Jenkinsfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/deploy/chp-api/Jenkinsfile b/deploy/chp-api/Jenkinsfile index 65360eb..2f2b894 100644 --- a/deploy/chp-api/Jenkinsfile +++ b/deploy/chp-api/Jenkinsfile @@ -36,7 +36,9 @@ pipeline { when { expression { return env.BUILD == 'true' }} steps { script { - docker.build(env.DOCKER_REPO_NAME, "--no-cache -f ./chp_api/Dockerfile ./") + sh 'cd chp_api' + docker.build(env.DOCKER_REPO_NAME, "--no-cache -f ./Dockerfile ./") + sh 'cd ..' sh 'docker login -u AWS -p $(aws ecr get-login-password --region us-east-1) 853771734544.dkr.ecr.us-east-1.amazonaws.com' docker.image(env.DOCKER_REPO_NAME).push("${BUILD_VERSION}") sh 'cp deploy/chp-api/configs/nginx.conf deploy/chp-api/nginx/' From de4a9c52c7647a03845b50b7e2cfa5ed61e6be51 Mon Sep 17 00:00:00 2001 From: yakaboskic <35247528+yakaboskic@users.noreply.github.com> Date: Tue, 7 May 2024 16:47:31 -0400 Subject: [PATCH 119/132] Update Jenkinsfile --- deploy/chp-api/Jenkinsfile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/deploy/chp-api/Jenkinsfile b/deploy/chp-api/Jenkinsfile index 2f2b894..77131dc 100644 --- a/deploy/chp-api/Jenkinsfile +++ b/deploy/chp-api/Jenkinsfile @@ -36,9 +36,10 @@ pipeline { when { expression { return env.BUILD == 'true' }} steps { script { - sh 'cd chp_api' - docker.build(env.DOCKER_REPO_NAME, "--no-cache -f ./Dockerfile ./") - sh 'cd ..' + dir('chp_api') { + script { + docker.build(env.DOCKER_REPO_NAME, "--no-cache -f ./Dockerfile ./") + } sh 'docker login -u AWS -p $(aws ecr get-login-password --region us-east-1) 853771734544.dkr.ecr.us-east-1.amazonaws.com' docker.image(env.DOCKER_REPO_NAME).push("${BUILD_VERSION}") sh 'cp deploy/chp-api/configs/nginx.conf deploy/chp-api/nginx/' From e7d39203a6d679fe283c0c5005a76627768c9bd2 Mon Sep 17 00:00:00 2001 From: yakaboskic <35247528+yakaboskic@users.noreply.github.com> Date: Tue, 7 May 2024 16:47:55 -0400 Subject: [PATCH 120/132] Update Jenkinsfile --- deploy/chp-api/Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/deploy/chp-api/Jenkinsfile b/deploy/chp-api/Jenkinsfile index 77131dc..799a5de 100644 --- a/deploy/chp-api/Jenkinsfile +++ b/deploy/chp-api/Jenkinsfile @@ -40,6 +40,7 @@ pipeline { script { docker.build(env.DOCKER_REPO_NAME, "--no-cache -f ./Dockerfile ./") } + } sh 'docker login -u AWS -p $(aws ecr get-login-password --region us-east-1) 853771734544.dkr.ecr.us-east-1.amazonaws.com' docker.image(env.DOCKER_REPO_NAME).push("${BUILD_VERSION}") sh 'cp deploy/chp-api/configs/nginx.conf deploy/chp-api/nginx/' From 4da6e175138d7a9b6300dfd8447cf3adcb4ac4d3 Mon Sep 17 00:00:00 2001 From: "CHP Dev Machine User (Chase, Greg, or Anthony)" Date: Wed, 8 May 2024 18:49:07 -0400 Subject: [PATCH 121/132] fixed a few bugs with the trapi 1.5 deployment --- ...0_alter_dispatchersetting_trapi_version.py | 18 +++ chp_api/dispatcher/models.py | 2 +- chp_api/gennifer/app_interface.py | 2 +- chp_api/gennifer/trapi_interface.py | 65 +++++---- requirements.txt | 123 ------------------ 5 files changed, 57 insertions(+), 153 deletions(-) create mode 100644 chp_api/dispatcher/migrations/0010_alter_dispatchersetting_trapi_version.py delete mode 100644 requirements.txt diff --git a/chp_api/dispatcher/migrations/0010_alter_dispatchersetting_trapi_version.py b/chp_api/dispatcher/migrations/0010_alter_dispatchersetting_trapi_version.py new file mode 100644 index 0000000..1952c00 --- /dev/null +++ b/chp_api/dispatcher/migrations/0010_alter_dispatchersetting_trapi_version.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.11 on 2024-05-08 22:06 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('dispatcher', '0009_rename_dispatchersettings_dispatchersetting'), + ] + + operations = [ + migrations.AlterField( + model_name='dispatchersetting', + name='trapi_version', + field=models.CharField(default='1.5', max_length=28), + ), + ] diff --git a/chp_api/dispatcher/models.py b/chp_api/dispatcher/models.py index 22d49cd..9b62d73 100644 --- a/chp_api/dispatcher/models.py +++ b/chp_api/dispatcher/models.py @@ -71,7 +71,7 @@ def load(cls): return obj class DispatcherSetting(Singleton): - trapi_version = models.CharField(max_length=28, default='1.4') + trapi_version = models.CharField(max_length=28, default='1.5') sri_node_normalizer_baseurl = models.URLField(max_length=128, default='https://nodenormalization-sri.renci.org') def __str__(self): diff --git a/chp_api/gennifer/app_interface.py b/chp_api/gennifer/app_interface.py index 3e9a717..49ab30f 100644 --- a/chp_api/gennifer/app_interface.py +++ b/chp_api/gennifer/app_interface.py @@ -9,7 +9,7 @@ def get_app_config(message: Union[Message, None]) -> GenniferConfig: def get_trapi_interface(get_app_config: GenniferConfig = get_app_config(None)): - return TrapiInterface(trapi_version='1.4') + return TrapiInterface(trapi_version='1.5') def get_meta_knowledge_graph() -> MetaKnowledgeGraph: diff --git a/chp_api/gennifer/trapi_interface.py b/chp_api/gennifer/trapi_interface.py index 522b75a..64f7710 100644 --- a/chp_api/gennifer/trapi_interface.py +++ b/chp_api/gennifer/trapi_interface.py @@ -6,7 +6,9 @@ import logging from typing import Tuple, Union +from pydantic import parse_obj_as from django.db.models import QuerySet +from reasoner_pydantic.utils import HashableMapping from django.core.exceptions import ObjectDoesNotExist from reasoner_pydantic import MetaKnowledgeGraph, Message, KnowledgeGraph from reasoner_pydantic.kgraph import RetrievalSource, Attribute @@ -25,7 +27,7 @@ def note(self, message, *args, **kwargs): APP_PATH = os.path.dirname(os.path.abspath(__file__)) class TrapiInterface: - def __init__(self, trapi_version: str = '1.4'): + def __init__(self, trapi_version: str = '1.5'): self.trapi_version = trapi_version def get_meta_knowledge_graph(self) -> MetaKnowledgeGraph: @@ -42,7 +44,7 @@ def get_name(self) -> str: def _get_sources(self): source_1 = RetrievalSource(resource_id = "infores:connections-hypothesis", resource_role="primary_knowledge_source") - return {source_1} + return [source_1] def _get_attributes(self, val, algorithm_instance, dataset): att_1 = Attribute( @@ -64,18 +66,14 @@ def _get_attributes(self, val, algorithm_instance, dataset): description=f'{dataset.title}: {dataset.description}', ) att_4 = Attribute( - attribute_type_id = 'primary_knowledge_source', - value='infores:connections-hypothesis', - value_url='https://github.com/di2ag/gennifer', - description='The Connections Hypothesis Provider from NCATS Translator.' + attribute_type_id = 'knowledge_level', + value='statistical_association' ) - return {att_1, att_2, att_3, att_4} + return [att_1, att_2, att_3, att_4] def _add_results( self, message, - node_bindings, - edge_bindings, qg_subject_id, subject_curies, subject_category, @@ -88,13 +86,15 @@ def _add_results( algorithms, datasets, ): + node_binding_group = [] + edge_binding_group = [] nodes = dict() edges = dict() val_id = 0 for subject_curie in subject_curies: for object_curie in object_curies: - nodes[subject_curie] = {"categories": [subject_category]} - nodes[object_curie] = {"categories": [object_category]} + nodes[subject_curie] = {"categories": [subject_category], "attributes" : []} + nodes[object_curie] = {"categories": [object_category], "attributes" : []} kg_edge_id = str(uuid.uuid4()) edges[kg_edge_id] = {"predicate": predicate, "subject": subject_curie, @@ -106,14 +106,19 @@ def _add_results( datasets[val_id], )} val_id += 1 - node_bindings[qg_subject_id].add(NodeBinding(id = subject_curie)) - node_bindings[qg_object_id].add(NodeBinding(id = object_curie)) - edge_bindings[qg_edge_id].add(EdgeBinding(id = kg_edge_id)) + node_bindings = {qg_subject_id: set(), qg_object_id: set()} + edge_bindings = {qg_edge_id : set()} + node_bindings[qg_subject_id].add(NodeBinding(id = subject_curie, attributes=[])) + node_bindings[qg_object_id].add(NodeBinding(id = object_curie, attributes=[])) + edge_bindings[qg_edge_id].add(EdgeBinding(id = kg_edge_id, attributes=[])) + node_binding_group.append(node_bindings) + edge_binding_group.append(edge_bindings) kgraph = KnowledgeGraph(nodes=nodes, edges=edges) if message.knowledge_graph is not None: message.knowledge_graph.update(kgraph) else: message.knowledge_graph = kgraph + return node_binding_group, edge_binding_group def _extract_qnode_info(self, qnode): return qnode.ids, qnode.categories[0] @@ -127,8 +132,8 @@ def get_response(self, message: Message, logger): subject_curies, subject_category = self._extract_qnode_info(message.query_graph.nodes[qg_subject_id]) object_curies, object_category = self._extract_qnode_info(message.query_graph.nodes[qg_object_id]) # annotation - node_bindings = {qg_subject_id: set(), qg_object_id: set()} - edge_bindings = {qg_edge_id : set()} + node_bindings = [] + edge_bindings = [] #TODO: Should probably offer support to return all results if subject_curies is not None and object_curies is not None: logger.info('Annotation edges detected') @@ -156,22 +161,23 @@ def get_response(self, message: Message, logger): vals = [r.edge_weight for r in results] algorithms = [r.study.algorithm_instance for r in results] datasets = [r.study.dataset for r in results] - self._add_results( + node_binding_group, edge_binding_group = self._add_results( message, - node_bindings, - edge_bindings, qg_subject_id, subject_curies, subject_category, predicate, qg_edge_id, - qg_object_id, - [curie], + object_mapping, + qg_object_id, + [curie], object_category, vals, algorithms, - datasets, + datasets ) + node_bindings.extend(node_binding_group) + edge_bindings.extend(edge_binding_group) elif subject_curies is not None: logger.info('Wildcard detected') for curie in subject_curies: @@ -194,10 +200,8 @@ def get_response(self, message: Message, logger): vals = [r.edge_weight for r in results] algorithms = [r.study.algorithm_instance for r in results] datasets = [r.study.dataset for r in results] - self._add_results( + node_binding_group, edge_binding_group = self._add_results( message, - node_bindings, - edge_bindings, qg_subject_id, subject_curies, subject_category, @@ -210,10 +214,15 @@ def get_response(self, message: Message, logger): algorithms, datasets, ) + node_bindings.extend(node_binding_group) + edge_bindings.extend(edge_binding_group) else: logger.info('No curies detected. Returning no results') return message - analysis = Analysis(resource_id='infores:connections-hypothesis', edge_bindings=edge_bindings) - result = Result(node_bindings=node_bindings, analyses=[analysis]) - message.results = Results(__root__ = {result}) + results = Results(__root__ = parse_obj_as(HashableMapping, {})) + for node_binding_dict, edge_binding_dict in zip(node_bindings, edge_bindings): + analysis = Analysis(resource_id='infores:connections-hypothesis', edge_bindings = edge_binding_dict, attributes=[]) + result = Result(node_bindings = node_binding_dict, analyses=[analysis]) + results.add(result) + message.results = results return message diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 2e5c27f..0000000 --- a/requirements.txt +++ /dev/null @@ -1,123 +0,0 @@ -amqp==5.2.0 -asgiref==3.8.1 -async-timeout==4.0.3 -attrs==23.2.0 -autocommand==2.2.2 -backports.csv==1.0.7 -backports.tarfile==1.1.1 -beautifulsoup4==4.12.3 -billiard==4.2.0 -bmt==1.4.0 -cattrs==23.2.3 -celery==5.4.0 -certifi==2024.2.2 -cffi==1.16.0 -charset-normalizer==3.3.2 -cheroot==10.0.1 -CherryPy==18.9.0 -click==8.1.7 -click-didyoumean==0.3.1 -click-plugins==1.1.1 -click-repl==0.3.0 -cryptography==42.0.5 -curies==0.7.9 -Deprecated==1.2.14 -deprecation==2.1.0 -Django==4.2.11 -django-cors-headers==4.3.1 -django-environ==0.11.2 -django-extensions==3.2.3 -django-filter==24.2 -django-hosts==6.0 -django-oauth-toolkit==2.3.0 -djangorestframework==3.15.1 -djangorestframework-simplejwt==5.3.1 -exceptiongroup==1.2.1 -feedparser==6.0.11 -flower==2.0.1 -future==1.0.0 -hbreader==0.9.1 -humanize==4.9.0 -idna==3.7 -importlib-metadata==7.0.0 -inflect==7.2.0 -iniconfig==2.0.0 -isodate==0.6.1 -jaraco.collections==5.0.1 -jaraco.context==5.3.0 -jaraco.functools==4.0.1 -jaraco.text==3.12.0 -joblib==1.4.0 -json-flattener==0.1.9 -jsonasobj2==1.0.4 -jsonschema==4.21.1 -jsonschema-specifications==2023.12.1 -jwcrypto==1.5.6 -kombu==5.3.7 -linkml-runtime==1.7.5 -lxml==5.2.1 -more-itertools==10.2.0 -mysqlclient==2.2.4 -nltk==3.8.1 -numpy==1.26.4 -oauthlib==3.2.2 -opentelemetry-api==1.24.0 -opentelemetry-instrumentation==0.45b0 -opentelemetry-instrumentation-django==0.45b0 -opentelemetry-instrumentation-wsgi==0.45b0 -opentelemetry-sdk==1.24.0 -opentelemetry-semantic-conventions==0.45b0 -opentelemetry-util-http==0.45b0 -packaging==24.0 -pandas==2.2.2 -Pattern==3.6 -pdfminer.six==20231228 -platformdirs==4.2.1 -pluggy==1.5.0 -portend==3.2.0 -prefixcommons==0.1.12 -prefixmaps==0.2.4 -prometheus_client==0.20.0 -prompt-toolkit==3.0.43 -psycopg2-binary==2.9.9 -pycparser==2.22 -pydantic==1.10.12 -PyJWT==2.8.0 -pyparsing==3.1.2 -pytest==8.1.1 -pytest-logging==2015.11.4 -python-dateutil==2.9.0.post0 -python-docx==1.1.0 -PyTrie==0.4.0 -pytz==2024.1 -PyYAML==6.0.1 -rdflib==7.0.0 -reasoner-pydantic==5.0.2 -redis==5.0.4 -referencing==0.34.0 -regex==2024.4.16 -requests==2.31.0 -requests-cache==1.2.0 -rpds-py==0.18.0 -scipy==1.13.0 -sgmllib3k==1.0.0 -six==1.16.0 -sortedcontainers==2.4.0 -soupsieve==2.5 -sqlparse==0.5.0 -stringcase==1.2.0 -tempora==5.5.1 -tomli==2.0.1 -tornado==6.4 -tqdm==4.66.2 -typeguard==4.2.1 -typing_extensions==4.11.0 -tzdata==2024.1 -url-normalize==1.4.3 -urllib3==2.2.1 -uWSGI==2.0.25.1 -vine==5.1.0 -wcwidth==0.2.13 -wrapt==1.16.0 -zc.lockfile==3.0.post1 -zipp==3.18.1 From ac6c98a99e315f268e4ec7aa303631b0177f9d42 Mon Sep 17 00:00:00 2001 From: "CHP Dev Machine User (Chase, Greg, or Anthony)" Date: Thu, 9 May 2024 11:10:54 -0400 Subject: [PATCH 122/132] set Dockerfile to run migrations in the event there are any migrations to run --- chp_api/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/chp_api/Dockerfile b/chp_api/Dockerfile index abcdc7d..95b6ba2 100644 --- a/chp_api/Dockerfile +++ b/chp_api/Dockerfile @@ -70,3 +70,5 @@ RUN chown -R chp_api:chp_api $APP_HOME \ # change to the app user USER chp_api + +CMD ["sh", "python manage.py migrate"] From 82439beea76e01bf600871b205900ad8df680f18 Mon Sep 17 00:00:00 2001 From: akadapa <134102641+akadapa@users.noreply.github.com> Date: Wed, 15 May 2024 13:03:54 -0500 Subject: [PATCH 123/132] Update values.yaml --- deploy/chp-api/values.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/deploy/chp-api/values.yaml b/deploy/chp-api/values.yaml index 3d75b9d..1869f18 100644 --- a/deploy/chp-api/values.yaml +++ b/deploy/chp-api/values.yaml @@ -80,7 +80,7 @@ ingress: tolerations: - key: "transltr" - value: "chp" + value: "chp-api" operator: "Equal" effect: "NoSchedule" @@ -93,7 +93,7 @@ affinity: - key: application operator: In values: - - chp + - chp-api topologyKey: "kubernetes.io/hostname" # this ensures pod only runs on node with label application=managed-app nodeAffinity: @@ -103,4 +103,4 @@ affinity: - key: application operator: In values: - - chp + - chp-api From e9b7c59e324ef1130b1479114592833b1a0656dd Mon Sep 17 00:00:00 2001 From: GregHyde Date: Wed, 15 May 2024 15:46:40 -0400 Subject: [PATCH 124/132] updated values --- deploy/chp-api/values.yaml | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/deploy/chp-api/values.yaml b/deploy/chp-api/values.yaml index 1869f18..bee66c9 100644 --- a/deploy/chp-api/values.yaml +++ b/deploy/chp-api/values.yaml @@ -28,10 +28,6 @@ app: djangoSuperuserEmail: "chp_admin@chp.com" staticfsFolder: "/var/www" staticfsDebug: "0" - OtelPythonLoggingAutoInstrumentationEnabled: "true" - OtelTracesExporter: "jaeger" - OtelExporterJaegerAgentHost: "jaeger-otel-agent.sri" - OtelExporterJaegerAgentPort: "6831" # database connection information db: @@ -80,7 +76,7 @@ ingress: tolerations: - key: "transltr" - value: "chp-api" + value: "managed-app" operator: "Equal" effect: "NoSchedule" @@ -90,10 +86,10 @@ affinity: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: matchExpressions: - - key: application + - key: app operator: In values: - - chp-api + - managed-app topologyKey: "kubernetes.io/hostname" # this ensures pod only runs on node with label application=managed-app nodeAffinity: @@ -103,4 +99,4 @@ affinity: - key: application operator: In values: - - chp-api + - managed-app From 349170ce715a90be223ea6741cd621059dc03e8a Mon Sep 17 00:00:00 2001 From: akadapa <134102641+akadapa@users.noreply.github.com> Date: Wed, 15 May 2024 15:26:10 -0500 Subject: [PATCH 125/132] Update values.yaml --- deploy/chp-api/values.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/deploy/chp-api/values.yaml b/deploy/chp-api/values.yaml index bee66c9..bfa0aae 100644 --- a/deploy/chp-api/values.yaml +++ b/deploy/chp-api/values.yaml @@ -76,7 +76,7 @@ ingress: tolerations: - key: "transltr" - value: "managed-app" + value: "chp" operator: "Equal" effect: "NoSchedule" @@ -86,10 +86,10 @@ affinity: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: matchExpressions: - - key: app + - key: applicaiton operator: In values: - - managed-app + - chp topologyKey: "kubernetes.io/hostname" # this ensures pod only runs on node with label application=managed-app nodeAffinity: @@ -99,4 +99,4 @@ affinity: - key: application operator: In values: - - managed-app + - chp From 22828ba2dbc400031ca8377b14356898b3121865 Mon Sep 17 00:00:00 2001 From: GregHydeDartmouth <60626258+GregHydeDartmouth@users.noreply.github.com> Date: Thu, 16 May 2024 09:51:16 -0400 Subject: [PATCH 126/132] Update Dockerfile put settings env variable explicitly in dockerfile. --- chp_api/Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/chp_api/Dockerfile b/chp_api/Dockerfile index 95b6ba2..0363063 100644 --- a/chp_api/Dockerfile +++ b/chp_api/Dockerfile @@ -64,6 +64,9 @@ COPY . . #COPY ./chp_db_fixture.json.gz $APP_HOME #COPY ./gunicorn.config.py $APP_HOME +# set DJANGO_SETTINGS_MODULE environment variable +ENV DJANGO_SETTINGS_MODULE=chp_api.settings + # chown all the files to the app user RUN chown -R chp_api:chp_api $APP_HOME \ && chmod 700 $APP_HOME/staticfiles From 3cfb83e45a3ea9f6736076cf4776b08b638c121c Mon Sep 17 00:00:00 2001 From: "CHP Dev Machine User (Chase, Greg, or Anthony)" Date: Thu, 6 Jun 2024 17:47:10 -0400 Subject: [PATCH 127/132] reverting explicit settings link. Should not be needed --- chp_api/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chp_api/Dockerfile b/chp_api/Dockerfile index 0363063..5c61142 100644 --- a/chp_api/Dockerfile +++ b/chp_api/Dockerfile @@ -65,7 +65,7 @@ COPY . . #COPY ./gunicorn.config.py $APP_HOME # set DJANGO_SETTINGS_MODULE environment variable -ENV DJANGO_SETTINGS_MODULE=chp_api.settings +#ENV DJANGO_SETTINGS_MODULE=chp_api.settings # chown all the files to the app user RUN chown -R chp_api:chp_api $APP_HOME \ From e90072ad712a5c1bca61acd5c8f8dba1be6e0e6e Mon Sep 17 00:00:00 2001 From: GregHydeDartmouth <60626258+GregHydeDartmouth@users.noreply.github.com> Date: Tue, 6 Aug 2024 11:05:36 -0400 Subject: [PATCH 128/132] Update urls.py --- chp_api/dispatcher/urls.py | 1 + 1 file changed, 1 insertion(+) diff --git a/chp_api/dispatcher/urls.py b/chp_api/dispatcher/urls.py index a5d1588..24e0751 100644 --- a/chp_api/dispatcher/urls.py +++ b/chp_api/dispatcher/urls.py @@ -23,6 +23,7 @@ path('query/', views.query.as_view()), path('query', views.query.as_view()), path('meta_knowledge_graph/', views.meta_knowledge_graph.as_view()), + path('meta_knowledge_graph', views.meta_knowledge_graph.as_view()), path('versions/', views.versions.as_view()), path('transactions/', views.TransactionList.as_view(), name='transaction-list'), path('recent/', views.RecentTransactionList.as_view(), name='recent-transaction-list'), From f669baf9d72ee4dd6f47a5d9f470776bc3f90703 Mon Sep 17 00:00:00 2001 From: GregHydeDartmouth <60626258+GregHydeDartmouth@users.noreply.github.com> Date: Wed, 11 Sep 2024 09:34:06 -0400 Subject: [PATCH 129/132] Update README.md --- README.md | 107 ++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 79 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index e5727fa..82acc3c 100644 --- a/README.md +++ b/README.md @@ -1,43 +1,94 @@ # Connections Hypothesis Provider API Documentation -## Description -Connections Hypothesis Provider (CHP) is a service built by Dartmouth College (PI – Dr. Eugene Santos) and Tufts University (Co-PI – Joseph Gormley) in collaboration with the National Center for Advancing Translational Sciences (NCATS). CHP aims to leverage clinical data along with structured biochemical knowledge to derive a computational representation of pathway structures and molecular components to support human and machine-driven interpretation, enable pathway-based biomarker discovery, and aid in the drug development process. -In its current version, CHP supports queries relating to genetic, therapeutic, and patient clinical features (e.g. tumor staging) contribution toward patient survival, as computed within the context of our test pilot: a robust breast cancer dataset from The Cancer Genome Atlas (TCGA). We are using this as a proving ground for our system’s basic operations as we work to incorporate structured pathway knowledge from Reactome and pathway analysis methods into the tool. - ## Introduction -Our system utilizes the Bayesian Knowledge Base (BKB), which is a directed probabilistic graphical model capable of modeling incomplete, cyclic and heterogenous knowledge. We currently expose a portion of our computational framework as a proving ground to reason over patterns that exist within the TCGA dataset, determine sensitivity of features to the underlying conditional probabilities, and guide hypothesis generation. Querying semantics are of the form P(Target=t | Evidence=e). To this end we’ve explored setting survival time targets specifying mutational profiles and drug treatments as evidence, though this process is generalizable to any TCGA feature type (e.g., tumor staging, gene copy number, age of diagnosis, etc.). -However, as we incorporate pathway knowledge from domain-standard resources like Reactome and overlay biochemical pathway traversal logic, we will extend these querying semantics to derive inferences relating to biochemical mechanisms. The short-term benefits of tacking on this difficult task are to provide mechanism-of-action level analysis over cellular behavior and clinical outcomes. The long-term benefits of this work is to characterize three categories of information and discovery critical to pathway science, pathway components, pathway topology, and pathway dynamics. -Queries are governed by the Translator Reasoner API (TRAPI) which support the Biolink Model ontology. We’ve constructed a TRAPI compliant schema that represents our probabilistic queries and is digestible by our service. Upon receiving TRAPI compliant queries we return a conditional probability pertaining to the query as well as the auditable features our system captured and their overall sensitivity to the conditional probability. These features can be used to guide future exploration of the dataset and be used to lead to novel conclusions over the data. - -## Terms and Definitions -The greater NCATS consortium uses a series of terms (that we have adopted) to convey meaning quickly. A link to those terms and their definitions are available here: https://docs.google.com/spreadsheets/d/1C8hKXacxtQC5UzXI4opQs1r4pBJ_5hqgXrZH_raYQ4w/edit#gid=1581951609 -We extend this list local to our KP (Look, here is an NCATS term right here!) with the following terms: -• Connections Hypothesis Provider – CHP -• The Cancer Genome Atlas – TCGA -• Bayesian Knowledge Base – BKB +Connections Hypothesis Provider (CHP) is a collaborative service developed by Dartmouth College and Tufts University, in partnership with the National Center for Advancing Translational Sciences (NCATS). CHP's mission is to utilize clinical data and structured biochemical knowledge to create computational representations of pathway structures and molecular components. This effort supports both human and machine-driven analysis, enabling pathway-based biomarker discovery and contributing to the drug development process. -## Smart API -CHP is registered with Smart API: https://smart-api.info/ui/855adaa128ce5aa58a091d99e520d396 +Currently, CHP serves as a platform for Gene Regulatory Network (GRN) discovery, allowing researchers to upload their own RNASeq data, or work with pre-existing datasets. Users can analyze, refine, and explore novel gene-to-gene regulatory relationships through our core discovery tool, GenNIFER, a web-based portal featuring state-of-the-art GRN inference algorithms. Additionally, the platform integrates with the Translator ecosystem, allowing users to contextualize their findings using existing knowledge sources. -## How To Use -We encourage anyone looking for tooling/instructions, to interface with our API, to the following repository, CHP Client, https://github.com/di2ag/chp_client. CHP Client is a lightweight Python client that interfaces CHP. It is meant to be an easy-to-use wrapper utility to both run and build TRAPI queries that the CHP web service will understand. +Through its integration with the [Knowledge Collaboratory](https://github.com/MaastrichtU-IDS/knowledge-collaboratory) team, GenNIFER also enables researchers to publish their findings back into the Translator ecosystem, facilitating further collaboration and discovery. -Our API is in active developement and is currently following [Translator Reasoner API standards 1.2.0](https://github.com/NCATSTranslator/ReasonerAPI) +This Docker repository contains the necessary build instructions to launch our tooling in support of the CHP API. Specifically, CHP API powers the following: +* [GenNIFER](https://github.com/di2ag/gennifer): our tool for GRN discovery. +* [Tissue-Gene Specificity Tool](https://github.com/di2ag/gene-specificity) – Our tool for assessing a gene’s expression specificity to a tissue. + +For more specifics about either application, see their respective repository READMEs. -Our API is currently live at: [https://chp-api.transltr.io](https://chp-api.transltr.io) +## Interacting with CHP API +The CHP API provides supporting data as a Knowledge Provider (KP) for the Translator consortium and can be interacted with from our build servers. For a list of knowledge that we support, see our meta knowledge graph. Further details about CHP API can be found in its [SmartAPI](http://smart-api.info/registry?q=412af63e15b73e5a30778aac84ce313f) registration. We also provide examples for how to interact with the individual tools in their own relevant repository. +### Build servers +* Production: https://chp-api.transltr.io +* Testing: https://chp-api.test.transltr.io +* Staging: https://chp-api.ci.transltr.io -## Open Endpoints +### Endpoints * [query](query.md) : `POST /query/` -* [predicates](predicates.md) : `GET /predicates/` -* [curies](curies.md) : `GET /curies/` - -## Other Notable Links -Our roadmap outlining or KP’s milestones and the progression of those milestones: https://github.com/di2ag/Connections-Hypothesis-Provider-Roadmap +* [predicates](predicates.md) : `GET /meta_knowledge_graph/` -Our NCATS Wiki Page: https://github.com/NCATSTranslator/Translator-All/wiki/Connections-Hypothesis-Provider +### Meta Knowledge Graph +
+ Click to view json example -A repository for our reasoning code: https://github.com/di2ag/chp + ```json + { + "nodes": { + "biolink:Gene": { + "id_prefixes": [ + "ENSEMBL" + ], + "attributes": null + }, + "biolink:GrossAnatomicalStructure": { + "id_prefixes": [ + "UBERON", + "EFO" + ], + "attributes": null + } + }, + "edges": [ + { + "subject": "biolink:Gene", + "predicate": "biolink:expressed_in", + "object": "biolink:GrossAnatomicalStructure", + "qualifiers": null, + "attributes": null, + "knowledge_types": null, + "association": null + }, + { + "subject": "biolink:GrossAnatomicalStructure", + "predicate": "biolink:expresses", + "object": "biolink:Gene", + "qualifiers": null, + "attributes": null, + "knowledge_types": null, + "association": null + }, + { + "subject": "biolink:Gene", + "predicate": "biolink:regulates", + "object": "biolink:Gene", + "qualifiers": null, + "attributes": null, + "knowledge_types": null, + "association": null + }, + { + "subject": "biolink:Gene", + "predicate": "biolink:regulated_by", + "object": "biolink:Gene", + "qualifiers": null, + "attributes": null, + "knowledge_types": null, + "association": null + } + ] +} +``` +
+### SmartAPI +CHP is registered with [SmartAPI](http://smart-api.info/registry?q=412af63e15b73e5a30778aac84ce313f). ## Contacts Dr. Eugene Santos (PI): Eugene.Santos.Jr@dartmouth.edu From 1f3d4af833bfe29d47b6163f47f430e11ec598bd Mon Sep 17 00:00:00 2001 From: GregHydeDartmouth <60626258+GregHydeDartmouth@users.noreply.github.com> Date: Thu, 12 Sep 2024 09:00:04 -0400 Subject: [PATCH 130/132] Update README.md changed contact --- README.md | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 82acc3c..e3d1f04 100644 --- a/README.md +++ b/README.md @@ -90,8 +90,5 @@ The CHP API provides supporting data as a Knowledge Provider (KP) for the Transl ### SmartAPI CHP is registered with [SmartAPI](http://smart-api.info/registry?q=412af63e15b73e5a30778aac84ce313f). -## Contacts -Dr. Eugene Santos (PI): Eugene.Santos.Jr@dartmouth.edu - -Joseph Gormley (Co-PI): jgormley@tuftsmedicalcenter.org - +## Contact for this code +Gregory Hyde (gregory.m.hyde.th@dartmouth.edu) From 9c4919427e19969c751cf563698df099fc2f34a1 Mon Sep 17 00:00:00 2001 From: GregHydeDartmouth <60626258+GregHydeDartmouth@users.noreply.github.com> Date: Thu, 12 Sep 2024 09:10:23 -0400 Subject: [PATCH 131/132] Update README.md --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index e3d1f04..49bf6bf 100644 --- a/README.md +++ b/README.md @@ -92,3 +92,8 @@ CHP is registered with [SmartAPI](http://smart-api.info/registry?q=412af63e15b73 ## Contact for this code Gregory Hyde (gregory.m.hyde.th@dartmouth.edu) + +## TRAPI and Biolink +Trapi = 1.5.0 + +Biolink = 4.2.0 From fed8758a4e88d1c1a2d2a72e03599fdecc69adb4 Mon Sep 17 00:00:00 2001 From: GregHydeDartmouth <60626258+GregHydeDartmouth@users.noreply.github.com> Date: Thu, 12 Sep 2024 09:15:54 -0400 Subject: [PATCH 132/132] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 49bf6bf..071a970 100644 --- a/README.md +++ b/README.md @@ -3,13 +3,13 @@ ## Introduction Connections Hypothesis Provider (CHP) is a collaborative service developed by Dartmouth College and Tufts University, in partnership with the National Center for Advancing Translational Sciences (NCATS). CHP's mission is to utilize clinical data and structured biochemical knowledge to create computational representations of pathway structures and molecular components. This effort supports both human and machine-driven analysis, enabling pathway-based biomarker discovery and contributing to the drug development process. -Currently, CHP serves as a platform for Gene Regulatory Network (GRN) discovery, allowing researchers to upload their own RNASeq data, or work with pre-existing datasets. Users can analyze, refine, and explore novel gene-to-gene regulatory relationships through our core discovery tool, GenNIFER, a web-based portal featuring state-of-the-art GRN inference algorithms. Additionally, the platform integrates with the Translator ecosystem, allowing users to contextualize their findings using existing knowledge sources. +Currently, CHP serves as a platform for Gene Regulatory Network (GRN) discovery, allowing researchers to upload their own RNASeq data, or work with pre-existing datasets. Users can analyze, refine, and explore novel gene-to-gene regulatory relationships through our core discovery tool, GenNIFER, a web-based portal featuring state-of-the-art GRN inferencing algorithms. Additionally, the platform integrates with the Translator ecosystem, allowing users to contextualize their findings using existing knowledge sources. Through its integration with the [Knowledge Collaboratory](https://github.com/MaastrichtU-IDS/knowledge-collaboratory) team, GenNIFER also enables researchers to publish their findings back into the Translator ecosystem, facilitating further collaboration and discovery. This Docker repository contains the necessary build instructions to launch our tooling in support of the CHP API. Specifically, CHP API powers the following: * [GenNIFER](https://github.com/di2ag/gennifer): our tool for GRN discovery. -* [Tissue-Gene Specificity Tool](https://github.com/di2ag/gene-specificity) – Our tool for assessing a gene’s expression specificity to a tissue. +* [Tissue-Gene Specificity Tool](https://github.com/di2ag/gene-specificity): our tool for assessing a gene’s expression specificity to a tissue. For more specifics about either application, see their respective repository READMEs.