diff --git a/.github/workflows/lint-check.yml b/.github/workflows/lint-check.yml index efee9f91a..5124317a7 100644 --- a/.github/workflows/lint-check.yml +++ b/.github/workflows/lint-check.yml @@ -1,7 +1,9 @@ name: Lint Check on: - push + pull_request: + # The branches below must be a subset of the branches above + branches: [ develop ] jobs: lint: diff --git a/.gitignore b/.gitignore index f4d38460f..1238b4a67 100644 --- a/.gitignore +++ b/.gitignore @@ -8,7 +8,7 @@ node_modules *.DS_Store deployment/helm deployment/compose -docker-compose.yaml +/docker-compose.yaml *.egg-info *.idea /build diff --git a/README.md b/README.md index 1806489ff..0ddee245c 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,9 @@ Can scaffold and maintain your cloud solution on top of Cloudharness without wri Kubernetes templates, with in place common utilities and applications already configured for you. What building your cloud solution with CloudHarness gives to you: + - Common framework and utilities to develop and deploy micro-service application + - Helm chart automatic generation - deployments - services @@ -23,11 +25,13 @@ What building your cloud solution with CloudHarness gives to you: - databases (postgreql) - access gatekeepers configuration - secrets and configmaps + * Automatic build and push of images * REST-API scaffolding building based on OpenApi * Continuous deployment script generation * Debug backend applications running on Kubernetes * Python cluster access utilities + * Prebuilt support applications and shared library to: * Log in and user management - based on Keycloak * Submit batch and asynchronous workflows - based on Argo @@ -43,17 +47,18 @@ What building your cloud solution with CloudHarness gives to you: The microservice architecture is a great to get code separation and flexible development, but may not be of easy implementation, especially for small development teams/projects. In particular, these questions may rise: - - How do I create a deployment for my microservices? - - How do I orchestrate my microservices? - - How to create consistent api documentation? - - Do I need to be an experienced devops to create a micro-service based application? - - Wouldn't it be nice to develop a plain database/backend/frontend application without infrastructure boilerplate but still be able to configure everything I want when needed? - - How to run batch operations like ETL processes easily and efficiently in a cloud environment? - - How to manage databases without being locked to a specific vendor solution? - - How to perform database backups? - - How to manage secret data? - - What about having a precounfigured account management application? - - Sooner rather than later I'll need an orchestration queue. Why not have that just ready to use? + +- How do I create a deployment for my microservices? +- How do I orchestrate my microservices? +- How to create consistent api documentation? +- Do I need to be an experienced devops to create a micro-service based application? +- Wouldn't it be nice to develop a plain database/backend/frontend application without infrastructure boilerplate but still be able to configure everything I want when needed? +- How to run batch operations like ETL processes easily and efficiently in a cloud environment? +- How to manage databases without being locked to a specific vendor solution? +- How to perform database backups? +- How to manage secret data? +- What about having a precounfigured account management application? +- Sooner rather than later I'll need an orchestration queue. Why not have that just ready to use? # Command line tools @@ -63,7 +68,7 @@ CloudHarness provides the following command line tools to help application scaff * `harness-application` - create a new CloudHarness REST application. * `harness-generate` - generates server and client code for all CloudHarness REST applications. * `harness-test` - run end to end tests - + # Get started ## Prerequisites @@ -71,22 +76,26 @@ CloudHarness provides the following command line tools to help application scaff ### Operative system Cloudharness can be used on all major operative systems. + - Linux: supported and tested - MacOS: supported and tested - Windows/WSL2: supported and tested - Windows native: mostly working, unsupported ### Python -Python 3.9 must be installed. + +Python 3.10+ must be installed. It is recommended to setup a virtual environment. With conda: + ```bash conda create --name ch python=3.12 conda activate ch ``` ### Docker + [Docker](https://www.docker.com) is required to build locally. ### Kubernetes command line client @@ -110,6 +119,7 @@ conda activate ch A node environment with npm is required for developing web applications and to run end to end tests. Recommended: + - node >= v14.0.0 - npm >= 8.0.0 @@ -120,26 +130,31 @@ A JRE is needed to run the code generators based on openapi-generator. For more info, see [here](https://openapi-generator.tech/docs/installation). ## CloudHarness command line tools + To use the cli tools, install requirements first: ```bash bash install.sh ``` -### Generate deployment -To generate a deployment, run `harness-deployment`. See [below](#Deployment) for more. +### Create a new REST application -### Create new REST application -To create a new REST application, run `harness-application` from the root of your solution. +`harness-application` is a command-line tool used to create new applications based on predefined code templates. It allows users to quickly scaffold applications with backend, frontend, and database configurations. +More information can be found [here](./docs/applications/harness-application.md). ### Generate server and client code from openapi -To (re)generate the code for your applications, run `harness-generate` from the root. -The script will look for all openapi applications, and regenerate the Flask server code and documentation. -Note: the script will eventually override any manually modified file. To avoid that, define a file openapi-generator-ignore. + +To (re)generate the code for your applications, run `harness-generate`. +`harness-generate` is a command-line tool used to generate client code, server stubs, and model libraries for applications. +More information can be found [here](./docs/applications/harness-generate.md) + +### Generate deployment + +To generate a deployment, run `harness-deployment`. See [below](#build-and-deploy) for more information. # Extend CloudHarness to build your project -CloudHarness is born to be extended. +CloudHarness is born to be extended. The quickest way to start is to install Cloud Harness, copy the *blueprint* folder and build from that with the cli tools, such as `harness-application`, `harness-generate`, `harness-deployment`. @@ -150,10 +165,11 @@ See the [developers documentation](docs/dev.md#start-your-project) for more info The script `harness-deployment` scans your applications and configurations to create the build and deploy artifacts. Created artifacts include: - - Helm chart (or docker compose configuration file) - - Skaffold build and run configuration - - Visual Studio Code debug and run configuration - - Codefresh pipeline yaml specification (optional) + +- Helm chart (or docker compose configuration file) +- Skaffold build and run configuration +- Visual Studio Code debug and run configuration +- Codefresh pipeline yaml specification (optional) With your project folder structure looking like @@ -193,5 +209,4 @@ Then, you can selectively add files related to configuration that you want to pe For more information about how to configure a deployment, see [here](./build-deploy/helm-configuration.md) - -[![Codefresh build status]( https://g.codefresh.io/api/badges/pipeline/tarelli/Cloudharness%2Funittests?type=cf-1&key=eyJhbGciOiJIUzI1NiJ9.NWFkNzMyNDIzNjQ1YWMwMDAxMTJkN2Rl.-gUEkJxH6NCCIRgSIgEikVDte-Q0BsGZKEs4uahgpzs)]( https://g.codefresh.io/pipelines/edit/new/builds?id=6034cfce1036693697cd602b&pipeline=unittests&projects=Cloudharness&projectId=6034cfb83bb11c399e85c71b) +[![Codefresh build status](https://g.codefresh.io/api/badges/pipeline/tarelli/Cloudharness%2Funittests?type=cf-1&key=eyJhbGciOiJIUzI1NiJ9.NWFkNzMyNDIzNjQ1YWMwMDAxMTJkN2Rl.-gUEkJxH6NCCIRgSIgEikVDte-Q0BsGZKEs4uahgpzs)](https://g.codefresh.io/pipelines/edit/new/builds?id=6034cfce1036693697cd602b&pipeline=unittests&projects=Cloudharness&projectId=6034cfb83bb11c399e85c71b) diff --git a/application-templates/django-app/api/genapi.sh b/application-templates/django-app/api/genapi.sh deleted file mode 100644 index aab1d30dd..000000000 --- a/application-templates/django-app/api/genapi.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -fastapi-codegen --input openapi.yaml --output app -t templates && mv app/main.py ../backend/ && mv app/models.py ../backend/openapi/ -rm -rf app - -echo Generated new models and main.py diff --git a/application-templates/django-app/.dockerignore b/application-templates/django-base/.dockerignore similarity index 100% rename from application-templates/django-app/.dockerignore rename to application-templates/django-base/.dockerignore diff --git a/application-templates/django-app/api/test_st.py b/application-templates/django-base/api/test_st.py similarity index 100% rename from application-templates/django-app/api/test_st.py rename to application-templates/django-base/api/test_st.py diff --git a/application-templates/django-app/backend/__APP_NAME__/__init__.py b/application-templates/django-base/backend/__APP_NAME__/__init__.py similarity index 100% rename from application-templates/django-app/backend/__APP_NAME__/__init__.py rename to application-templates/django-base/backend/__APP_NAME__/__init__.py diff --git a/application-templates/django-app/backend/__APP_NAME__/admin.py b/application-templates/django-base/backend/__APP_NAME__/admin.py similarity index 100% rename from application-templates/django-app/backend/__APP_NAME__/admin.py rename to application-templates/django-base/backend/__APP_NAME__/admin.py diff --git a/application-templates/django-app/backend/__APP_NAME__/apps.py b/application-templates/django-base/backend/__APP_NAME__/apps.py similarity index 100% rename from application-templates/django-app/backend/__APP_NAME__/apps.py rename to application-templates/django-base/backend/__APP_NAME__/apps.py diff --git a/application-templates/django-app/backend/__APP_NAME__/migrations/0001_initial.py b/application-templates/django-base/backend/__APP_NAME__/migrations/0001_initial.py similarity index 100% rename from application-templates/django-app/backend/__APP_NAME__/migrations/0001_initial.py rename to application-templates/django-base/backend/__APP_NAME__/migrations/0001_initial.py diff --git a/application-templates/django-app/backend/__APP_NAME__/migrations/__init__.py b/application-templates/django-base/backend/__APP_NAME__/migrations/__init__.py similarity index 100% rename from application-templates/django-app/backend/__APP_NAME__/migrations/__init__.py rename to application-templates/django-base/backend/__APP_NAME__/migrations/__init__.py diff --git a/application-templates/django-app/backend/__APP_NAME__/models.py b/application-templates/django-base/backend/__APP_NAME__/models.py similarity index 100% rename from application-templates/django-app/backend/__APP_NAME__/models.py rename to application-templates/django-base/backend/__APP_NAME__/models.py diff --git a/application-templates/django-app/backend/__APP_NAME__/tests.py b/application-templates/django-base/backend/__APP_NAME__/tests.py similarity index 100% rename from application-templates/django-app/backend/__APP_NAME__/tests.py rename to application-templates/django-base/backend/__APP_NAME__/tests.py diff --git a/application-templates/django-app/backend/__APP_NAME__/views.py b/application-templates/django-base/backend/__APP_NAME__/views.py similarity index 100% rename from application-templates/django-app/backend/__APP_NAME__/views.py rename to application-templates/django-base/backend/__APP_NAME__/views.py diff --git a/application-templates/django-app/backend/django_baseapp/__init__.py b/application-templates/django-base/backend/django_baseapp/__init__.py similarity index 100% rename from application-templates/django-app/backend/django_baseapp/__init__.py rename to application-templates/django-base/backend/django_baseapp/__init__.py diff --git a/application-templates/django-app/backend/django_baseapp/asgi.py b/application-templates/django-base/backend/django_baseapp/asgi.py similarity index 74% rename from application-templates/django-app/backend/django_baseapp/asgi.py rename to application-templates/django-base/backend/django_baseapp/asgi.py index 049df40e6..4ec66b7d7 100644 --- a/application-templates/django-app/backend/django_baseapp/asgi.py +++ b/application-templates/django-base/backend/django_baseapp/asgi.py @@ -11,7 +11,7 @@ from django.core.asgi import get_asgi_application -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "__APP_NAME__.settings") +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "django_baseapp.settings") application = get_asgi_application() @@ -21,6 +21,6 @@ init_services() # start the kafka event listener -from cloudharness_django.services.events import init_listner # noqa E402 +from cloudharness_django.services.events import init_listener # noqa E402 -init_listner() +init_listener() diff --git a/application-templates/django-app/backend/django_baseapp/migrations/0001_initial.py b/application-templates/django-base/backend/django_baseapp/migrations/0001_initial.py similarity index 100% rename from application-templates/django-app/backend/django_baseapp/migrations/0001_initial.py rename to application-templates/django-base/backend/django_baseapp/migrations/0001_initial.py diff --git a/application-templates/django-app/backend/django_baseapp/migrations/__init__.py b/application-templates/django-base/backend/django_baseapp/migrations/__init__.py similarity index 100% rename from application-templates/django-app/backend/django_baseapp/migrations/__init__.py rename to application-templates/django-base/backend/django_baseapp/migrations/__init__.py diff --git a/application-templates/django-app/backend/django_baseapp/models.py b/application-templates/django-base/backend/django_baseapp/models.py similarity index 100% rename from application-templates/django-app/backend/django_baseapp/models.py rename to application-templates/django-base/backend/django_baseapp/models.py diff --git a/application-templates/django-app/backend/django_baseapp/static/www/index.html b/application-templates/django-base/backend/django_baseapp/static/www/index.html similarity index 100% rename from application-templates/django-app/backend/django_baseapp/static/www/index.html rename to application-templates/django-base/backend/django_baseapp/static/www/index.html diff --git a/application-templates/django-app/backend/django_baseapp/templates/__APP_NAME__/index.html b/application-templates/django-base/backend/django_baseapp/templates/__APP_NAME__/index.html similarity index 100% rename from application-templates/django-app/backend/django_baseapp/templates/__APP_NAME__/index.html rename to application-templates/django-base/backend/django_baseapp/templates/__APP_NAME__/index.html diff --git a/application-templates/django-app/backend/django_baseapp/templates/__APP_NAME__/swagger-ui.html b/application-templates/django-base/backend/django_baseapp/templates/__APP_NAME__/swagger-ui.html similarity index 100% rename from application-templates/django-app/backend/django_baseapp/templates/__APP_NAME__/swagger-ui.html rename to application-templates/django-base/backend/django_baseapp/templates/__APP_NAME__/swagger-ui.html diff --git a/application-templates/django-app/backend/django_baseapp/views.py b/application-templates/django-base/backend/django_baseapp/views.py similarity index 100% rename from application-templates/django-app/backend/django_baseapp/views.py rename to application-templates/django-base/backend/django_baseapp/views.py diff --git a/application-templates/django-app/backend/django_baseapp/wsgi.py b/application-templates/django-base/backend/django_baseapp/wsgi.py similarity index 85% rename from application-templates/django-app/backend/django_baseapp/wsgi.py rename to application-templates/django-base/backend/django_baseapp/wsgi.py index e2d6fc099..fccfadef0 100644 --- a/application-templates/django-app/backend/django_baseapp/wsgi.py +++ b/application-templates/django-base/backend/django_baseapp/wsgi.py @@ -21,6 +21,6 @@ init_services() # start the kafka event listener -from cloudharness_django.services.events import init_listner # noqa E402 +from cloudharness_django.services.events import init_listener # noqa E402 -init_listner() +init_listener() diff --git a/application-templates/django-app/backend/manage.py b/application-templates/django-base/backend/manage.py similarity index 100% rename from application-templates/django-app/backend/manage.py rename to application-templates/django-base/backend/manage.py diff --git a/application-templates/django-app/backend/openapi/.gitkeep b/application-templates/django-base/backend/openapi/.gitkeep similarity index 100% rename from application-templates/django-app/backend/openapi/.gitkeep rename to application-templates/django-base/backend/openapi/.gitkeep diff --git a/application-templates/django-app/backend/persistent/.gitkeep b/application-templates/django-base/backend/persistent/.gitkeep similarity index 100% rename from application-templates/django-app/backend/persistent/.gitkeep rename to application-templates/django-base/backend/persistent/.gitkeep diff --git a/application-templates/django-app/backend/setup.py b/application-templates/django-base/backend/setup.py similarity index 100% rename from application-templates/django-app/backend/setup.py rename to application-templates/django-base/backend/setup.py diff --git a/application-templates/django-app/backend/static/www/.gitkeep b/application-templates/django-base/backend/static/www/.gitkeep similarity index 100% rename from application-templates/django-app/backend/static/www/.gitkeep rename to application-templates/django-base/backend/static/www/.gitkeep diff --git a/application-templates/django-app/deploy/values.yaml b/application-templates/django-base/deploy/values.yaml similarity index 100% rename from application-templates/django-app/deploy/values.yaml rename to application-templates/django-base/deploy/values.yaml diff --git a/application-templates/django-app/Dockerfile b/application-templates/django-fastapi/Dockerfile similarity index 88% rename from application-templates/django-app/Dockerfile rename to application-templates/django-fastapi/Dockerfile index f40e95601..05dbd5202 100644 --- a/application-templates/django-app/Dockerfile +++ b/application-templates/django-fastapi/Dockerfile @@ -7,11 +7,11 @@ ENV APP_DIR=/app WORKDIR ${APP_DIR} COPY frontend/package.json ${APP_DIR} -COPY frontend/package-lock.json ${APP_DIR} -RUN npm ci +COPY frontend/yarn.lock ${APP_DIR} +RUN yarn install --frozen-lockfile --timeout 60000 COPY frontend ${APP_DIR} -RUN npm run build +RUN yarn build ##### diff --git a/application-templates/django-app/README.md b/application-templates/django-fastapi/README.md similarity index 98% rename from application-templates/django-app/README.md rename to application-templates/django-fastapi/README.md index d77502b9d..b237db704 100644 --- a/application-templates/django-app/README.md +++ b/application-templates/django-fastapi/README.md @@ -31,7 +31,7 @@ See [backend/README.md#Develop] ### Frontend -Backend code is inside the *frontend* directory. +Frontend code is inside the *frontend* directory. Frontend is by default generated as a React web application, but no constraint about this specific technology. diff --git a/application-templates/django-fastapi/api/genapi.sh b/application-templates/django-fastapi/api/genapi.sh new file mode 100644 index 000000000..cc5737b4d --- /dev/null +++ b/application-templates/django-fastapi/api/genapi.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +ROOT_PATH=$(realpath "$(dirname "$BASH_SOURCE")/../../..") +harness-generate servers --app-name "__APP_NAME__" "$ROOT_PATH" \ No newline at end of file diff --git a/application-templates/django-app/api/openapi.yaml b/application-templates/django-fastapi/api/openapi.yaml similarity index 100% rename from application-templates/django-app/api/openapi.yaml rename to application-templates/django-fastapi/api/openapi.yaml diff --git a/application-templates/django-app/api/templates/main.jinja2 b/application-templates/django-fastapi/api/templates/main.jinja2 similarity index 100% rename from application-templates/django-app/api/templates/main.jinja2 rename to application-templates/django-fastapi/api/templates/main.jinja2 diff --git a/application-templates/django-app/backend/README.md b/application-templates/django-fastapi/backend/README.md similarity index 100% rename from application-templates/django-app/backend/README.md rename to application-templates/django-fastapi/backend/README.md diff --git a/application-templates/django-app/backend/__APP_NAME__/controllers/__init__.py b/application-templates/django-fastapi/backend/__APP_NAME__/controllers/__init__.py similarity index 100% rename from application-templates/django-app/backend/__APP_NAME__/controllers/__init__.py rename to application-templates/django-fastapi/backend/__APP_NAME__/controllers/__init__.py diff --git a/application-templates/django-app/backend/__APP_NAME__/controllers/test.py b/application-templates/django-fastapi/backend/__APP_NAME__/controllers/test.py similarity index 100% rename from application-templates/django-app/backend/__APP_NAME__/controllers/test.py rename to application-templates/django-fastapi/backend/__APP_NAME__/controllers/test.py diff --git a/application-templates/django-app/backend/django_baseapp/settings.py b/application-templates/django-fastapi/backend/django_baseapp/settings.py similarity index 100% rename from application-templates/django-app/backend/django_baseapp/settings.py rename to application-templates/django-fastapi/backend/django_baseapp/settings.py diff --git a/application-templates/django-app/backend/django_baseapp/urls.py b/application-templates/django-fastapi/backend/django_baseapp/urls.py similarity index 97% rename from application-templates/django-app/backend/django_baseapp/urls.py rename to application-templates/django-fastapi/backend/django_baseapp/urls.py index ecfee23ee..3032dd298 100644 --- a/application-templates/django-app/backend/django_baseapp/urls.py +++ b/application-templates/django-fastapi/backend/django_baseapp/urls.py @@ -1,4 +1,4 @@ -"""MNP Checkout URL Configuration +"""URL Configuration The `urlpatterns` list routes URLs to views. For more information please see: https://docs.djangoproject.com/en/3.2/topics/http/urls/ diff --git a/application-templates/django-app/backend/requirements.txt b/application-templates/django-fastapi/backend/requirements.txt similarity index 100% rename from application-templates/django-app/backend/requirements.txt rename to application-templates/django-fastapi/backend/requirements.txt diff --git a/application-templates/django-app/dev-setup.sh b/application-templates/django-fastapi/dev-setup.sh similarity index 100% rename from application-templates/django-app/dev-setup.sh rename to application-templates/django-fastapi/dev-setup.sh diff --git a/application-templates/django-ninja/Dockerfile b/application-templates/django-ninja/Dockerfile new file mode 100644 index 000000000..dc8e2223f --- /dev/null +++ b/application-templates/django-ninja/Dockerfile @@ -0,0 +1,35 @@ +ARG CLOUDHARNESS_FRONTEND_BUILD +ARG CLOUDHARNESS_DJANGO + +FROM $CLOUDHARNESS_FRONTEND_BUILD AS frontend + +ARG APP_DIR=/app + +WORKDIR ${APP_DIR} +COPY frontend/package.json . +COPY frontend/yarn.lock . +RUN yarn install --timeout 60000 + +COPY frontend . +RUN yarn build + +##### + +FROM $CLOUDHARNESS_DJANGO + +WORKDIR ${APP_DIR} +RUN mkdir -p ${APP_DIR}/static/www + +COPY backend/requirements.txt ${APP_DIR} +RUN --mount=type=cache,target=/root/.cache python -m pip install --upgrade pip &&\ + pip3 install --no-cache-dir -r requirements.txt --prefer-binary + +COPY backend/requirements.txt backend/setup.py ${APP_DIR} +RUN python3 -m pip install -e . + +COPY backend ${APP_DIR} +RUN python3 manage.py collectstatic --noinput + +COPY --from=frontend /app/dist ${APP_DIR}/static/www + +ENTRYPOINT uvicorn --workers ${WORKERS} --host 0.0.0.0 --port ${PORT} django_baseapp.asgi:application diff --git a/application-templates/django-ninja/README.md b/application-templates/django-ninja/README.md new file mode 100644 index 000000000..450c1da86 --- /dev/null +++ b/application-templates/django-ninja/README.md @@ -0,0 +1,114 @@ +# __APP_NAME__ + +Django-Ninja/React-based web application. +This application is constructed to be deployed inside a cloud-harness Kubernetes. +It can be also run locally for development and test purpose. + +The code is generated with the script `harness-application`. + +## Configuration + +### Accounts + +The CloudHarness Django application template comes with a configuration that can retrieve user account updates from Keycloak (accounts) +To enable this feature: +* log in into the accounts admin interface +* select in the left sidebar Events +* select the `Config` tab +* enable "metacell-admin-event-listener" under the `Events Config` - `Event Listeners` + +An other option is to enable the "metacell-admin-event-listener" through customizing the Keycloak realm.json from the CloudHarness repository. + +## Develop + +This application is composed of a Django-Ninja backend and a React frontend. + +### Backend + +Backend code is inside the *backend* directory. +See [backend/README.md#Develop] + +### Frontend + +Frontend code is inside the *frontend* directory. + +Frontend is by default generated as a React web application, but no constraint about this specific technology. + +See also [frontend/README.md] + +#### Generate API client stubs +All the api stubs are automatically generated in the [frontend/rest](frontend/rest) directory by `harness-application` +and `harness-generate`. + +To update frontend client stubs, run + +``` +harness-generate clients __APP_NAME__ -t +``` + +Stubs can also be updated using the `genapi.sh` from the api folder. + +## Local build & run + +### Install Python dependencies +1 - Clone cloud-harness into your project root folder + +2 - Run the dev setup script +``` +cd applications/__APP_NAME__ +source dev-setup.sh +``` + +### Prepare backend + +Create a Django local superuser account, this you only need to do on initial setup +```bash +cd backend +python3 manage.py migrate # to sync the database with the Django models +python3 manage.py collectstatic --noinput # to copy all assets to the static folder +python3 manage.py createsuperuser +# link the frontend dist to the django static folder, this is only needed once, frontend updates will automatically be applied +cd static/www +ln -s ../../../frontend/dist dist +``` + +### Run frontend + +- `yarn dev` Local dev with no backend (no or mock data, cookie required) +- `yarn start` Local dev with backend on localhost:8000 -- see next paragraph (cookie required) +- `yarn start:dev` Local dev with backend on the remote dev deployment (cookie required) +- `yarn start:local` Local dev with backend on the local dev deployment on mnp.local (cookie required) + +To obtain the login cookie, login in the application with the forwarded backend, copy the `kc-access` cookie and set it into localhost:9000 + +### Run backend application + +start the Django server + +```bash +ACCOUNTS_ADMIN_PASSWORD=metacell ACCOUNTS_ADMIN_USERNAME=admin CH_CURRENT_APP_NAME=__APP_NAME__ CH_VALUES_PATH=../../../deployment/helm/values.yaml DJANGO_SETTINGS_MODULE=django_baseapp.settings KUBERNETES_SERVICE_HOST=a uvicorn --host 0.0.0.0 --port 8000 django_baseapp.asgi:application +``` + +Before running this backend, have to: +- Run `harness-deployment ... -n [NAMESPACE] -i __APP_NAME__` with the setup +- port-forward keycloak and the database (see below) + +### Running local with port forwardings to a kubernetes cluster +When you create port forwards to microservices in your k8s cluster you want to forced your local backend server to initialize +the AuthService and EventService services. +This can be done by setting the `KUBERNETES_SERVICE_HOST` environment variable to a dummy or correct k8s service host. +The `KUBERNETES_SERVICE_HOST` switch will activate the creation of the keycloak client and client roles of this microservice. + +Run `port-forward.sh` to get the keycloak and database running. + +To access those have to map to the hosts file: + +``` +127.0.0.1 accounts.[NAMESPACE] __APP_NAME__-db +``` + +After running the backend on port 8000, run `yarn start` to get a frontend to it + +#### Vs code run configuration + +Run configuration is automatically generated for VS code (__APP_NAME__ backend) \ No newline at end of file diff --git a/application-templates/django-ninja/backend/__APP_NAME__/api/__init__.py b/application-templates/django-ninja/backend/__APP_NAME__/api/__init__.py new file mode 100644 index 000000000..ab1fdac83 --- /dev/null +++ b/application-templates/django-ninja/backend/__APP_NAME__/api/__init__.py @@ -0,0 +1,40 @@ +import time +from django.http import HttpRequest +from ninja import NinjaAPI +from ..exceptions import Http401, Http403 + + +api = NinjaAPI(title='__APP_NAME__ API', version='0.1.0') + + +@api.exception_handler(Http401) +def unauthorized(request, exc): + return api.create_response( + request, + {'message': 'Unauthorized'}, + status=401, + ) + + +@api.exception_handler(Http403) +def forbidden(request, exc): + return api.create_response( + request, + {'message': 'Forbidden'}, + status=403, + ) + + +@api.get('/ping', response={200: float}, tags=['test']) +def ping(request: HttpRequest): + return time.time() + + +@api.get('/live', response={200: str}, tags=['test']) +def live(request: HttpRequest): + return 'OK' + + +@api.get('/ready', response={200: str}, tags=['test']) +def ready(request: HttpRequest): + return 'OK' diff --git a/application-templates/django-ninja/backend/__APP_NAME__/exceptions.py b/application-templates/django-ninja/backend/__APP_NAME__/exceptions.py new file mode 100644 index 000000000..b5b053c1d --- /dev/null +++ b/application-templates/django-ninja/backend/__APP_NAME__/exceptions.py @@ -0,0 +1,6 @@ +class Http401(Exception): + pass + + +class Http403(Exception): + pass diff --git a/application-templates/django-ninja/backend/__APP_NAME__/schema.py b/application-templates/django-ninja/backend/__APP_NAME__/schema.py new file mode 100644 index 000000000..b3d9c8b32 --- /dev/null +++ b/application-templates/django-ninja/backend/__APP_NAME__/schema.py @@ -0,0 +1,3 @@ +from ninja import Schema + +# Create your schema here diff --git a/application-templates/django-ninja/backend/django_baseapp/settings.py b/application-templates/django-ninja/backend/django_baseapp/settings.py new file mode 100644 index 000000000..e54f20f0d --- /dev/null +++ b/application-templates/django-ninja/backend/django_baseapp/settings.py @@ -0,0 +1,167 @@ +""" +Django settings for the MNP Checkout project. + +Generated by 'django-admin startproject' using Django 3.2.12. + +For more information on this file, see +https://docs.djangoproject.com/en/3.2/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/3.2/ref/settings/ +""" + +import os +from pathlib import Path + +# Build paths inside the project like this: BASE_DIR / 'subdir'. +BASE_DIR = Path(__file__).resolve().parent.parent + + +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/3.2/howto/deployment/checklist/ + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = "django-insecure-81kv$0=07xac7r(pgz6ndb5t0at4-z@ae6&f@u6_3jo&9d#4kl" + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = False if os.environ.get("PRODUCTION", None) else True + +ALLOWED_HOSTS = [ + "*", +] + +# Application definition + +INSTALLED_APPS = [ + "django.contrib.admin", + "django.contrib.auth", + "django.contrib.contenttypes", + "django.contrib.sessions", + "django.contrib.messages", + "django.contrib.staticfiles", +] + +MIDDLEWARE = [ + "django.middleware.security.SecurityMiddleware", + "django.contrib.sessions.middleware.SessionMiddleware", + "django.middleware.common.CommonMiddleware", + 'django.middleware.csrf.CsrfViewMiddleware', + "django.contrib.auth.middleware.AuthenticationMiddleware", + "django.contrib.messages.middleware.MessageMiddleware", + "django.middleware.clickjacking.XFrameOptionsMiddleware", + "cloudharness.middleware.django.CloudharnessMiddleware", +] + + +ROOT_URLCONF = "django_baseapp.urls" + +TEMPLATES = [ + { + "BACKEND": "django.template.backends.django.DjangoTemplates", + "DIRS": [BASE_DIR / "templates"], + "APP_DIRS": True, + "OPTIONS": { + "context_processors": [ + "django.template.context_processors.debug", + "django.template.context_processors.request", + "django.contrib.auth.context_processors.auth", + "django.contrib.messages.context_processors.messages", + ], + }, + }, +] + +WSGI_APPLICATION = "django_baseapp.wsgi.application" + + +# Password validation +# https://docs.djangoproject.com/en/3.2/ref/settings/#auth-password-validators + +AUTH_PASSWORD_VALIDATORS = [ + { + "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator", + }, + { + "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator", + }, + { + "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator", + }, + { + "NAME": "django.contrib.auth.password_validation.NumericPasswordValidator", + }, +] + + +# Internationalization +# https://docs.djangoproject.com/en/3.2/topics/i18n/ + +LANGUAGE_CODE = "en-us" + +TIME_ZONE = "UTC" + +USE_I18N = True + +USE_L10N = True + +USE_TZ = True + +# Default primary key field type +# https://docs.djangoproject.com/en/3.2/ref/settings/#default-auto-field + +DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" + + +PROJECT_NAME = "__APP_NAME__".upper() + +# Persistent storage +PERSISTENT_ROOT = os.path.join(BASE_DIR, "persistent") + +# *********************************************************************** +# * __APP_NAME__ settings +# *********************************************************************** +from cloudharness.applications import get_configuration # noqa E402 +from cloudharness.utils.config import ALLVALUES_PATH, CloudharnessConfig # noqa E402 + +# *********************************************************************** +# * import base CloudHarness Django settings +# *********************************************************************** +from cloudharness_django.settings import * # noqa E402 + +# add the local apps +INSTALLED_APPS += [ + "__APP_NAME__", + "django_baseapp", + "ninja", +] + +# override django admin base template with a local template +# to add some custom styling +TEMPLATES[0]["DIRS"] = [BASE_DIR / "templates"] + +# Static files (CSS, JavaScript, Images) +MEDIA_ROOT = PERSISTENT_ROOT +STATIC_ROOT = os.path.join(BASE_DIR, "static") +MEDIA_URL = "/media/" +STATIC_URL = "/static/" + +# KC Client & roles +KC_CLIENT_NAME = PROJECT_NAME.lower() + +# __APP_NAME__ specific roles + +# Default KC roles +KC_ADMIN_ROLE = f"{KC_CLIENT_NAME}-administrator" # admin user +KC_MANAGER_ROLE = f"{KC_CLIENT_NAME}-manager" # manager user +KC_USER_ROLE = f"{KC_CLIENT_NAME}-user" # customer user +KC_ALL_ROLES = [ + KC_ADMIN_ROLE, + KC_MANAGER_ROLE, + KC_USER_ROLE, +] +KC_PRIVILEGED_ROLES = [ + KC_ADMIN_ROLE, + KC_MANAGER_ROLE, +] + +KC_DEFAULT_USER_ROLE = None # don't add the user role to the realm default role diff --git a/application-templates/django-ninja/backend/django_baseapp/urls.py b/application-templates/django-ninja/backend/django_baseapp/urls.py new file mode 100644 index 000000000..c95d49269 --- /dev/null +++ b/application-templates/django-ninja/backend/django_baseapp/urls.py @@ -0,0 +1,31 @@ +"""URL Configuration + +The `urlpatterns` list routes URLs to views. For more information please see: + https://docs.djangoproject.com/en/3.2/topics/http/urls/ +Examples: +Function views + 1. Add an import: from my_app import views + 2. Add a URL to urlpatterns: path('', views.home, name='home') +Class-based views + 1. Add an import: from other_app.views import Home + 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') +Including another URLconf + 1. Import the include() function: from django.urls import include, path + 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) +""" +import re +from django.conf import settings +from django.views.static import serve +from django.contrib import admin +from django.urls import path, re_path +from __APP_NAME__.api import api +from django_baseapp.views import index + + +urlpatterns = [ + path("admin/", admin.site.urls), + path("api/", api.urls), + re_path(r"^%s(?P.*)$" % re.escape(settings.MEDIA_URL.lstrip("/")), serve, kwargs=dict(document_root=settings.MEDIA_ROOT)), + re_path(r"^%s(?P.*)$" % re.escape(settings.STATIC_URL.lstrip("/")), serve, kwargs=dict(document_root=settings.STATIC_ROOT)), + re_path(r"^(?P.*)$", index, name="index"), +] diff --git a/application-templates/django-ninja/backend/requirements.txt b/application-templates/django-ninja/backend/requirements.txt new file mode 100644 index 000000000..b3575416e --- /dev/null +++ b/application-templates/django-ninja/backend/requirements.txt @@ -0,0 +1,2 @@ +pydantic==2.9.2 +django-ninja \ No newline at end of file diff --git a/application-templates/django-ninja/dev-setup.sh b/application-templates/django-ninja/dev-setup.sh new file mode 100644 index 000000000..d7d062d15 --- /dev/null +++ b/application-templates/django-ninja/dev-setup.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash + +CURRENT_PATH=$(pwd) +CH_DIRECTORY="__CLOUDHARNESS_PATH__" +INSTALL_PYTEST=false +CURRENT_DIRECTORY="$(pwd)" +APP_NAME="__APP_NAME__" + +pip_upgrade_error() { + echo "Unable to upgrade pip" + exit 1 +} + +install_error () { + echo "Unable to install $1" 1>&2 + exit 1 +} + +while getopts ch_directory:pytest arg; +do + case "$arg" in + ch_directory) CH_DIRECTORY=${OPTARG};; + pytest) INSTALL_PYTEST=true;; + esac +done + +pip install --upgrade pip || pip_upgrade_error + +# Install pip dependencies from cloudharness-base-debian image + +if $INSTALL_PYTEST; then + pip install pytest || install_error pytest +fi + +pip install -r "$CH_DIRECTORY/libraries/models/requirements.txt" || install_error "models requirements" +pip install -r "$CH_DIRECTORY/libraries/cloudharness-common/requirements.txt" || install_error "cloudharness-common requirements" +pip install -r "$CH_DIRECTORY/libraries/client/cloudharness_cli/requirements.txt" || install_error "cloudharness_cli requirements" + +pip install -e "$CH_DIRECTORY/libraries/models" || install_error models +pip install -e "$CH_DIRECTORY/libraries/cloudharness-common" || install_error cloudharness-common +pip install -e "$CH_DIRECTORY/libraries/client/cloudharness_cli" || install_error cloudharness_cli + +# Install pip dependencies from cloudharness-django image + +pip install -e "$CH_DIRECTORY/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django" || install_error cloudharness-django + +# Install application + +pip install -r "$CURRENT_DIRECTORY/backend/requirements.txt" || install_error "$APP_NAME dependencies" +pip install -e "$CURRENT_DIRECTORY/backend" || install_error "$APP_NAME" \ No newline at end of file diff --git a/application-templates/django-ninja/port-forward.sh b/application-templates/django-ninja/port-forward.sh new file mode 100755 index 000000000..63a06be84 --- /dev/null +++ b/application-templates/django-ninja/port-forward.sh @@ -0,0 +1,6 @@ +fuser -k 5432/tcp +fuser -k 8080/tcp +namespace=${1:-ch} +echo "Port forwarding for $namespace" +kubectl port-forward --namespace $namespace deployment/accounts 8080:8080 & +kubectl port-forward --namespace $namespace deployment/__APP_NAME__-db 5432:5432 & diff --git a/application-templates/webapp/frontend/package.json b/application-templates/webapp/frontend/package.json index 4d852c343..bd64303dc 100644 --- a/application-templates/webapp/frontend/package.json +++ b/application-templates/webapp/frontend/package.json @@ -3,8 +3,8 @@ "scripts": { "dev": "vite", "start": "DOMAIN=http://localhost:5000 vite", - "start:dev": "DOMAIN=https://test.ch.metacell.us vite", - "start:local": "DOMAIN=http://samples.ch vite", + "start:dev": "DOMAIN=https://__APP_NAME__.ch.metacell.us vite", + "start:local": "DOMAIN=http://__APP_NAME__.ch vite", "prebuild": "eslint .", "build": "vite build", "lint": "eslint src --report-unused-disable-directives --fix" diff --git a/application-templates/webapp/frontend/src/components/RestTest.tsx b/application-templates/webapp/frontend/src/components/RestTest.tsx index c1dc7c002..0123e48bd 100644 --- a/application-templates/webapp/frontend/src/components/RestTest.tsx +++ b/application-templates/webapp/frontend/src/components/RestTest.tsx @@ -1,7 +1,7 @@ import { useState, useEffect } from 'react'; -import { TestApi } from '../rest/apis/TestApi' +import { TestApi } from '../rest/__APP_NAME__/apis/TestApi' const test = new TestApi(); @@ -9,11 +9,11 @@ const test = new TestApi(); const RestTest = () => { const [result, setResult] = useState(null); useEffect(() => { - test.ping().then((r) => setResult(r), () => setResult( "API error")); + test.ping().then((r) => setResult(r), () => setResult("API error")); }, []); - - return result ?

Backend answered: { result }

:

Backend did not answer

+ + return result ?

Backend answered: {result}

:

Backend did not answer

} export default RestTest; \ No newline at end of file diff --git a/applications/accounts/.ch-manifest b/applications/accounts/.ch-manifest new file mode 100644 index 000000000..8862ec5bf --- /dev/null +++ b/applications/accounts/.ch-manifest @@ -0,0 +1,4 @@ +app-name: accounts +inferred: true +templates: [base, db-postgres] +version: '2' diff --git a/applications/accounts/.dockerignore b/applications/accounts/.dockerignore new file mode 100644 index 000000000..d0c6ee4d6 --- /dev/null +++ b/applications/accounts/.dockerignore @@ -0,0 +1,4 @@ +/dev +.ch-manifest +README.md +/deploy \ No newline at end of file diff --git a/applications/accounts/Dockerfile b/applications/accounts/Dockerfile index 4e61bef42..c14b452fc 100644 --- a/applications/accounts/Dockerfile +++ b/applications/accounts/Dockerfile @@ -1,4 +1,4 @@ -FROM quay.io/keycloak/keycloak:16.1.1 +FROM quay.io/keycloak/keycloak:16.1.0 # add kubectl USER root diff --git a/applications/accounts/deploy/resources/realm.json b/applications/accounts/deploy/resources/realm.json index c04b0b7fd..f25643b69 100644 --- a/applications/accounts/deploy/resources/realm.json +++ b/applications/accounts/deploy/resources/realm.json @@ -44,12 +44,18 @@ "resetPasswordAllowed": true, "editUsernameAllowed": true, "users": [ + {{- $j := 0}} {{- range $app := .Values.apps }} {{- if (hasKey $app.harness "accounts") }} + {{- if $j}},{{end}} + {{- if len $app.harness.accounts.users}} + {{- $j = add1 $j }} + {{- end }} {{- range $i, $user := $app.harness.accounts.users }}{{if $i}},{{end}} {{ include "deploy_accounts_utils.user" (dict "root" $ "app" $app "user" $user) }} {{- end }} {{- end }} + {{- end }} ], "roles": { @@ -82,14 +88,18 @@ } ], "client": { + {{- $k := 0}} {{- range $app := .Values.apps }} + {{- if (hasKey $app.harness "accounts") }} + {{- if $k}},{{end}} {{ $app.harness.name | quote }}: [ {{- range $i, $role := $app.harness.accounts.roles }} {{if $i}},{{end}} - {{ include "deploy_accounts_utils.role" (dict "root" $ "app" $app "role" $role) }} + {{- include "deploy_accounts_utils.role" (dict "root" $ "app" $app "role" $role) }} {{- end }} ] + {{- $k = add1 $k }} {{- end }} {{- end }} } diff --git a/applications/accounts/deploy/values-test.yaml b/applications/accounts/deploy/values-test.yaml new file mode 100644 index 000000000..a6fb91d05 --- /dev/null +++ b/applications/accounts/deploy/values-test.yaml @@ -0,0 +1,20 @@ + accounts: + roles: + - role1 + - role2 + - role3 + users: + - username: sample@testuser.com + clientRoles: + - role1 + realmRoles: + - administrator + - offline_access + - username: samples-test-user2 + email: sample2@testuser.com + password: test1 + clientRoles: + - role1 + realmRoles: + - offline_access + diff --git a/applications/accounts/deploy/values.yaml b/applications/accounts/deploy/values.yaml index 42e290cb6..3e32ce4c2 100644 --- a/applications/accounts/deploy/values.yaml +++ b/applications/accounts/deploy/values.yaml @@ -11,7 +11,7 @@ harness: cpu: "10m" limits: memory: "1024Mi" - cpu: "500m" + cpu: "2000m" service: auto: true port: 8080 diff --git a/applications/accounts/dev/disable-theme-cache.cli b/applications/accounts/dev/disable-theme-cache.cli new file mode 100644 index 000000000..4eca62b37 --- /dev/null +++ b/applications/accounts/dev/disable-theme-cache.cli @@ -0,0 +1,5 @@ +embed-server --std-out=echo --server-config=standalone-ha.xml +/subsystem=keycloak-server/theme=defaults/:write-attribute(name=cacheThemes,value=false) +/subsystem=keycloak-server/theme=defaults/:write-attribute(name=cacheTemplates,value=false) +/subsystem=keycloak-server/theme=defaults/:write-attribute(name=staticMaxAge,value=-1) +stop-embedded-server \ No newline at end of file diff --git a/applications/accounts/dev/docker-compose.yaml b/applications/accounts/dev/docker-compose.yaml new file mode 100644 index 000000000..fc043360b --- /dev/null +++ b/applications/accounts/dev/docker-compose.yaml @@ -0,0 +1,45 @@ +version: '3.2' + +services: + postgres: + image: postgres + environment: + POSTGRES_DB: keycloak + POSTGRES_USER: keycloak + POSTGRES_PASSWORD: password + PGDATA: /var/lib/postgresql/data/pgdata + volumes: + - pg_data:/var/lib/postgresql/data/pgdata + + keycloak: + image: quay.io/keycloak/keycloak:16.1.1 + environment: + DB_VENDOR: POSTGRES + DB_ADDR: postgres + DB_DATABASE: keycloak + DB_USER: keycloak + DB_SCHEMA: public + DB_PASSWORD: password + KEYCLOAK_USER: admin + KEYCLOAK_PASSWORD: Pa55w0rd + + ports: + - 8080:8080 + depends_on: + - postgres + volumes: + - type: bind + source: ../themes/custom + target: /opt/jboss/keycloak/themes/custom + # disable cache + - type: bind + source: ./disable-theme-cache.cli + target: /opt/jboss/startup-scripts/disable-theme-cache.cli + - type: bind + source: ../scripts/create_api_user.sh + target: /opt/jboss/startup-scripts/create_api_user.sh + - type: bind + source: ../plugins/metacell-admin-event-listener-bundle-1.0.0.ear + target: /opt/jboss/keycloak/standalone/deployments/metacell-admin-event-listener-bundle-1.0.0.ear +volumes: + pg_data: \ No newline at end of file diff --git a/applications/accounts/dev/realm.json b/applications/accounts/dev/realm.json new file mode 100644 index 000000000..045e68944 --- /dev/null +++ b/applications/accounts/dev/realm.json @@ -0,0 +1,669 @@ +{ + "id": "ch", + "realm": "ch", + "enabled": true, + "sslRequired": "none", + "loginTheme": "keycloak", + "accountTheme": "keycloak", + "adminTheme": "keycloak", + "emailTheme": "keycloak", + "registrationAllowed": true, + "registrationEmailAsUsername": false, + "rememberMe": true, + "verifyEmail": false, + "loginWithEmailAllowed": true, + "duplicateEmailsAllowed": false, + "resetPasswordAllowed": true, + "editUsernameAllowed": true, + "users": [ + ], + "roles": { + "realm": [ + { + "id": "70835ad6-1454-4bc5-86a4-f1597e776b75", + "name": "administrator", + "composite": false, + "clientRole": false, + "containerId": "ch", + "attributes": {} + }, + { + "id": "498353dd-88eb-4a5e-99b8-d912e0f20f23", + "name": "uma_authorization", + "description": "${role_uma_authorization}", + "composite": false, + "clientRole": false, + "containerId": "ch", + "attributes": {} + }, + { + "id": "f99970f1-958b-4bb8-8b39-0d7498b0ecc4", + "name": "offline_access", + "description": "${role_offline-access}", + "composite": false, + "clientRole": false, + "containerId": "ch", + "attributes": {} + } + ], + "client": { + } + }, + "clients": [ + { + "id": "9a6a2560-c6be-4493-8bd5-3fdc4522d82b", + "clientId": "rest-client", + "baseUrl": "http://accounts.ch.local", + "surrogateAuthRequired": false, + "enabled": true, + "clientAuthenticatorType": "client-secret", + "secret": "5678eb6e-9e2c-4ee5-bd54-34e7411339e8", + "redirectUris": [ + "*" + ], + "webOrigins": [ + "*" + ], + "standardFlowEnabled": true, + "directAccessGrantsEnabled": true, + "protocol": "openid-connect", + "attributes": { + "access.token.lifespan": "3600" + }, + "fullScopeAllowed": true, + "defaultClientScopes": [ + "web-origins", + "role_list", + "profile", + "roles", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ] + }, + { + "id": "accounts", + "clientId": "accounts", + "name": "accounts", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "secret": "GEdDm8xOhpVFXy8jrjzT", + "redirectUris": [ + "*" + ], + "webOrigins": [ + "*", + "+" + ], + "notBefore": 0, + "bearerOnly": false, + "consentRequired": false, + "standardFlowEnabled": true, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": true, + "serviceAccountsEnabled": false, + "publicClient": true, + "frontchannelLogout": false, + "protocol": "openid-connect", + "attributes": { + "backchannel.logout.session.required": "true", + "backchannel.logout.revoke.offline.tokens": "false" + }, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": true, + "nodeReRegistrationTimeout": -1, + "defaultClientScopes": [], + "optionalClientScopes": [] + }, + { + "id": "111caf43-3d26-484d-8dc9-7fa911ac221c", + "clientId": "web-client", + "baseUrl": "http://accounts.ch.local", + "surrogateAuthRequired": false, + "enabled": true, + "clientAuthenticatorType": "client-secret", + "secret": "452952ae-922c-4766-b912-7b106271e34b", + "redirectUris": [ + "*" + ], + "webOrigins": [ + "*", + "+" + ], + "standardFlowEnabled": true, + "directAccessGrantsEnabled": true, + "publicClient": true, + "protocol": "openid-connect", + "fullScopeAllowed": true, + "defaultClientScopes": [ + "web-origins", + "role_list", + "administrator-scope", + "profile", + "roles", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ] + } + ], + "clientScopes": [ + { + "id": "a8cddc84-c506-4196-8f2d-1bd5e8769f3c", + "name": "administrator-scope", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "true", + "display.on.consent.screen": "true" + }, + "protocolMappers": [ + { + "id": "45fc2547-1761-420b-b6a8-7dc882a51507", + "name": "administrator-audience", + "protocol": "openid-connect", + "protocolMapper": "oidc-audience-mapper", + "consentRequired": false, + "config": { + "included.client.audience": "web-client", + "id.token.claim": "true", + "access.token.claim": "true" + } + } + ] + }, + { + "id": "35c37cdc-6841-41e7-b90f-2964fc563998", + "name": "microprofile-jwt", + "description": "Microprofile - JWT built-in scope", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "true", + "display.on.consent.screen": "false" + }, + "protocolMappers": [ + { + "id": "611fb1bc-56cd-49d2-a11b-ddf05bd220db", + "name": "upn", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-property-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "username", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "upn", + "jsonType.label": "String" + } + }, + { + "id": "63850e7d-1031-447a-a8af-3df588a39350", + "name": "groups", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-realm-role-mapper", + "consentRequired": false, + "config": { + "multivalued": "true", + "user.attribute": "foo", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "groups", + "jsonType.label": "String" + } + } + ] + }, + { + "id": "dc927013-0448-4a29-ac72-7d6b019180d9", + "name": "web-origins", + "description": "OpenID Connect scope for add allowed web origins to the access token", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "false", + "display.on.consent.screen": "false", + "consent.screen.text": "" + }, + "protocolMappers": [ + { + "id": "3cc4569c-83b0-4bc9-af31-186c8081f8ac", + "name": "allowed web origins", + "protocol": "openid-connect", + "protocolMapper": "oidc-allowed-origins-mapper", + "consentRequired": false, + "config": {} + } + ] + }, + { + "id": "4bd583e6-9f6d-4846-9a94-2f02b1b4b1db", + "name": "roles", + "description": "OpenID Connect scope for add user roles to the access token", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "false", + "display.on.consent.screen": "true", + "consent.screen.text": "${rolesScopeConsentText}" + }, + "protocolMappers": [ + { + "id": "0d359e4f-3d4d-4ef3-88fd-2dd9f41da8cd", + "name": "client roles", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-client-role-mapper", + "consentRequired": false, + "config": { + "multivalued": "true", + "user.attribute": "foo", + "access.token.claim": "true", + "claim.name": "resource_access.${client_id}.roles", + "jsonType.label": "String" + } + }, + { + "id": "98ea5505-f703-49d2-b927-7715a7fc7a19", + "name": "realm roles", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-realm-role-mapper", + "consentRequired": false, + "config": { + "multivalued": "true", + "user.attribute": "foo", + "access.token.claim": "true", + "claim.name": "realm_access.roles", + "jsonType.label": "String" + } + }, + { + "id": "28b26ce3-7edc-47c2-982f-881f1c001ef3", + "name": "audience resolve", + "protocol": "openid-connect", + "protocolMapper": "oidc-audience-resolve-mapper", + "consentRequired": false, + "config": {} + } + ] + }, + { + "id": "e2606962-dd91-4926-af4e-cce6a036a04a", + "name": "phone", + "description": "OpenID Connect built-in scope: phone", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "true", + "display.on.consent.screen": "true", + "consent.screen.text": "${phoneScopeConsentText}" + }, + "protocolMappers": [ + { + "id": "c7e30f92-6026-4291-b526-3716662c26f1", + "name": "phone number verified", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "phoneNumberVerified", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "phone_number_verified", + "jsonType.label": "boolean" + } + }, + { + "id": "b1927570-c38d-49b8-9bbb-3cf9571f00be", + "name": "phone number", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "phoneNumber", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "phone_number", + "jsonType.label": "String" + } + } + ] + }, + { + "id": "6f532104-efc0-41d9-8fbc-9c78372d3f1b", + "name": "address", + "description": "OpenID Connect built-in scope: address", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "true", + "display.on.consent.screen": "true", + "consent.screen.text": "${addressScopeConsentText}" + }, + "protocolMappers": [ + { + "id": "548dd8e4-1ee8-4f7d-8934-439bdd1cc0bb", + "name": "address", + "protocol": "openid-connect", + "protocolMapper": "oidc-address-mapper", + "consentRequired": false, + "config": { + "user.attribute.formatted": "formatted", + "user.attribute.country": "country", + "user.attribute.postal_code": "postal_code", + "userinfo.token.claim": "true", + "user.attribute.street": "street", + "id.token.claim": "true", + "user.attribute.region": "region", + "access.token.claim": "true", + "user.attribute.locality": "locality" + } + } + ] + }, + { + "id": "b16d9232-a4e2-47d4-a368-5279a0d84913", + "name": "email", + "description": "OpenID Connect built-in scope: email", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "true", + "display.on.consent.screen": "true", + "consent.screen.text": "${emailScopeConsentText}" + }, + "protocolMappers": [ + { + "id": "4bd6701a-cc02-481e-83c5-e048ea5d83a9", + "name": "email", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-property-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "email", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "email", + "jsonType.label": "String" + } + }, + { + "id": "4cf00282-d385-456a-8943-4bdde6357c16", + "name": "email verified", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-property-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "emailVerified", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "email_verified", + "jsonType.label": "boolean" + } + } + ] + }, + { + "id": "1e9fa514-8ae1-4980-9ccc-2d2d2c43c7e6", + "name": "profile", + "description": "OpenID Connect built-in scope: profile", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "true", + "display.on.consent.screen": "true", + "consent.screen.text": "${profileScopeConsentText}" + }, + "protocolMappers": [ + { + "id": "2e186cd7-b7d5-4b63-b765-c77036183db6", + "name": "updated at", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "updatedAt", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "updated_at", + "jsonType.label": "String" + } + }, + { + "id": "86e94688-d91b-493b-809a-07005c7e6cab", + "name": "picture", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "picture", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "picture", + "jsonType.label": "String" + } + }, + { + "id": "8e65f9c7-a3c0-4bf6-9c4e-47be99464408", + "name": "zoneinfo", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "zoneinfo", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "zoneinfo", + "jsonType.label": "String" + } + }, + { + "id": "9eeaaeb3-93fc-439f-a8db-d6f3693a8ba1", + "name": "given name", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-property-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "firstName", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "given_name", + "jsonType.label": "String" + } + }, + { + "id": "34e60d98-fcde-49a2-b093-748464886a0d", + "name": "middle name", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "middleName", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "middle_name", + "jsonType.label": "String" + } + }, + { + "id": "08fa0341-5dd3-42e2-babb-1151c35b72c3", + "name": "nickname", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "nickname", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "nickname", + "jsonType.label": "String" + } + }, + { + "id": "9d9f1655-9b23-4e15-b244-aeffcb20c5ba", + "name": "gender", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "gender", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "gender", + "jsonType.label": "String" + } + }, + { + "id": "23b19dbb-5af2-494e-b462-e8f63d9266f4", + "name": "birthdate", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "birthdate", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "birthdate", + "jsonType.label": "String" + } + }, + { + "id": "b4644d65-ffbb-4e0b-8aac-238665af40dc", + "name": "locale", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "locale", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "locale", + "jsonType.label": "String" + } + }, + { + "id": "6366756e-bf69-4844-b127-60fa514ad768", + "name": "website", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "website", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "website", + "jsonType.label": "String" + } + }, + { + "id": "3d763f84-d417-4b4e-99e4-2b0e05bf861a", + "name": "family name", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-property-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "lastName", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "family_name", + "jsonType.label": "String" + } + }, + { + "id": "d05efa25-5348-4a14-9550-69791df4ec5e", + "name": "full name", + "protocol": "openid-connect", + "protocolMapper": "oidc-full-name-mapper", + "consentRequired": false, + "config": { + "id.token.claim": "true", + "access.token.claim": "true", + "userinfo.token.claim": "true" + } + }, + { + "id": "0d66e664-6b0c-45de-ba88-b2b86b23cacc", + "name": "profile", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "profile", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "profile", + "jsonType.label": "String" + } + }, + { + "id": "17d3b93d-993b-4768-892c-0b20f8462be3", + "name": "username", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-property-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "username", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "preferred_username", + "jsonType.label": "String" + } + } + ] + }, + { + "id": "a46716b3-8da1-4657-b703-13a5cd472c92", + "name": "role_list", + "description": "SAML role list", + "protocol": "saml", + "attributes": { + "consent.screen.text": "${samlRoleListScopeConsentText}", + "display.on.consent.screen": "true" + }, + "protocolMappers": [ + { + "id": "0ab50259-1e8b-40bd-9686-fb9a54dfc37d", + "name": "role list", + "protocol": "saml", + "protocolMapper": "saml-role-list-mapper", + "consentRequired": false, + "config": { + "single": "false", + "attribute.nameformat": "Basic", + "attribute.name": "Role" + } + } + ] + }, + { + "id": "0f9bd78c-129e-4f87-9cf7-8b68b628ea1b", + "name": "offline_access", + "description": "OpenID Connect built-in scope: offline_access", + "protocol": "openid-connect", + "attributes": { + "consent.screen.text": "${offlineAccessScopeConsentText}", + "display.on.consent.screen": "true" + } + } + ], + "keycloakVersion": "9.0.2" +} diff --git a/applications/accounts/scripts/create_api_user.sh b/applications/accounts/scripts/create_api_user.sh old mode 100644 new mode 100755 diff --git a/applications/argo/.ch-manifest b/applications/argo/.ch-manifest new file mode 100644 index 000000000..6271cbc03 --- /dev/null +++ b/applications/argo/.ch-manifest @@ -0,0 +1,4 @@ +app-name: argo +inferred: true +templates: [base] +version: '2' diff --git a/applications/common/.ch-manifest b/applications/common/.ch-manifest new file mode 100644 index 000000000..10271ee52 --- /dev/null +++ b/applications/common/.ch-manifest @@ -0,0 +1,4 @@ +app-name: common +inferred: true +templates: [base, flask-server] +version: '2' diff --git a/applications/common/server/setup.py b/applications/common/server/setup.py index 1935137a4..fdfc3ea89 100644 --- a/applications/common/server/setup.py +++ b/applications/common/server/setup.py @@ -4,7 +4,7 @@ from setuptools import setup, find_packages NAME = "openapi_server" -VERSION = "2.4.0" +VERSION = "2.5.0" # To install the library, run the following # diff --git a/applications/events/.ch-manifest b/applications/events/.ch-manifest new file mode 100644 index 000000000..5cdb28f07 --- /dev/null +++ b/applications/events/.ch-manifest @@ -0,0 +1,4 @@ +app-name: events +inferred: true +templates: [base] +version: '2' diff --git a/applications/jupyterhub/.ch-manifest b/applications/jupyterhub/.ch-manifest new file mode 100644 index 000000000..4f42b848c --- /dev/null +++ b/applications/jupyterhub/.ch-manifest @@ -0,0 +1,4 @@ +app-name: jupyterhub +inferred: true +templates: [base] +version: '2' diff --git a/applications/neo4j/.ch-manifest b/applications/neo4j/.ch-manifest new file mode 100644 index 000000000..63ef59d41 --- /dev/null +++ b/applications/neo4j/.ch-manifest @@ -0,0 +1,4 @@ +app-name: neo4j +inferred: true +templates: [base, db-neo4j] +version: '2' diff --git a/applications/nfsserver/.ch-manifest b/applications/nfsserver/.ch-manifest new file mode 100644 index 000000000..b982761e3 --- /dev/null +++ b/applications/nfsserver/.ch-manifest @@ -0,0 +1,4 @@ +app-name: nfsserver +inferred: true +templates: [base] +version: '2' diff --git a/applications/nfsserver/deploy/values.yaml b/applications/nfsserver/deploy/values.yaml index 8bf13056d..713ee0ec2 100644 --- a/applications/nfsserver/deploy/values.yaml +++ b/applications/nfsserver/deploy/values.yaml @@ -4,6 +4,7 @@ harness: auto: false deployment: auto: false + image: gcr.io/metacellllc/cloudharness/nfsserver:1.0 # nfs server pvc disk size (/exports) diff --git a/applications/notifications/.ch-manifest b/applications/notifications/.ch-manifest new file mode 100644 index 000000000..cf24fc505 --- /dev/null +++ b/applications/notifications/.ch-manifest @@ -0,0 +1,4 @@ +app-name: notifications +inferred: true +templates: [base, flask-server] +version: '2' diff --git a/applications/notifications/server/setup.py b/applications/notifications/server/setup.py index f625329c1..590a1fd2a 100644 --- a/applications/notifications/server/setup.py +++ b/applications/notifications/server/setup.py @@ -4,7 +4,7 @@ from setuptools import setup, find_packages NAME = "notifications" -VERSION = "2.4.0" +VERSION = "2.5.0" # To install the library, run the following # diff --git a/applications/samples/.ch-manifest b/applications/samples/.ch-manifest new file mode 100644 index 000000000..cd34091ee --- /dev/null +++ b/applications/samples/.ch-manifest @@ -0,0 +1,4 @@ +app-name: samples +inferred: true +templates: [base, webapp, flask-server] +version: '2' diff --git a/applications/samples/deploy/values-test.yaml b/applications/samples/deploy/values-test.yaml index ebf25ca3a..e897ad996 100644 --- a/applications/samples/deploy/values-test.yaml +++ b/applications/samples/deploy/values-test.yaml @@ -5,6 +5,7 @@ harness: - events - common - jupyterhub + - volumemanager accounts: roles: - role1 diff --git a/applications/sentry/.ch-manifest b/applications/sentry/.ch-manifest new file mode 100644 index 000000000..4acc26d53 --- /dev/null +++ b/applications/sentry/.ch-manifest @@ -0,0 +1,4 @@ +app-name: sentry +inferred: true +templates: [base, db-postgres] +version: '2' diff --git a/applications/volumemanager/.ch-manifest b/applications/volumemanager/.ch-manifest new file mode 100644 index 000000000..f994722e4 --- /dev/null +++ b/applications/volumemanager/.ch-manifest @@ -0,0 +1,4 @@ +app-name: volumemanager +inferred: true +templates: [base, flask-server] +version: '2' diff --git a/applications/volumemanager/server/setup.py b/applications/volumemanager/server/setup.py index ea5b3b377..21b33fc11 100644 --- a/applications/volumemanager/server/setup.py +++ b/applications/volumemanager/server/setup.py @@ -4,7 +4,7 @@ from setuptools import setup, find_packages NAME = "volumemanager" -VERSION = "2.4.0" +VERSION = "2.5.0" # To install the library, run the following # diff --git a/applications/workflows/.ch-manifest b/applications/workflows/.ch-manifest new file mode 100644 index 000000000..16cb6bbde --- /dev/null +++ b/applications/workflows/.ch-manifest @@ -0,0 +1,4 @@ +app-name: workflows +inferred: true +templates: [base, flask-server] +version: '2' diff --git a/applications/workflows/server/setup.py b/applications/workflows/server/setup.py index 230cdd9d4..93f8036b3 100644 --- a/applications/workflows/server/setup.py +++ b/applications/workflows/server/setup.py @@ -4,7 +4,7 @@ from setuptools import setup, find_packages NAME = "workflows_api" -VERSION = "2.4.0" +VERSION = "2.5.0" # To install the library, run the following # diff --git a/ch-166.patch b/ch-166.patch new file mode 100644 index 000000000..1603e0a07 --- /dev/null +++ b/ch-166.patch @@ -0,0 +1,14 @@ +diff --git a/tools/deployment-cli-tools/ch_cli_tools/codefresh.py b/tools/deployment-cli-tools/ch_cli_tools/codefresh.py +index 8bcf2b79..3ea43e31 100644 +--- a/tools/deployment-cli-tools/ch_cli_tools/codefresh.py ++++ b/tools/deployment-cli-tools/ch_cli_tools/codefresh.py +@@ -175,8 +175,7 @@ def create_codefresh_deployment_scripts(root_paths, envs=(), include=(), exclude + + if app_config and app_config.dependencies and app_config.dependencies.git: + for dep in app_config.dependencies.git: +- step_name = f"clone_{basename(dep.url).replace('.', '_')}_{basename(dockerfile_relative_to_root).replace('.', '_')}" +- steps[CD_BUILD_STEP_DEPENDENCIES]['steps'][step_name] = clone_step_spec(dep, dockerfile_relative_to_root) ++ steps[CD_BUILD_STEP_DEPENDENCIES]['steps'][f"clone_{basename(dep.url).replace(".", "_")}_{basename(dockerfile_relative_to_root).replace(".", "_")}"] = clone_step_spec(dep, dockerfile_relative_to_root) + + build = None + if build_step in steps: diff --git a/deployment-configuration/codefresh-template-dev.yaml b/deployment-configuration/codefresh-template-dev.yaml index 516200484..a12f30f44 100644 --- a/deployment-configuration/codefresh-template-dev.yaml +++ b/deployment-configuration/codefresh-template-dev.yaml @@ -120,6 +120,7 @@ steps: image: "${{test-e2e}}" fail_fast: false commands: + - npx puppeteer browsers install chrome - yarn test scale: {} when: diff --git a/deployment-configuration/codefresh-template-test.yaml b/deployment-configuration/codefresh-template-test.yaml index b727d53cd..45febee47 100644 --- a/deployment-configuration/codefresh-template-test.yaml +++ b/deployment-configuration/codefresh-template-test.yaml @@ -112,6 +112,7 @@ steps: image: "${{test-e2e}}" fail_fast: false commands: + - npx puppeteer browsers install chrome - yarn test scale: {} hooks: diff --git a/deployment-configuration/compose/Chart.yaml b/deployment-configuration/compose/Chart.yaml index 1b396ffe1..e71aca492 100644 --- a/deployment-configuration/compose/Chart.yaml +++ b/deployment-configuration/compose/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v1 -appVersion: 2.4.0-compose +appVersion: 2.5.0-compose description: CloudHarness Docker compose Helm Chart maintainers: - {email: filippo@metacell.us, name: Filippo Ledda} diff --git a/deployment-configuration/compose/values.yaml b/deployment-configuration/compose/values.yaml index 0fa87e29f..023ab9f5c 100644 --- a/deployment-configuration/compose/values.yaml +++ b/deployment-configuration/compose/values.yaml @@ -21,7 +21,7 @@ apps: {} env: # -- Cloud Harness version - name: CH_VERSION - value: 2.4.0 + value: 2.5.0 proxy: timeout: # -- Timeout for proxy connections in seconds. diff --git a/deployment-configuration/helm/Chart.yaml b/deployment-configuration/helm/Chart.yaml index 31dbc5ff6..09aa97e5b 100644 --- a/deployment-configuration/helm/Chart.yaml +++ b/deployment-configuration/helm/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v1 -appVersion: 2.4.0 +appVersion: 2.5.0 description: CloudHarness Helm Chart maintainers: - {email: filippo@metacell.us, name: Filippo Ledda} diff --git a/deployment-configuration/helm/templates/auto-gatekeepers.yaml b/deployment-configuration/helm/templates/auto-gatekeepers.yaml index 898995cd6..8e1c3ac0f 100644 --- a/deployment-configuration/helm/templates/auto-gatekeepers.yaml +++ b/deployment-configuration/helm/templates/auto-gatekeepers.yaml @@ -1,6 +1,16 @@ {{/* Secured Services/Deployments */}} +{{- define "check_no_wildcard_uri" }} +{{- $check := true }} +{{- range .uri_role_mapping }} + {{- if eq .uri "/*" }} + {{- $check = false }} + {{- end }} +{{- end }} +{{- $check }} +{{- end }} {{- define "deploy_utils.securedservice" }} {{- $tls := not (not .root.Values.tls) }} +{{- $noWildcards := include "check_no_wildcard_uri" (dict "uri_role_mapping" .app.harness.uri_role_mapping) }} apiVersion: v1 kind: ConfigMap metadata: @@ -15,7 +25,7 @@ data: client-secret: {{ .root.Values.apps.accounts.webclient.secret }} secure-cookie: {{ $tls }} forbidden-page: /templates/access-denied.html.tmpl - enable-default-deny: {{ eq (.app.harness.secured | toString) "true" }} + enable-default-deny: {{ $noWildcards }} listen: 0.0.0.0:8080 enable-refresh-tokens: true server-write-timeout: {{ .app.harness.proxy.timeout.send | default .root.Values.proxy.timeout.send | default 180 }}s @@ -99,7 +109,6 @@ metadata: name: "{{ .app.harness.service.name }}-gk" labels: app: "{{ .app.harness.service.name }}-gk" - spec: replicas: 1 selector: @@ -115,7 +124,7 @@ spec: {{ include "deploy_utils.etcHosts" .root | indent 6 }} containers: - name: {{ .app.harness.service.name | quote }} - image: "quay.io/gogatekeeper/gatekeeper:1.3.8" + image: "quay.io/gogatekeeper/gatekeeper:2.14.3" imagePullPolicy: IfNotPresent {{ if .root.Values.local }} securityContext: diff --git a/deployment-configuration/helm/templates/auto-secrets.yaml b/deployment-configuration/helm/templates/auto-secrets.yaml index a0a37a2f8..0a1ccc3b0 100644 --- a/deployment-configuration/helm/templates/auto-secrets.yaml +++ b/deployment-configuration/helm/templates/auto-secrets.yaml @@ -1,5 +1,4 @@ {{- define "deploy_utils.secret" }} -{{- if .app.harness.secrets }} {{- $secret_name := printf "%s" .app.harness.deployment.name }} apiVersion: v1 kind: Secret @@ -9,42 +8,53 @@ metadata: labels: app: {{ .app.harness.deployment.name }} type: Opaque - {{- $secret := (lookup "v1" "Secret" .root.Values.namespace $secret_name) }} - {{- if $secret }} -# secret already exists - {{- if not (compact (values .app.harness.secrets)) }} -# secret values are null, copy from the existing secret -data: - {{- range $k, $v := $secret.data }} - {{ $k }}: {{ $v }} - {{- end }} - {{- else }} -# there are non default values in values.yaml, use these +{{- $secret := (lookup "v1" "Secret" .root.Values.namespace $secret_name) }} +{{/*- $secret := dict "data" (dict "test" "test") */}} stringData: - {{- range $k, $v := .app.harness.secrets }} - {{ $k }}: {{ $v | default (randAlphaNum 20) }} - {{- end }} - {{- end }} - {{- else }} -# secret doesn't exist -stringData: - {{- range $k, $v := .app.harness.secrets }} - {{ $k }}: {{ $v | default (randAlphaNum 20) }} + updated: {{ now | quote }} # Added because in case of update, if no field is updated, alla data is erased +{{- if $secret }} + {{- range $k, $v := .app.harness.secrets }} + {{- if $v }} + {{- if eq (typeOf $v) "string" }} + {{- if ne $v "?" }} + # Update/set value to value in values.yaml if specified + {{ $k }}: {{ $v | quote }} + {{- else }} + # Refresh at any deployment for ? (pure random) value + {{ $k }}: {{ randAlphaNum 20 | quote }} + {{- end }} + {{- else }} + # Type not recognized: setting to a empty string" + {{ $k }}-formatnotrecognized: {{ $v }} + {{ $k }}: "" + {{- end }} + {{- else if eq (typeOf $secret.data) (typeOf dict) }} + # Value empty or null in the values.yaml + {{- if not (hasKey $secret.data $k) }} + # Create a random secret value if not specified in values.yaml if it is not set and it is not already in the deployed secret (static random secret) */}} + {{ $k }}: {{ randAlphaNum 20 | quote }} + {{- else }} + # confirm previous value from the secret (static random secret already set, do nothing)} + {{- end}} {{- end }} + {{- end }} # range end +{{- else }} +# New secret + {{- range $k, $v := .app.harness.secrets }} + {{ $k }}: {{ $v | default (randAlphaNum 20) | quote }} {{- end }} {{- end }} --- {{- end }} ---- {{- range $app := .Values.apps }} ---- + {{- if $app.harness.secrets }}{{- if ne (len $app.harness.secrets) 0 }} {{- include "deploy_utils.secret" (dict "root" $ "app" $app) }} + {{- end }}{{- end }} {{- range $subapp := $app }} {{- if contains "map" (typeOf $subapp) }} - {{- if hasKey $subapp "harness" }} ---- + {{- if hasKey $subapp "harness" }}{{- if $app.harness.secrets }}{{- if ne (len $app.harness.secrets) 0 }} {{- include "deploy_utils.secret" (dict "root" $ "app" $subapp) }} - {{- end }} + {{- end }}{{- end }}{{- end }} {{- end }} {{- end }} {{- end }} \ No newline at end of file diff --git a/deployment-configuration/helm/values.yaml b/deployment-configuration/helm/values.yaml index 86ada06f8..75a14f3ab 100644 --- a/deployment-configuration/helm/values.yaml +++ b/deployment-configuration/helm/values.yaml @@ -21,7 +21,7 @@ apps: {} env: # -- Cloud Harness version - name: CH_VERSION - value: 2.4.0 + value: 2.5.0 privenv: # -- Defines a secret as private environment variable that is injected in containers. - name: CH_SECRET diff --git a/deployment-configuration/vscode-django-app-debug-template.json b/deployment-configuration/vscode-django-fastapi-debug-template.json similarity index 100% rename from deployment-configuration/vscode-django-app-debug-template.json rename to deployment-configuration/vscode-django-fastapi-debug-template.json diff --git a/deployment-configuration/vscode-django-ninja-debug-template.json b/deployment-configuration/vscode-django-ninja-debug-template.json new file mode 100644 index 000000000..198d465d6 --- /dev/null +++ b/deployment-configuration/vscode-django-ninja-debug-template.json @@ -0,0 +1,24 @@ +{ + "args": [ + "--host", + "0.0.0.0", + "--port", + "8000", + "django_baseapp.asgi:application" + ], + "console": "integratedTerminal", + "cwd": "${workspaceFolder}/applications/__APP_NAME__/backend", + "env": { + "ACCOUNTS_ADMIN_PASSWORD": "metacell", + "ACCOUNTS_ADMIN_USERNAME": "admin", + "CH_CURRENT_APP_NAME": "__APP_NAME__", + "CH_VALUES_PATH": "${workspaceFolder}/deployment/helm/values.yaml", + "DJANGO_SETTINGS_MODULE": "django_baseapp.settings", + "KUBERNETES_SERVICE_HOST": "ssdds" + }, + "justMyCode": false, + "module": "uvicorn", + "name": "__APP_NAME__ backend", + "request": "launch", + "type": "debugpy" + } \ No newline at end of file diff --git a/deployment/codefresh-test.yaml b/deployment/codefresh-test.yaml index f2b0a8217..21bef9232 100644 --- a/deployment/codefresh-test.yaml +++ b/deployment/codefresh-test.yaml @@ -33,8 +33,8 @@ steps: working_directory: . commands: - bash cloud-harness/install.sh - - 'harness-deployment . -n test-${{NAMESPACE_BASENAME}} -d ${{DOMAIN}} -r ${{REGISTRY}} - -rs ${{REGISTRY_SECRET}} -e test --write-env -N ' + - harness-deployment . -n test-${{NAMESPACE_BASENAME}} -d ${{DOMAIN}} -r ${{REGISTRY}} + -rs ${{REGISTRY_SECRET}} -e test --write-env -N -i samples - cat deployment/.env >> ${{CF_VOLUME_PATH}}/env_vars_to_export - cat ${{CF_VOLUME_PATH}}/env_vars_to_export prepare_deployment_view: @@ -124,49 +124,6 @@ steps: type: parallel stage: build steps: - nfsserver: - type: build - stage: build - dockerfile: Dockerfile - registry: '${{CODEFRESH_REGISTRY}}' - buildkit: true - build_arguments: - - DOMAIN=${{DOMAIN}} - - NOCACHE=${{CF_BUILD_ID}} - - REGISTRY=${{REGISTRY}}/cloudharness/ - image_name: cloudharness/nfsserver - title: Nfsserver - working_directory: ./applications/nfsserver - tag: '${{NFSSERVER_TAG}}' - when: - condition: - any: - buildDoesNotExist: includes('${{NFSSERVER_TAG_EXISTS}}', '{{NFSSERVER_TAG_EXISTS}}') - == true - forceNoCache: includes('${{NFSSERVER_TAG_FORCE_BUILD}}', '{{NFSSERVER_TAG_FORCE_BUILD}}') - == false - notifications: - type: build - stage: build - dockerfile: Dockerfile - registry: '${{CODEFRESH_REGISTRY}}' - buildkit: true - build_arguments: - - DOMAIN=${{DOMAIN}} - - NOCACHE=${{CF_BUILD_ID}} - - REGISTRY=${{REGISTRY}}/cloudharness/ - - CLOUDHARNESS_BASE=${{REGISTRY}}/cloudharness/cloudharness-base:${{CLOUDHARNESS_BASE_TAG}} - image_name: cloudharness/notifications - title: Notifications - working_directory: ./applications/notifications/server - tag: '${{NOTIFICATIONS_TAG}}' - when: - condition: - any: - buildDoesNotExist: includes('${{NOTIFICATIONS_TAG_EXISTS}}', '{{NOTIFICATIONS_TAG_EXISTS}}') - == true - forceNoCache: includes('${{NOTIFICATIONS_TAG_FORCE_BUILD}}', '{{NOTIFICATIONS_TAG_FORCE_BUILD}}') - == false accounts: type: build stage: build @@ -210,27 +167,6 @@ steps: == true forceNoCache: includes('${{VOLUMEMANAGER_TAG_FORCE_BUILD}}', '{{VOLUMEMANAGER_TAG_FORCE_BUILD}}') == false - sentry: - type: build - stage: build - dockerfile: Dockerfile - registry: '${{CODEFRESH_REGISTRY}}' - buildkit: true - build_arguments: - - DOMAIN=${{DOMAIN}} - - NOCACHE=${{CF_BUILD_ID}} - - REGISTRY=${{REGISTRY}}/cloudharness/ - image_name: cloudharness/sentry - title: Sentry - working_directory: ./applications/sentry - tag: '${{SENTRY_TAG}}' - when: - condition: - any: - buildDoesNotExist: includes('${{SENTRY_TAG_EXISTS}}', '{{SENTRY_TAG_EXISTS}}') - == true - forceNoCache: includes('${{SENTRY_TAG_FORCE_BUILD}}', '{{SENTRY_TAG_FORCE_BUILD}}') - == false jupyterhub: type: build stage: build @@ -476,11 +412,6 @@ steps: custom_value_files: - ./deployment/helm/values.yaml custom_values: - - apps_notifications_harness_secrets_email-user=${{EMAIL-USER}} - - apps_notifications_harness_secrets_email-password=${{EMAIL-PASSWORD}} - - apps_sentry_harness_secrets_email-server=${{EMAIL-SERVER}} - - apps_sentry_harness_secrets_email-user=${{EMAIL-USER}} - - apps_sentry_harness_secrets_email-password=${{EMAIL-PASSWORD}} - apps_samples_harness_secrets_asecret=${{ASECRET}} build_test_images: title: Build test images @@ -537,10 +468,8 @@ steps: commands: - kubectl config use-context ${{CLUSTER_NAME}} - kubectl config set-context --current --namespace=test-${{NAMESPACE_BASENAME}} - - kubectl rollout status deployment/notifications - kubectl rollout status deployment/accounts - kubectl rollout status deployment/volumemanager - - kubectl rollout status deployment/sentry - kubectl rollout status deployment/argo-server-gk - kubectl rollout status deployment/samples - kubectl rollout status deployment/samples-gk @@ -556,18 +485,6 @@ steps: commands: - echo $APP_NAME scale: - volumemanager_api_test: - title: volumemanager api test - volumes: - - '${{CF_REPO_NAME}}/applications/volumemanager:/home/test' - - '${{CF_REPO_NAME}}/deployment/helm/values.yaml:/opt/cloudharness/resources/allvalues.yaml' - environment: - - APP_URL=https://volumemanager.${{DOMAIN}}/api - - USERNAME=volumes@testuser.com - - PASSWORD=test - commands: - - st --pre-run cloudharness_test.apitest_init run api/openapi.yaml --base-url - https://volumemanager.${{DOMAIN}}/api -c all samples_api_test: title: samples api test volumes: @@ -617,6 +534,7 @@ steps: image: '${{REGISTRY}}/cloudharness/test-e2e:${{TEST_E2E_TAG}}' fail_fast: false commands: + - npx puppeteer browsers install chrome - yarn test scale: jupyterhub_e2e_test: diff --git a/deployment/docker-compose.yaml b/deployment/docker-compose.yaml new file mode 100644 index 000000000..05df8c244 --- /dev/null +++ b/deployment/docker-compose.yaml @@ -0,0 +1,313 @@ +# Source: cloudharness/templates/auto-compose.yaml +version: '3.7' + +services: + traefik: + image: traefik:v2.10 + container_name: traefik + networks: + - ch + command: + - --log.level=INFO + - --api.insecure=true + - --providers.docker=true + - --providers.docker.exposedbydefault=false + - --entrypoints.web.address=:80 + - --entrypoints.websecure.address=:443 + - --providers.file.directory=/etc/traefik/dynamic_conf + ports: + - 80:80 + - 443:443 + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + - ./traefik/certs/:/certs/:ro + - ./traefik/traefik.yaml:/etc/traefik/dynamic_conf/conf.yml:ro + accounts: + networks: + - ch + image: cloudharness/accounts + + expose: + - 8080 + deploy: + mode: replicated + replicas: 1 + resources: + limits: + cpus: 0.500 + memory: 1024M + reservations: + cpus: 0.010 + memory: 512M + healthcheck: + test: [CMD, curl, -f, http://127.0.0.1:8080/auth/realms/ch/account] + interval: 1s + timeout: 3s + retries: 30 + environment: + - CH_CURRENT_APP_NAME=accounts + - CH_VERSION=2.4.0 + - CH_ACCOUNTS_SUBDOMAIN=accounts + - CH_ACCOUNTS_NAME=accounts + - CH_JUPYTERHUB_SUBDOMAIN=hub + - CH_JUPYTERHUB_NAME=jupyterhub + - CH_ARGO_SUBDOMAIN=argo + - CH_ARGO_NAME=argo + - CH_SAMPLES_SUBDOMAIN=samples + - CH_SAMPLES_PORT=80 + - CH_SAMPLES_NAME=samples + - CH_COMMON_SUBDOMAIN=common + - CH_COMMON_NAME=common + - CH_EVENTS_SUBDOMAIN=events + - CH_EVENTS_NAME=events + - CH_WORKFLOWS_SUBDOMAIN=workflows + - CH_WORKFLOWS_NAME=workflows + - CH_DOMAIN=ch + - CH_IMAGE_REGISTRY= + - CH_IMAGE_TAG= + - CH_ACCOUNTS_CLIENT_SECRET=5678eb6e-9e2c-4ee5-bd54-34e7411339e8 + - CH_ACCOUNTS_REALM=ch + - CH_ACCOUNTS_AUTH_DOMAIN=accounts.ch + - CH_ACCOUNTS_CLIENT_ID=rest-client + - DOMAIN=ch + - KEYCLOAK_IMPORT=/tmp/realm.json + - KEYCLOAK_USER=admin + - KEYCLOAK_PASSWORD=metacell + - PROXY_ADDRESS_FORWARDING=true + - DB_VENDOR=POSTGRES + - DB_ADDR=keycloak-postgres + - DB_DATABASE=auth_db + - DB_USER=user + - DB_PASSWORD=password + - JAVA_OPTS=-server -Xms64m -Xmx896m -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m + -Djava.net.preferIPv4Stack=true -Djboss.modules.system.pkgs=org.jboss.byteman + -Djava.awt.headless=true --add-exports=java.base/sun.nio.ch=ALL-UNNAMED --add-exports=jdk.unsupported/sun.misc=ALL-UNNAMED + --add-exports=jdk.unsupported/sun.reflect=ALL-UNNAMED + volumes: + - ./compose/allvalues.yaml:/opt/cloudharness/resources/allvalues.yaml:ro + - ./compose/resources/generated/auth/api_user_password:/opt/cloudharness/resources/auth/api_user_password + - type: bind + source: ./compose/resources/generated/accounts/realm.json + target: /tmp/realm.json + labels: + - traefik.enable=true + - traefik.http.services.accounts.loadbalancer.server.port=8080 + # - "traefik.http.middlewares.redirect-middleware.redirectscheme.scheme=https" + # - "traefik.http.routers..middlewares=redirect-middleware" + - traefik.http.routers.accounts.rule=Host(`accounts.ch`) + - traefik.http.routers.accounts.entrypoints=web + # Database type postgres named keycloak-postgres + keycloak-postgres: + networks: + ch: + image: postgres:10.4 + expose: + - '5432' + deploy: + resources: + limits: + cpus: 1.000 + memory: 2G + reservations: + cpus: 0.100 + memory: 512M + volumes: + - type: volume + source: keycloak-postgres + target: /data/db + - type: volume + source: dshm-keycloak-postgres + target: /dev/shm + environment: + - POSTGRES_DB=auth_db + - POSTGRES_USER=user + - POSTGRES_PASSWORD=password + - PGDATA=/data/db/pgdata + common: + networks: + - ch + image: cloudharness/common + + expose: + - 8080 + deploy: + mode: replicated + replicas: 1 + resources: + limits: + cpus: 0.200 + memory: 256M + reservations: + cpus: 0.050 + memory: 128M + # entrypoint: python /usr/src/app/common/__main__.py + environment: + - CH_CURRENT_APP_NAME=common + - CH_VERSION=2.4.0 + - CH_ACCOUNTS_SUBDOMAIN=accounts + - CH_ACCOUNTS_NAME=accounts + - CH_JUPYTERHUB_SUBDOMAIN=hub + - CH_JUPYTERHUB_NAME=jupyterhub + - CH_ARGO_SUBDOMAIN=argo + - CH_ARGO_NAME=argo + - CH_SAMPLES_SUBDOMAIN=samples + - CH_SAMPLES_PORT=80 + - CH_SAMPLES_NAME=samples + - CH_COMMON_SUBDOMAIN=common + - CH_COMMON_NAME=common + - CH_EVENTS_SUBDOMAIN=events + - CH_EVENTS_NAME=events + - CH_WORKFLOWS_SUBDOMAIN=workflows + - CH_WORKFLOWS_NAME=workflows + - CH_DOMAIN=ch + - CH_IMAGE_REGISTRY= + - CH_IMAGE_TAG= + - CH_ACCOUNTS_CLIENT_SECRET=5678eb6e-9e2c-4ee5-bd54-34e7411339e8 + - CH_ACCOUNTS_REALM=ch + - CH_ACCOUNTS_AUTH_DOMAIN=accounts.ch + - CH_ACCOUNTS_CLIENT_ID=rest-client + - DOMAIN=ch + volumes: + - ./compose/allvalues.yaml:/opt/cloudharness/resources/allvalues.yaml:ro + labels: + - traefik.enable=true + - traefik.http.services.common.loadbalancer.server.port=8080 + # - "traefik.http.middlewares.redirect-middleware.redirectscheme.scheme=https" + # - "traefik.http.routers..middlewares=redirect-middleware" + - traefik.http.routers.common.rule=Host(`common.ch`) + - traefik.http.routers.common.entrypoints=web + samples: + networks: + - ch + image: cloudharness/samples + + expose: + - 8080 + deploy: + mode: replicated + replicas: 1 + resources: + limits: + cpus: 0.500 + memory: 500M + reservations: + cpus: 0.010 + memory: 32M + # entrypoint: python /usr/src/app/samples/__main__.py + environment: + - CH_CURRENT_APP_NAME=samples + - CH_VERSION=2.4.0 + - CH_ACCOUNTS_SUBDOMAIN=accounts + - CH_ACCOUNTS_NAME=accounts + - CH_JUPYTERHUB_SUBDOMAIN=hub + - CH_JUPYTERHUB_NAME=jupyterhub + - CH_ARGO_SUBDOMAIN=argo + - CH_ARGO_NAME=argo + - CH_SAMPLES_SUBDOMAIN=samples + - CH_SAMPLES_PORT=80 + - CH_SAMPLES_NAME=samples + - CH_COMMON_SUBDOMAIN=common + - CH_COMMON_NAME=common + - CH_EVENTS_SUBDOMAIN=events + - CH_EVENTS_NAME=events + - CH_WORKFLOWS_SUBDOMAIN=workflows + - CH_WORKFLOWS_NAME=workflows + - CH_DOMAIN=ch + - CH_IMAGE_REGISTRY= + - CH_IMAGE_TAG= + - CH_ACCOUNTS_CLIENT_SECRET=5678eb6e-9e2c-4ee5-bd54-34e7411339e8 + - CH_ACCOUNTS_REALM=ch + - CH_ACCOUNTS_AUTH_DOMAIN=accounts.ch + - CH_ACCOUNTS_CLIENT_ID=rest-client + - DOMAIN=ch + - WORKERS=3 + + + links: + - workflows:workflows.ch +# - events:events.ch + - common:common.ch +# - jupyterhub:jupyterhub.ch + volumes: + - ./compose/allvalues.yaml:/opt/cloudharness/resources/allvalues.yaml:ro + - ./compose/resources/generated/auth/asecret:/opt/cloudharness/resources/auth/asecret + - type: volume + source: my-shared-volume + target: /tmp/myvolume + - type: bind + source: ./compose/resources/generated/samples/myConfig.json + target: /tmp/resources/myConfig.json + - type: bind + source: ./compose/resources/generated/samples/example.yaml + target: /usr/src/app/important_config.yaml + labels: + - traefik.enable=true + - traefik.http.services.samples.loadbalancer.server.port=8080 + # - "traefik.http.middlewares.redirect-middleware.redirectscheme.scheme=https" + # - "traefik.http.routers..middlewares=redirect-middleware" + - traefik.http.routers.samples.rule=Host(`samples.ch`) + - traefik.http.routers.samples.entrypoints=web + workflows: + networks: + - ch + image: cloudharness/workflows + + expose: + - 8080 + deploy: + mode: replicated + replicas: 1 + resources: + limits: + cpus: 0.500 + memory: 500M + reservations: + cpus: 0.010 + memory: 32M + # entrypoint: python /usr/src/app/workflows_api/__main__.py + environment: + - CH_CURRENT_APP_NAME=workflows + - CH_VERSION=2.4.0 + - CH_ACCOUNTS_SUBDOMAIN=accounts + - CH_ACCOUNTS_NAME=accounts + - CH_JUPYTERHUB_SUBDOMAIN=hub + - CH_JUPYTERHUB_NAME=jupyterhub + - CH_ARGO_SUBDOMAIN=argo + - CH_ARGO_NAME=argo + - CH_SAMPLES_SUBDOMAIN=samples + - CH_SAMPLES_PORT=80 + - CH_SAMPLES_NAME=samples + - CH_COMMON_SUBDOMAIN=common + - CH_COMMON_NAME=common + - CH_EVENTS_SUBDOMAIN=events + - CH_EVENTS_NAME=events + - CH_WORKFLOWS_SUBDOMAIN=workflows + - CH_WORKFLOWS_NAME=workflows + - CH_DOMAIN=ch + - CH_IMAGE_REGISTRY= + - CH_IMAGE_TAG= + - CH_ACCOUNTS_CLIENT_SECRET=5678eb6e-9e2c-4ee5-bd54-34e7411339e8 + - CH_ACCOUNTS_REALM=ch + - CH_ACCOUNTS_AUTH_DOMAIN=accounts.ch + - CH_ACCOUNTS_CLIENT_ID=rest-client + - DOMAIN=ch + + + volumes: + - ./compose/allvalues.yaml:/opt/cloudharness/resources/allvalues.yaml:ro + labels: + - traefik.enable=true + - traefik.http.services.workflows.loadbalancer.server.port=8080 + # - "traefik.http.middlewares.redirect-middleware.redirectscheme.scheme=https" + # - "traefik.http.routers..middlewares=redirect-middleware" + - traefik.http.routers.workflows.rule=Host(`workflows.ch`) + - traefik.http.routers.workflows.entrypoints=web + +# Network definition +networks: + ch: + name: ch_network +volumes: + keycloak-postgres: + dshm-keycloak-postgres: + my-shared-volume: diff --git a/deployment/secret.yaml b/deployment/secret.yaml new file mode 100644 index 000000000..ff1ad4f3a --- /dev/null +++ b/deployment/secret.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Secret +metadata: + annotations: + meta.helm.sh/release-name: mnp + meta.helm.sh/release-namespace: mnp + labels: + app: accounts + app.kubernetes.io/managed-by: Helm + name: accounts + namespace: mnp +stringData: + pino: gianni +data: + api_user_password: ZGRkYTNjMTYxOTRhZTc2MjE5YzNiYjdjYjRiN2FlMDMzYThiN2ZlNQ== +type: Opaque diff --git a/docs/applications/harness-application.md b/docs/applications/harness-application.md index e21103983..fdbb5f6ca 100644 --- a/docs/applications/harness-application.md +++ b/docs/applications/harness-application.md @@ -1,53 +1,103 @@ -# Use harness-application to create a new application from templates +# Use harness-application to create a new application -## Choosing Templates +## Overview -If you create a new application, you can choose templates that are used to generate the application scaffold. +`harness-application` is a command-line tool used to create new applications from predefined code templates. It allows users to quickly scaffold applications with backend, frontend, and database configurations. -Running `harness-application --help` will list the currently available templates: +## Usage +```sh +harness-application [name] [-t TEMPLATE] ``` -usage: harness-application [-h] [-t TEMPLATES] name -Creates a new Application. +## Arguments + +- `name` *(required)* – The name of the application to be created. + +## Options + +- `-h, --help` – Displays the help message and exits. +- `-t TEMPLATES, --template TEMPLATES` – Specifies one or more templates to use when creating the application. + +## Choosing Templates -positional arguments: - name Application name +When creating a new application, you can choose templates that define its structure and components. Running `harness-application --help` will list the currently available templates: -optional arguments: - -h, --help show this help message and exit - -t TEMPLATES, --template TEMPLATES - Add a template name. Available templates: - base (always included) - flask-server (backend flask app based on openapi) - webapp (webapp including backend and frontend) - db-postgres - db-neo4j - db-mongo - django-app (fastapi django backend based on openapi) +```sh +usage: harness-application [-h] [-t TEMPLATES] name ``` ## Available Templates ### Base -* The `base` template is always included and used as foundation for any other template. +- The `base` template is always included and serves as the foundation for any other template. + +### Backend Templates + +#### Flask Server + +- The `flask-server` template consists of a backend built using [Flask](https://flask.palletsprojects.com/en/1.1.x/). + - Uses [Connexion](https://github.com/zalando/connexion) to map OpenAPI definitions to Flask routes. + - Served by [Gunicorn](https://gunicorn.org/) with 2 synchronous workers by default. + - Supports customization of the worker count and type. + +#### Django + +- The `django-fastapi` consists of a backend based on [FastAPI](https://fastapi.tiangolo.com/) and [Django](https://www.djangoproject.com/). + - Uses the [FastAPI code generator](https://github.com/koxudaxi/fastapi-code-generator) to map OpenAPI definitions. + - Served by [Uvicorn](https://www.uvicorn.org/) with 2 workers by default. +- The `django-ninja` consists of a backend based on [Django Ninja](https://django-ninja.dev/) + - Provides automatic OpenAPI schema generation. + - Supports Django's built-in ORM for seamless database integration. + - High performance due to Pydantic-based data validation. + - Simplifies request parsing and authentication. -### Flask Server -* It consists of a single backend, a Python [Flask](https://flask.palletsprojects.com/en/1.1.x/) application. -* The [Connexion](https://github.com/zalando/connexion) library maps the OpenAPI definition to Flask routing. -* Per default, [Gunicorn](https://gunicorn.org/) serves the Flask app with 2 synchronous workers. Depending on the application requirements, you can update the number of workers or choose a different [worker type](https://docs.gunicorn.org/en/stable/design.html). +### Full-Stack Templates +#### Webapp -### Webapp +- The `webapp` template extends the `base` template by adding a [React](https://reactjs.org/) frontend. + - The frontend bundle is served by the Python backend. + - React is used by default, but other frontend technologies can be integrated. -* The `webapp` template consists builds upon the `base` template extends it by a [React](https://reactjs.org/) frontend application. -* The generated frontend bundle is served by the Python backend. -* Per default, React is used as a frontend application, but you are free to choose a different frontend technology. +### Database Templates +- `db-postgres` – [PostgreSQL](https://www.postgresql.org/), a relational database. +- `db-neo4j` – [Neo4J](https://neo4j.com/), a graph database. +- `db-mongo` – [MongoDB](https://www.mongodb.com/), a NoSQL document-based database. -### Databases +## Examples + +### Create a New Flask-Based Microservice Application + +```sh +harness-application myapp +``` + +### Create a Full-Stack Web Application + +```sh +harness-application myapp -t webapp +``` + +### Create a Web Application with a Mongo Database + +```sh +harness-application myapp -t webapp -t db-mongo +``` + +### Display Help Information + +```sh +harness-application --help +``` -Additionally, you can choose one of the following database templates: -* `db-postgres` - [PostgreSQL](https://www.postgresql.org/), a relational database -* `db-neo4j`- [Neo4J](https://neo4j.com/), a graph database -* `db-mongo` - [MongoDB](https://www.mongodb.com/), a NoSQL document-based database +## Notes -### Django -* It consists of a single backend, a Python [FastAPI](https://fastapi.tiangolo.com/) application. -* The [FastAPI code generator](https://github.com/koxudaxi/fastapi-code-generator) maps the OpenAPI definition to FastAPI routing. -* The [Django framework](https://www.djangoproject.com/) encourages rapid development and clean, pragmatic design. -* Per default, [Uvicorn](https://www.uvicorn.org/) serves the FastAPI app with 2 workers. Depending on the application requirements, you can update the number of workers. +- Multiple templates can be specified by concatenating the `-t` parameter. +- The tool generates the necessary scaffolding for the chosen templates. +- Ensure you have the required dependencies installed before running the generated application. +- For more information, run `harness-application --help` or check out the additional documentation: + - [Applications README](./docs/applications/README.md) + - [Developer Guide](./docs/dev.md) diff --git a/docs/applications/harness-generate.md b/docs/applications/harness-generate.md new file mode 100644 index 000000000..c065cbe93 --- /dev/null +++ b/docs/applications/harness-generate.md @@ -0,0 +1,96 @@ +# Use harness-generate to generate server and client stubs + +To (re)generate the code for your applications, run `harness-generate`. +`harness-generate` is a command-line tool used to generate client code, server stubs, and model libraries for applications. It walks through the filesystem inside the `./applications` folder to create and update application scaffolding. The tool supports different generation modes and allows for both interactive and non-interactive usage. + +## Usage + +```sh +harness-generate [mode] [-h] [-i] [-a APP_NAME] [-cn CLIENT_NAME] [-t | -p] [path] +``` + +## harness-generate Arguments + +- `path` *(optional)* – The base path of the application. If provided, the `-a/--app-name` flag is ignored. + +## harness-generate Options + +- `-h, --help` – Displays the help message and exits. +- `-i, --interactive` – Asks for confirmation before generating code. +- `-a APP_NAME, --app-name APP_NAME` – Specifies the application name to generate clients for. +- `-cn CLIENT_NAME, --client-name CLIENT_NAME` – Specifies a prefix for the client name. +- `-t, --ts-only` – Generates only TypeScript clients. +- `-p, --python-only` – Generates only Python clients. + +## Generation Modes + +`harness-generate` supports the following modes: + +- **all** – Generates both server stubs and client libraries. +- **clients** – Generates only client libraries. +- **servers** – Generates only server stubs. +- **models** – Regenerates only model libraries. + +## harness-generate Examples + +### Generate Client and Server stubs for all applications + +```sh +harness-generate all +``` + +### Generate Client and Server stubs for a Specific Application + +```sh +harness-generate all -a myApp +``` + +### Generate Only Client Libraries + +```sh +harness-generate clients +``` + +### Generate Only Server Stubs + +```sh +harness-generate servers +``` + +### Regenerate Only Model Libraries (deprecated) + +```sh +harness-generate models +``` + +### Generate TypeScript Clients Only and Server stubs + +```sh +harness-generate all -t +``` + +### Generate Python Clients Only and Server stubs + +```sh +harness-generate all -p +``` + +### Interactive Mode + +```sh +harness-generate all -i +``` + +## harness-generate Notes + +- The tool scans the `./applications` directory for available applications. +- If `path` is provided, `-a/--app-name` is ignored. +- The `models` mode is a special flag used when regenerating only model libraries (deprecated). +- The tool supports interactive mode to confirm before generating clients. +- Use either `-t` or `-p`, but not both simultaneously. + +For further details, run: + +```sh +harness-generate --help +``` diff --git a/docs/applications/secrets.md b/docs/applications/secrets.md index dfd72256d..5cac227ad 100644 --- a/docs/applications/secrets.md +++ b/docs/applications/secrets.md @@ -20,13 +20,16 @@ harness: secrets: unsecureSecret: secureSecret: - random-secret: "" + random-static-secret: "" + random-dynamic-secret: ? ``` Secret values are initialized in three different ways: * Set the secret's value (as in `unsecureSecret`). Do that only if you aware of what you are doing as the value may be pushed in the git(hub) repository. * Leave the secret's value `null` (as in `secureSecret`) to configure manually later in the ci/cd pipeline. -* Use the "" (empty string) value (as in `random-secret`) to let cloudharness generate a random value for you. +* Use the "" (empty string) value (as in `random-static-secret`) to let CloudHarness generate a random value for you. + This secret won't be updated after being set by any of the CloudHarness automations, so has to be managed through `kubectl` directly. +* Use the `?` value (as in `random-dynamic-secret`) to get a new random value for every deployment upgrade Secret editing/maintenance alternatives: * CI/CD Codefresh support: all `null` and `` secrets will be added to the codefresh deployment file(s) and can be set/overwritten through the codefresh variable configuration diff --git a/docs/dev.md b/docs/dev.md index 280818627..cdd91c980 100644 --- a/docs/dev.md +++ b/docs/dev.md @@ -80,6 +80,7 @@ The code is organized around the idea that there is a module by artifact that ca deployment-cli-tools ├── ch_cli_tools │   ├── codefresh.py # Code Fresh configuration generation +│ ├── common_types.py # Commmon classes needed across multiple scripts/modules │   ├── helm.py # Helm chart files generation │   ├── __init__.py # Defines logging level and some global constants │   ├── models.py # Currently empty file @@ -106,23 +107,59 @@ First the skeleton of the application is generated (the directories, basic files The following code fragment from the `harness-application` script shows how the skeleton is produced: ```python -if "django-app" in args.templates and "webapp" not in templates: - templates = ["base", "webapp"] + templates +def main(): + # ... + + templates = normalize_templates(templates) + + if TemplateType.WEBAPP in templates: + handle_webapp_template(app_name, app_path) + + if TemplateType.SERVER in templates: + handle_server_template(app_path) + for template_name in templates: - if template_name == 'server': - with tempfile.TemporaryDirectory() as tmp_dirname: - copymergedir(os.path.join(CH_ROOT, APPLICATION_TEMPLATE_PATH, template_name), tmp_dirname) # <1> - merge_configuration_directories(app_path, tmp_dirname) - generate_server(app_path, tmp_dirname) - for base_path in (CH_ROOT, os.getcwd()): - template_path = os.path.join(base_path, APPLICATION_TEMPLATE_PATH, template_name) - if os.path.exists(template_path): - merge_configuration_directories(template_path, app_path) # <1> + merge_template_directories(template_name, app_path) + +# ... + +def normalize_templates(templates): + normalized_templates = list(templates) + + if TemplateType.DJANGO_APP in normalized_templates and TemplateType.WEBAPP not in normalized_templates: + django_app_index = normalized_templates.index(TemplateType.DJANGO_APP) + normalized_templates.insert(django_app_index, TemplateType.WEBAPP) + + has_database_template = any(template in TemplateType.database_templates() for template in normalized_templates) + if TemplateType.DJANGO_APP in normalize_templates and not has_database_template: + django_app_index = normalized_templates.index(TemplateType.DJANGO_APP) + normalized_templates.insert(django_app_index, TemplateType.DB_POSTGRES) + + return normalized_templates + +# ... + +def handle_server_template(app_path): + with tempfile.TemporaryDirectory() as tmp_dirname: + tmp_path = pathlib.Path(tmp_dirname) + server_template_path = pathlib.Path(CH_ROOT)/APPLICATION_TEMPLATE_PATH/TemplateType.SERVER + + copymergedir(server_template_path, tmp_path) + merge_configuration_directories(app_path, tmp_path) + generate_server(app_path, tmp_path) + +#... + +def merge_template_directories(template_name, app_path): + for base_path in (pathlib.Path(CH_ROOT), pathlib.Path.cwd()): + template_path = base_path/APPLICATION_TEMPLATE_PATH/template_name + if template_path.exists(): + merge_configuration_directories(template_path, app_path) ``` -First, if `django-app` is defined as a template for the application, and the `webapp` template is not set, then `base` and `webapp` are added to the list of templates. -Then, depending on the template name, a template directory is merged with the code of the application that will be developed (if it exists), as seen in `<1>`. -The templates for each type of application is described by the constant `APPLICATION_TEPLATE_PATH` and points to [`application-templates`](../application-templates/). +First, if `django-app` is defined as a template for the application, and the `webapp` template is not set and/or there is no database template, then `webapp` and/or `db-postgres` are added to the list of templates (using the `TemplateType` string enum). +Then, depending on the template name, a template directory is merged with the code of the application that will be developed (if it exists). +The templates for each type of application is described by the constant `APPLICATION_TEMPLATE_PATH` and points to [`application-templates`](../application-templates/). Based on the name of the template used for the application generation, the actual template with the same name is searched in this path, and copied/merged in the application target folder. The constant, as well as many other constants, are located in [`cloudharness_utils.constants`](../libraries/cloudharness-utils/cloudharness_utils/constants.py). This file is part of the CloudHarness runtime. @@ -143,12 +180,14 @@ Those constants defines several aspects of CloudHarness. For example, we can see there what base Docker image will be considered depending on what's configured for your application, where will be located the deployment files, from where the applications to generate/pick should be generated, where are located the templates for each kind of generation target, as well as where the configuration for codefresh should be looked for. Once the skeleton of the application is generated considering some templates, the code of the REST API is generated from the OpenAPI specification. -The generation relies on two functions: `generate_server` and `generate_fastapi_server` and `generate_ts_client`. +The generation relies on the functions: `generate_server` and `generate_fastapi_server` and `generate_ts_client`. Those functions are defined in the [`openapi.py`](../tools/deployment-cli-tools/ch_cli_tools/openapi.py) module. -This module and those functions use `openapi-generator-cli` to generate the code for the backend and/or the frontend. +This module and those functions use `openapi-generator-cli` and `fastapi-codegen` to generate the code for the backend and/or the frontend. With this generation, and depending on the templates used, some fine tuning or performed in the code/files generated. For example, some placeholders are replaced depending on the name of the application, or depending on the module in which the application is generated. +As final steps a `.ch-manifest` file is created in the root of the application which contains details about the app name and templates used in generation for use by [`harness-generate`](../tools/deployment-cli-tools/harness-generate) and `harness-generate` is run to ensure all server stubs and client code is in place. + #### How to extend it? Here is some scenarios that would need to modify or impact this part of CloudHarness: @@ -165,35 +204,41 @@ Here is some scenarios that would need to modify or impact this part of CloudHar ### Generation of the base application skeleton The (re-)generation REST API is obtain through the [`harness-generate`](../tools/deployment-cli-tools/harness-generate) command. -The command parses the name of the application, gets the necessary dependencies (the java OpenAPI generator cli), and generates the REST model, the servers stubs and well as the clients code from the OpenAPI specifications. +The command parses the `.ch-manifest` file (inferring and creating one if needed), gets the necessary dependencies (the java OpenAPI generator cli), and generates the REST model, the servers stubs and well as the clients code from the OpenAPI specifications. -The generation of the REST model is done by the `generate_model(...)` function, the generation of the server stub is done by the `generate_servers(...)` function, while the clients generation is done by the `generate_clients(...)` function. +The generation of the REST model is done by the `generate_model(...)` function, the generation of the server stub is done by either the `generate_servers(...)` function, while the clients generation is done by the `generate_clients(...)` function. All of these functions are located in the `harness-generate` script. Under the hood, the `generate_servers(...)` function uses the `generate_fastapi_server(...)` and the `generate_server(...)` function that are defined in the [`openapi.py`](../tools/deployment-cli-tools/ch_cli_tools/openapi.py) module. -The generation of one type of servers over another one is bound to the existence of a `genapi.sh` file: +The generation of one type of servers over another one is based on the template used for generation (if the manifest does not exist, the template is inferred by the existance/non-existance of the `genapi.sh` file): ```python -def generate_servers(root_path, interactive=False): +def generate_servers(root_path, should_generate, app_name): # ... - if os.path.exists(os.path.join(application_root, "api", "genapi.sh")): - # fastapi server --> use the genapi.sh script - generate_fastapi_server(application_root) - else: - generate_server(application_root) + for openapi_file in openapi_files: + #... + + if TemplateType.DJANGO_APP in manifest.templates: + generate_fastapi_server(app_path) + + if TemplateType.FLASK_SERVER in manifest.templates: + generate_server(app_path) ``` The `generate_clients(...)` function also uses `generate_python_client(...)` and `generate_ts_client(...)` from the [`openapi.py`](../tools/deployment-cli-tools/ch_cli_tools/openapi.py) module. -The `generate_ts_client(...)` function is called only if there is folder named `frontend` in the application directory structure: +The `generate_ts_client(...)` function is called only if the manifest templates contains `webapp` (if the manifest does not exist then the use of `webapp` is inferred by the existance/non-existance of a `frontend` directory in the application directory structure), and flags can be used to limit generation to just python or typescript clients: ```python def generate_clients(root_path, client_lib_name=LIB_NAME, interactive=False): # ... - app_dir = os.path.dirname(os.path.dirname(openapi_file)) - generate_python_client(app_name, openapi_file, - client_src_path, lib_name=client_lib_name) - if os.path.exists(os.path.join(app_dir, 'frontend')): - generate_ts_client(openapi_file) + for openapi_file in openapi_files: + #... + + if ClientType.PYTHON_CLIENT in client_types: + generate_python_client(manifest.app_name, openapi_file, client_src_path, lib_name=client_lib_name) + + if TemplateType.WEBAPP in manifest.templates and ClientType.TS_CLIENT in client_types: + generate_ts_client(openapi_file) ``` ### Generation of the application deployment files diff --git a/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/admin.py b/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/admin.py index 493384e07..ea3d423d6 100644 --- a/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/admin.py +++ b/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/admin.py @@ -4,7 +4,7 @@ from django.contrib.auth.models import User, Group from admin_extra_buttons.api import ExtraButtonsMixin, button - +from .models import Member from cloudharness_django.services import get_user_service # Register your models here. @@ -13,16 +13,22 @@ admin.site.unregister(Group) +class MemberAdmin(admin.StackedInline): + model = Member + + class CHUserAdmin(ExtraButtonsMixin, UserAdmin): + inlines = [MemberAdmin] + def has_add_permission(self, request): - return settings.DEBUG + return settings.DEBUG or settings.USER_CHANGE_ENABLED def has_change_permission(self, request, obj=None): - return settings.DEBUG + return settings.DEBUG or settings.USER_CHANGE_ENABLED def has_delete_permission(self, request, obj=None): - return settings.DEBUG + return settings.DEBUG or settings.USER_CHANGE_ENABLED @button() def sync_keycloak(self, request): diff --git a/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/services/user.py b/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/services/user.py index fc49ba368..189e6516d 100644 --- a/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/services/user.py +++ b/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/services/user.py @@ -100,7 +100,9 @@ def sync_kc_user(self, kc_user, is_superuser=False, delete=False): user, created = User.objects.get_or_create(username=kc_user["username"]) - Member.objects.get_or_create(user=user, kc_id=kc_user["id"]) + member, created = Member.objects.get_or_create(user=user) + member.kc_id = kc_user["id"] + member.save() user = self._map_kc_user(user, kc_user, is_superuser, delete) user.save() return user diff --git a/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/settings.py b/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/settings.py index b9efd5726..d5df35168 100644 --- a/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/settings.py +++ b/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/settings.py @@ -26,6 +26,8 @@ 'cloudharness_django.middleware.BearerTokenMiddleware', ] +USER_CHANGE_ENABLED = False + # test if the kubernetes CH all values exists, if so then set up specific k8s stuff # IMPROTANT NOTE: # when testing/debugging with Kafka then copy the deployment/helm/values.yaml to the ALLVALUES_PATH @@ -48,7 +50,7 @@ except: # no current app found, fall back to the default settings, there is a god change that # we are running on a developers local machine - log.warning("Error setting current app configuration, continuing...") + log.warning("Error setting current app configuration, was `harness-deployment` executed? Continuing...") current_app = applications.ApplicationConfiguration({ "name": app_name, diff --git a/libraries/client/cloudharness_cli/cloudharness_cli/__init__.py b/libraries/client/cloudharness_cli/cloudharness_cli/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/libraries/client/cloudharness_cli/setup.py b/libraries/client/cloudharness_cli/setup.py index 07fbb77a1..15dd3d4a2 100644 --- a/libraries/client/cloudharness_cli/setup.py +++ b/libraries/client/cloudharness_cli/setup.py @@ -12,7 +12,7 @@ from setuptools import setup, find_packages # noqa: H301 NAME = "cloudharness-cli" -VERSION = "2.4.0" +VERSION = "2.5.0" # To install the library, run the following # # python setup.py install diff --git a/libraries/client/cloudharness_cli/test/ninjatest/__init__.py b/libraries/client/cloudharness_cli/test/ninjatest/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/libraries/client/cloudharness_cli/test/ninjatest/test_test_api.py b/libraries/client/cloudharness_cli/test/ninjatest/test_test_api.py new file mode 100644 index 000000000..6d93d9a42 --- /dev/null +++ b/libraries/client/cloudharness_cli/test/ninjatest/test_test_api.py @@ -0,0 +1,52 @@ +# coding: utf-8 + +""" + ninjatest API + + No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + + The version of the OpenAPI document: 0.1.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +import unittest + +from cloudharness_cli.ninjatest.api.test_api import TestApi + + +class TestTestApi(unittest.TestCase): + """TestApi unit test stubs""" + + def setUp(self) -> None: + self.api = TestApi() + + def tearDown(self) -> None: + pass + + def test_ninjatest_api_live(self) -> None: + """Test case for ninjatest_api_live + + Live + """ + pass + + def test_ninjatest_api_ping(self) -> None: + """Test case for ninjatest_api_ping + + Ping + """ + pass + + def test_ninjatest_api_ready(self) -> None: + """Test case for ninjatest_api_ready + + Ready + """ + pass + + +if __name__ == '__main__': + unittest.main() diff --git a/libraries/cloudharness-common/setup.py b/libraries/cloudharness-common/setup.py index 071845071..d6aa84336 100644 --- a/libraries/cloudharness-common/setup.py +++ b/libraries/cloudharness-common/setup.py @@ -3,7 +3,7 @@ NAME = "cloudharness" -VERSION = "2.4.0" +VERSION = "2.5.0" # To install the library, run the following # # python setup.py install diff --git a/libraries/cloudharness-utils/setup.py b/libraries/cloudharness-utils/setup.py index 6a2b4b481..91d3e31d8 100644 --- a/libraries/cloudharness-utils/setup.py +++ b/libraries/cloudharness-utils/setup.py @@ -11,7 +11,7 @@ NAME = "cloudharness_utils" -VERSION = "2.4.0" +VERSION = "2.5.0" # To install the library, run the following # # python setup.py install diff --git a/libraries/models/setup.py b/libraries/models/setup.py index 7b9ca1f90..e2171e3fc 100644 --- a/libraries/models/setup.py +++ b/libraries/models/setup.py @@ -8,7 +8,7 @@ HERE = dn(realpath(__file__)) NAME = "cloudharness_model" -VERSION = "2.4.0" +VERSION = "2.5.0" REQUIREMENTS = [ "Jinja2 >= 3.1.3", "oyaml >= 1.0", diff --git a/tools/cloudharness-test/setup.py b/tools/cloudharness-test/setup.py index 5f07dfd6a..c82959f0e 100644 --- a/tools/cloudharness-test/setup.py +++ b/tools/cloudharness-test/setup.py @@ -12,7 +12,7 @@ NAME = "cloudharness-test" -VERSION = "2.3.0" +VERSION = "2.5.0" # To install the library, run the following # # python setup.py install diff --git a/tools/deployment-cli-tools/PKG-INFO b/tools/deployment-cli-tools/PKG-INFO new file mode 100644 index 000000000..ddcea8a61 --- /dev/null +++ b/tools/deployment-cli-tools/PKG-INFO @@ -0,0 +1,18 @@ +Metadata-Version: 2.1 +Name: cloudharness-cli-tools +Version: 2.5.0 +Summary: CloudHarness deploy and code generation tools +Home-page: +Author-email: cloudharness@metacell.us +Keywords: Cloud,Kubernetes,Helm,Deploy +Requires-Dist: ruamel.yaml +Requires-Dist: oyaml +Requires-Dist: docker +Requires-Dist: six +Requires-Dist: cloudharness_model +Requires-Dist: cloudharness_utils +Requires-Dist: dirhash +Requires-Dist: StrEnum; python_version < "3.11" + + CloudHarness deploy library + diff --git a/tools/deployment-cli-tools/ch_cli_tools/application_builders.py b/tools/deployment-cli-tools/ch_cli_tools/application_builders.py new file mode 100644 index 000000000..329ab580d --- /dev/null +++ b/tools/deployment-cli-tools/ch_cli_tools/application_builders.py @@ -0,0 +1,289 @@ +import abc +import json +import logging +import pathlib +import shutil +import subprocess +import tempfile +from .common_types import TemplateType +from .openapi import generate_fastapi_server, generate_flask_server, generate_openapi_from_ninja_schema, generate_ts_client +from .utils import copymergedir, get_json_template, merge_configuration_directories, replace_in_dict, replace_in_file, replaceindir, to_python_module +from . import CH_ROOT +from cloudharness_utils.constants import APPLICATION_TEMPLATE_PATH + + +class ApplicationBuilder(abc.ABC): + APP_NAME_PLACEHOLDER = '__APP_NAME__' + CLOUDHARNESS_PATH_PLACEHOLDER = '__CLOUDHARNESS_PATH__' + + def __init__(self, app_name: str, app_path: pathlib.Path): + self.app_name = app_name + self.app_path = app_path + + @abc.abstractmethod + def handles(self, templates: list[str]) -> bool: + pass + + @abc.abstractmethod + def handle_pre_merge(self) -> None: + pass + + @abc.abstractmethod + def handle_merge(self) -> None: + pass + + @abc.abstractmethod + def handle_post_merge(self) -> None: + pass + + def run_command(self, *command: str, cwd: pathlib.Path = None) -> None: + if not cwd: + cwd = self.app_path + + logging.info(f'Running command: {" ".join(map(str, command))}') + subprocess.run(command, cwd=cwd) + + def merge_template_directories(self, template_name: str) -> None: + for base_path in (self.ch_root, pathlib.Path.cwd()): + template_path = base_path / APPLICATION_TEMPLATE_PATH / template_name + if template_path.exists(): + merge_configuration_directories(template_path, self.app_path) + + @property + def frontend_path(self): + return self.app_path / 'frontend' + + @property + def backend_path(self): + return self.app_path / 'backend' + + @property + def api_path(self): + return self.app_path / 'api' + + @property + def ch_root(self): + return pathlib.Path(CH_ROOT) + + @property + def app_template_path(self): + return self.ch_root / APPLICATION_TEMPLATE_PATH + + +class WebAppBuilder(ApplicationBuilder): + def handles(self, templates): + return TemplateType.WEBAPP in templates + + def handle_pre_merge(self): + if self.frontend_path.exists(): + shutil.rmtree(self.frontend_path) + logging.info('Creating vite app') + self.create_vite_skaffold(self.frontend_path) + + def handle_merge(self): + self.merge_template_directories(TemplateType.WEBAPP) + + def handle_post_merge(self): + backend_dockerfile_path = self.backend_path / 'Dockerfile' + backend_dockerfile_path.unlink(missing_ok=True) + logging.info('Installing frontend dependencies') + self.install_frontend_dependencies() + logging.info('Generating ts client') + generate_ts_client(self.api_path / 'openapi.yaml', self.app_name) + + def create_vite_skaffold(self, frontend_path: pathlib.Path) -> None: + self.run_command( + 'yarn', 'create', 'vite', self.app_name, + '--template', 'react-ts', + ) + shutil.move(self.app_path / self.app_name, frontend_path) + + def install_frontend_dependencies(self) -> None: + self.run_command('yarn', 'install', cwd=self.frontend_path) + + +class ServerAppBuilder(ApplicationBuilder): + def handles(self, templates): + return TemplateType.SERVER in templates + + def handle_pre_merge(self): + with tempfile.TemporaryDirectory() as tmp_dirname: + tmp_path = pathlib.Path(tmp_dirname) + server_template_path = self.app_template_path / TemplateType.SERVER + + copymergedir(server_template_path, tmp_path) + merge_configuration_directories(self.app_path, tmp_path) + generate_flask_server(self.app_name, tmp_path) + + def handle_merge(self): + self.merge_template_directories(TemplateType.SERVER) + + def handle_post_merge(self): + pass + + +class FlaskServerAppBuilder(ApplicationBuilder): + def handles(self, templates): + return TemplateType.FLASK_SERVER in templates + + def handle_pre_merge(self): + pass + + def handle_merge(self): + self.merge_template_directories(TemplateType.FLASK_SERVER) + + def handle_post_merge(self): + generate_flask_server(self.app_path) + + +class BaseDjangoAppBuilder(ApplicationBuilder): + @abc.abstractmethod + def handle_merge(self): + self.merge_template_directories('django-base') + + @abc.abstractmethod + def handle_post_merge(self): + replace_in_file( + self.app_path / 'deploy' / 'values.yaml', + f'{self.APP_NAME_PLACEHOLDER}:{self.APP_NAME_PLACEHOLDER}', + f'{self.python_app_name}:{self.python_app_name}', + ) + replace_in_file(self.app_path / 'dev-setup.sh', self.APP_NAME_PLACEHOLDER, self.app_name) + replace_in_file(self.app_path / 'dev-setup.sh', self.CLOUDHARNESS_PATH_PLACEHOLDER, self.ch_root) + + self.create_django_app_vscode_debug_configuration() + + def create_django_app_vscode_debug_configuration(self): + + vscode_launch_path = pathlib.Path('.vscode/launch.json') + configuration_name = f'{self.app_name} backend' + + launch_config = get_json_template(vscode_launch_path, True) + + launch_config['configurations'] = [ + configuration for configuration in launch_config['configurations'] + if configuration['name'] != configuration_name + ] + + debug_config = get_json_template(self.debug_template_file, True) + debug_config = replace_in_dict(debug_config, self.APP_NAME_PLACEHOLDER, self.app_name) + + launch_config['configurations'].append(debug_config) + + vscode_launch_path.parent.mkdir(parents=True, exist_ok=True) + with vscode_launch_path.open('w') as f: + json.dump(launch_config, f, indent=2, sort_keys=True) + + @property + def python_app_name(self): + return to_python_module(self.app_name) + + @property + @abc.abstractmethod + def debug_template_file(self) -> str: + raise NotImplementedError() + + +class DjangoFastApiBuilder(BaseDjangoAppBuilder): + debug_template_file = 'vscode-django-fastapi-debug-template.json' + + def handles(self, templates): + return TemplateType.DJANGO_FASTAPI in templates + + def handle_pre_merge(self): + pass + + def handle_merge(self): + super().handle_merge() + self.merge_template_directories(TemplateType.DJANGO_FASTAPI) + + def handle_post_merge(self): + super().handle_post_merge() + + replace_in_file( + self.api_path / 'templates' / 'main.jinja2', + self.APP_NAME_PLACEHOLDER, + self.python_app_name, + ) + replace_in_file(self.api_path / 'genapi.sh', self.APP_NAME_PLACEHOLDER, self.app_name) + generate_fastapi_server(self.app_path) + + (self.backend_path / self.APP_NAME_PLACEHOLDER / '__main__.py').unlink(missing_ok=True) + + +class DjangoNinjaBuilder(BaseDjangoAppBuilder): + debug_template_file = 'vscode-django-ninja-debug-template.json' + + def handles(self, templates): + return TemplateType.DJANGO_NINJA in templates + + def handle_pre_merge(self): + pass + + def handle_merge(self): + super().handle_merge() + self.merge_template_directories(TemplateType.DJANGO_NINJA) + + def handle_post_merge(self): + super().handle_post_merge() + logging.info('Generating openapi from ninja schema') + generate_openapi_from_ninja_schema(self.app_name, self.app_path) + + +class AppBuilderPipeline(ApplicationBuilder): + def __init__(self, app_name: str, app_path: pathlib.Path, templates: list[str]): + super().__init__(app_name, app_path) + self.templates = templates + self.app_builders: dict[str, ApplicationBuilder] = { + TemplateType.WEBAPP: WebAppBuilder(app_name, app_path), + TemplateType.SERVER: ServerAppBuilder(app_name, app_path), + TemplateType.FLASK_SERVER: FlaskServerAppBuilder(app_name, app_path), + TemplateType.DJANGO_FASTAPI: DjangoFastApiBuilder(app_name, app_path), + TemplateType.DJANGO_NINJA: DjangoNinjaBuilder(app_name, app_path), + } + + def handles(self, templates): + return templates == self.templates + + def handle_pre_merge(self): + pre_merge_template_order = [ + TemplateType.FLASK_SERVER, + TemplateType.DJANGO_FASTAPI, + TemplateType.DJANGO_NINJA, + TemplateType.WEBAPP, + TemplateType.SERVER, + ] + + app_builders = [ + self.app_builders[template] for template in pre_merge_template_order + if self.app_builders[template].handles(self.templates) + ] + + for app_builder in app_builders: + app_builder.handle_pre_merge() + + def handle_merge(self): + for template in self.templates: + run_merge = ( + app_builder.handle_merge + if (app_builder := self.app_builders.get(template, None)) + else lambda: self.merge_template_directories(template) + ) + run_merge() + + def handle_post_merge(self): + post_merge_template_order = [ + TemplateType.FLASK_SERVER, + TemplateType.DJANGO_FASTAPI, + TemplateType.DJANGO_NINJA, + TemplateType.WEBAPP, + TemplateType.SERVER, + ] + + app_builders = [ + self.app_builders[template] for template in post_merge_template_order + if self.app_builders[template].handles(self.templates) + ] + + for app_builder in app_builders: + app_builder.handle_post_merge() diff --git a/tools/deployment-cli-tools/ch_cli_tools/codefresh.py b/tools/deployment-cli-tools/ch_cli_tools/codefresh.py index 8bcf2b79b..4f9b740d8 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/codefresh.py +++ b/tools/deployment-cli-tools/ch_cli_tools/codefresh.py @@ -173,6 +173,9 @@ def codefresh_steps_from_base_path(base_path, build_step, fixed_context=None, in # Skip excluded apps continue + if app_config and not helm_values.apps[app_key].get('build', True): + continue + if app_config and app_config.dependencies and app_config.dependencies.git: for dep in app_config.dependencies.git: step_name = f"clone_{basename(dep.url).replace('.', '_')}_{basename(dockerfile_relative_to_root).replace('.', '_')}" @@ -319,11 +322,13 @@ def add_unit_test_step(app_config: ApplicationHarnessConfig): steps = codefresh["steps"] if CD_E2E_TEST_STEP in steps and not steps[CD_E2E_TEST_STEP]["scale"]: del steps[CD_E2E_TEST_STEP] - del steps[CD_BUILD_STEP_TEST]["steps"]["test-e2e"] + if CD_BUILD_STEP_TEST in steps and 'test-e2e' in steps[CD_BUILD_STEP_TEST]["steps"]: + del steps[CD_BUILD_STEP_TEST]["steps"]["test-e2e"] if CD_API_TEST_STEP in steps and not steps[CD_API_TEST_STEP]["scale"]: del steps[CD_API_TEST_STEP] - del steps[CD_BUILD_STEP_TEST]["steps"]["test-api"] + if CD_BUILD_STEP_TEST in steps and 'test-api' in steps[CD_BUILD_STEP_TEST]["steps"]: + del steps[CD_BUILD_STEP_TEST]["steps"]["test-api"] if CD_BUILD_STEP_TEST in steps and not steps[CD_BUILD_STEP_TEST]["steps"]: del steps[CD_BUILD_STEP_TEST] diff --git a/tools/deployment-cli-tools/ch_cli_tools/common_types.py b/tools/deployment-cli-tools/ch_cli_tools/common_types.py new file mode 100644 index 000000000..eb1e5feb7 --- /dev/null +++ b/tools/deployment-cli-tools/ch_cli_tools/common_types.py @@ -0,0 +1,54 @@ +import copy +from dataclasses import dataclass +from typing import Union + + +try: + from enum import StrEnum +except ImportError: + from strenum import StrEnum + + +class TemplateType(StrEnum): + BASE = 'base' + FLASK_SERVER = 'flask-server' + WEBAPP = 'webapp' + DB_POSTGRES = 'db-postgres' + DB_NEO4J = 'db-neo4j' + DB_MONGO = 'db-mongo' + DJANGO_FASTAPI = 'django-fastapi' + DJANGO_NINJA = 'django-ninja' + SERVER = 'server' + + @classmethod + def database_templates(cls): + return [cls.DB_POSTGRES, cls.DB_NEO4J, cls.DB_MONGO] + + @classmethod + def django_templates(cls) -> list[str]: + return [cls.DJANGO_FASTAPI, cls.DJANGO_NINJA] + + +@dataclass +class CloudHarnessManifest(): + app_name: str + inferred: bool + templates: list[str] + version: str = '2' + + @classmethod + def from_dict(cls, data: dict) -> 'CloudHarnessManifest': + return cls( + app_name=data['app-name'], + version=data['version'], + inferred=data['inferred'], + templates=data['templates'], + ) + + def to_dict(self) -> dict: + return { + 'app-name': self.app_name, + 'version': self.version, + 'inferred': self.inferred, + 'templates': [str(template) for template in self.templates], + } diff --git a/tools/deployment-cli-tools/ch_cli_tools/helm.py b/tools/deployment-cli-tools/ch_cli_tools/helm.py index b81d0f191..0c34df8b3 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/helm.py +++ b/tools/deployment-cli-tools/ch_cli_tools/helm.py @@ -190,13 +190,15 @@ def create_app_values_spec(self, app_name, app_path, base_image_name=None, helm_ else: build_dependencies = [] - if len(image_paths) > 0: + deployment_values = values.get(KEY_HARNESS, {}).get(KEY_DEPLOYMENT, {}) + deployment_image = deployment_values.get('image', None) or values.get('image', None) + values['build'] = not bool(deployment_image) # Used by skaffold and ci/cd to determine if the image should be built + if len(image_paths) > 0 and not deployment_image: image_name = image_name_from_dockerfile_path(os.path.relpath( image_paths[0], os.path.dirname(app_path)), base_image_name) - values['image'] = self.image_tag( image_name, build_context_path=app_path, dependencies=build_dependencies) - elif KEY_HARNESS in values and not values[KEY_HARNESS].get(KEY_DEPLOYMENT, {}).get('image', None) and values[ + elif KEY_HARNESS in values and not deployment_image and values[ KEY_HARNESS].get(KEY_DEPLOYMENT, {}).get('auto', False): raise Exception(f"At least one Dockerfile must be specified on application {app_name}. " f"Specify harness.deployment.image value if you intend to use a prebuilt image.") diff --git a/tools/deployment-cli-tools/ch_cli_tools/manifest.py b/tools/deployment-cli-tools/ch_cli_tools/manifest.py new file mode 100644 index 000000000..0425d806d --- /dev/null +++ b/tools/deployment-cli-tools/ch_cli_tools/manifest.py @@ -0,0 +1,136 @@ +import abc +import copy +import logging +import pathlib +from typing import Iterable +from ruamel.yaml.error import YAMLError +from .common_types import CloudHarnessManifest, TemplateType +from .utils import load_yaml, save_yaml + + +def get_manifest(app_path: pathlib.Path) -> CloudHarnessManifest: + manifest_file = app_path / '.ch-manifest' + + try: + manifest_data = load_yaml(manifest_file) + return CloudHarnessManifest.from_dict(manifest_data) + except (FileNotFoundError, YAMLError): + logging.info(f'Could not load manifest file {manifest_file}, inferring manifest from app structure...') + manifest = CloudHarnessManifest( + app_name=app_path.name, + inferred=True, + templates=infer_templates(app_path), + ) + save_yaml(manifest_file, manifest.to_dict()) + return manifest + + +def load_manifest(manifest_file: pathlib.Path) -> dict: + manifest_data = load_yaml(manifest_file) + migrated_data = migrate_manifest_data(manifest_data) + + if manifest_data != migrated_data: + save_yaml(manifest_file, migrated_data) + + return migrated_data + + +def migrate_manifest_data(data: dict) -> dict: + data = copy.deepcopy(data) + data_version = data['version'] + migrations = [ + migration for migration in _MIGRATIONS_LIST + if data_version < migration.change_version + ] + + for migration in migrations: + migration.migrate(data) + + return data + + +def infer_templates(app_path: pathlib.Path) -> list[str]: + return [ + TemplateType.BASE, + *infer_webapp_template(app_path), + *infer_server_template(app_path), + *infer_database_template(app_path), + ] + + +def infer_webapp_template(app_path: pathlib.Path) -> Iterable[str]: + frontend_path = app_path / 'frontend' + if frontend_path.exists(): + yield TemplateType.WEBAPP + + +def infer_server_template(app_path: pathlib.Path) -> Iterable[str]: + backend_path = app_path / 'backend' + manage_path = backend_path / 'manage.py' + + if manage_path.exists(): + yield from infer_django_template(backend_path) + return + + server_path = app_path / 'server' + if server_path.exists() or backend_path.exists(): + yield TemplateType.FLASK_SERVER + + +def infer_django_template(backend_path: pathlib.Path) -> Iterable[str]: + requirements_path = backend_path / 'requirements.txt' + requirements = requirements_path.read_text() + + if 'django-ninja' in requirements: + yield TemplateType.DJANGO_NINJA + else: + yield TemplateType.DJANGO_FASTAPI + + +def infer_database_template(app_path: pathlib.Path) -> Iterable[str]: + values_file = app_path / 'deploy' / 'values.yaml' + + try: + values_data = load_yaml(values_file) + database_config = values_data['harness']['database'] + if not database_config['auto']: + return + + database_type = database_config['type'] + database_type_to_template_map = { + 'mongo': TemplateType.DB_MONGO, + 'neo4j': TemplateType.DB_NEO4J, + 'postgres': TemplateType.DB_POSTGRES, + } + + if database_type in database_type_to_template_map: + yield database_type_to_template_map[database_type] + + except (FileNotFoundError, YAMLError, KeyError): + pass + + +class ManifestMigration(abc.ABC): + @property + @abc.abstractmethod + def change_version(self) -> str: + ... + + @abc.abstractmethod + def migrate(data: dict) -> None: + ... + + +class NameChangeFromDjangoAppToDjangoFastapi(ManifestMigration): + change_version = '2' + + def migrate(data): + data['templates'] = [ + template if template != 'django-app' else 'django-fastapi' + for template in data['templates'] + ] + + +_MIGRATIONS_LIST: list[ManifestMigration] = [ + NameChangeFromDjangoAppToDjangoFastapi(), +] diff --git a/tools/deployment-cli-tools/ch_cli_tools/openapi.py b/tools/deployment-cli-tools/ch_cli_tools/openapi.py index 4df7b8834..bc3eda816 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/openapi.py +++ b/tools/deployment-cli-tools/ch_cli_tools/openapi.py @@ -1,15 +1,23 @@ +import enum +import functools import glob import json import logging +import operator import os +import pathlib import shutil import subprocess import sys +from typing import Callable, Optional import urllib.request from os.path import dirname as dn, join +from ch_cli_tools.common_types import TemplateType +from ch_cli_tools.manifest import get_manifest + from . import HERE -from .utils import replaceindir, to_python_module +from .utils import confirm, copymergedir, replace_in_file, replaceindir, to_python_module, get_apps_paths CODEGEN = os.path.join(HERE, 'bin', 'openapi-generator-cli.jar') APPLICATIONS_SRC_PATH = os.path.join('applications') @@ -19,22 +27,67 @@ OPENAPI_GEN_URL = 'https://repo1.maven.org/maven2/org/openapitools/openapi-generator-cli/7.7.0/openapi-generator-cli-7.7.0.jar' -def generate_server(app_path, overrides_folder=""): +class ClientType(enum.Flag): + TS_CLIENT = enum.auto() + PYTHON_CLIENT = enum.auto() + + @classmethod + def all(cls): + return functools.reduce(operator.or_, cls) + + +def generate_flask_server(app_path: pathlib.Path, overrides_folder: Optional[pathlib.Path] = None) -> None: get_dependencies() - openapi_dir = os.path.join(app_path, 'api') - openapi_file = glob.glob(os.path.join(openapi_dir, '*.yaml'))[0] - out_name = f"backend" if not os.path.exists( - f"{app_path}/server") else f"server" - out_path = f"{app_path}/{out_name}" - command = f"java -jar {CODEGEN} generate -i {openapi_file} -g python-flask -o {out_path} " \ - f"-c {openapi_dir}/config.json " + \ - (f"-t {overrides_folder}" if overrides_folder else "") - os.system(command) + try: + openapi_directory = app_path / 'api' + openapi_file = next(openapi_directory.glob('*.yaml')) -def generate_fastapi_server(app_path): - command = f"cd {app_path}/api && bash genapi.sh" - os.system(command) + server_path = app_path / 'server' + backend_path = app_path / 'backend' + out_path = server_path if server_path.exists() else backend_path + + command = [ + 'java', '-jar', CODEGEN, 'generate', + '-i', openapi_file, + '-g', 'python-flask', + '-o', out_path, + '-c', openapi_directory / 'config.json', + ] + if overrides_folder: + command += ['-t', overrides_folder] + + subprocess.run(command) + except: + logging.error(f'An error occurred while generating the server stubs for {app_path.name}', exc_info=True) + + +def generate_fastapi_server(app_path: pathlib.Path) -> None: + # Install the fastapi code generator here as it comes with potential problematic dependencies + subprocess.check_call([sys.executable, "-m", "pip", "install", "fastapi-code-generator"]) + api_directory = app_path / 'api' + backend_directory = app_path / 'backend' + temp_directory = api_directory / 'app' + + command = [ + 'fastapi-codegen', + '--input', api_directory / 'openapi.yaml', + '--output', temp_directory, + '-t', api_directory / 'templates', + ] + subprocess.run(command) + + source_main = temp_directory / 'main.py' + destination_main = backend_directory / 'main.py' + source_main.replace(destination_main) + + source_models = temp_directory / 'models.py' + destination_models = backend_directory / 'openapi' / 'models.py' + source_models.replace(destination_models) + + temp_directory.rmdir() + + logging.info('Generated new models and main.py') def generate_model(base_path=ROOT): @@ -79,18 +132,61 @@ def generate_python_client(module, openapi_file, client_src_path, lib_name=LIB_N os.system(command) -def generate_ts_client(openapi_file): +def generate_ts_client(openapi_file, app_name=""): get_dependencies() - out_dir = f"{os.path.dirname(os.path.dirname(openapi_file))}/frontend/src/rest" + out_dir = f"{os.path.dirname(os.path.dirname(openapi_file))}/frontend/src/rest/{app_name}" command = f"java -jar {CODEGEN} generate " \ f"-i {openapi_file} " \ f"-g typescript-fetch " \ - f"-o {out_dir}" + f"-o {out_dir} "\ + f"--additional-properties=prefixParameterInterfaces=false" os.system(command) replaceindir(out_dir, "http://localhost", '') +def json2yaml(json_filename, yaml_file=None): + import yaml + if yaml_file is None: + yaml_file = str(json_filename).replace('.json', '.yaml') + with open(json_filename, 'r') as json_filename: + data = json.load(json_filename) + with open(yaml_file, 'w') as yaml_file: + yaml.dump(data, yaml_file) + + +def generate_openapi_from_ninja_schema(app_name: str, app_path: pathlib.Path) -> None: + # check if cloudharness_django python library is installed + python_module = to_python_module(app_name) + try: + import cloudharness_django # noqa + # dynamically import python_module + __import__(python_module) + except ImportError: + if confirm('Runtime env is not installed. Do you want to install it?'): + subprocess.check_call(["sh", "dev-setup.sh"], cwd=app_path) + else: + logging.error('Runtime env is not installed. Cound not generate openapi files for Django Ninja.') + return + logging.info(f"Generating openapi files for Django Ninja for application {app_name}") + out_path = app_path / 'api' / 'openapi.json' + + manage_path = app_path / 'backend' / 'manage.py' + command = [ + 'python', manage_path, 'export_openapi_schema', + '--settings', 'django_baseapp.settings', + '--api', f'{python_module}.api.api', + '--output', out_path, + '--indent', '2', + ] + + subprocess.run(command) + + replace_in_file(out_path, f'{app_name}_api_', '') + + json2yaml(out_path) + + def get_dependencies(): """ Checks if java is installed @@ -109,3 +205,149 @@ def get_dependencies(): if not os.path.exists(cdir): os.makedirs(cdir) urllib.request.urlretrieve(OPENAPI_GEN_URL, CODEGEN) + + +def generate_models( + root_path: pathlib.Path, + should_generate: Callable[[str], bool], +) -> None: + """ + Generates the main model + """ + library_models_path = root_path / 'libraries' / 'models' + + if not library_models_path.exists(): + return + + if not should_generate('the main model'): + return + + generate_model() + + +def generate_servers( + root_path: pathlib.Path, + should_generate: Callable[[str], bool], + app_name: Optional[str], +) -> None: + """ + Generates server stubs + """ + apps_path = get_apps_paths(root_path, app_name) + + for app_path in apps_path: + manifest = get_manifest(app_path) + + if not should_generate(f'Should we regenerate the server stubs for {app_path.name} ?'): + continue + + if TemplateType.DJANGO_FASTAPI in manifest.templates: + generate_fastapi_server(app_path) + + if TemplateType.FLASK_SERVER in manifest.templates: + generate_flask_server(app_path) + + +def generate_clients( + root_path: pathlib.Path, + should_generate: Callable[[str], bool], + app_name: Optional[str], + client_lib_name: str, + client_types: ClientType, +) -> None: + """ + Generates client stubs + """ + apps_path = get_apps_paths(root_path, app_name) + + logging.info('Generating client libraries for %s', str(client_types)) + + if client_lib_name: + client_src_path = root_path / 'libraries' / 'client' / client_lib_name + + for app_path in apps_path: + manifest = get_manifest(app_path) + + if not should_generate(f'Should we regenerate the client libraries for {app_path.name} ?'): + continue + + if TemplateType.DJANGO_NINJA in manifest.templates: + generate_openapi_from_ninja_schema(manifest.app_name, app_path) + + for openapi_file in app_path.glob('api/*.yaml'): + if ClientType.PYTHON_CLIENT in client_types and client_lib_name: + generate_python_client(manifest.app_name, openapi_file, client_src_path, lib_name=client_lib_name) + + if TemplateType.WEBAPP in manifest.templates and ClientType.TS_CLIENT in client_types: + ts_client_name = app_name if app_name else manifest.app_name + generate_ts_client(openapi_file, ts_client_name) + + if client_lib_name: + aggregate_packages(client_src_path, client_lib_name) + + +def aggregate_packages(client_source_path: pathlib.Path, lib_name=LIB_NAME): + client_source_path.mkdir(parents=True, exist_ok=True) + + client_docs_path = client_source_path / 'docs' + client_docs_path.mkdir(exist_ok=True) + + client_test_path = client_source_path / 'test' + client_test_path.mkdir(exist_ok=True) + + client_readme_file = client_source_path / 'README.md' + client_readme_file.unlink(missing_ok=True) + + client_requirements_file = client_source_path / 'requirements.txt' + client_requirements_file.unlink(missing_ok=True) + + client_test_requirements_file = client_source_path / 'test-requirements.txt' + client_test_requirements_file.unlink(missing_ok=True) + + requirements_lines_seen = set() + test_requirements_lines_seen = set() + + for temp_module_path in client_source_path.glob('tmp-*/'): + module = ( + temp_module_path + .name + .removeprefix('tmp-') + .replace('-', '_') + ) + + code_destination_directory = client_source_path / lib_name / module + copymergedir(temp_module_path / lib_name / module, code_destination_directory) + copymergedir(temp_module_path / f'{lib_name}.{module}', code_destination_directory) # Fixes a bug with nested packages + + module_docs_path = client_docs_path / module + module_docs_path.mkdir(parents=True, exist_ok=True) + copymergedir(client_source_path / temp_module_path.name / 'docs', module_docs_path) + + module_tests_path = client_source_path / 'test' / module + copymergedir(temp_module_path / 'test', module_tests_path) + + readme_file = temp_module_path / 'README.md' + if not readme_file.exists(): + logging.warning(f'Readme file not found: {readme_file}.') + continue + + with client_readme_file.open('+a') as out_file, readme_file.open('r') as in_file: + file_data = in_file.read() + updated_file_data = file_data.replace('docs/', f'docs/{module}/') + out_file.write(updated_file_data) + + # FIXME: Different package versions will remain in the output file + requirements_file = temp_module_path / 'requirements.txt' + with requirements_file.open('r') as in_file, client_requirements_file.open('+a') as out_file: + unseen_lines = [line for line in in_file if line not in requirements_lines_seen] + requirements_lines_seen.update(unseen_lines) + out_file.writelines(unseen_lines) + + # FIXME: Different package versions will remain in the output file + test_requirements_file = temp_module_path / 'test-requirements.txt' + with test_requirements_file.open('r') as in_file, client_test_requirements_file.open('+a') as out_file: + unseen_lines = [line for line in in_file if line not in test_requirements_lines_seen] + test_requirements_lines_seen.update(unseen_lines) + out_file.writelines(unseen_lines) + + shutil.rmtree(temp_module_path) diff --git a/tools/deployment-cli-tools/ch_cli_tools/skaffold.py b/tools/deployment-cli-tools/ch_cli_tools/skaffold.py index 9609ad86e..b8730971a 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/skaffold.py +++ b/tools/deployment-cli-tools/ch_cli_tools/skaffold.py @@ -81,6 +81,8 @@ def process_build_dockerfile( if app_name is None: app_name = app_name_from_path(basename(dockerfile_path)) app_key = app_name.replace("-", "_") + if app_key in helm_values.apps and not helm_values.apps[app_key]['build']: + return if app_name in helm_values[KEY_TASK_IMAGES] or app_key in helm_values.apps: context_path = relpath_if(root_path, output_path) if global_context else relpath_if(dockerfile_path, output_path) @@ -157,6 +159,8 @@ def process_build_dockerfile( # app_image_tag, app_relative_to_skaffold, build_requirements) process_build_dockerfile(dockerfile_path, root_path, requirements=build_requirements, app_name=app_name) app = apps[app_key] + if not app['build']: + continue if app[KEY_HARNESS][KEY_DEPLOYMENT]['image']: release_config['artifactOverrides']['apps'][app_key] = \ { diff --git a/tools/deployment-cli-tools/ch_cli_tools/utils.py b/tools/deployment-cli-tools/ch_cli_tools/utils.py index 85fcb5bba..8f9f5e45f 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/utils.py +++ b/tools/deployment-cli-tools/ch_cli_tools/utils.py @@ -1,8 +1,10 @@ +import contextlib +import pathlib import socket import glob import subprocess -from typing import Any +from typing import Any, Union import requests import os from functools import cache @@ -174,17 +176,23 @@ def replaceindir(root_src_dir, source, replace): if not any(file_.endswith(ext) for ext in REPLACE_TEXT_FILES_EXTENSIONS): continue - src_file = join(src_dir, file_) + src_file = pathlib.Path(src_dir) / file_ replace_in_file(src_file, source, replace) -def replace_in_file(src_file, source, replace): - if src_file.endswith('.py') or basename(src_file) == 'Dockerfile': - replace = to_python_module(replace) - with fileinput.FileInput(src_file, inplace=True) as file: +def confirm(question): + answer = input(f"{question} (Y/n): ").casefold() + return answer == "y" if answer else True + + +def replace_in_file(src_file: pathlib.Path, source: str, replacement) -> None: + if src_file.name.endswith('.py') or src_file.name == 'Dockerfile': + replacement = to_python_module(str(replacement)) + + with fileinput.input(src_file, inplace=True) as file: try: for line in file: - print(line.replace(source, replace), end='') + print(line.replace(source, str(replacement)), end='') except UnicodeDecodeError: pass @@ -205,29 +213,28 @@ def replace_value(value: Any) -> Any: } -def copymergedir(root_src_dir, root_dst_dir): +def copymergedir(source_root_directory: pathlib.Path, destination_root_directory: pathlib.Path) -> None: """ Does copy and merge (shutil.copytree requires that the destination does not exist) - :param root_src_dir: - :param root_dst_dir: + :param source_root_directory: + :param destination_root_directory: :return: """ - logging.info('Copying directory %s to %s', root_src_dir, root_dst_dir) - for src_dir, dirs, files in os.walk(root_src_dir): + logging.info(f'Copying directory {source_root_directory} to {destination_root_directory}') + + for source_directory, _, files in os.walk(source_root_directory): # source_root_directory.walk() from Python 3.12 + source_directory = pathlib.Path(source_directory) + destination_directory = destination_root_directory / source_directory.relative_to(source_root_directory) + destination_directory.mkdir(parents=True, exist_ok=True) + + for file in files: + source_file = source_directory / file + destination_file = destination_directory / file - dst_dir = src_dir.replace(root_src_dir, root_dst_dir, 1) - if not exists(dst_dir): - os.makedirs(dst_dir) - for file_ in files: - src_file = join(src_dir, file_) - dst_file = join(dst_dir, file_) - if exists(dst_file): - os.remove(dst_file) try: - shutil.copy(src_file, dst_dir) + source_file.replace(destination_file) except: - logging.warning("Error copying file %s to %s.", - src_file, dst_dir) + logging.warning(f'Error copying file {source_file} to {destination_file}.') def movedircontent(root_src_dir, root_dst_dir): @@ -256,52 +263,69 @@ def movedircontent(root_src_dir, root_dst_dir): shutil.rmtree(root_src_dir) -def merge_configuration_directories(source, dest): - if source == dest: +def merge_configuration_directories(source: Union[str, pathlib.Path], destination: Union[str, pathlib.Path]) -> None: + source_path, destination_path = pathlib.Path(source), pathlib.Path(destination) + + if source_path == destination_path: + return + + if not source_path.exists(): + logging.warning("Trying to merge the not existing directory: %s", source) return - if not exists(source): - logging.warning( - "Trying to merge the not existing directory: %s", source) + + if not destination_path.exists(): + shutil.copytree(source_path, destination_path, ignore=shutil.ignore_patterns(*EXCLUDE_PATHS)) return - if not exists(dest): - shutil.copytree( - source, dest, ignore=shutil.ignore_patterns(*EXCLUDE_PATHS)) + + for source_directory, _, files in os.walk(source_path): # source_path.walk() from Python 3.12 + _merge_configuration_directory(source_path, destination_path, pathlib.Path(source_directory), files) + + +def _merge_configuration_directory( + source: pathlib.Path, + destination: pathlib.Path, + source_directory: pathlib.Path, + files: list[str] +) -> None: + if any(path in str(source_directory) for path in EXCLUDE_PATHS): return - for src_dir, dirs, files in os.walk(source): - if any(path in src_dir for path in EXCLUDE_PATHS): + destination_directory = destination / source_directory.relative_to(source) + destination_directory.mkdir(exist_ok=True) + + non_build_files = (file for file in files if file not in BUILD_FILENAMES) + + for file_name in non_build_files: + source_file_path = source_directory / file_name + destination_file_path = destination_directory / file_name + + _merge_configuration_file(source_file_path, destination_file_path) + + +def _merge_configuration_file(source_file_path: pathlib.Path, destination_file_path: pathlib.Path) -> None: + if not exists(destination_file_path): + shutil.copy2(source_file_path, destination_file_path) + return + + merge_operations = [ + (file_is_yaml, merge_yaml_files), + (file_is_json, merge_json_files), + ] + + for can_merge_file, merge_files in merge_operations: + if not can_merge_file(source_file_path.name): continue - dst_dir = src_dir.replace(source, dest, 1) - if not exists(dst_dir): - os.makedirs(dst_dir) - for fname in files: - if fname in BUILD_FILENAMES: - continue - fpath = join(src_dir, fname) - frel = relpath(fpath, start=source) - fdest = join(dest, frel) - if not exists(fdest): - shutil.copy2(fpath, fdest) - elif file_is_yaml(fpath): - - try: - merge_yaml_files(fpath, fdest) - logging.info( - f"Merged/overridden file content of {fdest} with {fpath}") - except Exception as e: - logging.warning(f"Overwriting file {fdest} with {fpath}") - shutil.copy2(fpath, fdest) - elif file_is_json(fpath): - try: - merge_json_files(fpath, fdest) - logging.info( - f"Merged/overridden file content of {fdest} with {fpath}") - except Exception as e: - logging.warning(f"Overwriting file {fdest} with {fpath}") - shutil.copy2(fpath, fdest) - else: - logging.warning(f"Overwriting file {fdest} with {fpath}") - shutil.copy2(fpath, fdest) + + try: + merge_files(source_file_path, destination_file_path) + logging.info(f'Merged/overridden file content of {destination_file_path} with {source_file_path}') + except: + break + + return + + logging.warning(f'Overwriting file {destination_file_path} with {source_file_path}') + shutil.copy2(source_file_path, destination_file_path) def merge_yaml_files(fname, fdest): @@ -456,3 +480,25 @@ def get_git_commit_hash(path): ['git', 'rev-parse', '--short', 'HEAD'], cwd=path).decode("utf-8").strip() except: return None + + +def load_yaml(yaml_file: pathlib.Path) -> dict: + with yaml_file.open('r') as file: + return yaml.load(file) + + +def save_yaml(yaml_file: pathlib.Path, data: dict) -> None: + with yaml_file.open('w') as file: + yaml.dump(data, file) + + +def get_apps_paths(root, app_name) -> tuple[str]: + apps_path = [] + + if app_name: + logging.info('### Generating server stubs for %s ###', app_name) + apps_path = [path for path in root.glob(f'applications/{app_name}') if path.is_dir()] + else: + logging.info('### Generating server stubs for all applications ###') + apps_path = [path for path in root.glob('applications/*') if path.is_dir()] + return apps_path diff --git a/tools/deployment-cli-tools/harness-application b/tools/deployment-cli-tools/harness-application index d3601f7a5..4ae81c9b4 100644 --- a/tools/deployment-cli-tools/harness-application +++ b/tools/deployment-cli-tools/harness-application @@ -1,26 +1,17 @@ #!/usr/bin/env python -import json import pathlib import sys -import os import re -import shutil -import tempfile -import subprocess import logging import argparse +from typing import Union -from ch_cli_tools import CH_ROOT -from cloudharness_utils.constants import APPLICATION_TEMPLATE_PATH -from ch_cli_tools.openapi import generate_server, generate_fastapi_server, APPLICATIONS_SRC_PATH, generate_ts_client -from ch_cli_tools.utils import merge_configuration_directories, replaceindir, replace_in_file, \ - to_python_module, copymergedir, get_json_template, replace_in_dict - -try: - from enum import StrEnum -except ImportError: - from strenum import StrEnum +from ch_cli_tools.openapi import APPLICATIONS_SRC_PATH +from ch_cli_tools.utils import confirm, replaceindir, replace_in_file, save_yaml, \ + to_python_module +from ch_cli_tools.common_types import CloudHarnessManifest, TemplateType +from ch_cli_tools.application_builders import AppBuilderPipeline # Only allow lowercased alphabetical characters separated by "-". name_pattern = re.compile("[a-z]+((-)?[a-z])?") @@ -28,47 +19,25 @@ name_pattern = re.compile("[a-z]+((-)?[a-z])?") PLACEHOLDER = '__APP_NAME__' -class TemplateType(StrEnum): - BASE = 'base' - FLASK_SERVER = 'flask-server' - WEBAPP = 'webapp' - DB_POSTGRES = 'db-postgres' - DB_NEO4J = 'db-neo4j' - DB_MONGO = 'db-mongo' - DJANGO_APP = 'django-app' - SERVER = 'server' - - -def main(): +def main() -> None: app_name, templates = get_command_line_arguments() - app_path = os.path.join(APPLICATIONS_SRC_PATH, app_name) - os.makedirs(app_path, exist_ok=True) - - if TemplateType.DJANGO_APP in templates and TemplateType.WEBAPP not in templates: - templates = [TemplateType.BASE, TemplateType.WEBAPP] + templates - - if TemplateType.WEBAPP in templates: - handle_webapp_template(app_name, app_path) + app_path = pathlib.Path(APPLICATIONS_SRC_PATH) / app_name + app_path.mkdir(exist_ok=True) - if TemplateType.SERVER in templates: - handle_server_template(app_path) + templates = normalize_templates(templates) - for template_name in templates: - merge_template_directories(template_name, app_path) + pipeline = AppBuilderPipeline(app_name, app_path, templates) - if TemplateType.FLASK_SERVER in templates: - handle_flask_server_template(app_path) - - replace_in_file(os.path.join(app_path, 'api/config.json'), PLACEHOLDER, to_python_module(app_name)) - - if TemplateType.DJANGO_APP in templates: - handle_django_app_template(app_name, app_path) + pipeline.handle_pre_merge() + pipeline.handle_merge() + replace_in_file(app_path / 'api' / 'config.json', PLACEHOLDER, to_python_module(app_name)) replaceindir(app_path, PLACEHOLDER, app_name) - if TemplateType.WEBAPP in templates: - handle_webapp_template_cleanup(app_path) + pipeline.handle_post_merge() + + create_manifest_file(app_path, app_name, templates) def get_command_line_arguments() -> tuple[str, list[str]]: @@ -89,7 +58,8 @@ def get_command_line_arguments() -> tuple[str, list[str]]: - db-postgres - db-neo4j - db-mongo - - django-app (fastapi django backend based on openapi) + - django-fastapi (fastapi django backend based on openapi) + - django-ninja (django ninja backend) """) args, unknown = parser.parse_known_args(sys.argv[1:]) @@ -112,78 +82,36 @@ def get_command_line_arguments() -> tuple[str, list[str]]: return args.name, args.templates -def handle_webapp_template(app_name: str, app_path: str) -> None: - if os.path.exists(os.path.join(app_path, 'frontend')): - shutil.rmtree(os.path.join(app_path, 'frontend')) - cmd = ["yarn", "create", "vite", app_name, "--template", "react-ts"] - logging.info(f"Running command: {' '.join(cmd)}") - subprocess.run(cmd, cwd=app_path) - shutil.move(os.path.join(app_path, app_name), os.path.join(app_path, 'frontend')) - generate_ts_client(openapi_file=os.path.join(app_path, 'api/openapi.yaml')) - - -def handle_webapp_template_cleanup(app_path: str) -> None: - try: - os.remove(os.path.join(app_path, 'backend', 'Dockerfile')) - except FileNotFoundError: - # backend dockerfile not found, continue - pass +def normalize_templates(templates: list[str]) -> list[str]: + templates = list(templates) + def django_template_index(): + return next(index for index, template in enumerate(templates) if template in TemplateType.django_templates()) -def handle_server_template(app_path: str) -> None: - with tempfile.TemporaryDirectory() as tmp_dirname: - copymergedir(os.path.join(CH_ROOT, APPLICATION_TEMPLATE_PATH, TemplateType.SERVER), tmp_dirname) - merge_configuration_directories(app_path, tmp_dirname) - generate_server(app_path, tmp_dirname) + has_django_template = any(template in TemplateType.django_templates() for template in templates) + if TemplateType.WEBAPP not in templates: + if (confirm(f'Do you want to generate Vite frontend application?')): + templates.insert(django_template_index(), TemplateType.WEBAPP) + has_database_template = any(template in TemplateType.database_templates() for template in templates) + if has_django_template and not has_database_template: + if (confirm(f'Do you want to use a postgres database?')): + templates.insert(django_template_index(), TemplateType.DB_POSTGRES) -def handle_flask_server_template(app_path: str) -> None: - generate_server(app_path) + return templates -def handle_django_app_template(app_name: str, app_path: str) -> None: - replace_in_file(os.path.join(app_path, 'api/templates/main.jinja2'), PLACEHOLDER, to_python_module(app_name)) - generate_fastapi_server(app_path) - replace_in_file( - os.path.join(app_path, 'deploy/values.yaml'), - f"{PLACEHOLDER}:{PLACEHOLDER}", - f"{to_python_module(app_name)}:{to_python_module(app_name)}" +def create_manifest_file(app_path: pathlib.Path, app_name: str, templates: list[Union[str, TemplateType]]) -> None: + manifest_file = app_path / '.ch-manifest' + manifest = CloudHarnessManifest( + app_name=app_name, + version='1', + inferred=False, + templates=[str(template) for template in templates], ) - replace_in_file(os.path.join(app_path, "dev-setup.sh"), PLACEHOLDER, app_name) - create_django_app_vscode_debug_configuration(app_name) - try: - os.remove(os.path.join(app_path, 'backend', "__APP_NAME__", "__main__.py")) - except FileNotFoundError: - # backend dockerfile not found, continue - pass - - -def create_django_app_vscode_debug_configuration(app_name: str): - vscode_launch_path = pathlib.Path('.vscode/launch.json') - configuration_name = f'{app_name} backend' - - launch_config = get_json_template(vscode_launch_path, True) - - launch_config['configurations'] = [ - configuration for configuration in launch_config['configurations'] - if configuration['name'] != configuration_name - ] - - debug_config = get_json_template('vscode-django-app-debug-template.json', True) - debug_config = replace_in_dict(debug_config, PLACEHOLDER, app_name) - - launch_config['configurations'].append(debug_config) - - vscode_launch_path.parent.mkdir(parents=True, exist_ok=True) - with vscode_launch_path.open('w') as f: - json.dump(launch_config, f, indent=2, sort_keys=True) - -def merge_template_directories(template_name: str, app_path: str) -> None: - for base_path in (CH_ROOT, os.getcwd()): - template_path = os.path.join(base_path, APPLICATION_TEMPLATE_PATH, template_name) - if os.path.exists(template_path): - merge_configuration_directories(template_path, app_path) + logging.info('Creating manifest file') + save_yaml(manifest_file, manifest.to_dict()) if __name__ == "__main__": diff --git a/tools/deployment-cli-tools/harness-generate b/tools/deployment-cli-tools/harness-generate old mode 100644 new mode 100755 index 12d40e8a3..e698b716c --- a/tools/deployment-cli-tools/harness-generate +++ b/tools/deployment-cli-tools/harness-generate @@ -1,176 +1,125 @@ #!/usr/bin/env python -import glob -import os -import shutil -import sys +import argparse +from dataclasses import dataclass +import enum +import functools +import operator +import pathlib import logging +from typing import Optional -from ch_cli_tools.openapi import LIB_NAME, generate_python_client, generate_server, generate_fastapi_server, \ - get_dependencies, generate_ts_client, generate_model -from ch_cli_tools.utils import copymergedir - -HERE = os.path.dirname(os.path.realpath(__file__)) -ROOT = os.path.dirname(HERE) - - -def get_openapi_file_paths(root_path): - return [path for path in glob.glob(root_path + '/applications/*/api/*.yaml')] - - -def get_application_paths(openapi_files): - return [os.path.basename(os.path.dirname(os.path.dirname(path))) for path in openapi_files] - - -def generate_servers(root_path, interactive=False, server=None): - """ - Generates server stubs - """ - openapi_files = get_openapi_file_paths(root_path) - modules = get_application_paths(openapi_files) - for i in range(len(modules)): - if not interactive or input("Do you want to generate " + openapi_files[i] + "? [Y/n]").upper() != 'N': - openapi_file = openapi_files[i] - application_root = os.path.dirname(os.path.dirname(openapi_file)) - appname = os.path.basename(application_root) - if server and server != appname: - continue - if os.path.exists(os.path.join(application_root, "api", "genapi.sh")): - # fastapi server --> use the genapi.sh script - generate_fastapi_server(application_root) - else: - generate_server(application_root) - - -def aggregate_packages(client_src_path, lib_name=LIB_NAME): - DOCS_PATH = os.path.join(client_src_path, 'docs') - TEST_PATH = os.path.join(client_src_path, 'test') - README = os.path.join(client_src_path, 'README.md') - REQUIREMENTS = os.path.join(client_src_path, 'requirements.txt') - TEST_REQUIREMENTS = os.path.join(client_src_path, 'test-requirements.txt') - - if not os.path.exists(DOCS_PATH): - os.makedirs(DOCS_PATH) - if not os.path.exists(TEST_PATH): - os.makedirs(TEST_PATH) - if os.path.exists(README): - os.remove(README) - if os.path.exists(REQUIREMENTS): - os.remove(REQUIREMENTS) - if os.path.exists(TEST_REQUIREMENTS): - os.remove(TEST_REQUIREMENTS) - - req_lines_seen = set() - test_req_lines_seen = set() - - for MODULE_TMP_PATH in glob.glob(client_src_path + '/tmp-*'): - module = MODULE_TMP_PATH.split( - f'{lib_name}/tmp-')[-1].replace('-', '_') - - # Moves package - - code_dest_dir = os.path.join(client_src_path, lib_name, module) - copymergedir(os.path.join(MODULE_TMP_PATH, - lib_name, module), code_dest_dir) - copymergedir(f"{MODULE_TMP_PATH}/{lib_name}.{module}", - code_dest_dir) # Fixes a a bug with nested packages - - # Adds Docs - module_doc_path = os.path.join(DOCS_PATH, module) - if not os.path.exists(module_doc_path): - os.mkdir(module_doc_path) - copymergedir(f"{client_src_path}/tmp-{module}/docs", module_doc_path) - - # Adds Tests - module_test_path = os.path.join(client_src_path, 'test', module) - copymergedir(os.path.join(MODULE_TMP_PATH, 'test'), module_test_path) - - # Merges Readme - readme_file = f"{MODULE_TMP_PATH}/README.md" - if not os.path.exists(readme_file): - logging.warning("Readme file not found: %s.", readme_file) - continue - with open(README, 'a+') as outfile: - with open(readme_file) as infile: - filedata = infile.read() - fd = filedata.replace('docs/', f'docs/{module}/') - outfile.write(fd) - - # Merges Requirements - # FIXME: Different package versions will remain in the output file - - requirements_file = f"{MODULE_TMP_PATH}/requirements.txt" - outfile = open(REQUIREMENTS, "a+") - for line in open(requirements_file, "r"): - if line not in req_lines_seen: - outfile.write(line) - req_lines_seen.add(line) - outfile.close() - - # Merges Test Requirements - # FIXME: Different package versions will remain in the output file - test_requirements_file = f"{MODULE_TMP_PATH}/test-requirements.txt" - outfile = open(TEST_REQUIREMENTS, "a+") - for line in open(test_requirements_file, "r"): - if line not in test_req_lines_seen: - outfile.write(line) - test_req_lines_seen.add(line) - outfile.close() - - # Removes Tmp Files - shutil.rmtree(MODULE_TMP_PATH) - - -def generate_clients(root_path, client_lib_name=LIB_NAME, interactive=False): - """ - Generates client stubs - """ - if interactive and input("Do you want to generate client libraries? [Y/n]").upper() == 'N': - return - - openapi_files = get_openapi_file_paths(root_path) - applications = get_application_paths(openapi_files) - - client_src_path = os.path.join( - root_path, 'libraries/client', client_lib_name) - for i in range(len(applications)): - app_name = applications[i] - openapi_file = openapi_files[i] - app_dir = os.path.dirname(os.path.dirname(openapi_file)) - generate_python_client(app_name, openapi_file, - client_src_path, lib_name=client_lib_name) - if os.path.exists(os.path.join(app_dir, 'frontend')): - generate_ts_client(openapi_file) - - aggregate_packages(client_src_path, client_lib_name) +from ch_cli_tools.openapi import LIB_NAME, ClientType, generate_clients, generate_models, generate_servers, get_dependencies +from ch_cli_tools.utils import confirm -if __name__ == "__main__": - import argparse - - parser = argparse.ArgumentParser( - description='Walk filesystem inside ./applications create application scaffolding.') - parser.add_argument('path', metavar='path', default=ROOT, type=str, - help='Base path of the application.') - parser.add_argument('-cn', '--client-name', dest='client_name', action="store", default=LIB_NAME, - help='Specify image registry prefix') - parser.add_argument('-i', '--interactive', dest='interactive', action="store_true", - help='Asks before generate') - parser.add_argument('-s', '--server', dest='server', action="store", - help='Generate only a specific server (provide application name) stubs', default=()) - parser.add_argument('-c', '--clients', dest='clients', action="store_true", - help='Generate only client libraries') - parser.add_argument('-m', '--models', dest='models', action="store_true", - help='Generate only model library') - args, unknown = parser.parse_known_args(sys.argv[1:]) - - root_path = os.path.join(os.getcwd(), args.path) if not os.path.isabs( - args.path) else args.path - +def main(): + args = get_command_line_arguments() get_dependencies() - if args.models and os.path.exists(os.path.join(root_path, "libraries/models")) and (not args.interactive or input("Do you want to generate the main model? [Y/n]").upper() != 'N'): - generate_model() - if not (args.clients or args.models) or args.server: - generate_servers(root_path, interactive=args.interactive, server=args.server) - if not (args.server or args.models) or args.clients: - generate_clients(root_path, args.client_name, interactive=args.interactive) + + root_path = args.path.absolute() if args.path else pathlib.Path.cwd().absolute() + app_name = root_path.name if args.path else args.app_name + if args.path and args.app_name: + logging.warning('Ignoring app_name flag because path was provided') + + # Check if applications folder exists, if not, go up until it's found + while not (root_path / 'applications').exists(): + if root_path == root_path.parent: + logging.error('Could not find applications folder') + return + root_path = root_path.parent + + should_generate = should_generate_interactive if args.is_interactive else lambda _: True + + if args.generate_models: + generate_models(root_path, should_generate) + + if args.generate_servers: + generate_servers(root_path, should_generate, app_name) + + if args.generate_clients: + generate_clients(root_path, should_generate, app_name, args.client_name, args.client_types) + + +class GenerationMode(enum.Flag): + CLIENTS = enum.auto() + MODELS = enum.auto() + SERVERS = enum.auto() + + @classmethod + def all(cls): + return functools.reduce(operator.or_, [cls.CLIENTS, cls.SERVERS]) + + +@dataclass(frozen=True) +class CommandLineArguments: + path: pathlib.Path + app_name: Optional[str] + is_interactive: bool + generation_mode: GenerationMode + client_name: Optional[str] = None + client_types: ClientType = ClientType.all() + + @property + def generate_models(self): + return GenerationMode.MODELS in self.generation_mode + + @property + def generate_servers(self): + return GenerationMode.SERVERS in self.generation_mode + + @property + def generate_clients(self): + return GenerationMode.CLIENTS in self.generation_mode + + +def get_command_line_arguments() -> CommandLineArguments: + parser = argparse.ArgumentParser(description='Walks the filesystem inside the ./applications folder to create and update applications scaffolding.') + + common_arguments = argparse.ArgumentParser(add_help=False) + common_arguments.add_argument('path', metavar='path', nargs='?', default=None, type=pathlib.Path, + help='Base path of the application. If used, the -a/--app_name flag is ignored.') + common_arguments.add_argument('-i', '--interactive', dest='is_interactive', action="store_true", + help='Asks before generate') + common_arguments.add_argument('-a', '--app-name', dest='app_name', action="store", default=None, + help='Generate only for a specific application') + + clients_arguments = argparse.ArgumentParser(add_help=False) + clients_arguments.add_argument('-cn', '--client-name', dest='client_name', action='store', default=None, + help='specify client prefix name') + client_type_group = clients_arguments.add_mutually_exclusive_group(required=False) + client_type_group.add_argument('-t', '--ts-only', dest='client_types', action='store_const', const=ClientType.TS_CLIENT, + help='Generate only typescript clients') + client_type_group.add_argument('-p', '--python-only', dest='client_types', action='store_const', const=ClientType.PYTHON_CLIENT, + help='Generate only python clients') + clients_arguments.set_defaults(client_types=ClientType.all()) + + subparsers = parser.add_subparsers(title='generation modes', required=True) + + all_parser = subparsers.add_parser('all', parents=[common_arguments, clients_arguments], + help='Generate server stubs and client libraries') + all_parser.set_defaults(generation_mode=GenerationMode.all()) + + clients_parser = subparsers.add_parser('clients', parents=[common_arguments, clients_arguments], + help='Generate only client libraries') + clients_parser.set_defaults(generation_mode=GenerationMode.CLIENTS) + + servers_parser = subparsers.add_parser('servers', parents=[common_arguments], + help='Generate only server stubs') + servers_parser.set_defaults(generation_mode=GenerationMode.SERVERS) + + models_parser = subparsers.add_parser('models', parents=[common_arguments], + help='Special flag, used to regenerate only model libraries') + models_parser.set_defaults(generation_mode=GenerationMode.MODELS) + + args = parser.parse_args() + + return CommandLineArguments(**args.__dict__) + + +def should_generate_interactive(resource: str) -> bool: + return confirm(f'Do you want to generate {resource}?') + + +if __name__ == "__main__": + main() diff --git a/tools/deployment-cli-tools/setup.py b/tools/deployment-cli-tools/setup.py index 280cd716b..1b9b1bf2d 100644 --- a/tools/deployment-cli-tools/setup.py +++ b/tools/deployment-cli-tools/setup.py @@ -12,7 +12,7 @@ NAME = "cloudharness-cli-tools" -VERSION = "2.3.0" +VERSION = "2.5.0" # To install the library, run the following # # python setup.py install @@ -27,7 +27,6 @@ 'six', 'cloudharness_model', 'cloudharness_utils', - 'fastapi-code-generator', 'dirhash', "StrEnum ; python_version < '3.11'", ] diff --git a/tools/deployment-cli-tools/tests/resources/applications/myapp/deploy/values-nobuild.yaml b/tools/deployment-cli-tools/tests/resources/applications/myapp/deploy/values-nobuild.yaml new file mode 100644 index 000000000..f61836364 --- /dev/null +++ b/tools/deployment-cli-tools/tests/resources/applications/myapp/deploy/values-nobuild.yaml @@ -0,0 +1,5 @@ +harness: + dependencies: + build: [] + deployment: + image: "custom-image" \ No newline at end of file diff --git a/tools/deployment-cli-tools/tests/test_codefresh.py b/tools/deployment-cli-tools/tests/test_codefresh.py index 0cda8eca4..bf1282c95 100644 --- a/tools/deployment-cli-tools/tests/test_codefresh.py +++ b/tools/deployment-cli-tools/tests/test_codefresh.py @@ -282,3 +282,39 @@ def test_create_codefresh_configuration_tests(): finally: shutil.rmtree(BUILD_MERGE_DIR) + + +def test_create_codefresh_configuration_nobuild(): + values = create_helm_chart( + [RESOURCES], + output_path=OUT, + include=['myapp'], + exclude=['events'], + domain="my.local", + namespace='test', + env=['dev', 'nobuild'], + local=False, + tag=1, + registry='reg' + ) + + root_paths = preprocess_build_overrides( + root_paths=[CLOUD_HARNESS_PATH, RESOURCES], + helm_values=values, + merge_build_path=BUILD_MERGE_DIR + ) + + build_included = [app['harness']['name'] + for app in values['apps'].values() if 'harness' in app] + + cf = create_codefresh_deployment_scripts(root_paths, include=build_included, + envs=['dev', 'nobuild'], + base_image_name=values['name'], + helm_values=values, save=False) + l1_steps = cf['steps'] + steps = l1_steps["build_application_images"]["steps"] + assert len(steps) == 1 + assert "myapp" not in steps + assert "myapp-mytask" in steps + assert "publish_myapp" not in l1_steps["publish"]["steps"] + assert "publish_myapp-mytask" in l1_steps["publish"]["steps"] diff --git a/tools/deployment-cli-tools/tests/test_helm.py b/tools/deployment-cli-tools/tests/test_helm.py index 35d903a11..c0a1a1486 100644 --- a/tools/deployment-cli-tools/tests/test_helm.py +++ b/tools/deployment-cli-tools/tests/test_helm.py @@ -17,13 +17,6 @@ def test_collect_helm_values(tmp_path): exclude=['events'], domain="my.local", namespace='test', env='dev', local=False, tag=1, registry='reg') - # Auto values - assert values[KEY_APPS]['myapp'][KEY_HARNESS]['deployment']['image'] == 'reg/cloudharness/myapp:1' - assert values.apps['myapp'].harness.deployment.image == 'reg/cloudharness/myapp:1' - assert values[KEY_APPS]['myapp'][KEY_HARNESS]['name'] == 'myapp' - assert values[KEY_APPS]['legacy'][KEY_HARNESS]['name'] == 'legacy' - assert values[KEY_APPS]['accounts'][KEY_HARNESS]['deployment']['image'] == 'reg/cloudharness/accounts:1' - # First level include apps assert 'samples' in values[KEY_APPS] assert 'myapp' in values[KEY_APPS] @@ -41,6 +34,14 @@ def test_collect_helm_values(tmp_path): # Explicit exclude overrides include assert 'events' not in values[KEY_APPS] + # Auto values + assert values[KEY_APPS]['myapp'][KEY_HARNESS]['deployment']['image'] == 'reg/cloudharness/myapp:1' + assert values[KEY_APPS]['myapp']['build'] == True + assert values.apps['myapp'].harness.deployment.image == 'reg/cloudharness/myapp:1' + assert values[KEY_APPS]['myapp'][KEY_HARNESS]['name'] == 'myapp' + assert values[KEY_APPS]['legacy'][KEY_HARNESS]['name'] == 'legacy' + assert values[KEY_APPS]['accounts'][KEY_HARNESS]['deployment']['image'] == 'reg/cloudharness/accounts:1' + # Base values kept assert values[KEY_APPS]['accounts'][KEY_HARNESS]['subdomain'] == 'accounts' @@ -79,6 +80,15 @@ def test_collect_helm_values(tmp_path): assert 'cloudharness-base-debian' not in values[KEY_TASK_IMAGES] +def test_collect_nobuild(tmp_path): + out_folder = tmp_path / 'test_collect_helm_values' + values = create_helm_chart([RESOURCES], output_path=out_folder, include=['myapp'], + exclude=['events'], domain="my.local", + namespace='test', env='nobuild', local=False, tag=1, registry='reg') + assert values[KEY_APPS]['myapp'][KEY_HARNESS]['deployment']['image'] == 'custom-image' + assert values[KEY_APPS]['myapp']['build'] == False + + def test_collect_helm_values_noreg_noinclude(tmp_path): out_path = tmp_path / 'test_collect_helm_values_noreg_noinclude' values = create_helm_chart([CLOUDHARNESS_ROOT, RESOURCES], output_path=out_path, domain="my.local", diff --git a/tools/deployment-cli-tools/tests/test_skaffold.py b/tools/deployment-cli-tools/tests/test_skaffold.py index 3c545d214..232fe7d85 100644 --- a/tools/deployment-cli-tools/tests/test_skaffold.py +++ b/tools/deployment-cli-tools/tests/test_skaffold.py @@ -180,3 +180,37 @@ def test_create_skaffold_configuration_with_conflicting_dependencies_requirement myapp_config = release['overrides']['apps']['myapp2'] assert myapp_config['harness']['deployment']['args'][0] == '/usr/src/app/myapp_code/__main__.py' + + +def test_create_skaffold_configuration_nobuild(): + values = create_helm_chart( + [RESOURCES], + output_path=OUT, + include=['myapp'], + domain="my.local", + namespace='test', + env='nobuild', + local=False, + tag=1, + registry='reg' + ) + + BUILD_DIR = "/tmp/build" + root_paths = preprocess_build_overrides( + root_paths=[CLOUDHARNESS_ROOT, RESOURCES], + helm_values=values, + merge_build_path=BUILD_DIR + ) + + sk = create_skaffold_configuration( + root_paths=root_paths, + helm_values=values, + output_path=OUT + ) + releases = sk['deploy']['helm']['releases'] + + assert len(sk['build']['artifacts']) == 1 + assert len(releases) == 1 # Ensure we only found 1 deployment (for myapp) + + release = releases[0] + assert 'myapp' not in release['overrides']['apps']