From 1d4ed6819d54fd0db6d6dda880dafdd05ebd0054 Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Mon, 30 Sep 2024 14:59:49 +0100 Subject: [PATCH 01/93] CH-151 Refactor merge configuration directories to improve readability --- .../ch_cli_tools/utils.py | 92 +++++++++++-------- 1 file changed, 52 insertions(+), 40 deletions(-) diff --git a/tools/deployment-cli-tools/ch_cli_tools/utils.py b/tools/deployment-cli-tools/ch_cli_tools/utils.py index 85fcb5bb..e80c0de6 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/utils.py +++ b/tools/deployment-cli-tools/ch_cli_tools/utils.py @@ -1,4 +1,5 @@ +import contextlib import socket import glob import subprocess @@ -256,52 +257,63 @@ def movedircontent(root_src_dir, root_dst_dir): shutil.rmtree(root_src_dir) -def merge_configuration_directories(source, dest): - if source == dest: +def merge_configuration_directories(source, destination): + if source == destination: return + if not exists(source): - logging.warning( - "Trying to merge the not existing directory: %s", source) + logging.warning("Trying to merge the not existing directory: %s", source) return - if not exists(dest): - shutil.copytree( - source, dest, ignore=shutil.ignore_patterns(*EXCLUDE_PATHS)) + + if not exists(destination): + shutil.copytree(source, destination, ignore=shutil.ignore_patterns(*EXCLUDE_PATHS)) return - for src_dir, dirs, files in os.walk(source): - if any(path in src_dir for path in EXCLUDE_PATHS): + for source_directory, _, files in os.walk(source): + _merge_configuration_directory(source, destination, source_directory, files) + + +def _merge_configuration_directory(source: str, destination: str, source_directory: str, files: list[str]) -> None: + if any(path in source_directory for path in EXCLUDE_PATHS): + return + + destination_directory = source_directory.replace(source, destination, 1) + if not exists(destination_directory): + os.makedirs(destination_directory) + + non_build_files = (file for file in files if file not in BUILD_FILENAMES) + + for file_name in non_build_files: + source_file_path = join(source_directory, file_name) + destination_file_path = join(destination_directory, file_name) + + _merge_configuration_file(source_file_path, destination_file_path) + + +def _merge_configuration_file(source_file_path: str, destination_file_path: str) -> None: + if not exists(destination_file_path): + shutil.copy2(source_file_path, destination_file_path) + return + + merge_operations = [ + (file_is_yaml, merge_yaml_files), + (file_is_json, merge_json_files), + ] + + for file_is_expected_type, merge_files in merge_operations: + if not file_is_expected_type(source_file_path): continue - dst_dir = src_dir.replace(source, dest, 1) - if not exists(dst_dir): - os.makedirs(dst_dir) - for fname in files: - if fname in BUILD_FILENAMES: - continue - fpath = join(src_dir, fname) - frel = relpath(fpath, start=source) - fdest = join(dest, frel) - if not exists(fdest): - shutil.copy2(fpath, fdest) - elif file_is_yaml(fpath): - - try: - merge_yaml_files(fpath, fdest) - logging.info( - f"Merged/overridden file content of {fdest} with {fpath}") - except Exception as e: - logging.warning(f"Overwriting file {fdest} with {fpath}") - shutil.copy2(fpath, fdest) - elif file_is_json(fpath): - try: - merge_json_files(fpath, fdest) - logging.info( - f"Merged/overridden file content of {fdest} with {fpath}") - except Exception as e: - logging.warning(f"Overwriting file {fdest} with {fpath}") - shutil.copy2(fpath, fdest) - else: - logging.warning(f"Overwriting file {fdest} with {fpath}") - shutil.copy2(fpath, fdest) + + try: + merge_files(source_file_path, destination_file_path) + logging.info(f'Merged/overridden file content of {destination_file_path} with {source_file_path}') + except: + break + + return + + logging.warning(f'Overwriting file {destination_file_path} with {source_file_path}') + shutil.copy2(source_file_path, destination_file_path) def merge_yaml_files(fname, fdest): From 10114cd3963043c4441cd83ccb1a7c48f586ccf3 Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Mon, 30 Sep 2024 15:25:00 +0100 Subject: [PATCH 02/93] CH-151 Add postgres db as another base template of the django app --- tools/deployment-cli-tools/harness-application | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment-cli-tools/harness-application b/tools/deployment-cli-tools/harness-application index d3601f7a..a1cbeb50 100644 --- a/tools/deployment-cli-tools/harness-application +++ b/tools/deployment-cli-tools/harness-application @@ -46,7 +46,7 @@ def main(): os.makedirs(app_path, exist_ok=True) if TemplateType.DJANGO_APP in templates and TemplateType.WEBAPP not in templates: - templates = [TemplateType.BASE, TemplateType.WEBAPP] + templates + templates = [TemplateType.BASE, TemplateType.WEBAPP, TemplateType.DB_POSTGRES] + templates if TemplateType.WEBAPP in templates: handle_webapp_template(app_name, app_path) From 4925ec556e0cbf29e9eed69c5988a1040e99aad1 Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Mon, 30 Sep 2024 16:19:21 +0100 Subject: [PATCH 03/93] CH-151 Update django app dockerfile to use yarn instead of npm --- application-templates/django-app/Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/application-templates/django-app/Dockerfile b/application-templates/django-app/Dockerfile index f40e9560..05dbd520 100644 --- a/application-templates/django-app/Dockerfile +++ b/application-templates/django-app/Dockerfile @@ -7,11 +7,11 @@ ENV APP_DIR=/app WORKDIR ${APP_DIR} COPY frontend/package.json ${APP_DIR} -COPY frontend/package-lock.json ${APP_DIR} -RUN npm ci +COPY frontend/yarn.lock ${APP_DIR} +RUN yarn install --frozen-lockfile --timeout 60000 COPY frontend ${APP_DIR} -RUN npm run build +RUN yarn build ##### From 47a7eb1d13f9a347b03999100c8c44e1112b61c4 Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Thu, 3 Oct 2024 10:49:52 +0100 Subject: [PATCH 04/93] CH-151 Add initial install of yarn dependencies to harness-application --- .../deployment-cli-tools/harness-application | 30 ++++++++++++++----- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/tools/deployment-cli-tools/harness-application b/tools/deployment-cli-tools/harness-application index a1cbeb50..7c30b01e 100644 --- a/tools/deployment-cli-tools/harness-application +++ b/tools/deployment-cli-tools/harness-application @@ -113,13 +113,29 @@ def get_command_line_arguments() -> tuple[str, list[str]]: def handle_webapp_template(app_name: str, app_path: str) -> None: - if os.path.exists(os.path.join(app_path, 'frontend')): - shutil.rmtree(os.path.join(app_path, 'frontend')) - cmd = ["yarn", "create", "vite", app_name, "--template", "react-ts"] - logging.info(f"Running command: {' '.join(cmd)}") - subprocess.run(cmd, cwd=app_path) - shutil.move(os.path.join(app_path, app_name), os.path.join(app_path, 'frontend')) - generate_ts_client(openapi_file=os.path.join(app_path, 'api/openapi.yaml')) + app_path = pathlib.Path(app_path) + frontend_app_path = app_path / 'frontend' + + if os.path.exists(frontend_app_path): + shutil.rmtree(frontend_app_path) + + create_vite_skaffold(app_name, app_path) + install_frontend_dependencies(app_path) + + shutil.move(app_path / app_name, frontend_app_path) + generate_ts_client(openapi_file=app_path / 'api' / 'openapi.yaml') + + +def create_vite_skaffold(app_name: str, app_path: pathlib.Path) -> None: + command = ['yarn', 'create', 'vite', app_name, '--template', 'react-ts'] + logging.info(f'Running command: {" ".join(command)}') + subprocess.run(command, cwd=app_path) + + +def install_frontend_dependencies(app_path: pathlib.Path) -> None: + command = ['yarn', 'install'] + logging.info(f'Running command: {" ".join(command)}') + subprocess.run(command, cwd=app_path) def handle_webapp_template_cleanup(app_path: str) -> None: From 230adbae31643607840a97d28d8195880d9930de Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Thu, 3 Oct 2024 13:48:52 +0100 Subject: [PATCH 05/93] CH-151 Refactor harness-generate --- tools/deployment-cli-tools/harness-generate | 86 ++++++++++++++------- 1 file changed, 58 insertions(+), 28 deletions(-) diff --git a/tools/deployment-cli-tools/harness-generate b/tools/deployment-cli-tools/harness-generate index 12d40e8a..ef5e9250 100644 --- a/tools/deployment-cli-tools/harness-generate +++ b/tools/deployment-cli-tools/harness-generate @@ -1,10 +1,14 @@ #!/usr/bin/env python +import argparse +from dataclasses import dataclass import glob import os +import pathlib import shutil import sys import logging +from typing import Any, Optional from ch_cli_tools.openapi import LIB_NAME, generate_python_client, generate_server, generate_fastapi_server, \ get_dependencies, generate_ts_client, generate_model @@ -14,6 +18,59 @@ HERE = os.path.dirname(os.path.realpath(__file__)) ROOT = os.path.dirname(HERE) +@dataclass(frozen=True) +class CommandLineArguments: + path: pathlib.Path + client_name: str + server_name: Optional[str] + generate_clients: bool + generate_models: bool + is_interactive: bool + + +def main(): + args = get_command_line_arguments() + + root_path = os.path.join(os.getcwd(), args.path) if not os.path.isabs( + args.path) else args.path + + get_dependencies() + if args.generate_models and os.path.exists(os.path.join(root_path, "libraries/models")) and (not args.is_interactive or input("Do you want to generate the main model? [Y/n]").upper() != 'N'): + generate_model() + if not (args.generate_clients or args.generate_models) or args.server_name: + generate_servers(root_path, interactive=args.is_interactive, server=args.server_name) + if not (args.server_name or args.generate_models) or args.generate_clients: + generate_clients(root_path, args.client_name, interactive=args.is_interactive) + + +def get_command_line_arguments() -> CommandLineArguments: + parser = argparse.ArgumentParser(description='Walk filesystem inside ./applications create application scaffolding.') + + parser.add_argument('path', metavar='path', default=ROOT, type=str, + help='Base path of the application.') + parser.add_argument('-cn', '--client-name', dest='client_name', action="store", default=LIB_NAME, + help='Specify image registry prefix') + parser.add_argument('-i', '--interactive', dest='interactive', action="store_true", + help='Asks before generate') + parser.add_argument('-s', '--server', dest='server', action="store", default=None, + help='Generate only a specific server (provide application name) stubs') + parser.add_argument('-c', '--clients', dest='clients', action="store_true", + help='Generate only client libraries') + parser.add_argument('-m', '--models', dest='models', action="store_true", + help='Generate only model library') + + args, _ = parser.parse_known_args(sys.argv[1:]) + + return CommandLineArguments( + path=args.path, + client_name=args.client_name, + server_name=args.server, + generate_clients=args.clients, + generate_models=args.models, + is_interactive=args.interactive, + ) + + def get_openapi_file_paths(root_path): return [path for path in glob.glob(root_path + '/applications/*/api/*.yaml')] @@ -146,31 +203,4 @@ def generate_clients(root_path, client_lib_name=LIB_NAME, interactive=False): if __name__ == "__main__": - import argparse - - parser = argparse.ArgumentParser( - description='Walk filesystem inside ./applications create application scaffolding.') - parser.add_argument('path', metavar='path', default=ROOT, type=str, - help='Base path of the application.') - parser.add_argument('-cn', '--client-name', dest='client_name', action="store", default=LIB_NAME, - help='Specify image registry prefix') - parser.add_argument('-i', '--interactive', dest='interactive', action="store_true", - help='Asks before generate') - parser.add_argument('-s', '--server', dest='server', action="store", - help='Generate only a specific server (provide application name) stubs', default=()) - parser.add_argument('-c', '--clients', dest='clients', action="store_true", - help='Generate only client libraries') - parser.add_argument('-m', '--models', dest='models', action="store_true", - help='Generate only model library') - args, unknown = parser.parse_known_args(sys.argv[1:]) - - root_path = os.path.join(os.getcwd(), args.path) if not os.path.isabs( - args.path) else args.path - - get_dependencies() - if args.models and os.path.exists(os.path.join(root_path, "libraries/models")) and (not args.interactive or input("Do you want to generate the main model? [Y/n]").upper() != 'N'): - generate_model() - if not (args.clients or args.models) or args.server: - generate_servers(root_path, interactive=args.interactive, server=args.server) - if not (args.server or args.models) or args.clients: - generate_clients(root_path, args.client_name, interactive=args.interactive) + main() From 7044fc093a3e315a3c4792e6fe35ccc319ee04e5 Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Thu, 3 Oct 2024 14:54:22 +0100 Subject: [PATCH 06/93] CH-151 Refactor harness-generate --- tools/deployment-cli-tools/harness-generate | 114 +++++++++++--------- 1 file changed, 62 insertions(+), 52 deletions(-) diff --git a/tools/deployment-cli-tools/harness-generate b/tools/deployment-cli-tools/harness-generate index ef5e9250..b38c2ee3 100644 --- a/tools/deployment-cli-tools/harness-generate +++ b/tools/deployment-cli-tools/harness-generate @@ -8,7 +8,7 @@ import pathlib import shutil import sys import logging -from typing import Any, Optional +from typing import Any, Callable, Optional from ch_cli_tools.openapi import LIB_NAME, generate_python_client, generate_server, generate_fastapi_server, \ get_dependencies, generate_ts_client, generate_model @@ -30,17 +30,22 @@ class CommandLineArguments: def main(): args = get_command_line_arguments() + get_dependencies() - root_path = os.path.join(os.getcwd(), args.path) if not os.path.isabs( - args.path) else args.path + path_arg = pathlib.Path(args.path) + root_path = path_arg if path_arg.is_absolute else path_arg.absolute + library_models_path = root_path / 'libraries' / 'models' - get_dependencies() - if args.generate_models and os.path.exists(os.path.join(root_path, "libraries/models")) and (not args.is_interactive or input("Do you want to generate the main model? [Y/n]").upper() != 'N'): + should_generate = should_generate_interactive if args.is_interactive else lambda _: True + + if args.generate_models and library_models_path.exists and should_generate('the main model'): generate_model() - if not (args.generate_clients or args.generate_models) or args.server_name: - generate_servers(root_path, interactive=args.is_interactive, server=args.server_name) - if not (args.server_name or args.generate_models) or args.generate_clients: - generate_clients(root_path, args.client_name, interactive=args.is_interactive) + + if args.server_name or not (args.generate_clients or args.generate_models): + generate_servers(root_path, should_generate, args.server_name) + + if args.generate_clients or not (args.server_name or args.generate_models): + generate_clients(root_path, should_generate, args.client_name) def get_command_line_arguments() -> CommandLineArguments: @@ -71,32 +76,61 @@ def get_command_line_arguments() -> CommandLineArguments: ) -def get_openapi_file_paths(root_path): - return [path for path in glob.glob(root_path + '/applications/*/api/*.yaml')] +def should_generate_interactive(resource: str) -> bool: + user_input = input(f'Do you want to generate {resource}? [Y/n]').casefold() + return user_input == 'y' -def get_application_paths(openapi_files): - return [os.path.basename(os.path.dirname(os.path.dirname(path))) for path in openapi_files] - -def generate_servers(root_path, interactive=False, server=None): +def generate_servers(root_path: pathlib.Path, should_generate: Callable[[str], bool], server: Optional[str]): """ Generates server stubs """ openapi_files = get_openapi_file_paths(root_path) - modules = get_application_paths(openapi_files) - for i in range(len(modules)): - if not interactive or input("Do you want to generate " + openapi_files[i] + "? [Y/n]").upper() != 'N': - openapi_file = openapi_files[i] - application_root = os.path.dirname(os.path.dirname(openapi_file)) - appname = os.path.basename(application_root) - if server and server != appname: - continue - if os.path.exists(os.path.join(application_root, "api", "genapi.sh")): - # fastapi server --> use the genapi.sh script - generate_fastapi_server(application_root) - else: - generate_server(application_root) + + for openapi_file in openapi_files: + if not should_generate(openapi_file): + continue + + application_root = openapi_file.parent.parent + app_name = application_root.name + + if server and server != app_name: + continue + + genapi_path = application_root / 'api' / 'genapi.sh' + if genapi_path.exists: + # fastapi server --> use the genapi.sh script + generate_fastapi_server(application_root) + else: + generate_server(application_root) + + +def generate_clients(root_path: pathlib.Path, should_generate: Callable[[str], bool], client_lib_name: str): + """ + Generates client stubs + """ + if not should_generate('client libraries'): + return + + openapi_files = get_openapi_file_paths(root_path) + client_src_path = root_path / 'libraries' / 'client' / client_lib_name + + for openapi_file in openapi_files: + app_name = openapi_file.parent.parent.name + + generate_python_client(app_name, openapi_file, client_src_path, lib_name=client_lib_name) + + frontend_path = openapi_file.parent.parent / 'frontend' + if frontend_path.exists: + generate_ts_client(openapi_file) + + aggregate_packages(str(client_src_path), client_lib_name) + + +def get_openapi_file_paths(root_path) -> list[pathlib.Path]: + search_path = root_path / 'applications' / '*' / 'api' / '*.yaml' + return [pathlib.Path(path) for path in glob.glob(str(search_path))] def aggregate_packages(client_src_path, lib_name=LIB_NAME): @@ -178,29 +212,5 @@ def aggregate_packages(client_src_path, lib_name=LIB_NAME): shutil.rmtree(MODULE_TMP_PATH) -def generate_clients(root_path, client_lib_name=LIB_NAME, interactive=False): - """ - Generates client stubs - """ - if interactive and input("Do you want to generate client libraries? [Y/n]").upper() == 'N': - return - - openapi_files = get_openapi_file_paths(root_path) - applications = get_application_paths(openapi_files) - - client_src_path = os.path.join( - root_path, 'libraries/client', client_lib_name) - for i in range(len(applications)): - app_name = applications[i] - openapi_file = openapi_files[i] - app_dir = os.path.dirname(os.path.dirname(openapi_file)) - generate_python_client(app_name, openapi_file, - client_src_path, lib_name=client_lib_name) - if os.path.exists(os.path.join(app_dir, 'frontend')): - generate_ts_client(openapi_file) - - aggregate_packages(client_src_path, client_lib_name) - - if __name__ == "__main__": main() From 618d0643a3a68e9feabf5f61a5790fa9f17dfc44 Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Thu, 3 Oct 2024 16:45:40 +0100 Subject: [PATCH 07/93] CH-151 Refactor merge_configuration_directories to use pathlib --- .../ch_cli_tools/utils.py | 39 +++++++++++-------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/tools/deployment-cli-tools/ch_cli_tools/utils.py b/tools/deployment-cli-tools/ch_cli_tools/utils.py index e80c0de6..efe9fbd9 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/utils.py +++ b/tools/deployment-cli-tools/ch_cli_tools/utils.py @@ -1,5 +1,6 @@ import contextlib +import pathlib import socket import glob import subprocess @@ -257,40 +258,46 @@ def movedircontent(root_src_dir, root_dst_dir): shutil.rmtree(root_src_dir) -def merge_configuration_directories(source, destination): +def merge_configuration_directories(source: str, destination: str) -> None: if source == destination: return - if not exists(source): + source_path, destination_path = pathlib.Path(source), pathlib.Path(destination) + + if not source_path.exists(): logging.warning("Trying to merge the not existing directory: %s", source) return - if not exists(destination): - shutil.copytree(source, destination, ignore=shutil.ignore_patterns(*EXCLUDE_PATHS)) + if not destination_path.exists(): + shutil.copytree(source_path, destination_path, ignore=shutil.ignore_patterns(*EXCLUDE_PATHS)) return - for source_directory, _, files in os.walk(source): - _merge_configuration_directory(source, destination, source_directory, files) + for source_directory, _, files in source_path.walk(): + _merge_configuration_directory(source_path, destination_path, source_directory, files) -def _merge_configuration_directory(source: str, destination: str, source_directory: str, files: list[str]) -> None: - if any(path in source_directory for path in EXCLUDE_PATHS): +def _merge_configuration_directory( + source: pathlib.Path, + destination: pathlib.Path, + source_directory: pathlib.Path, + files: list[str] +) -> None: + if any(path in str(source_directory) for path in EXCLUDE_PATHS): return - destination_directory = source_directory.replace(source, destination, 1) - if not exists(destination_directory): - os.makedirs(destination_directory) + destination_directory = destination / source_directory.relative_to(source) + destination_directory.mkdir(exist_ok=True) non_build_files = (file for file in files if file not in BUILD_FILENAMES) for file_name in non_build_files: - source_file_path = join(source_directory, file_name) - destination_file_path = join(destination_directory, file_name) + source_file_path = source_directory / file_name + destination_file_path = destination_directory / file_name _merge_configuration_file(source_file_path, destination_file_path) -def _merge_configuration_file(source_file_path: str, destination_file_path: str) -> None: +def _merge_configuration_file(source_file_path: pathlib.Path, destination_file_path: pathlib.Path) -> None: if not exists(destination_file_path): shutil.copy2(source_file_path, destination_file_path) return @@ -300,8 +307,8 @@ def _merge_configuration_file(source_file_path: str, destination_file_path: str) (file_is_json, merge_json_files), ] - for file_is_expected_type, merge_files in merge_operations: - if not file_is_expected_type(source_file_path): + for can_merge_file, merge_files in merge_operations: + if not can_merge_file(source_file_path.name): continue try: From 55fd63dd1618c00039eeb0db9d24cce66639f82a Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Thu, 3 Oct 2024 16:47:45 +0100 Subject: [PATCH 08/93] CH-151 Refactor harness-generate to use pathlib glob --- tools/deployment-cli-tools/harness-generate | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tools/deployment-cli-tools/harness-generate b/tools/deployment-cli-tools/harness-generate index b38c2ee3..4b95d820 100644 --- a/tools/deployment-cli-tools/harness-generate +++ b/tools/deployment-cli-tools/harness-generate @@ -8,7 +8,7 @@ import pathlib import shutil import sys import logging -from typing import Any, Callable, Optional +from typing import Callable, Optional from ch_cli_tools.openapi import LIB_NAME, generate_python_client, generate_server, generate_fastapi_server, \ get_dependencies, generate_ts_client, generate_model @@ -128,9 +128,8 @@ def generate_clients(root_path: pathlib.Path, should_generate: Callable[[str], b aggregate_packages(str(client_src_path), client_lib_name) -def get_openapi_file_paths(root_path) -> list[pathlib.Path]: - search_path = root_path / 'applications' / '*' / 'api' / '*.yaml' - return [pathlib.Path(path) for path in glob.glob(str(search_path))] +def get_openapi_file_paths(root_path: pathlib.Path) -> list[pathlib.Path]: + return [path for path in root_path.glob('applications/*/api/*.yaml')] def aggregate_packages(client_src_path, lib_name=LIB_NAME): From 9a1148a50d0b3b75a3123ab480f0eeb5f4b5ba9b Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Fri, 4 Oct 2024 14:00:48 +0100 Subject: [PATCH 09/93] CH-151 Refactor harness-application to only use pathlib over os.path --- .../deployment-cli-tools/harness-application | 64 ++++++++----------- 1 file changed, 28 insertions(+), 36 deletions(-) diff --git a/tools/deployment-cli-tools/harness-application b/tools/deployment-cli-tools/harness-application index 7c30b01e..b2064ccf 100644 --- a/tools/deployment-cli-tools/harness-application +++ b/tools/deployment-cli-tools/harness-application @@ -3,7 +3,6 @@ import json import pathlib import sys -import os import re import shutil import tempfile @@ -42,8 +41,8 @@ class TemplateType(StrEnum): def main(): app_name, templates = get_command_line_arguments() - app_path = os.path.join(APPLICATIONS_SRC_PATH, app_name) - os.makedirs(app_path, exist_ok=True) + app_path = pathlib.Path(APPLICATIONS_SRC_PATH) / app_name + app_path.mkdir(exist_ok=True) if TemplateType.DJANGO_APP in templates and TemplateType.WEBAPP not in templates: templates = [TemplateType.BASE, TemplateType.WEBAPP, TemplateType.DB_POSTGRES] + templates @@ -60,7 +59,7 @@ def main(): if TemplateType.FLASK_SERVER in templates: handle_flask_server_template(app_path) - replace_in_file(os.path.join(app_path, 'api/config.json'), PLACEHOLDER, to_python_module(app_name)) + replace_in_file(app_path/'api'/'config.json', PLACEHOLDER, to_python_module(app_name)) if TemplateType.DJANGO_APP in templates: handle_django_app_template(app_name, app_path) @@ -112,18 +111,17 @@ def get_command_line_arguments() -> tuple[str, list[str]]: return args.name, args.templates -def handle_webapp_template(app_name: str, app_path: str) -> None: - app_path = pathlib.Path(app_path) - frontend_app_path = app_path / 'frontend' +def handle_webapp_template(app_name: str, app_path: pathlib.Path) -> None: + frontend_app_path = app_path/'frontend' - if os.path.exists(frontend_app_path): + if frontend_app_path.exists(): shutil.rmtree(frontend_app_path) create_vite_skaffold(app_name, app_path) install_frontend_dependencies(app_path) - shutil.move(app_path / app_name, frontend_app_path) - generate_ts_client(openapi_file=app_path / 'api' / 'openapi.yaml') + shutil.move(app_path/app_name, frontend_app_path) + generate_ts_client(openapi_file=app_path/'api'/'openapi.yaml') def create_vite_skaffold(app_name: str, app_path: pathlib.Path) -> None: @@ -138,40 +136,34 @@ def install_frontend_dependencies(app_path: pathlib.Path) -> None: subprocess.run(command, cwd=app_path) -def handle_webapp_template_cleanup(app_path: str) -> None: - try: - os.remove(os.path.join(app_path, 'backend', 'Dockerfile')) - except FileNotFoundError: - # backend dockerfile not found, continue - pass +def handle_webapp_template_cleanup(app_path: pathlib.Path) -> None: + backend_dockerfile_path = app_path/'backend'/'Dockerfile' + backend_dockerfile_path.unlink(missing_ok=True) -def handle_server_template(app_path: str) -> None: +def handle_server_template(app_path: pathlib.Path) -> None: with tempfile.TemporaryDirectory() as tmp_dirname: - copymergedir(os.path.join(CH_ROOT, APPLICATION_TEMPLATE_PATH, TemplateType.SERVER), tmp_dirname) + server_template_path = pathlib.Path(CH_ROOT)/APPLICATION_TEMPLATE_PATH/TemplateType.SERVER + copymergedir(str(server_template_path), tmp_dirname) merge_configuration_directories(app_path, tmp_dirname) generate_server(app_path, tmp_dirname) -def handle_flask_server_template(app_path: str) -> None: +def handle_flask_server_template(app_path: pathlib.Path) -> None: generate_server(app_path) -def handle_django_app_template(app_name: str, app_path: str) -> None: - replace_in_file(os.path.join(app_path, 'api/templates/main.jinja2'), PLACEHOLDER, to_python_module(app_name)) +def handle_django_app_template(app_name: str, app_path: pathlib.Path) -> None: + python_app_name = to_python_module(app_name) + + replace_in_file(app_path/'api'/'templates'/'main.jinja2', PLACEHOLDER, python_app_name) generate_fastapi_server(app_path) - replace_in_file( - os.path.join(app_path, 'deploy/values.yaml'), - f"{PLACEHOLDER}:{PLACEHOLDER}", - f"{to_python_module(app_name)}:{to_python_module(app_name)}" - ) - replace_in_file(os.path.join(app_path, "dev-setup.sh"), PLACEHOLDER, app_name) + + replace_in_file(app_path/'deploy'/'values.yaml', f'{PLACEHOLDER}:{PLACEHOLDER}', f'{python_app_name}:{python_app_name}') + replace_in_file(app_path/'dev-setup.sh', PLACEHOLDER, app_name) create_django_app_vscode_debug_configuration(app_name) - try: - os.remove(os.path.join(app_path, 'backend', "__APP_NAME__", "__main__.py")) - except FileNotFoundError: - # backend dockerfile not found, continue - pass + + (app_path/'backend'/'__APP_NAME__'/'__main__.py').unlink(missing_ok=True) def create_django_app_vscode_debug_configuration(app_name: str): @@ -195,10 +187,10 @@ def create_django_app_vscode_debug_configuration(app_name: str): json.dump(launch_config, f, indent=2, sort_keys=True) -def merge_template_directories(template_name: str, app_path: str) -> None: - for base_path in (CH_ROOT, os.getcwd()): - template_path = os.path.join(base_path, APPLICATION_TEMPLATE_PATH, template_name) - if os.path.exists(template_path): +def merge_template_directories(template_name: str, app_path: pathlib.Path) -> None: + for base_path in (pathlib.Path(CH_ROOT), pathlib.Path.cwd()): + template_path = base_path/APPLICATION_TEMPLATE_PATH/template_name + if template_path.exists(): merge_configuration_directories(template_path, app_path) From a334992c0c475451be74c7327acd489d52b86383 Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Fri, 4 Oct 2024 14:21:01 +0100 Subject: [PATCH 10/93] CH-151 Refactor replace_in_file to work with pathlib --- tools/deployment-cli-tools/ch_cli_tools/utils.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tools/deployment-cli-tools/ch_cli_tools/utils.py b/tools/deployment-cli-tools/ch_cli_tools/utils.py index efe9fbd9..f34c3b8b 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/utils.py +++ b/tools/deployment-cli-tools/ch_cli_tools/utils.py @@ -176,17 +176,18 @@ def replaceindir(root_src_dir, source, replace): if not any(file_.endswith(ext) for ext in REPLACE_TEXT_FILES_EXTENSIONS): continue - src_file = join(src_dir, file_) + src_file = pathlib.Path(src_dir)/file_ replace_in_file(src_file, source, replace) -def replace_in_file(src_file, source, replace): - if src_file.endswith('.py') or basename(src_file) == 'Dockerfile': - replace = to_python_module(replace) - with fileinput.FileInput(src_file, inplace=True) as file: +def replace_in_file(src_file: pathlib.Path, source: str, replacement: str) -> None: + if src_file.name.endswith('.py') or src_file.name == 'Dockerfile': + replacement = to_python_module(replacement) + + with fileinput.input(src_file, inplace=True) as file: try: for line in file: - print(line.replace(source, replace), end='') + print(line.replace(source, replacement), end='') except UnicodeDecodeError: pass From 94f12de0e3e4643725f87bafc83d5b331742743b Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Fri, 4 Oct 2024 14:21:39 +0100 Subject: [PATCH 11/93] CH-151 Add missing method calls for checking path existance --- tools/deployment-cli-tools/harness-generate | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/deployment-cli-tools/harness-generate b/tools/deployment-cli-tools/harness-generate index 4b95d820..e9d4c40b 100644 --- a/tools/deployment-cli-tools/harness-generate +++ b/tools/deployment-cli-tools/harness-generate @@ -38,7 +38,7 @@ def main(): should_generate = should_generate_interactive if args.is_interactive else lambda _: True - if args.generate_models and library_models_path.exists and should_generate('the main model'): + if args.generate_models and library_models_path.exists() and should_generate('the main model'): generate_model() if args.server_name or not (args.generate_clients or args.generate_models): @@ -99,7 +99,7 @@ def generate_servers(root_path: pathlib.Path, should_generate: Callable[[str], b continue genapi_path = application_root / 'api' / 'genapi.sh' - if genapi_path.exists: + if genapi_path.exists(): # fastapi server --> use the genapi.sh script generate_fastapi_server(application_root) else: @@ -122,7 +122,7 @@ def generate_clients(root_path: pathlib.Path, should_generate: Callable[[str], b generate_python_client(app_name, openapi_file, client_src_path, lib_name=client_lib_name) frontend_path = openapi_file.parent.parent / 'frontend' - if frontend_path.exists: + if frontend_path.exists(): generate_ts_client(openapi_file) aggregate_packages(str(client_src_path), client_lib_name) From 87d2cb7fdc3c7dfe8cf79f1deb173abfe96eda0d Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Fri, 4 Oct 2024 14:22:51 +0100 Subject: [PATCH 12/93] CH-151 Refactor openapi functions to use pathlib and directly call fastapi-codegen rather than genapi.sh --- .../ch_cli_tools/openapi.py | 54 ++++++++++++++----- 1 file changed, 41 insertions(+), 13 deletions(-) diff --git a/tools/deployment-cli-tools/ch_cli_tools/openapi.py b/tools/deployment-cli-tools/ch_cli_tools/openapi.py index 4df7b883..70bdf0b8 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/openapi.py +++ b/tools/deployment-cli-tools/ch_cli_tools/openapi.py @@ -2,9 +2,11 @@ import json import logging import os +import pathlib import shutil import subprocess import sys +from typing import Optional import urllib.request from os.path import dirname as dn, join @@ -19,22 +21,48 @@ OPENAPI_GEN_URL = 'https://repo1.maven.org/maven2/org/openapitools/openapi-generator-cli/7.7.0/openapi-generator-cli-7.7.0.jar' -def generate_server(app_path, overrides_folder=""): +def generate_server(app_path: pathlib.Path, overrides_folder: Optional[str]=None) -> None: get_dependencies() - openapi_dir = os.path.join(app_path, 'api') - openapi_file = glob.glob(os.path.join(openapi_dir, '*.yaml'))[0] - out_name = f"backend" if not os.path.exists( - f"{app_path}/server") else f"server" - out_path = f"{app_path}/{out_name}" - command = f"java -jar {CODEGEN} generate -i {openapi_file} -g python-flask -o {out_path} " \ - f"-c {openapi_dir}/config.json " + \ - (f"-t {overrides_folder}" if overrides_folder else "") - os.system(command) + openapi_directory = app_path/'api' + openapi_file = next(openapi_directory.glob('*.yaml')) -def generate_fastapi_server(app_path): - command = f"cd {app_path}/api && bash genapi.sh" - os.system(command) + server_path = app_path/'server' + backend_path = app_path/'backend' + out_path = server_path if server_path.exists() else backend_path + + command = [ + 'java', '-jar', CODEGEN, 'generate', + '-i', openapi_file, + '-g', 'python-flask', + '-o', out_path, + '-c', openapi_directory/'config.json', + ] + if overrides_folder: + command += ['-t', overrides_folder] + + subprocess.run(command) + + +def generate_fastapi_server(app_path: pathlib.Path) -> None: + api_path = app_path/'api' + tmp_app_path = api_path/'app' + + command = [ + 'fastapi-codegen', + '--input', api_path/'openapi.yaml', + '--output', tmp_app_path, + '-t', api_path/'templates', + ] + subprocess.run(command) + + backend_path = app_path/'backend' + shutil.move(tmp_app_path/'main.py', backend_path) + shutil.move(tmp_app_path/'models.py', backend_path/'openapi') + + tmp_app_path.rmdir() + + logging.info('Generated new models and main.py') def generate_model(base_path=ROOT): From 15618cb8cb08f22c46d0bf7c04170d2537353bf9 Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Fri, 4 Oct 2024 14:35:01 +0100 Subject: [PATCH 13/93] CH-151 Replace --server option with --app_name option in harness-generate --- tools/deployment-cli-tools/harness-generate | 28 ++++++++++----------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/tools/deployment-cli-tools/harness-generate b/tools/deployment-cli-tools/harness-generate index e9d4c40b..f4388897 100644 --- a/tools/deployment-cli-tools/harness-generate +++ b/tools/deployment-cli-tools/harness-generate @@ -22,7 +22,7 @@ ROOT = os.path.dirname(HERE) class CommandLineArguments: path: pathlib.Path client_name: str - server_name: Optional[str] + app_name: Optional[str] generate_clients: bool generate_models: bool is_interactive: bool @@ -34,17 +34,17 @@ def main(): path_arg = pathlib.Path(args.path) root_path = path_arg if path_arg.is_absolute else path_arg.absolute - library_models_path = root_path / 'libraries' / 'models' + library_models_path = root_path/'libraries'/'models' should_generate = should_generate_interactive if args.is_interactive else lambda _: True if args.generate_models and library_models_path.exists() and should_generate('the main model'): generate_model() - if args.server_name or not (args.generate_clients or args.generate_models): - generate_servers(root_path, should_generate, args.server_name) + if args.app_name or not (args.generate_clients or args.generate_models): + generate_servers(root_path, should_generate, args.app_name) - if args.generate_clients or not (args.server_name or args.generate_models): + if args.generate_clients or not (args.app_name or args.generate_models): generate_clients(root_path, should_generate, args.client_name) @@ -57,8 +57,8 @@ def get_command_line_arguments() -> CommandLineArguments: help='Specify image registry prefix') parser.add_argument('-i', '--interactive', dest='interactive', action="store_true", help='Asks before generate') - parser.add_argument('-s', '--server', dest='server', action="store", default=None, - help='Generate only a specific server (provide application name) stubs') + parser.add_argument('-a', '--app_name', dest='app_name', action="store", default=None, + help='Generate only for a specific application') parser.add_argument('-c', '--clients', dest='clients', action="store_true", help='Generate only client libraries') parser.add_argument('-m', '--models', dest='models', action="store_true", @@ -69,7 +69,7 @@ def get_command_line_arguments() -> CommandLineArguments: return CommandLineArguments( path=args.path, client_name=args.client_name, - server_name=args.server, + app_name=args.app_name, generate_clients=args.clients, generate_models=args.models, is_interactive=args.interactive, @@ -82,7 +82,7 @@ def should_generate_interactive(resource: str) -> bool: return user_input == 'y' -def generate_servers(root_path: pathlib.Path, should_generate: Callable[[str], bool], server: Optional[str]): +def generate_servers(root_path: pathlib.Path, should_generate: Callable[[str], bool], app_name: Optional[str]): """ Generates server stubs """ @@ -93,12 +93,12 @@ def generate_servers(root_path: pathlib.Path, should_generate: Callable[[str], b continue application_root = openapi_file.parent.parent - app_name = application_root.name + current_app_name = application_root.name - if server and server != app_name: + if app_name and current_app_name != app_name: continue - genapi_path = application_root / 'api' / 'genapi.sh' + genapi_path = application_root/'api'/'genapi.sh' if genapi_path.exists(): # fastapi server --> use the genapi.sh script generate_fastapi_server(application_root) @@ -114,14 +114,14 @@ def generate_clients(root_path: pathlib.Path, should_generate: Callable[[str], b return openapi_files = get_openapi_file_paths(root_path) - client_src_path = root_path / 'libraries' / 'client' / client_lib_name + client_src_path = root_path/'libraries'/'client'/client_lib_name for openapi_file in openapi_files: app_name = openapi_file.parent.parent.name generate_python_client(app_name, openapi_file, client_src_path, lib_name=client_lib_name) - frontend_path = openapi_file.parent.parent / 'frontend' + frontend_path = openapi_file.parent.parent/'frontend' if frontend_path.exists(): generate_ts_client(openapi_file) From 7e194ac4ea7e8db96aee02a5155076d00483a0ef Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Fri, 4 Oct 2024 15:56:49 +0100 Subject: [PATCH 14/93] CH-151 Refactor to add in a specific generate_models method for other upcoming changes --- tools/deployment-cli-tools/harness-generate | 24 ++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/tools/deployment-cli-tools/harness-generate b/tools/deployment-cli-tools/harness-generate index f4388897..003464ac 100644 --- a/tools/deployment-cli-tools/harness-generate +++ b/tools/deployment-cli-tools/harness-generate @@ -34,12 +34,11 @@ def main(): path_arg = pathlib.Path(args.path) root_path = path_arg if path_arg.is_absolute else path_arg.absolute - library_models_path = root_path/'libraries'/'models' should_generate = should_generate_interactive if args.is_interactive else lambda _: True - if args.generate_models and library_models_path.exists() and should_generate('the main model'): - generate_model() + if args.generate_models: + generate_model(root_path, should_generate) if args.app_name or not (args.generate_clients or args.generate_models): generate_servers(root_path, should_generate, args.app_name) @@ -82,7 +81,22 @@ def should_generate_interactive(resource: str) -> bool: return user_input == 'y' -def generate_servers(root_path: pathlib.Path, should_generate: Callable[[str], bool], app_name: Optional[str]): +def generate_models(root_path: pathlib.Path, should_generate: Callable[[str], bool]) -> None: + """ + Generates the main model + """ + library_models_path = root_path/'libraries'/'models' + + if not library_models_path.exists(): + return + + if not should_generate('the main model'): + return + + generate_model() + + +def generate_servers(root_path: pathlib.Path, should_generate: Callable[[str], bool], app_name: Optional[str]) -> None: """ Generates server stubs """ @@ -106,7 +120,7 @@ def generate_servers(root_path: pathlib.Path, should_generate: Callable[[str], b generate_server(application_root) -def generate_clients(root_path: pathlib.Path, should_generate: Callable[[str], bool], client_lib_name: str): +def generate_clients(root_path: pathlib.Path, should_generate: Callable[[str], bool], client_lib_name: str) -> None: """ Generates client stubs """ From e37927305a4b4bd275ec1845841b7c77e0100ef4 Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Fri, 4 Oct 2024 18:05:12 +0100 Subject: [PATCH 15/93] CH-151 Update the harness-generate command line arguments to allow for more flexibility of options --- .../ch_cli_tools/openapi.py | 23 ++++--- tools/deployment-cli-tools/harness-generate | 67 +++++++++++-------- 2 files changed, 54 insertions(+), 36 deletions(-) diff --git a/tools/deployment-cli-tools/ch_cli_tools/openapi.py b/tools/deployment-cli-tools/ch_cli_tools/openapi.py index 70bdf0b8..15945221 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/openapi.py +++ b/tools/deployment-cli-tools/ch_cli_tools/openapi.py @@ -45,22 +45,27 @@ def generate_server(app_path: pathlib.Path, overrides_folder: Optional[str]=None def generate_fastapi_server(app_path: pathlib.Path) -> None: - api_path = app_path/'api' - tmp_app_path = api_path/'app' + api_directory = app_path/'api' + backend_directory = app_path/'backend' + temp_directory = api_directory/'app' command = [ 'fastapi-codegen', - '--input', api_path/'openapi.yaml', - '--output', tmp_app_path, - '-t', api_path/'templates', + '--input', api_directory/'openapi.yaml', + '--output', temp_directory, + '-t', api_directory/'templates', ] subprocess.run(command) - backend_path = app_path/'backend' - shutil.move(tmp_app_path/'main.py', backend_path) - shutil.move(tmp_app_path/'models.py', backend_path/'openapi') + source_main = temp_directory/'main.py' + destination_main = backend_directory/'main.py' + source_main.replace(destination_main) + + source_models = temp_directory/'models.py' + destination_models = backend_directory/'openapi'/'models.py' + source_models.replace(destination_models) - tmp_app_path.rmdir() + temp_directory.rmdir() logging.info('Generated new models and main.py') diff --git a/tools/deployment-cli-tools/harness-generate b/tools/deployment-cli-tools/harness-generate index 003464ac..4d3df727 100644 --- a/tools/deployment-cli-tools/harness-generate +++ b/tools/deployment-cli-tools/harness-generate @@ -21,11 +21,13 @@ ROOT = os.path.dirname(HERE) @dataclass(frozen=True) class CommandLineArguments: path: pathlib.Path - client_name: str app_name: Optional[str] - generate_clients: bool - generate_models: bool is_interactive: bool + generate_clients: bool = False + generate_models: bool = False + generate_servers: bool = False + client_name: Optional[str] = None + client_types: ClientType = ClientType.all() def main(): @@ -33,50 +35,61 @@ def main(): get_dependencies() path_arg = pathlib.Path(args.path) - root_path = path_arg if path_arg.is_absolute else path_arg.absolute + root_path = path_arg if path_arg.is_absolute() else path_arg.absolute() should_generate = should_generate_interactive if args.is_interactive else lambda _: True if args.generate_models: - generate_model(root_path, should_generate) + generate_models(root_path, should_generate) - if args.app_name or not (args.generate_clients or args.generate_models): + if args.generate_servers: generate_servers(root_path, should_generate, args.app_name) - if args.generate_clients or not (args.app_name or args.generate_models): + if args.generate_clients: + assert args.client_name is not None generate_clients(root_path, should_generate, args.client_name) def get_command_line_arguments() -> CommandLineArguments: parser = argparse.ArgumentParser(description='Walk filesystem inside ./applications create application scaffolding.') - parser.add_argument('path', metavar='path', default=ROOT, type=str, - help='Base path of the application.') - parser.add_argument('-cn', '--client-name', dest='client_name', action="store", default=LIB_NAME, - help='Specify image registry prefix') - parser.add_argument('-i', '--interactive', dest='interactive', action="store_true", + common_arguments = argparse.ArgumentParser(add_help=False) + common_arguments.add_argument('path', metavar='path', default=ROOT, type=str, + help='Base path of the application.') + common_arguments.add_argument('-i', '--interactive', dest='is_interactive', action="store_true", help='Asks before generate') - parser.add_argument('-a', '--app_name', dest='app_name', action="store", default=None, + common_arguments.add_argument('-a', '--app_name', dest='app_name', action="store", default=None, help='Generate only for a specific application') - parser.add_argument('-c', '--clients', dest='clients', action="store_true", - help='Generate only client libraries') - parser.add_argument('-m', '--models', dest='models', action="store_true", - help='Generate only model library') - args, _ = parser.parse_known_args(sys.argv[1:]) + clients_arguments = argparse.ArgumentParser(add_help=False) + clients_arguments.add_argument('-cn', '--client-name', dest='client_name', action='store', default=LIB_NAME, + help='specify image registry prefix') - return CommandLineArguments( - path=args.path, - client_name=args.client_name, - app_name=args.app_name, - generate_clients=args.clients, - generate_models=args.models, - is_interactive=args.interactive, - ) + subparsers = parser.add_subparsers(title='generation modes', required=True) + + all_parser = subparsers.add_parser('all', parents=[common_arguments, clients_arguments], + help='Generate models, server stubs and client libraries') + all_parser.set_defaults(generate_models=True, generate_servers=True, generate_clients=True) + + models_parser = subparsers.add_parser('models', parents=[common_arguments], + help='Generate only model library') + models_parser.set_defaults(generate_models=True) + + servers_parser = subparsers.add_parser('servers', parents=[common_arguments], + help='Generate only server stubs') + servers_parser.set_defaults(generate_servers=True) + + clients_parser = subparsers.add_parser('clients', parents=[common_arguments, clients_arguments], + help='Generate only client libraries') + clients_parser.set_defaults(generate_clients=True) + + args = parser.parse_args() + + return CommandLineArguments(**args.__dict__) def should_generate_interactive(resource: str) -> bool: - user_input = input(f'Do you want to generate {resource}? [Y/n]').casefold() + user_input = input(f'Do you want to generate {resource}? [Y/n] ').casefold() return user_input == 'y' From e4ef0aeb9da4b55a05cbef0923b6071046c23d66 Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Mon, 7 Oct 2024 10:59:41 +0100 Subject: [PATCH 16/93] CH-151 Add flags to control which client types are generated by harness-generate --- tools/deployment-cli-tools/harness-generate | 27 ++++++++++++++++++--- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/tools/deployment-cli-tools/harness-generate b/tools/deployment-cli-tools/harness-generate index 4d3df727..21f7569c 100644 --- a/tools/deployment-cli-tools/harness-generate +++ b/tools/deployment-cli-tools/harness-generate @@ -2,7 +2,10 @@ import argparse from dataclasses import dataclass +import enum +import functools import glob +import operator import os import pathlib import shutil @@ -18,6 +21,15 @@ HERE = os.path.dirname(os.path.realpath(__file__)) ROOT = os.path.dirname(HERE) +class ClientType(enum.Flag): + TS_CLIENT = enum.auto() + PYTHON_CLIENT = enum.auto() + + @classmethod + def all(cls): + return functools.reduce(operator.or_, cls) + + @dataclass(frozen=True) class CommandLineArguments: path: pathlib.Path @@ -47,7 +59,7 @@ def main(): if args.generate_clients: assert args.client_name is not None - generate_clients(root_path, should_generate, args.client_name) + generate_clients(root_path, should_generate, args.client_name, args.client_types) def get_command_line_arguments() -> CommandLineArguments: @@ -64,6 +76,12 @@ def get_command_line_arguments() -> CommandLineArguments: clients_arguments = argparse.ArgumentParser(add_help=False) clients_arguments.add_argument('-cn', '--client-name', dest='client_name', action='store', default=LIB_NAME, help='specify image registry prefix') + client_type_group = clients_arguments.add_mutually_exclusive_group(required=False) + client_type_group.add_argument('-t', '--ts-only', dest='client_types', action='store_const', const=ClientType.TS_CLIENT, + help='Generate only typescript clients') + client_type_group.add_argument('-p', '--python-only', dest='client_types', action='store_const', const=ClientType.PYTHON_CLIENT, + help='Generate only python clients') + clients_arguments.set_defaults(client_types=ClientType.all()) subparsers = parser.add_subparsers(title='generation modes', required=True) @@ -133,7 +151,7 @@ def generate_servers(root_path: pathlib.Path, should_generate: Callable[[str], b generate_server(application_root) -def generate_clients(root_path: pathlib.Path, should_generate: Callable[[str], bool], client_lib_name: str) -> None: +def generate_clients(root_path: pathlib.Path, should_generate: Callable[[str], bool], client_lib_name: str, client_types: ClientType) -> None: """ Generates client stubs """ @@ -146,10 +164,11 @@ def generate_clients(root_path: pathlib.Path, should_generate: Callable[[str], b for openapi_file in openapi_files: app_name = openapi_file.parent.parent.name - generate_python_client(app_name, openapi_file, client_src_path, lib_name=client_lib_name) + if ClientType.PYTHON_CLIENT in client_types: + generate_python_client(app_name, openapi_file, client_src_path, lib_name=client_lib_name) frontend_path = openapi_file.parent.parent/'frontend' - if frontend_path.exists(): + if frontend_path.exists() and ClientType.TS_CLIENT in client_types: generate_ts_client(openapi_file) aggregate_packages(str(client_src_path), client_lib_name) From da8556c71fcf9d684cbcc5cb9a5ec783697261eb Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Mon, 7 Oct 2024 12:09:31 +0100 Subject: [PATCH 17/93] CH-151 Remove dependency on os.path in harness-generate --- .../ch_cli_tools/openapi.py | 2 +- .../ch_cli_tools/utils.py | 47 +++-- .../deployment-cli-tools/harness-application | 10 +- tools/deployment-cli-tools/harness-generate | 178 +++++++++--------- 4 files changed, 117 insertions(+), 120 deletions(-) diff --git a/tools/deployment-cli-tools/ch_cli_tools/openapi.py b/tools/deployment-cli-tools/ch_cli_tools/openapi.py index 15945221..418e56b3 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/openapi.py +++ b/tools/deployment-cli-tools/ch_cli_tools/openapi.py @@ -21,7 +21,7 @@ OPENAPI_GEN_URL = 'https://repo1.maven.org/maven2/org/openapitools/openapi-generator-cli/7.7.0/openapi-generator-cli-7.7.0.jar' -def generate_server(app_path: pathlib.Path, overrides_folder: Optional[str]=None) -> None: +def generate_server(app_path: pathlib.Path, overrides_folder: Optional[pathlib.Path]=None) -> None: get_dependencies() openapi_directory = app_path/'api' diff --git a/tools/deployment-cli-tools/ch_cli_tools/utils.py b/tools/deployment-cli-tools/ch_cli_tools/utils.py index f34c3b8b..141f141d 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/utils.py +++ b/tools/deployment-cli-tools/ch_cli_tools/utils.py @@ -4,7 +4,7 @@ import socket import glob import subprocess -from typing import Any +from typing import Any, Union import requests import os from functools import cache @@ -208,29 +208,28 @@ def replace_value(value: Any) -> Any: } -def copymergedir(root_src_dir, root_dst_dir): +def copymergedir(source_root_directory: pathlib.Path, destination_root_directory: pathlib.Path) -> None: """ Does copy and merge (shutil.copytree requires that the destination does not exist) - :param root_src_dir: - :param root_dst_dir: + :param source_root_directory: + :param destination_root_directory: :return: """ - logging.info('Copying directory %s to %s', root_src_dir, root_dst_dir) - for src_dir, dirs, files in os.walk(root_src_dir): + logging.info(f'Copying directory {source_root_directory} to {destination_root_directory}') + + for source_directory, _, files in source_root_directory.walk(): + + destination_directory = destination_root_directory/source_directory.relative_to(source_root_directory) + destination_directory.mkdir(parents=True, exist_ok=True) + + for file in files: + source_file = source_directory/file + destination_file = destination_directory/file - dst_dir = src_dir.replace(root_src_dir, root_dst_dir, 1) - if not exists(dst_dir): - os.makedirs(dst_dir) - for file_ in files: - src_file = join(src_dir, file_) - dst_file = join(dst_dir, file_) - if exists(dst_file): - os.remove(dst_file) try: - shutil.copy(src_file, dst_dir) + source_file.replace(destination_file) except: - logging.warning("Error copying file %s to %s.", - src_file, dst_dir) + logging.warning(f'Error copying file {source_file} to {destination_file}.') def movedircontent(root_src_dir, root_dst_dir): @@ -259,11 +258,11 @@ def movedircontent(root_src_dir, root_dst_dir): shutil.rmtree(root_src_dir) -def merge_configuration_directories(source: str, destination: str) -> None: - if source == destination: - return - +def merge_configuration_directories(source: Union[str, pathlib.Path], destination: Union[str, pathlib.Path]) -> None: source_path, destination_path = pathlib.Path(source), pathlib.Path(destination) + + if source_path == destination_path: + return if not source_path.exists(): logging.warning("Trying to merge the not existing directory: %s", source) @@ -286,14 +285,14 @@ def _merge_configuration_directory( if any(path in str(source_directory) for path in EXCLUDE_PATHS): return - destination_directory = destination / source_directory.relative_to(source) + destination_directory = destination/source_directory.relative_to(source) destination_directory.mkdir(exist_ok=True) non_build_files = (file for file in files if file not in BUILD_FILENAMES) for file_name in non_build_files: - source_file_path = source_directory / file_name - destination_file_path = destination_directory / file_name + source_file_path = source_directory/file_name + destination_file_path = destination_directory/file_name _merge_configuration_file(source_file_path, destination_file_path) diff --git a/tools/deployment-cli-tools/harness-application b/tools/deployment-cli-tools/harness-application index b2064ccf..84c26755 100644 --- a/tools/deployment-cli-tools/harness-application +++ b/tools/deployment-cli-tools/harness-application @@ -41,7 +41,7 @@ class TemplateType(StrEnum): def main(): app_name, templates = get_command_line_arguments() - app_path = pathlib.Path(APPLICATIONS_SRC_PATH) / app_name + app_path = pathlib.Path(APPLICATIONS_SRC_PATH)/app_name app_path.mkdir(exist_ok=True) if TemplateType.DJANGO_APP in templates and TemplateType.WEBAPP not in templates: @@ -143,10 +143,12 @@ def handle_webapp_template_cleanup(app_path: pathlib.Path) -> None: def handle_server_template(app_path: pathlib.Path) -> None: with tempfile.TemporaryDirectory() as tmp_dirname: + tmp_path = pathlib.Path(tmp_dirname) server_template_path = pathlib.Path(CH_ROOT)/APPLICATION_TEMPLATE_PATH/TemplateType.SERVER - copymergedir(str(server_template_path), tmp_dirname) - merge_configuration_directories(app_path, tmp_dirname) - generate_server(app_path, tmp_dirname) + + copymergedir(server_template_path, tmp_path) + merge_configuration_directories(app_path, tmp_path) + generate_server(app_path, tmp_path) def handle_flask_server_template(app_path: pathlib.Path) -> None: diff --git a/tools/deployment-cli-tools/harness-generate b/tools/deployment-cli-tools/harness-generate index 21f7569c..9f2778dc 100644 --- a/tools/deployment-cli-tools/harness-generate +++ b/tools/deployment-cli-tools/harness-generate @@ -4,12 +4,9 @@ import argparse from dataclasses import dataclass import enum import functools -import glob import operator -import os import pathlib import shutil -import sys import logging from typing import Callable, Optional @@ -17,9 +14,6 @@ from ch_cli_tools.openapi import LIB_NAME, generate_python_client, generate_serv get_dependencies, generate_ts_client, generate_model from ch_cli_tools.utils import copymergedir -HERE = os.path.dirname(os.path.realpath(__file__)) -ROOT = os.path.dirname(HERE) - class ClientType(enum.Flag): TS_CLIENT = enum.auto() @@ -46,8 +40,7 @@ def main(): args = get_command_line_arguments() get_dependencies() - path_arg = pathlib.Path(args.path) - root_path = path_arg if path_arg.is_absolute() else path_arg.absolute() + root_path = args.path if args.path.is_absolute() else args.path.absolute() should_generate = should_generate_interactive if args.is_interactive else lambda _: True @@ -59,14 +52,14 @@ def main(): if args.generate_clients: assert args.client_name is not None - generate_clients(root_path, should_generate, args.client_name, args.client_types) + generate_clients(root_path, should_generate, args.app_name, args.client_name, args.client_types) def get_command_line_arguments() -> CommandLineArguments: parser = argparse.ArgumentParser(description='Walk filesystem inside ./applications create application scaffolding.') common_arguments = argparse.ArgumentParser(add_help=False) - common_arguments.add_argument('path', metavar='path', default=ROOT, type=str, + common_arguments.add_argument('path', metavar='path', nargs='?', default=pathlib.Path.cwd(), type=pathlib.Path, help='Base path of the application.') common_arguments.add_argument('-i', '--interactive', dest='is_interactive', action="store_true", help='Asks before generate') @@ -112,7 +105,10 @@ def should_generate_interactive(resource: str) -> bool: return user_input == 'y' -def generate_models(root_path: pathlib.Path, should_generate: Callable[[str], bool]) -> None: +def generate_models( + root_path: pathlib.Path, + should_generate: Callable[[str], bool], +) -> None: """ Generates the main model """ @@ -127,22 +123,26 @@ def generate_models(root_path: pathlib.Path, should_generate: Callable[[str], bo generate_model() -def generate_servers(root_path: pathlib.Path, should_generate: Callable[[str], bool], app_name: Optional[str]) -> None: +def generate_servers( + root_path: pathlib.Path, + should_generate: Callable[[str], bool], + app_name: Optional[str], +) -> None: """ Generates server stubs """ openapi_files = get_openapi_file_paths(root_path) for openapi_file in openapi_files: - if not should_generate(openapi_file): - continue - application_root = openapi_file.parent.parent current_app_name = application_root.name if app_name and current_app_name != app_name: continue + if not should_generate(f'server stubs for {openapi_file}'): + continue + genapi_path = application_root/'api'/'genapi.sh' if genapi_path.exists(): # fastapi server --> use the genapi.sh script @@ -151,7 +151,13 @@ def generate_servers(root_path: pathlib.Path, should_generate: Callable[[str], b generate_server(application_root) -def generate_clients(root_path: pathlib.Path, should_generate: Callable[[str], bool], client_lib_name: str, client_types: ClientType) -> None: +def generate_clients( + root_path: pathlib.Path, + should_generate: Callable[[str], bool], + app_name: Optional[str], + client_lib_name: str, + client_types: ClientType, +) -> None: """ Generates client stubs """ @@ -162,99 +168,89 @@ def generate_clients(root_path: pathlib.Path, should_generate: Callable[[str], b client_src_path = root_path/'libraries'/'client'/client_lib_name for openapi_file in openapi_files: - app_name = openapi_file.parent.parent.name + current_app_name = openapi_file.parent.parent.name + if app_name and current_app_name != app_name: + continue if ClientType.PYTHON_CLIENT in client_types: - generate_python_client(app_name, openapi_file, client_src_path, lib_name=client_lib_name) + generate_python_client(current_app_name, openapi_file, client_src_path, lib_name=client_lib_name) frontend_path = openapi_file.parent.parent/'frontend' if frontend_path.exists() and ClientType.TS_CLIENT in client_types: generate_ts_client(openapi_file) - aggregate_packages(str(client_src_path), client_lib_name) + aggregate_packages(client_src_path, client_lib_name) def get_openapi_file_paths(root_path: pathlib.Path) -> list[pathlib.Path]: return [path for path in root_path.glob('applications/*/api/*.yaml')] -def aggregate_packages(client_src_path, lib_name=LIB_NAME): - DOCS_PATH = os.path.join(client_src_path, 'docs') - TEST_PATH = os.path.join(client_src_path, 'test') - README = os.path.join(client_src_path, 'README.md') - REQUIREMENTS = os.path.join(client_src_path, 'requirements.txt') - TEST_REQUIREMENTS = os.path.join(client_src_path, 'test-requirements.txt') - - if not os.path.exists(DOCS_PATH): - os.makedirs(DOCS_PATH) - if not os.path.exists(TEST_PATH): - os.makedirs(TEST_PATH) - if os.path.exists(README): - os.remove(README) - if os.path.exists(REQUIREMENTS): - os.remove(REQUIREMENTS) - if os.path.exists(TEST_REQUIREMENTS): - os.remove(TEST_REQUIREMENTS) - - req_lines_seen = set() - test_req_lines_seen = set() - - for MODULE_TMP_PATH in glob.glob(client_src_path + '/tmp-*'): - module = MODULE_TMP_PATH.split( - f'{lib_name}/tmp-')[-1].replace('-', '_') - - # Moves package - - code_dest_dir = os.path.join(client_src_path, lib_name, module) - copymergedir(os.path.join(MODULE_TMP_PATH, - lib_name, module), code_dest_dir) - copymergedir(f"{MODULE_TMP_PATH}/{lib_name}.{module}", - code_dest_dir) # Fixes a a bug with nested packages - - # Adds Docs - module_doc_path = os.path.join(DOCS_PATH, module) - if not os.path.exists(module_doc_path): - os.mkdir(module_doc_path) - copymergedir(f"{client_src_path}/tmp-{module}/docs", module_doc_path) - - # Adds Tests - module_test_path = os.path.join(client_src_path, 'test', module) - copymergedir(os.path.join(MODULE_TMP_PATH, 'test'), module_test_path) - - # Merges Readme - readme_file = f"{MODULE_TMP_PATH}/README.md" - if not os.path.exists(readme_file): - logging.warning("Readme file not found: %s.", readme_file) +def aggregate_packages(client_source_path: pathlib.Path, lib_name=LIB_NAME): + client_source_path.mkdir(parents=True, exist_ok=True) + + client_docs_path = client_source_path/'docs' + client_docs_path.mkdir(exist_ok=True) + + client_test_path = client_source_path/'test' + client_test_path.mkdir(exist_ok=True) + + client_readme_file = client_source_path/'README.md' + client_readme_file.unlink(missing_ok=True) + + client_requirements_file = client_source_path/'requirements.txt' + client_requirements_file.unlink(missing_ok=True) + + client_test_requirements_file = client_source_path/'test-requirements.txt' + client_test_requirements_file.unlink(missing_ok=True) + + requirements_lines_seen = set() + test_requirements_lines_seen = set() + + for temp_module_path in client_source_path.glob('tmp-*/'): + module = ( + temp_module_path + .name + .removeprefix('tmp-') + .replace('-', '_') + ) + + code_destination_directory = client_source_path/lib_name/module + copymergedir(temp_module_path/lib_name/module, code_destination_directory) + copymergedir(temp_module_path/f'{lib_name}.{module}', code_destination_directory) # Fixes a bug with nested packages + + module_docs_path = client_docs_path/module + module_docs_path.mkdir(parents=True, exist_ok=True) + copymergedir(client_source_path/temp_module_path.name/'docs', module_docs_path) + + module_tests_path = client_source_path/'test'/module + copymergedir(temp_module_path/'test', module_tests_path) + + readme_file = temp_module_path/'README.md' + if not readme_file.exists(): + logging.warning(f'Readme file not found: {readme_file}.') continue - with open(README, 'a+') as outfile: - with open(readme_file) as infile: - filedata = infile.read() - fd = filedata.replace('docs/', f'docs/{module}/') - outfile.write(fd) - # Merges Requirements - # FIXME: Different package versions will remain in the output file + with client_readme_file.open('+a') as out_file, readme_file.open('r') as in_file: + file_data = in_file.read() + updated_file_data = file_data.replace('docs/', f'docs/{module}/') + out_file.write(updated_file_data) - requirements_file = f"{MODULE_TMP_PATH}/requirements.txt" - outfile = open(REQUIREMENTS, "a+") - for line in open(requirements_file, "r"): - if line not in req_lines_seen: - outfile.write(line) - req_lines_seen.add(line) - outfile.close() + # FIXME: Different package versions will remain in the output file + requirements_file = temp_module_path/'requirements.txt' + with requirements_file.open('r') as in_file, client_requirements_file.open('+a') as out_file: + unseen_lines = [line for line in in_file if line not in requirements_lines_seen] + requirements_lines_seen.update(unseen_lines) + out_file.writelines(unseen_lines) - # Merges Test Requirements # FIXME: Different package versions will remain in the output file - test_requirements_file = f"{MODULE_TMP_PATH}/test-requirements.txt" - outfile = open(TEST_REQUIREMENTS, "a+") - for line in open(test_requirements_file, "r"): - if line not in test_req_lines_seen: - outfile.write(line) - test_req_lines_seen.add(line) - outfile.close() - - # Removes Tmp Files - shutil.rmtree(MODULE_TMP_PATH) + test_requirements_file = temp_module_path/'test-requirements.txt' + with test_requirements_file.open('r') as in_file, client_test_requirements_file.open('+a') as out_file: + unseen_lines = [line for line in in_file if line not in test_requirements_lines_seen] + test_requirements_lines_seen.update(unseen_lines) + out_file.writelines(unseen_lines) + + shutil.rmtree(temp_module_path) if __name__ == "__main__": From 6bc98a51098c237eb10cdd625c7131914c160854 Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Mon, 7 Oct 2024 14:03:24 +0100 Subject: [PATCH 18/93] CH-151 Refactor harness-generate options to use flag enum for generation mode --- tools/deployment-cli-tools/harness-generate | 34 ++++++++++++++++----- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/tools/deployment-cli-tools/harness-generate b/tools/deployment-cli-tools/harness-generate index 9f2778dc..fb02bdf4 100644 --- a/tools/deployment-cli-tools/harness-generate +++ b/tools/deployment-cli-tools/harness-generate @@ -22,6 +22,16 @@ class ClientType(enum.Flag): @classmethod def all(cls): return functools.reduce(operator.or_, cls) + + +class GenerationMode(enum.Flag): + CLIENTS = enum.auto() + MODELS = enum.auto() + SERVERS = enum.auto() + + @classmethod + def all(cls): + return functools.reduce(operator.or_, cls) @dataclass(frozen=True) @@ -29,12 +39,22 @@ class CommandLineArguments: path: pathlib.Path app_name: Optional[str] is_interactive: bool - generate_clients: bool = False - generate_models: bool = False - generate_servers: bool = False + generation_mode: GenerationMode client_name: Optional[str] = None client_types: ClientType = ClientType.all() + @property + def generate_models(self): + return GenerationMode.MODELS in self.generation_mode + + @property + def generate_servers(self): + return GenerationMode.SERVERS in self.generation_mode + + @property + def generate_clients(self): + return GenerationMode.CLIENTS in self.generation_mode + def main(): args = get_command_line_arguments() @@ -80,19 +100,19 @@ def get_command_line_arguments() -> CommandLineArguments: all_parser = subparsers.add_parser('all', parents=[common_arguments, clients_arguments], help='Generate models, server stubs and client libraries') - all_parser.set_defaults(generate_models=True, generate_servers=True, generate_clients=True) + all_parser.set_defaults(generation_mode=GenerationMode.all()) models_parser = subparsers.add_parser('models', parents=[common_arguments], help='Generate only model library') - models_parser.set_defaults(generate_models=True) + models_parser.set_defaults(generation_mode=GenerationMode.MODELS) servers_parser = subparsers.add_parser('servers', parents=[common_arguments], help='Generate only server stubs') - servers_parser.set_defaults(generate_servers=True) + servers_parser.set_defaults(generation_mode=GenerationMode.SERVERS) clients_parser = subparsers.add_parser('clients', parents=[common_arguments, clients_arguments], help='Generate only client libraries') - clients_parser.set_defaults(generate_clients=True) + clients_parser.set_defaults(generation_mode=GenerationMode.CLIENTS) args = parser.parse_args() From 98eec41b4053b74a69963c9a9b858a893dc5d70d Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Mon, 7 Oct 2024 15:00:35 +0100 Subject: [PATCH 19/93] CH-151 Create .ch-manifest file when creating a harness application --- .../deployment-cli-tools/ch_cli_tools/utils.py | 10 ++++++++++ tools/deployment-cli-tools/harness-application | 17 ++++++++++++++++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/tools/deployment-cli-tools/ch_cli_tools/utils.py b/tools/deployment-cli-tools/ch_cli_tools/utils.py index 141f141d..9e67876e 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/utils.py +++ b/tools/deployment-cli-tools/ch_cli_tools/utils.py @@ -475,3 +475,13 @@ def get_git_commit_hash(path): ['git', 'rev-parse', '--short', 'HEAD'], cwd=path).decode("utf-8").strip() except: return None + + +def load_yaml(yaml_file: pathlib.Path) -> dict: + with yaml_file.open('r') as file: + return yaml.load(file) + + +def save_yaml(yaml_file: pathlib.Path, data: dict) -> None: + with yaml_file.open('w') as file: + yaml.dump(data, file) diff --git a/tools/deployment-cli-tools/harness-application b/tools/deployment-cli-tools/harness-application index 84c26755..92b639d8 100644 --- a/tools/deployment-cli-tools/harness-application +++ b/tools/deployment-cli-tools/harness-application @@ -9,11 +9,12 @@ import tempfile import subprocess import logging import argparse +from typing import Union from ch_cli_tools import CH_ROOT from cloudharness_utils.constants import APPLICATION_TEMPLATE_PATH from ch_cli_tools.openapi import generate_server, generate_fastapi_server, APPLICATIONS_SRC_PATH, generate_ts_client -from ch_cli_tools.utils import merge_configuration_directories, replaceindir, replace_in_file, \ +from ch_cli_tools.utils import merge_configuration_directories, replaceindir, replace_in_file, save_yaml, \ to_python_module, copymergedir, get_json_template, replace_in_dict try: @@ -69,6 +70,8 @@ def main(): if TemplateType.WEBAPP in templates: handle_webapp_template_cleanup(app_path) + create_manifest_file(app_path, app_name, templates) + def get_command_line_arguments() -> tuple[str, list[str]]: parser = argparse.ArgumentParser(description='Creates a new Application.') @@ -189,6 +192,18 @@ def create_django_app_vscode_debug_configuration(app_name: str): json.dump(launch_config, f, indent=2, sort_keys=True) +def create_manifest_file(app_path: pathlib.Path, app_name: str, templates: list[Union[str, TemplateType]]) -> None: + manifest_file = app_path/'.ch-manifest' + manifest_data = { + 'app-name': app_name, + 'templates': [str(template) for template in templates], + 'version': '1', + } + + logging.info('Creating manifest file') + save_yaml(manifest_file, manifest_data) + + def merge_template_directories(template_name: str, app_path: pathlib.Path) -> None: for base_path in (pathlib.Path(CH_ROOT), pathlib.Path.cwd()): template_path = base_path/APPLICATION_TEMPLATE_PATH/template_name From 8d56b89766db6e83e9f20adae8f4275e9ed3eee6 Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Mon, 7 Oct 2024 16:31:52 +0100 Subject: [PATCH 20/93] CH-151 Add cloud harness manifest support to harness-generate --- .../ch_cli_tools/common_types.py | 44 ++++++++ .../deployment-cli-tools/harness-application | 31 ++---- tools/deployment-cli-tools/harness-generate | 100 +++++++++++++++--- 3 files changed, 137 insertions(+), 38 deletions(-) create mode 100644 tools/deployment-cli-tools/ch_cli_tools/common_types.py diff --git a/tools/deployment-cli-tools/ch_cli_tools/common_types.py b/tools/deployment-cli-tools/ch_cli_tools/common_types.py new file mode 100644 index 00000000..ec03ea40 --- /dev/null +++ b/tools/deployment-cli-tools/ch_cli_tools/common_types.py @@ -0,0 +1,44 @@ +from dataclasses import dataclass +from typing import Union + + +try: + from enum import StrEnum +except ImportError: + from strenum import StrEnum + + +class TemplateType(StrEnum): + BASE = 'base' + FLASK_SERVER = 'flask-server' + WEBAPP = 'webapp' + DB_POSTGRES = 'db-postgres' + DB_NEO4J = 'db-neo4j' + DB_MONGO = 'db-mongo' + DJANGO_APP = 'django-app' + SERVER = 'server' + + +@dataclass +class CloudHarnessManifest(): + app_name: str + version: str + inferred: bool + templates: list[str] + + @classmethod + def from_dict(cls, data: dict) -> 'CloudHarnessManifest': + return cls( + app_name=data['app-name'], + version=data['version'], + inferred=data['inferred'], + templates=data['templates'], + ) + + def to_dict(self) -> dict: + return { + 'app-name': self.app_name, + 'version': self.version, + 'inferred': self.inferred, + 'templates': [str(template) for template in self.templates], + } \ No newline at end of file diff --git a/tools/deployment-cli-tools/harness-application b/tools/deployment-cli-tools/harness-application index 92b639d8..3743471c 100644 --- a/tools/deployment-cli-tools/harness-application +++ b/tools/deployment-cli-tools/harness-application @@ -16,29 +16,13 @@ from cloudharness_utils.constants import APPLICATION_TEMPLATE_PATH from ch_cli_tools.openapi import generate_server, generate_fastapi_server, APPLICATIONS_SRC_PATH, generate_ts_client from ch_cli_tools.utils import merge_configuration_directories, replaceindir, replace_in_file, save_yaml, \ to_python_module, copymergedir, get_json_template, replace_in_dict - -try: - from enum import StrEnum -except ImportError: - from strenum import StrEnum +from ch_cli_tools.common_types import CloudHarnessManifest, TemplateType # Only allow lowercased alphabetical characters separated by "-". name_pattern = re.compile("[a-z]+((-)?[a-z])?") PLACEHOLDER = '__APP_NAME__' - -class TemplateType(StrEnum): - BASE = 'base' - FLASK_SERVER = 'flask-server' - WEBAPP = 'webapp' - DB_POSTGRES = 'db-postgres' - DB_NEO4J = 'db-neo4j' - DB_MONGO = 'db-mongo' - DJANGO_APP = 'django-app' - SERVER = 'server' - - def main(): app_name, templates = get_command_line_arguments() @@ -194,14 +178,15 @@ def create_django_app_vscode_debug_configuration(app_name: str): def create_manifest_file(app_path: pathlib.Path, app_name: str, templates: list[Union[str, TemplateType]]) -> None: manifest_file = app_path/'.ch-manifest' - manifest_data = { - 'app-name': app_name, - 'templates': [str(template) for template in templates], - 'version': '1', - } + manifest = CloudHarnessManifest( + app_name=app_name, + version='1', + inferred=False, + templates=[str(template) for template in templates], + ) logging.info('Creating manifest file') - save_yaml(manifest_file, manifest_data) + save_yaml(manifest_file, manifest.to_dict()) def merge_template_directories(template_name: str, app_path: pathlib.Path) -> None: diff --git a/tools/deployment-cli-tools/harness-generate b/tools/deployment-cli-tools/harness-generate index fb02bdf4..66512698 100644 --- a/tools/deployment-cli-tools/harness-generate +++ b/tools/deployment-cli-tools/harness-generate @@ -9,10 +9,12 @@ import pathlib import shutil import logging from typing import Callable, Optional +from ruamel.yaml.error import YAMLError from ch_cli_tools.openapi import LIB_NAME, generate_python_client, generate_server, generate_fastapi_server, \ get_dependencies, generate_ts_client, generate_model -from ch_cli_tools.utils import copymergedir +from ch_cli_tools.utils import copymergedir, load_yaml, save_yaml +from ch_cli_tools.common_types import CloudHarnessManifest, TemplateType class ClientType(enum.Flag): @@ -154,21 +156,20 @@ def generate_servers( openapi_files = get_openapi_file_paths(root_path) for openapi_file in openapi_files: - application_root = openapi_file.parent.parent - current_app_name = application_root.name + app_path = openapi_file.parent.parent + manifest = get_manifest(app_path) - if app_name and current_app_name != app_name: + if app_name and manifest.app_name != app_name: continue if not should_generate(f'server stubs for {openapi_file}'): continue - genapi_path = application_root/'api'/'genapi.sh' - if genapi_path.exists(): - # fastapi server --> use the genapi.sh script - generate_fastapi_server(application_root) - else: - generate_server(application_root) + if TemplateType.DJANGO_APP in manifest.templates: + generate_fastapi_server(app_path) + + if TemplateType.FLASK_SERVER in manifest.templates: + generate_server(app_path) def generate_clients( @@ -188,15 +189,16 @@ def generate_clients( client_src_path = root_path/'libraries'/'client'/client_lib_name for openapi_file in openapi_files: - current_app_name = openapi_file.parent.parent.name - if app_name and current_app_name != app_name: + app_path = openapi_file.parent.parent + manifest = get_manifest(app_path) + + if app_name and manifest.app_name != app_name: continue if ClientType.PYTHON_CLIENT in client_types: - generate_python_client(current_app_name, openapi_file, client_src_path, lib_name=client_lib_name) + generate_python_client(manifest.app_name, openapi_file, client_src_path, lib_name=client_lib_name) - frontend_path = openapi_file.parent.parent/'frontend' - if frontend_path.exists() and ClientType.TS_CLIENT in client_types: + if TemplateType.WEBAPP in manifest.templates and ClientType.TS_CLIENT in client_types: generate_ts_client(openapi_file) aggregate_packages(client_src_path, client_lib_name) @@ -273,5 +275,73 @@ def aggregate_packages(client_source_path: pathlib.Path, lib_name=LIB_NAME): shutil.rmtree(temp_module_path) +def get_manifest(app_path: pathlib.Path) -> CloudHarnessManifest: + manifest_file = app_path/'.ch-manifest' + + try: + manifest_data = load_yaml(manifest_file) + manifest = CloudHarnessManifest.from_dict(manifest_data) + except (FileNotFoundError, YAMLError): + logging.info(f'Could not find manifest file {manifest_file}, inferring manifest from app structure...') + manifest = CloudHarnessManifest( + app_name=app_path.name, + version='1', + inferred=True, + templates=infer_templates(app_path), + ) + save_yaml(manifest_file, manifest.to_dict()) + + return manifest + + +def infer_templates(app_path: pathlib.Path) -> list[str]: + templates = [TemplateType.BASE] + + infer_webapp_template(app_path, templates) + infer_server_template(app_path, templates) + infer_database_template(app_path, templates) + + return templates + + +def infer_webapp_template(app_path: pathlib.Path, templates: list[str]) -> None: + frontend_path = app_path/'frontend' + if frontend_path.exists(): + templates.append(TemplateType.WEBAPP) + + +def infer_server_template(app_path: pathlib.Path, templates: list[str]) -> None: + genapi_path = app_path/'api'/'genapi.sh' + + if genapi_path.exists(): + templates.append(TemplateType.DJANGO_APP) + return + + server_path = app_path/'server' + backend_path = app_path/'backend' + if server_path.exists() or backend_path.exists(): + templates.append(TemplateType.FLASK_SERVER) + + +def infer_database_template(app_path: pathlib.Path, templates: list[str]) -> None: + values_file = app_path/'deploy'/'values.yaml' + + try: + values_data = load_yaml(values_file) + database_config = values_data['harness']['database'] + if not database_config['auto']: + return + + database_type = database_config['type'] + if database_type == 'mongo': + templates.append(TemplateType.DB_MONGO) + if database_type == 'neo4j': + templates.append(TemplateType.DB_NEO4J) + if database_type == 'postgres': + templates.append(TemplateType.DB_POSTGRES) + except (FileNotFoundError, YAMLError, KeyError): + pass + + if __name__ == "__main__": main() From f689061644885d3e72c739f857a532ee9f5a6ace Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Tue, 8 Oct 2024 10:12:32 +0100 Subject: [PATCH 21/93] CH-151 Update genapi.sh to call harness-generate --- application-templates/django-app/api/genapi.sh | 6 ++---- tools/deployment-cli-tools/harness-application | 4 +++- tools/deployment-cli-tools/harness-generate | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/application-templates/django-app/api/genapi.sh b/application-templates/django-app/api/genapi.sh index aab1d30d..cc5737b4 100644 --- a/application-templates/django-app/api/genapi.sh +++ b/application-templates/django-app/api/genapi.sh @@ -1,6 +1,4 @@ #!/bin/bash -fastapi-codegen --input openapi.yaml --output app -t templates && mv app/main.py ../backend/ && mv app/models.py ../backend/openapi/ -rm -rf app - -echo Generated new models and main.py +ROOT_PATH=$(realpath "$(dirname "$BASH_SOURCE")/../../..") +harness-generate servers --app-name "__APP_NAME__" "$ROOT_PATH" \ No newline at end of file diff --git a/tools/deployment-cli-tools/harness-application b/tools/deployment-cli-tools/harness-application index 3743471c..c78b0edd 100644 --- a/tools/deployment-cli-tools/harness-application +++ b/tools/deployment-cli-tools/harness-application @@ -145,7 +145,9 @@ def handle_flask_server_template(app_path: pathlib.Path) -> None: def handle_django_app_template(app_name: str, app_path: pathlib.Path) -> None: python_app_name = to_python_module(app_name) - replace_in_file(app_path/'api'/'templates'/'main.jinja2', PLACEHOLDER, python_app_name) + api_path = app_path/'api' + replace_in_file(api_path/'templates'/'main.jinja2', PLACEHOLDER, python_app_name) + replace_in_file(api_path/'genapi.sh', PLACEHOLDER, app_name) generate_fastapi_server(app_path) replace_in_file(app_path/'deploy'/'values.yaml', f'{PLACEHOLDER}:{PLACEHOLDER}', f'{python_app_name}:{python_app_name}') diff --git a/tools/deployment-cli-tools/harness-generate b/tools/deployment-cli-tools/harness-generate index 66512698..d5ed78f6 100644 --- a/tools/deployment-cli-tools/harness-generate +++ b/tools/deployment-cli-tools/harness-generate @@ -85,7 +85,7 @@ def get_command_line_arguments() -> CommandLineArguments: help='Base path of the application.') common_arguments.add_argument('-i', '--interactive', dest='is_interactive', action="store_true", help='Asks before generate') - common_arguments.add_argument('-a', '--app_name', dest='app_name', action="store", default=None, + common_arguments.add_argument('-a', '--app-name', dest='app_name', action="store", default=None, help='Generate only for a specific application') clients_arguments = argparse.ArgumentParser(add_help=False) From 38bbc1b68578ae69bdab96818aa2a2f1dbbd4933 Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Tue, 8 Oct 2024 11:28:32 +0100 Subject: [PATCH 22/93] CH-151 Move dependency installation and client generation to after template merge --- .../deployment-cli-tools/harness-application | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/tools/deployment-cli-tools/harness-application b/tools/deployment-cli-tools/harness-application index c78b0edd..21c36e3d 100644 --- a/tools/deployment-cli-tools/harness-application +++ b/tools/deployment-cli-tools/harness-application @@ -52,7 +52,7 @@ def main(): replaceindir(app_path, PLACEHOLDER, app_name) if TemplateType.WEBAPP in templates: - handle_webapp_template_cleanup(app_path) + handle_webapp_template_post_merge(app_path) create_manifest_file(app_path, app_name, templates) @@ -99,34 +99,34 @@ def get_command_line_arguments() -> tuple[str, list[str]]: def handle_webapp_template(app_name: str, app_path: pathlib.Path) -> None: - frontend_app_path = app_path/'frontend' + frontend_path = app_path/'frontend' - if frontend_app_path.exists(): - shutil.rmtree(frontend_app_path) + if frontend_path.exists(): + shutil.rmtree(frontend_path) - create_vite_skaffold(app_name, app_path) - install_frontend_dependencies(app_path) - - shutil.move(app_path/app_name, frontend_app_path) - generate_ts_client(openapi_file=app_path/'api'/'openapi.yaml') + create_vite_skaffold(app_name, app_path, frontend_path) -def create_vite_skaffold(app_name: str, app_path: pathlib.Path) -> None: +def create_vite_skaffold(app_name: str, app_path: pathlib.Path, frontend_path: pathlib.Path) -> None: command = ['yarn', 'create', 'vite', app_name, '--template', 'react-ts'] logging.info(f'Running command: {" ".join(command)}') subprocess.run(command, cwd=app_path) + shutil.move(app_path/app_name, frontend_path) -def install_frontend_dependencies(app_path: pathlib.Path) -> None: +def install_frontend_dependencies(frontend_path: pathlib.Path) -> None: command = ['yarn', 'install'] logging.info(f'Running command: {" ".join(command)}') - subprocess.run(command, cwd=app_path) + subprocess.run(command, cwd=frontend_path) -def handle_webapp_template_cleanup(app_path: pathlib.Path) -> None: +def handle_webapp_template_post_merge(app_path: pathlib.Path) -> None: backend_dockerfile_path = app_path/'backend'/'Dockerfile' backend_dockerfile_path.unlink(missing_ok=True) + install_frontend_dependencies(app_path/'frontend') + generate_ts_client(openapi_file=app_path/'api'/'openapi.yaml') + def handle_server_template(app_path: pathlib.Path) -> None: with tempfile.TemporaryDirectory() as tmp_dirname: From 90f84548d0d4fdb53adfb5a5e19ecf483a5d8458 Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Tue, 8 Oct 2024 11:28:59 +0100 Subject: [PATCH 23/93] CH-151 Call harness-generate after application harnessing --- tools/deployment-cli-tools/harness-application | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tools/deployment-cli-tools/harness-application b/tools/deployment-cli-tools/harness-application index 21c36e3d..77b8a0d4 100644 --- a/tools/deployment-cli-tools/harness-application +++ b/tools/deployment-cli-tools/harness-application @@ -55,6 +55,7 @@ def main(): handle_webapp_template_post_merge(app_path) create_manifest_file(app_path, app_name, templates) + call_harness_generate(app_path, app_name) def get_command_line_arguments() -> tuple[str, list[str]]: @@ -191,6 +192,13 @@ def create_manifest_file(app_path: pathlib.Path, app_name: str, templates: list[ save_yaml(manifest_file, manifest.to_dict()) +def call_harness_generate(app_path: pathlib.Path, app_name: str): + logging.info('Running initial harness generate...') + root_path = app_path.parent.parent + command = ['harness-generate', 'all', '--ts-only', '--app-name', app_name, root_path] + subprocess.run(command) + + def merge_template_directories(template_name: str, app_path: pathlib.Path) -> None: for base_path in (pathlib.Path(CH_ROOT), pathlib.Path.cwd()): template_path = base_path/APPLICATION_TEMPLATE_PATH/template_name From f5725a5ba097e3f6f2e9867d39ab98a15d7fbec9 Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Tue, 8 Oct 2024 12:30:32 +0100 Subject: [PATCH 24/93] CH-151 Refactor to clean up some loose ends --- .../ch_cli_tools/common_types.py | 3 ++ .../deployment-cli-tools/harness-application | 20 ++++++++-- tools/deployment-cli-tools/harness-generate | 37 +++++++++---------- 3 files changed, 38 insertions(+), 22 deletions(-) diff --git a/tools/deployment-cli-tools/ch_cli_tools/common_types.py b/tools/deployment-cli-tools/ch_cli_tools/common_types.py index ec03ea40..d724be9e 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/common_types.py +++ b/tools/deployment-cli-tools/ch_cli_tools/common_types.py @@ -18,6 +18,9 @@ class TemplateType(StrEnum): DJANGO_APP = 'django-app' SERVER = 'server' + def database_templates(self): + return [self.DB_POSTGRES, self.DB_NEO4J, self.DB_MONGO] + @dataclass class CloudHarnessManifest(): diff --git a/tools/deployment-cli-tools/harness-application b/tools/deployment-cli-tools/harness-application index 77b8a0d4..897f2465 100644 --- a/tools/deployment-cli-tools/harness-application +++ b/tools/deployment-cli-tools/harness-application @@ -23,14 +23,13 @@ name_pattern = re.compile("[a-z]+((-)?[a-z])?") PLACEHOLDER = '__APP_NAME__' -def main(): +def main() -> None: app_name, templates = get_command_line_arguments() app_path = pathlib.Path(APPLICATIONS_SRC_PATH)/app_name app_path.mkdir(exist_ok=True) - if TemplateType.DJANGO_APP in templates and TemplateType.WEBAPP not in templates: - templates = [TemplateType.BASE, TemplateType.WEBAPP, TemplateType.DB_POSTGRES] + templates + templates = normalize_templates(templates) if TemplateType.WEBAPP in templates: handle_webapp_template(app_name, app_path) @@ -99,6 +98,21 @@ def get_command_line_arguments() -> tuple[str, list[str]]: return args.name, args.templates +def normalize_templates(templates: list[str]) -> list[str]: + normalized_templates = list(templates) + + if TemplateType.DJANGO_APP in normalized_templates and TemplateType.WEBAPP not in normalized_templates: + django_app_index = normalized_templates.index(TemplateType.DJANGO_APP) + normalized_templates.insert(django_app_index, TemplateType.WEBAPP) + + has_database_template = any(template in TemplateType.database_templates() for template in normalized_templates) + if TemplateType.DJANGO_APP in normalize_templates and not has_database_template: + django_app_index = normalized_templates.index(TemplateType.DJANGO_APP) + normalized_templates.insert(django_app_index, TemplateType.DB_POSTGRES) + + return normalized_templates + + def handle_webapp_template(app_name: str, app_path: pathlib.Path) -> None: frontend_path = app_path/'frontend' diff --git a/tools/deployment-cli-tools/harness-generate b/tools/deployment-cli-tools/harness-generate index d5ed78f6..00270839 100644 --- a/tools/deployment-cli-tools/harness-generate +++ b/tools/deployment-cli-tools/harness-generate @@ -17,6 +17,24 @@ from ch_cli_tools.utils import copymergedir, load_yaml, save_yaml from ch_cli_tools.common_types import CloudHarnessManifest, TemplateType +def main(): + args = get_command_line_arguments() + get_dependencies() + + root_path = args.path.absolute() + should_generate = should_generate_interactive if args.is_interactive else lambda _: True + + if args.generate_models: + generate_models(root_path, should_generate) + + if args.generate_servers: + generate_servers(root_path, should_generate, args.app_name) + + if args.generate_clients: + assert args.client_name is not None + generate_clients(root_path, should_generate, args.app_name, args.client_name, args.client_types) + + class ClientType(enum.Flag): TS_CLIENT = enum.auto() PYTHON_CLIENT = enum.auto() @@ -58,25 +76,6 @@ class CommandLineArguments: return GenerationMode.CLIENTS in self.generation_mode -def main(): - args = get_command_line_arguments() - get_dependencies() - - root_path = args.path if args.path.is_absolute() else args.path.absolute() - - should_generate = should_generate_interactive if args.is_interactive else lambda _: True - - if args.generate_models: - generate_models(root_path, should_generate) - - if args.generate_servers: - generate_servers(root_path, should_generate, args.app_name) - - if args.generate_clients: - assert args.client_name is not None - generate_clients(root_path, should_generate, args.app_name, args.client_name, args.client_types) - - def get_command_line_arguments() -> CommandLineArguments: parser = argparse.ArgumentParser(description='Walk filesystem inside ./applications create application scaffolding.') From 8da4f16c0281d5ee6245f9d4109709581cd009db Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Tue, 8 Oct 2024 12:30:53 +0100 Subject: [PATCH 25/93] CH-151 Update documentation --- docs/dev.md | 107 +++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 76 insertions(+), 31 deletions(-) diff --git a/docs/dev.md b/docs/dev.md index 28081862..cdd91c98 100644 --- a/docs/dev.md +++ b/docs/dev.md @@ -80,6 +80,7 @@ The code is organized around the idea that there is a module by artifact that ca deployment-cli-tools ├── ch_cli_tools │   ├── codefresh.py # Code Fresh configuration generation +│ ├── common_types.py # Commmon classes needed across multiple scripts/modules │   ├── helm.py # Helm chart files generation │   ├── __init__.py # Defines logging level and some global constants │   ├── models.py # Currently empty file @@ -106,23 +107,59 @@ First the skeleton of the application is generated (the directories, basic files The following code fragment from the `harness-application` script shows how the skeleton is produced: ```python -if "django-app" in args.templates and "webapp" not in templates: - templates = ["base", "webapp"] + templates +def main(): + # ... + + templates = normalize_templates(templates) + + if TemplateType.WEBAPP in templates: + handle_webapp_template(app_name, app_path) + + if TemplateType.SERVER in templates: + handle_server_template(app_path) + for template_name in templates: - if template_name == 'server': - with tempfile.TemporaryDirectory() as tmp_dirname: - copymergedir(os.path.join(CH_ROOT, APPLICATION_TEMPLATE_PATH, template_name), tmp_dirname) # <1> - merge_configuration_directories(app_path, tmp_dirname) - generate_server(app_path, tmp_dirname) - for base_path in (CH_ROOT, os.getcwd()): - template_path = os.path.join(base_path, APPLICATION_TEMPLATE_PATH, template_name) - if os.path.exists(template_path): - merge_configuration_directories(template_path, app_path) # <1> + merge_template_directories(template_name, app_path) + +# ... + +def normalize_templates(templates): + normalized_templates = list(templates) + + if TemplateType.DJANGO_APP in normalized_templates and TemplateType.WEBAPP not in normalized_templates: + django_app_index = normalized_templates.index(TemplateType.DJANGO_APP) + normalized_templates.insert(django_app_index, TemplateType.WEBAPP) + + has_database_template = any(template in TemplateType.database_templates() for template in normalized_templates) + if TemplateType.DJANGO_APP in normalize_templates and not has_database_template: + django_app_index = normalized_templates.index(TemplateType.DJANGO_APP) + normalized_templates.insert(django_app_index, TemplateType.DB_POSTGRES) + + return normalized_templates + +# ... + +def handle_server_template(app_path): + with tempfile.TemporaryDirectory() as tmp_dirname: + tmp_path = pathlib.Path(tmp_dirname) + server_template_path = pathlib.Path(CH_ROOT)/APPLICATION_TEMPLATE_PATH/TemplateType.SERVER + + copymergedir(server_template_path, tmp_path) + merge_configuration_directories(app_path, tmp_path) + generate_server(app_path, tmp_path) + +#... + +def merge_template_directories(template_name, app_path): + for base_path in (pathlib.Path(CH_ROOT), pathlib.Path.cwd()): + template_path = base_path/APPLICATION_TEMPLATE_PATH/template_name + if template_path.exists(): + merge_configuration_directories(template_path, app_path) ``` -First, if `django-app` is defined as a template for the application, and the `webapp` template is not set, then `base` and `webapp` are added to the list of templates. -Then, depending on the template name, a template directory is merged with the code of the application that will be developed (if it exists), as seen in `<1>`. -The templates for each type of application is described by the constant `APPLICATION_TEPLATE_PATH` and points to [`application-templates`](../application-templates/). +First, if `django-app` is defined as a template for the application, and the `webapp` template is not set and/or there is no database template, then `webapp` and/or `db-postgres` are added to the list of templates (using the `TemplateType` string enum). +Then, depending on the template name, a template directory is merged with the code of the application that will be developed (if it exists). +The templates for each type of application is described by the constant `APPLICATION_TEMPLATE_PATH` and points to [`application-templates`](../application-templates/). Based on the name of the template used for the application generation, the actual template with the same name is searched in this path, and copied/merged in the application target folder. The constant, as well as many other constants, are located in [`cloudharness_utils.constants`](../libraries/cloudharness-utils/cloudharness_utils/constants.py). This file is part of the CloudHarness runtime. @@ -143,12 +180,14 @@ Those constants defines several aspects of CloudHarness. For example, we can see there what base Docker image will be considered depending on what's configured for your application, where will be located the deployment files, from where the applications to generate/pick should be generated, where are located the templates for each kind of generation target, as well as where the configuration for codefresh should be looked for. Once the skeleton of the application is generated considering some templates, the code of the REST API is generated from the OpenAPI specification. -The generation relies on two functions: `generate_server` and `generate_fastapi_server` and `generate_ts_client`. +The generation relies on the functions: `generate_server` and `generate_fastapi_server` and `generate_ts_client`. Those functions are defined in the [`openapi.py`](../tools/deployment-cli-tools/ch_cli_tools/openapi.py) module. -This module and those functions use `openapi-generator-cli` to generate the code for the backend and/or the frontend. +This module and those functions use `openapi-generator-cli` and `fastapi-codegen` to generate the code for the backend and/or the frontend. With this generation, and depending on the templates used, some fine tuning or performed in the code/files generated. For example, some placeholders are replaced depending on the name of the application, or depending on the module in which the application is generated. +As final steps a `.ch-manifest` file is created in the root of the application which contains details about the app name and templates used in generation for use by [`harness-generate`](../tools/deployment-cli-tools/harness-generate) and `harness-generate` is run to ensure all server stubs and client code is in place. + #### How to extend it? Here is some scenarios that would need to modify or impact this part of CloudHarness: @@ -165,35 +204,41 @@ Here is some scenarios that would need to modify or impact this part of CloudHar ### Generation of the base application skeleton The (re-)generation REST API is obtain through the [`harness-generate`](../tools/deployment-cli-tools/harness-generate) command. -The command parses the name of the application, gets the necessary dependencies (the java OpenAPI generator cli), and generates the REST model, the servers stubs and well as the clients code from the OpenAPI specifications. +The command parses the `.ch-manifest` file (inferring and creating one if needed), gets the necessary dependencies (the java OpenAPI generator cli), and generates the REST model, the servers stubs and well as the clients code from the OpenAPI specifications. -The generation of the REST model is done by the `generate_model(...)` function, the generation of the server stub is done by the `generate_servers(...)` function, while the clients generation is done by the `generate_clients(...)` function. +The generation of the REST model is done by the `generate_model(...)` function, the generation of the server stub is done by either the `generate_servers(...)` function, while the clients generation is done by the `generate_clients(...)` function. All of these functions are located in the `harness-generate` script. Under the hood, the `generate_servers(...)` function uses the `generate_fastapi_server(...)` and the `generate_server(...)` function that are defined in the [`openapi.py`](../tools/deployment-cli-tools/ch_cli_tools/openapi.py) module. -The generation of one type of servers over another one is bound to the existence of a `genapi.sh` file: +The generation of one type of servers over another one is based on the template used for generation (if the manifest does not exist, the template is inferred by the existance/non-existance of the `genapi.sh` file): ```python -def generate_servers(root_path, interactive=False): +def generate_servers(root_path, should_generate, app_name): # ... - if os.path.exists(os.path.join(application_root, "api", "genapi.sh")): - # fastapi server --> use the genapi.sh script - generate_fastapi_server(application_root) - else: - generate_server(application_root) + for openapi_file in openapi_files: + #... + + if TemplateType.DJANGO_APP in manifest.templates: + generate_fastapi_server(app_path) + + if TemplateType.FLASK_SERVER in manifest.templates: + generate_server(app_path) ``` The `generate_clients(...)` function also uses `generate_python_client(...)` and `generate_ts_client(...)` from the [`openapi.py`](../tools/deployment-cli-tools/ch_cli_tools/openapi.py) module. -The `generate_ts_client(...)` function is called only if there is folder named `frontend` in the application directory structure: +The `generate_ts_client(...)` function is called only if the manifest templates contains `webapp` (if the manifest does not exist then the use of `webapp` is inferred by the existance/non-existance of a `frontend` directory in the application directory structure), and flags can be used to limit generation to just python or typescript clients: ```python def generate_clients(root_path, client_lib_name=LIB_NAME, interactive=False): # ... - app_dir = os.path.dirname(os.path.dirname(openapi_file)) - generate_python_client(app_name, openapi_file, - client_src_path, lib_name=client_lib_name) - if os.path.exists(os.path.join(app_dir, 'frontend')): - generate_ts_client(openapi_file) + for openapi_file in openapi_files: + #... + + if ClientType.PYTHON_CLIENT in client_types: + generate_python_client(manifest.app_name, openapi_file, client_src_path, lib_name=client_lib_name) + + if TemplateType.WEBAPP in manifest.templates and ClientType.TS_CLIENT in client_types: + generate_ts_client(openapi_file) ``` ### Generation of the application deployment files From 1756efccc95039369411d9fb1689912089edb282 Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Tue, 8 Oct 2024 12:43:48 +0100 Subject: [PATCH 26/93] CH-151 Fix linting issues --- .../ch_cli_tools/common_types.py | 2 +- .../ch_cli_tools/openapi.py | 28 +++++----- .../ch_cli_tools/utils.py | 26 +++++----- .../deployment-cli-tools/harness-application | 35 +++++++------ tools/deployment-cli-tools/harness-generate | 52 +++++++++---------- 5 files changed, 72 insertions(+), 71 deletions(-) diff --git a/tools/deployment-cli-tools/ch_cli_tools/common_types.py b/tools/deployment-cli-tools/ch_cli_tools/common_types.py index d724be9e..b52bf1ae 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/common_types.py +++ b/tools/deployment-cli-tools/ch_cli_tools/common_types.py @@ -44,4 +44,4 @@ def to_dict(self) -> dict: 'version': self.version, 'inferred': self.inferred, 'templates': [str(template) for template in self.templates], - } \ No newline at end of file + } diff --git a/tools/deployment-cli-tools/ch_cli_tools/openapi.py b/tools/deployment-cli-tools/ch_cli_tools/openapi.py index 418e56b3..7de31d8c 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/openapi.py +++ b/tools/deployment-cli-tools/ch_cli_tools/openapi.py @@ -21,14 +21,14 @@ OPENAPI_GEN_URL = 'https://repo1.maven.org/maven2/org/openapitools/openapi-generator-cli/7.7.0/openapi-generator-cli-7.7.0.jar' -def generate_server(app_path: pathlib.Path, overrides_folder: Optional[pathlib.Path]=None) -> None: +def generate_server(app_path: pathlib.Path, overrides_folder: Optional[pathlib.Path] = None) -> None: get_dependencies() - openapi_directory = app_path/'api' + openapi_directory = app_path / 'api' openapi_file = next(openapi_directory.glob('*.yaml')) - server_path = app_path/'server' - backend_path = app_path/'backend' + server_path = app_path / 'server' + backend_path = app_path / 'backend' out_path = server_path if server_path.exists() else backend_path command = [ @@ -36,7 +36,7 @@ def generate_server(app_path: pathlib.Path, overrides_folder: Optional[pathlib.P '-i', openapi_file, '-g', 'python-flask', '-o', out_path, - '-c', openapi_directory/'config.json', + '-c', openapi_directory / 'config.json', ] if overrides_folder: command += ['-t', overrides_folder] @@ -45,24 +45,24 @@ def generate_server(app_path: pathlib.Path, overrides_folder: Optional[pathlib.P def generate_fastapi_server(app_path: pathlib.Path) -> None: - api_directory = app_path/'api' - backend_directory = app_path/'backend' - temp_directory = api_directory/'app' + api_directory = app_path / 'api' + backend_directory = app_path / 'backend' + temp_directory = api_directory / 'app' command = [ 'fastapi-codegen', - '--input', api_directory/'openapi.yaml', + '--input', api_directory / 'openapi.yaml', '--output', temp_directory, - '-t', api_directory/'templates', + '-t', api_directory / 'templates', ] subprocess.run(command) - source_main = temp_directory/'main.py' - destination_main = backend_directory/'main.py' + source_main = temp_directory / 'main.py' + destination_main = backend_directory / 'main.py' source_main.replace(destination_main) - source_models = temp_directory/'models.py' - destination_models = backend_directory/'openapi'/'models.py' + source_models = temp_directory / 'models.py' + destination_models = backend_directory / 'openapi' / 'models.py' source_models.replace(destination_models) temp_directory.rmdir() diff --git a/tools/deployment-cli-tools/ch_cli_tools/utils.py b/tools/deployment-cli-tools/ch_cli_tools/utils.py index 9e67876e..3f3f5659 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/utils.py +++ b/tools/deployment-cli-tools/ch_cli_tools/utils.py @@ -176,7 +176,7 @@ def replaceindir(root_src_dir, source, replace): if not any(file_.endswith(ext) for ext in REPLACE_TEXT_FILES_EXTENSIONS): continue - src_file = pathlib.Path(src_dir)/file_ + src_file = pathlib.Path(src_dir) / file_ replace_in_file(src_file, source, replace) @@ -219,12 +219,12 @@ def copymergedir(source_root_directory: pathlib.Path, destination_root_directory for source_directory, _, files in source_root_directory.walk(): - destination_directory = destination_root_directory/source_directory.relative_to(source_root_directory) + destination_directory = destination_root_directory / source_directory.relative_to(source_root_directory) destination_directory.mkdir(parents=True, exist_ok=True) for file in files: - source_file = source_directory/file - destination_file = destination_directory/file + source_file = source_directory / file + destination_file = destination_directory / file try: source_file.replace(destination_file) @@ -263,11 +263,11 @@ def merge_configuration_directories(source: Union[str, pathlib.Path], destinatio if source_path == destination_path: return - + if not source_path.exists(): logging.warning("Trying to merge the not existing directory: %s", source) return - + if not destination_path.exists(): shutil.copytree(source_path, destination_path, ignore=shutil.ignore_patterns(*EXCLUDE_PATHS)) return @@ -284,16 +284,16 @@ def _merge_configuration_directory( ) -> None: if any(path in str(source_directory) for path in EXCLUDE_PATHS): return - - destination_directory = destination/source_directory.relative_to(source) + + destination_directory = destination / source_directory.relative_to(source) destination_directory.mkdir(exist_ok=True) non_build_files = (file for file in files if file not in BUILD_FILENAMES) for file_name in non_build_files: - source_file_path = source_directory/file_name - destination_file_path = destination_directory/file_name - + source_file_path = source_directory / file_name + destination_file_path = destination_directory / file_name + _merge_configuration_file(source_file_path, destination_file_path) @@ -301,7 +301,7 @@ def _merge_configuration_file(source_file_path: pathlib.Path, destination_file_p if not exists(destination_file_path): shutil.copy2(source_file_path, destination_file_path) return - + merge_operations = [ (file_is_yaml, merge_yaml_files), (file_is_json, merge_json_files), @@ -318,7 +318,7 @@ def _merge_configuration_file(source_file_path: pathlib.Path, destination_file_p break return - + logging.warning(f'Overwriting file {destination_file_path} with {source_file_path}') shutil.copy2(source_file_path, destination_file_path) diff --git a/tools/deployment-cli-tools/harness-application b/tools/deployment-cli-tools/harness-application index 897f2465..948f30b3 100644 --- a/tools/deployment-cli-tools/harness-application +++ b/tools/deployment-cli-tools/harness-application @@ -23,10 +23,11 @@ name_pattern = re.compile("[a-z]+((-)?[a-z])?") PLACEHOLDER = '__APP_NAME__' + def main() -> None: app_name, templates = get_command_line_arguments() - app_path = pathlib.Path(APPLICATIONS_SRC_PATH)/app_name + app_path = pathlib.Path(APPLICATIONS_SRC_PATH) / app_name app_path.mkdir(exist_ok=True) templates = normalize_templates(templates) @@ -43,7 +44,7 @@ def main() -> None: if TemplateType.FLASK_SERVER in templates: handle_flask_server_template(app_path) - replace_in_file(app_path/'api'/'config.json', PLACEHOLDER, to_python_module(app_name)) + replace_in_file(app_path / 'api' / 'config.json', PLACEHOLDER, to_python_module(app_name)) if TemplateType.DJANGO_APP in templates: handle_django_app_template(app_name, app_path) @@ -114,7 +115,7 @@ def normalize_templates(templates: list[str]) -> list[str]: def handle_webapp_template(app_name: str, app_path: pathlib.Path) -> None: - frontend_path = app_path/'frontend' + frontend_path = app_path / 'frontend' if frontend_path.exists(): shutil.rmtree(frontend_path) @@ -126,7 +127,7 @@ def create_vite_skaffold(app_name: str, app_path: pathlib.Path, frontend_path: p command = ['yarn', 'create', 'vite', app_name, '--template', 'react-ts'] logging.info(f'Running command: {" ".join(command)}') subprocess.run(command, cwd=app_path) - shutil.move(app_path/app_name, frontend_path) + shutil.move(app_path / app_name, frontend_path) def install_frontend_dependencies(frontend_path: pathlib.Path) -> None: @@ -136,17 +137,17 @@ def install_frontend_dependencies(frontend_path: pathlib.Path) -> None: def handle_webapp_template_post_merge(app_path: pathlib.Path) -> None: - backend_dockerfile_path = app_path/'backend'/'Dockerfile' + backend_dockerfile_path = app_path / 'backend' / 'Dockerfile' backend_dockerfile_path.unlink(missing_ok=True) - install_frontend_dependencies(app_path/'frontend') - generate_ts_client(openapi_file=app_path/'api'/'openapi.yaml') + install_frontend_dependencies(app_path / 'frontend') + generate_ts_client(openapi_file=app_path / 'api' / 'openapi.yaml') def handle_server_template(app_path: pathlib.Path) -> None: with tempfile.TemporaryDirectory() as tmp_dirname: tmp_path = pathlib.Path(tmp_dirname) - server_template_path = pathlib.Path(CH_ROOT)/APPLICATION_TEMPLATE_PATH/TemplateType.SERVER + server_template_path = pathlib.Path(CH_ROOT) / APPLICATION_TEMPLATE_PATH / TemplateType.SERVER copymergedir(server_template_path, tmp_path) merge_configuration_directories(app_path, tmp_path) @@ -160,16 +161,16 @@ def handle_flask_server_template(app_path: pathlib.Path) -> None: def handle_django_app_template(app_name: str, app_path: pathlib.Path) -> None: python_app_name = to_python_module(app_name) - api_path = app_path/'api' - replace_in_file(api_path/'templates'/'main.jinja2', PLACEHOLDER, python_app_name) - replace_in_file(api_path/'genapi.sh', PLACEHOLDER, app_name) + api_path = app_path / 'api' + replace_in_file(api_path / 'templates' / 'main.jinja2', PLACEHOLDER, python_app_name) + replace_in_file(api_path / 'genapi.sh', PLACEHOLDER, app_name) generate_fastapi_server(app_path) - - replace_in_file(app_path/'deploy'/'values.yaml', f'{PLACEHOLDER}:{PLACEHOLDER}', f'{python_app_name}:{python_app_name}') - replace_in_file(app_path/'dev-setup.sh', PLACEHOLDER, app_name) + + replace_in_file(app_path / 'deploy' / 'values.yaml', f'{PLACEHOLDER}:{PLACEHOLDER}', f'{python_app_name}:{python_app_name}') + replace_in_file(app_path / 'dev-setup.sh', PLACEHOLDER, app_name) create_django_app_vscode_debug_configuration(app_name) - (app_path/'backend'/'__APP_NAME__'/'__main__.py').unlink(missing_ok=True) + (app_path / 'backend' / '__APP_NAME__' / '__main__.py').unlink(missing_ok=True) def create_django_app_vscode_debug_configuration(app_name: str): @@ -194,7 +195,7 @@ def create_django_app_vscode_debug_configuration(app_name: str): def create_manifest_file(app_path: pathlib.Path, app_name: str, templates: list[Union[str, TemplateType]]) -> None: - manifest_file = app_path/'.ch-manifest' + manifest_file = app_path / '.ch-manifest' manifest = CloudHarnessManifest( app_name=app_name, version='1', @@ -215,7 +216,7 @@ def call_harness_generate(app_path: pathlib.Path, app_name: str): def merge_template_directories(template_name: str, app_path: pathlib.Path) -> None: for base_path in (pathlib.Path(CH_ROOT), pathlib.Path.cwd()): - template_path = base_path/APPLICATION_TEMPLATE_PATH/template_name + template_path = base_path / APPLICATION_TEMPLATE_PATH / template_name if template_path.exists(): merge_configuration_directories(template_path, app_path) diff --git a/tools/deployment-cli-tools/harness-generate b/tools/deployment-cli-tools/harness-generate index 00270839..c51edf25 100644 --- a/tools/deployment-cli-tools/harness-generate +++ b/tools/deployment-cli-tools/harness-generate @@ -42,7 +42,7 @@ class ClientType(enum.Flag): @classmethod def all(cls): return functools.reduce(operator.or_, cls) - + class GenerationMode(enum.Flag): CLIENTS = enum.auto() @@ -83,9 +83,9 @@ def get_command_line_arguments() -> CommandLineArguments: common_arguments.add_argument('path', metavar='path', nargs='?', default=pathlib.Path.cwd(), type=pathlib.Path, help='Base path of the application.') common_arguments.add_argument('-i', '--interactive', dest='is_interactive', action="store_true", - help='Asks before generate') + help='Asks before generate') common_arguments.add_argument('-a', '--app-name', dest='app_name', action="store", default=None, - help='Generate only for a specific application') + help='Generate only for a specific application') clients_arguments = argparse.ArgumentParser(add_help=False) clients_arguments.add_argument('-cn', '--client-name', dest='client_name', action='store', default=LIB_NAME, @@ -133,7 +133,7 @@ def generate_models( """ Generates the main model """ - library_models_path = root_path/'libraries'/'models' + library_models_path = root_path / 'libraries' / 'models' if not library_models_path.exists(): return @@ -185,7 +185,7 @@ def generate_clients( return openapi_files = get_openapi_file_paths(root_path) - client_src_path = root_path/'libraries'/'client'/client_lib_name + client_src_path = root_path / 'libraries' / 'client' / client_lib_name for openapi_file in openapi_files: app_path = openapi_file.parent.parent @@ -210,19 +210,19 @@ def get_openapi_file_paths(root_path: pathlib.Path) -> list[pathlib.Path]: def aggregate_packages(client_source_path: pathlib.Path, lib_name=LIB_NAME): client_source_path.mkdir(parents=True, exist_ok=True) - client_docs_path = client_source_path/'docs' + client_docs_path = client_source_path / 'docs' client_docs_path.mkdir(exist_ok=True) - client_test_path = client_source_path/'test' + client_test_path = client_source_path / 'test' client_test_path.mkdir(exist_ok=True) - client_readme_file = client_source_path/'README.md' + client_readme_file = client_source_path / 'README.md' client_readme_file.unlink(missing_ok=True) - client_requirements_file = client_source_path/'requirements.txt' + client_requirements_file = client_source_path / 'requirements.txt' client_requirements_file.unlink(missing_ok=True) - client_test_requirements_file = client_source_path/'test-requirements.txt' + client_test_requirements_file = client_source_path / 'test-requirements.txt' client_test_requirements_file.unlink(missing_ok=True) requirements_lines_seen = set() @@ -236,18 +236,18 @@ def aggregate_packages(client_source_path: pathlib.Path, lib_name=LIB_NAME): .replace('-', '_') ) - code_destination_directory = client_source_path/lib_name/module - copymergedir(temp_module_path/lib_name/module, code_destination_directory) - copymergedir(temp_module_path/f'{lib_name}.{module}', code_destination_directory) # Fixes a bug with nested packages + code_destination_directory = client_source_path / lib_name / module + copymergedir(temp_module_path / lib_name / module, code_destination_directory) + copymergedir(temp_module_path / f'{lib_name}.{module}', code_destination_directory) # Fixes a bug with nested packages - module_docs_path = client_docs_path/module + module_docs_path = client_docs_path / module module_docs_path.mkdir(parents=True, exist_ok=True) - copymergedir(client_source_path/temp_module_path.name/'docs', module_docs_path) + copymergedir(client_source_path / temp_module_path.name / 'docs', module_docs_path) - module_tests_path = client_source_path/'test'/module - copymergedir(temp_module_path/'test', module_tests_path) + module_tests_path = client_source_path / 'test' / module + copymergedir(temp_module_path / 'test', module_tests_path) - readme_file = temp_module_path/'README.md' + readme_file = temp_module_path / 'README.md' if not readme_file.exists(): logging.warning(f'Readme file not found: {readme_file}.') continue @@ -258,14 +258,14 @@ def aggregate_packages(client_source_path: pathlib.Path, lib_name=LIB_NAME): out_file.write(updated_file_data) # FIXME: Different package versions will remain in the output file - requirements_file = temp_module_path/'requirements.txt' + requirements_file = temp_module_path / 'requirements.txt' with requirements_file.open('r') as in_file, client_requirements_file.open('+a') as out_file: unseen_lines = [line for line in in_file if line not in requirements_lines_seen] requirements_lines_seen.update(unseen_lines) out_file.writelines(unseen_lines) # FIXME: Different package versions will remain in the output file - test_requirements_file = temp_module_path/'test-requirements.txt' + test_requirements_file = temp_module_path / 'test-requirements.txt' with test_requirements_file.open('r') as in_file, client_test_requirements_file.open('+a') as out_file: unseen_lines = [line for line in in_file if line not in test_requirements_lines_seen] test_requirements_lines_seen.update(unseen_lines) @@ -275,7 +275,7 @@ def aggregate_packages(client_source_path: pathlib.Path, lib_name=LIB_NAME): def get_manifest(app_path: pathlib.Path) -> CloudHarnessManifest: - manifest_file = app_path/'.ch-manifest' + manifest_file = app_path / '.ch-manifest' try: manifest_data = load_yaml(manifest_file) @@ -304,26 +304,26 @@ def infer_templates(app_path: pathlib.Path) -> list[str]: def infer_webapp_template(app_path: pathlib.Path, templates: list[str]) -> None: - frontend_path = app_path/'frontend' + frontend_path = app_path / 'frontend' if frontend_path.exists(): templates.append(TemplateType.WEBAPP) def infer_server_template(app_path: pathlib.Path, templates: list[str]) -> None: - genapi_path = app_path/'api'/'genapi.sh' + genapi_path = app_path / 'api' / 'genapi.sh' if genapi_path.exists(): templates.append(TemplateType.DJANGO_APP) return - server_path = app_path/'server' - backend_path = app_path/'backend' + server_path = app_path / 'server' + backend_path = app_path / 'backend' if server_path.exists() or backend_path.exists(): templates.append(TemplateType.FLASK_SERVER) def infer_database_template(app_path: pathlib.Path, templates: list[str]) -> None: - values_file = app_path/'deploy'/'values.yaml' + values_file = app_path / 'deploy' / 'values.yaml' try: values_data = load_yaml(values_file) From 3ca70a3d47eef37e4d4dbe6542af8859ba4d3a36 Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Wed, 9 Oct 2024 17:04:56 +0100 Subject: [PATCH 27/93] CH-151 Fix bugs in normalize templates function --- tools/deployment-cli-tools/ch_cli_tools/common_types.py | 5 +++-- tools/deployment-cli-tools/harness-application | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/tools/deployment-cli-tools/ch_cli_tools/common_types.py b/tools/deployment-cli-tools/ch_cli_tools/common_types.py index b52bf1ae..63f1893d 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/common_types.py +++ b/tools/deployment-cli-tools/ch_cli_tools/common_types.py @@ -18,8 +18,9 @@ class TemplateType(StrEnum): DJANGO_APP = 'django-app' SERVER = 'server' - def database_templates(self): - return [self.DB_POSTGRES, self.DB_NEO4J, self.DB_MONGO] + @classmethod + def database_templates(cls): + return [cls.DB_POSTGRES, cls.DB_NEO4J, cls.DB_MONGO] @dataclass diff --git a/tools/deployment-cli-tools/harness-application b/tools/deployment-cli-tools/harness-application index 948f30b3..c79fee3f 100644 --- a/tools/deployment-cli-tools/harness-application +++ b/tools/deployment-cli-tools/harness-application @@ -107,7 +107,7 @@ def normalize_templates(templates: list[str]) -> list[str]: normalized_templates.insert(django_app_index, TemplateType.WEBAPP) has_database_template = any(template in TemplateType.database_templates() for template in normalized_templates) - if TemplateType.DJANGO_APP in normalize_templates and not has_database_template: + if TemplateType.DJANGO_APP in normalized_templates and not has_database_template: django_app_index = normalized_templates.index(TemplateType.DJANGO_APP) normalized_templates.insert(django_app_index, TemplateType.DB_POSTGRES) From 8dbce238fa6755cc2dc2033e0d6e77be4ed07091 Mon Sep 17 00:00:00 2001 From: Filippo Ledda Date: Fri, 11 Oct 2024 10:54:12 +0200 Subject: [PATCH 28/93] Fix backwards compatibility with Python 3.9 --- tools/deployment-cli-tools/ch_cli_tools/utils.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/deployment-cli-tools/ch_cli_tools/utils.py b/tools/deployment-cli-tools/ch_cli_tools/utils.py index 3f3f5659..53f36953 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/utils.py +++ b/tools/deployment-cli-tools/ch_cli_tools/utils.py @@ -217,8 +217,8 @@ def copymergedir(source_root_directory: pathlib.Path, destination_root_directory """ logging.info(f'Copying directory {source_root_directory} to {destination_root_directory}') - for source_directory, _, files in source_root_directory.walk(): - + for source_directory, _, files in os.walk(source_root_directory): # source_root_directory.walk() from Python 3.12 + source_directory = pathlib.Path(source_directory) destination_directory = destination_root_directory / source_directory.relative_to(source_root_directory) destination_directory.mkdir(parents=True, exist_ok=True) @@ -272,8 +272,8 @@ def merge_configuration_directories(source: Union[str, pathlib.Path], destinatio shutil.copytree(source_path, destination_path, ignore=shutil.ignore_patterns(*EXCLUDE_PATHS)) return - for source_directory, _, files in source_path.walk(): - _merge_configuration_directory(source_path, destination_path, source_directory, files) + for source_directory, _, files in os.walk(source_path): # source_path.walk() from Python 3.12 + _merge_configuration_directory(source_path, destination_path, pathlib.Path(source_directory), files) def _merge_configuration_directory( From 2c11b31d14ea118b09716f7db8f47dee6548589a Mon Sep 17 00:00:00 2001 From: Filippo Ledda Date: Tue, 15 Oct 2024 09:41:31 +0200 Subject: [PATCH 29/93] Reduce linting checks --- .github/workflows/lint-check.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/lint-check.yml b/.github/workflows/lint-check.yml index efee9f91..5124317a 100644 --- a/.github/workflows/lint-check.yml +++ b/.github/workflows/lint-check.yml @@ -1,7 +1,9 @@ name: Lint Check on: - push + pull_request: + # The branches below must be a subset of the branches above + branches: [ develop ] jobs: lint: From d671a285ea1f288d352ba2cd4f74ba9eea278503 Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Thu, 31 Oct 2024 11:42:10 +0000 Subject: [PATCH 30/93] CH-157 Move harness-application builder steps to a more structured pipeline approach --- .../ch_cli_tools/application_builders.py | 245 ++++++++++++++++++ .../deployment-cli-tools/harness-application | 108 +------- 2 files changed, 250 insertions(+), 103 deletions(-) create mode 100644 tools/deployment-cli-tools/ch_cli_tools/application_builders.py diff --git a/tools/deployment-cli-tools/ch_cli_tools/application_builders.py b/tools/deployment-cli-tools/ch_cli_tools/application_builders.py new file mode 100644 index 00000000..3ce72785 --- /dev/null +++ b/tools/deployment-cli-tools/ch_cli_tools/application_builders.py @@ -0,0 +1,245 @@ +import abc +import json +import logging +import pathlib +import shutil +import subprocess +import tempfile +from .common_types import TemplateType +from .openapi import generate_fastapi_server, generate_server, generate_ts_client +from .utils import copymergedir, get_json_template, merge_configuration_directories, replace_in_dict, replace_in_file, to_python_module +from . import CH_ROOT +from cloudharness_utils.constants import APPLICATION_TEMPLATE_PATH + + +class ApplicationBuilder(abc.ABC): + APP_NAME_PLACEHOLDER = '__APP_NAME__' + + def __init__(self, app_name: str, app_path: pathlib.Path): + self.app_name = app_name + self.app_path = app_path + + @abc.abstractmethod + def handles(self, templates: list[str]) -> bool: + pass + + @abc.abstractmethod + def handle_pre_merge(self) -> None: + pass + + @abc.abstractmethod + def handle_merge(self) -> None: + pass + + @abc.abstractmethod + def handle_post_merge(self) -> None: + pass + + def run_command(self, *command: str, cwd: pathlib.Path = None) -> None: + if not cwd: + cwd = self.app_path + + logging.info(f'Running command: {" ".join(map(str, command))}') + subprocess.run(command, cwd=cwd) + + def merge_template_directories(self, template_name: str) -> None: + for base_path in (self.ch_root, pathlib.Path.cwd()): + template_path = base_path / APPLICATION_TEMPLATE_PATH / template_name + if template_path.exists(): + merge_configuration_directories(template_path, self.app_path) + + @property + def frontend_path(self): + return self.app_path / 'frontend' + + @property + def backend_path(self): + return self.app_path / 'backend' + + @property + def api_path(self): + return self.app_path / 'api' + + @property + def ch_root(self): + return pathlib.Path(CH_ROOT) + + @property + def app_template_path(self): + return self.ch_root / APPLICATION_TEMPLATE_PATH + + +class WebAppBuilder(ApplicationBuilder): + def handles(self, templates): + return TemplateType.WEBAPP in templates + + def handle_pre_merge(self): + if self.frontend_path.exists(): + shutil.rmtree(self.frontend_path) + + self.create_vite_skaffold(self.frontend_path) + + def handle_merge(self): + self.merge_template_directories(TemplateType.WEBAPP) + + def handle_post_merge(self): + backend_dockerfile_path = self.backend_path / 'Dockerfile' + backend_dockerfile_path.unlink(missing_ok=True) + + self.install_frontend_dependencies() + generate_ts_client(self.api_path / 'openapi.yaml') + + def create_vite_skaffold(self, frontend_path: pathlib.Path) -> None: + self.run_command( + 'yarn', 'create', 'vite', self.app_name, + '--template', 'react-ts', + ) + shutil.move(self.app_path / self.app_name, frontend_path) + + def install_frontend_dependencies(self) -> None: + self.run_command('yarn', 'install', cwd=self.frontend_path) + + +class ServerAppBuilder(ApplicationBuilder): + def handles(self, templates): + return TemplateType.SERVER in templates + + def handle_pre_merge(self): + with tempfile.TemporaryDirectory() as tmp_dirname: + tmp_path = pathlib.Path(tmp_dirname) + server_template_path = self.app_template_path / TemplateType.SERVER + + copymergedir(server_template_path, tmp_path) + merge_configuration_directories(self.app_path, tmp_path) + generate_server(self.app_name, tmp_path) + + def handle_merge(self): + self.merge_template_directories(TemplateType.SERVER) + + def handle_post_merge(self): + pass + + +class FlaskServerAppBuilder(ApplicationBuilder): + def handles(self, templates): + return TemplateType.FLASK_SERVER in templates + + def handle_pre_merge(self): + pass + + def handle_merge(self): + self.merge_template_directories(TemplateType.FLASK_SERVER) + + def handle_post_merge(self): + generate_server(self.app_path) + + +class DjangoAppBuilder(ApplicationBuilder): + def handles(self, templates): + return TemplateType.DJANGO_APP in templates + + def handle_pre_merge(self): + pass + + def handle_merge(self): + self.merge_template_directories(TemplateType.DJANGO_APP) + + def handle_post_merge(self): + replace_in_file( + self.api_path / 'templates' / 'main.jinja2', + self.APP_NAME_PLACEHOLDER, + self.python_app_name, + ) + replace_in_file(self.api_path / 'genapi.sh', self.APP_NAME_PLACEHOLDER, self.app_name) + generate_fastapi_server(self.app_path) + + replace_in_file( + self.app_path / 'deploy' / 'values.yaml', + f'{self.APP_NAME_PLACEHOLDER}:{self.APP_NAME_PLACEHOLDER}', + f'{self.python_app_name}:{self.python_app_name}', + ) + replace_in_file(self.app_path / 'dev-setup.sh', self.APP_NAME_PLACEHOLDER, self.app_name) + + self.create_django_app_vscode_debug_configuration() + + (self.backend_path / self.APP_NAME_PLACEHOLDER / '__main__.py').unlink(missing_ok=True) + + @property + def python_app_name(self): + return to_python_module(self.app_name) + + def create_django_app_vscode_debug_configuration(self): + vscode_launch_path = pathlib.Path('.vscode/launch.json') + configuration_name = f'{self.app_name} backend' + + launch_config = get_json_template(vscode_launch_path, True) + + launch_config['configurations'] = [ + configuration for configuration in launch_config['configurations'] + if configuration['name'] != configuration_name + ] + + debug_config = get_json_template('vscode-django-app-debug-template.json', True) + debug_config = replace_in_dict(debug_config, self.APP_NAME_PLACEHOLDER, self.app_name) + + launch_config['configurations'].append(debug_config) + + vscode_launch_path.parent.mkdir(parents=True, exist_ok=True) + with vscode_launch_path.open('w') as f: + json.dump(launch_config, f, indent=2, sort_keys=True) + + +class AppBuilderPipeline(ApplicationBuilder): + def __init__(self, app_name: str, app_path: pathlib.Path, templates: list[str]): + super().__init__(app_name, app_path) + self.templates = templates + self.app_builders: dict[str, ApplicationBuilder] = { + TemplateType.WEBAPP: WebAppBuilder(app_name, app_path), + TemplateType.SERVER: ServerAppBuilder(app_name, app_path), + TemplateType.FLASK_SERVER: FlaskServerAppBuilder(app_name, app_path), + TemplateType.DJANGO_APP: DjangoAppBuilder(app_name, app_path), + } + + def handles(self, templates): + return templates == self.templates + + def handle_pre_merge(self): + pre_merge_template_order = [ + TemplateType.FLASK_SERVER, + TemplateType.DJANGO_APP, + TemplateType.WEBAPP, + TemplateType.SERVER, + ] + + app_builders = [ + self.app_builders[template] for template in pre_merge_template_order + if self.app_builders[template].handles(self.templates) + ] + + for app_builder in app_builders: + app_builder.handle_pre_merge() + + def handle_merge(self): + for template in self.templates: + run_merge = ( + app_builder.handle_merge + if (app_builder := self.app_builders.get(template, None)) + else lambda: self.merge_template_directories(template) + ) + run_merge() + + def handle_post_merge(self): + post_merge_template_order = [ + TemplateType.FLASK_SERVER, + TemplateType.DJANGO_APP, + TemplateType.WEBAPP, + TemplateType.SERVER, + ] + + app_builders = [ + self.app_builders[template] for template in post_merge_template_order + if self.app_builders[template].handles(self.templates) + ] + + for app_builder in app_builders: + app_builder.handle_post_merge() diff --git a/tools/deployment-cli-tools/harness-application b/tools/deployment-cli-tools/harness-application index c79fee3f..5cca3d75 100644 --- a/tools/deployment-cli-tools/harness-application +++ b/tools/deployment-cli-tools/harness-application @@ -17,6 +17,7 @@ from ch_cli_tools.openapi import generate_server, generate_fastapi_server, APPLI from ch_cli_tools.utils import merge_configuration_directories, replaceindir, replace_in_file, save_yaml, \ to_python_module, copymergedir, get_json_template, replace_in_dict from ch_cli_tools.common_types import CloudHarnessManifest, TemplateType +from ch_cli_tools.application_builders import AppBuilderPipeline # Only allow lowercased alphabetical characters separated by "-". name_pattern = re.compile("[a-z]+((-)?[a-z])?") @@ -32,27 +33,15 @@ def main() -> None: templates = normalize_templates(templates) - if TemplateType.WEBAPP in templates: - handle_webapp_template(app_name, app_path) + pipeline = AppBuilderPipeline(app_name, app_path, templates) - if TemplateType.SERVER in templates: - handle_server_template(app_path) - - for template_name in templates: - merge_template_directories(template_name, app_path) - - if TemplateType.FLASK_SERVER in templates: - handle_flask_server_template(app_path) + pipeline.handle_pre_merge() + pipeline.handle_merge() replace_in_file(app_path / 'api' / 'config.json', PLACEHOLDER, to_python_module(app_name)) - - if TemplateType.DJANGO_APP in templates: - handle_django_app_template(app_name, app_path) - replaceindir(app_path, PLACEHOLDER, app_name) - if TemplateType.WEBAPP in templates: - handle_webapp_template_post_merge(app_path) + pipeline.handle_post_merge() create_manifest_file(app_path, app_name, templates) call_harness_generate(app_path, app_name) @@ -114,86 +103,6 @@ def normalize_templates(templates: list[str]) -> list[str]: return normalized_templates -def handle_webapp_template(app_name: str, app_path: pathlib.Path) -> None: - frontend_path = app_path / 'frontend' - - if frontend_path.exists(): - shutil.rmtree(frontend_path) - - create_vite_skaffold(app_name, app_path, frontend_path) - - -def create_vite_skaffold(app_name: str, app_path: pathlib.Path, frontend_path: pathlib.Path) -> None: - command = ['yarn', 'create', 'vite', app_name, '--template', 'react-ts'] - logging.info(f'Running command: {" ".join(command)}') - subprocess.run(command, cwd=app_path) - shutil.move(app_path / app_name, frontend_path) - - -def install_frontend_dependencies(frontend_path: pathlib.Path) -> None: - command = ['yarn', 'install'] - logging.info(f'Running command: {" ".join(command)}') - subprocess.run(command, cwd=frontend_path) - - -def handle_webapp_template_post_merge(app_path: pathlib.Path) -> None: - backend_dockerfile_path = app_path / 'backend' / 'Dockerfile' - backend_dockerfile_path.unlink(missing_ok=True) - - install_frontend_dependencies(app_path / 'frontend') - generate_ts_client(openapi_file=app_path / 'api' / 'openapi.yaml') - - -def handle_server_template(app_path: pathlib.Path) -> None: - with tempfile.TemporaryDirectory() as tmp_dirname: - tmp_path = pathlib.Path(tmp_dirname) - server_template_path = pathlib.Path(CH_ROOT) / APPLICATION_TEMPLATE_PATH / TemplateType.SERVER - - copymergedir(server_template_path, tmp_path) - merge_configuration_directories(app_path, tmp_path) - generate_server(app_path, tmp_path) - - -def handle_flask_server_template(app_path: pathlib.Path) -> None: - generate_server(app_path) - - -def handle_django_app_template(app_name: str, app_path: pathlib.Path) -> None: - python_app_name = to_python_module(app_name) - - api_path = app_path / 'api' - replace_in_file(api_path / 'templates' / 'main.jinja2', PLACEHOLDER, python_app_name) - replace_in_file(api_path / 'genapi.sh', PLACEHOLDER, app_name) - generate_fastapi_server(app_path) - - replace_in_file(app_path / 'deploy' / 'values.yaml', f'{PLACEHOLDER}:{PLACEHOLDER}', f'{python_app_name}:{python_app_name}') - replace_in_file(app_path / 'dev-setup.sh', PLACEHOLDER, app_name) - create_django_app_vscode_debug_configuration(app_name) - - (app_path / 'backend' / '__APP_NAME__' / '__main__.py').unlink(missing_ok=True) - - -def create_django_app_vscode_debug_configuration(app_name: str): - vscode_launch_path = pathlib.Path('.vscode/launch.json') - configuration_name = f'{app_name} backend' - - launch_config = get_json_template(vscode_launch_path, True) - - launch_config['configurations'] = [ - configuration for configuration in launch_config['configurations'] - if configuration['name'] != configuration_name - ] - - debug_config = get_json_template('vscode-django-app-debug-template.json', True) - debug_config = replace_in_dict(debug_config, PLACEHOLDER, app_name) - - launch_config['configurations'].append(debug_config) - - vscode_launch_path.parent.mkdir(parents=True, exist_ok=True) - with vscode_launch_path.open('w') as f: - json.dump(launch_config, f, indent=2, sort_keys=True) - - def create_manifest_file(app_path: pathlib.Path, app_name: str, templates: list[Union[str, TemplateType]]) -> None: manifest_file = app_path / '.ch-manifest' manifest = CloudHarnessManifest( @@ -214,12 +123,5 @@ def call_harness_generate(app_path: pathlib.Path, app_name: str): subprocess.run(command) -def merge_template_directories(template_name: str, app_path: pathlib.Path) -> None: - for base_path in (pathlib.Path(CH_ROOT), pathlib.Path.cwd()): - template_path = base_path / APPLICATION_TEMPLATE_PATH / template_name - if template_path.exists(): - merge_configuration_directories(template_path, app_path) - - if __name__ == "__main__": main() From ad10cd86164af72aefa185eb47e3f063a8f6596a Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Fri, 1 Nov 2024 12:54:02 +0000 Subject: [PATCH 31/93] CH-157 Rename template from django-app to django-fastapi --- .../ch_cli_tools/application_builders.py | 14 ++++++------- .../ch_cli_tools/common_types.py | 19 ++++++++++++++++-- .../deployment-cli-tools/harness-application | 20 +++++++++---------- tools/deployment-cli-tools/harness-generate | 8 +++++--- 4 files changed, 39 insertions(+), 22 deletions(-) diff --git a/tools/deployment-cli-tools/ch_cli_tools/application_builders.py b/tools/deployment-cli-tools/ch_cli_tools/application_builders.py index 3ce72785..0f5bca0e 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/application_builders.py +++ b/tools/deployment-cli-tools/ch_cli_tools/application_builders.py @@ -134,15 +134,15 @@ def handle_post_merge(self): generate_server(self.app_path) -class DjangoAppBuilder(ApplicationBuilder): +class DjangoFastApiBuilder(ApplicationBuilder): def handles(self, templates): - return TemplateType.DJANGO_APP in templates - + return TemplateType.DJANGO_FASTAPI in templates + def handle_pre_merge(self): pass def handle_merge(self): - self.merge_template_directories(TemplateType.DJANGO_APP) + self.merge_template_directories(TemplateType.DJANGO_FASTAPI) def handle_post_merge(self): replace_in_file( @@ -197,7 +197,7 @@ def __init__(self, app_name: str, app_path: pathlib.Path, templates: list[str]): TemplateType.WEBAPP: WebAppBuilder(app_name, app_path), TemplateType.SERVER: ServerAppBuilder(app_name, app_path), TemplateType.FLASK_SERVER: FlaskServerAppBuilder(app_name, app_path), - TemplateType.DJANGO_APP: DjangoAppBuilder(app_name, app_path), + TemplateType.DJANGO_FASTAPI: DjangoFastApiBuilder(app_name, app_path), } def handles(self, templates): @@ -206,7 +206,7 @@ def handles(self, templates): def handle_pre_merge(self): pre_merge_template_order = [ TemplateType.FLASK_SERVER, - TemplateType.DJANGO_APP, + TemplateType.DJANGO_FASTAPI, TemplateType.WEBAPP, TemplateType.SERVER, ] @@ -231,7 +231,7 @@ def handle_merge(self): def handle_post_merge(self): post_merge_template_order = [ TemplateType.FLASK_SERVER, - TemplateType.DJANGO_APP, + TemplateType.DJANGO_FASTAPI, TemplateType.WEBAPP, TemplateType.SERVER, ] diff --git a/tools/deployment-cli-tools/ch_cli_tools/common_types.py b/tools/deployment-cli-tools/ch_cli_tools/common_types.py index 63f1893d..99b9d41c 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/common_types.py +++ b/tools/deployment-cli-tools/ch_cli_tools/common_types.py @@ -1,3 +1,4 @@ +import copy from dataclasses import dataclass from typing import Union @@ -15,7 +16,7 @@ class TemplateType(StrEnum): DB_POSTGRES = 'db-postgres' DB_NEO4J = 'db-neo4j' DB_MONGO = 'db-mongo' - DJANGO_APP = 'django-app' + DJANGO_FASTAPI = 'django-fastapi' SERVER = 'server' @classmethod @@ -26,9 +27,9 @@ def database_templates(cls): @dataclass class CloudHarnessManifest(): app_name: str - version: str inferred: bool templates: list[str] + version: str = '2' @classmethod def from_dict(cls, data: dict) -> 'CloudHarnessManifest': @@ -46,3 +47,17 @@ def to_dict(self) -> dict: 'inferred': self.inferred, 'templates': [str(template) for template in self.templates], } + + @classmethod + def migrate(cls, data: dict) -> tuple[dict, bool]: + data_copy = copy.deepcopy(data) + update_manifest = False + + if data_copy['version'] < '2': + update_manifest = True + data_copy['templates'] = [ + template if template != 'django-app' else 'django-fastapi' + for template in data_copy['templates'] + ] + + return data_copy, update_manifest \ No newline at end of file diff --git a/tools/deployment-cli-tools/harness-application b/tools/deployment-cli-tools/harness-application index 5cca3d75..2a65fe37 100644 --- a/tools/deployment-cli-tools/harness-application +++ b/tools/deployment-cli-tools/harness-application @@ -65,7 +65,7 @@ def get_command_line_arguments() -> tuple[str, list[str]]: - db-postgres - db-neo4j - db-mongo - - django-app (fastapi django backend based on openapi) + - django-fastapi (fastapi django backend based on openapi) """) args, unknown = parser.parse_known_args(sys.argv[1:]) @@ -89,18 +89,18 @@ def get_command_line_arguments() -> tuple[str, list[str]]: def normalize_templates(templates: list[str]) -> list[str]: - normalized_templates = list(templates) + templates = list(templates) - if TemplateType.DJANGO_APP in normalized_templates and TemplateType.WEBAPP not in normalized_templates: - django_app_index = normalized_templates.index(TemplateType.DJANGO_APP) - normalized_templates.insert(django_app_index, TemplateType.WEBAPP) + if TemplateType.DJANGO_FASTAPI in templates and TemplateType.WEBAPP not in templates: + django_app_index = templates.index(TemplateType.DJANGO_FASTAPI) + templates.insert(django_app_index, TemplateType.WEBAPP) - has_database_template = any(template in TemplateType.database_templates() for template in normalized_templates) - if TemplateType.DJANGO_APP in normalized_templates and not has_database_template: - django_app_index = normalized_templates.index(TemplateType.DJANGO_APP) - normalized_templates.insert(django_app_index, TemplateType.DB_POSTGRES) + has_database_template = any(template in TemplateType.database_templates() for template in templates) + if TemplateType.DJANGO_FASTAPI in templates and not has_database_template: + django_app_index = templates.index(TemplateType.DJANGO_FASTAPI) + templates.insert(django_app_index, TemplateType.DB_POSTGRES) - return normalized_templates + return templates def create_manifest_file(app_path: pathlib.Path, app_name: str, templates: list[Union[str, TemplateType]]) -> None: diff --git a/tools/deployment-cli-tools/harness-generate b/tools/deployment-cli-tools/harness-generate index c51edf25..00b80019 100644 --- a/tools/deployment-cli-tools/harness-generate +++ b/tools/deployment-cli-tools/harness-generate @@ -164,7 +164,7 @@ def generate_servers( if not should_generate(f'server stubs for {openapi_file}'): continue - if TemplateType.DJANGO_APP in manifest.templates: + if TemplateType.DJANGO_FASTAPI in manifest.templates: generate_fastapi_server(app_path) if TemplateType.FLASK_SERVER in manifest.templates: @@ -279,12 +279,14 @@ def get_manifest(app_path: pathlib.Path) -> CloudHarnessManifest: try: manifest_data = load_yaml(manifest_file) + manifest_data, update_manifest = CloudHarnessManifest.migrate(manifest_data) + if update_manifest: + save_yaml(manifest_file, manifest_data) manifest = CloudHarnessManifest.from_dict(manifest_data) except (FileNotFoundError, YAMLError): logging.info(f'Could not find manifest file {manifest_file}, inferring manifest from app structure...') manifest = CloudHarnessManifest( app_name=app_path.name, - version='1', inferred=True, templates=infer_templates(app_path), ) @@ -313,7 +315,7 @@ def infer_server_template(app_path: pathlib.Path, templates: list[str]) -> None: genapi_path = app_path / 'api' / 'genapi.sh' if genapi_path.exists(): - templates.append(TemplateType.DJANGO_APP) + templates.append(TemplateType.DJANGO_FASTAPI) return server_path = app_path / 'server' From 85b863a31b2abeb6ab342e7a4e6ebc68b697160e Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Fri, 1 Nov 2024 12:55:45 +0000 Subject: [PATCH 32/93] CH-157 Add app builder for django-ninja applications using a shared base with django-fastapi --- .../{django-app => django-base}/.dockerignore | 0 .../api/test_st.py | 0 .../backend/__APP_NAME__/__init__.py | 0 .../backend/__APP_NAME__/admin.py | 0 .../backend/__APP_NAME__/apps.py | 0 .../__APP_NAME__/migrations/0001_initial.py | 0 .../__APP_NAME__/migrations/__init__.py | 0 .../backend/__APP_NAME__/models.py | 0 .../backend/__APP_NAME__/tests.py | 0 .../backend/__APP_NAME__/views.py | 0 .../backend/django_baseapp/__init__.py | 0 .../backend/django_baseapp/asgi.py | 0 .../django_baseapp/migrations/0001_initial.py | 0 .../django_baseapp/migrations/__init__.py | 0 .../backend/django_baseapp/models.py | 0 .../django_baseapp/static/www/index.html | 0 .../templates/__APP_NAME__/index.html | 0 .../templates/__APP_NAME__/swagger-ui.html | 0 .../backend/django_baseapp/views.py | 0 .../backend/django_baseapp/wsgi.py | 0 .../backend/manage.py | 0 .../backend/openapi/.gitkeep | 0 .../backend/persistent/.gitkeep | 0 .../backend/setup.py | 0 .../backend/static/www/.gitkeep | 0 .../deploy/values.yaml | 0 .../{django-app => django-fastapi}/Dockerfile | 0 .../{django-app => django-fastapi}/README.md | 0 .../api/genapi.sh | 0 .../api/openapi.yaml | 0 .../api/templates/main.jinja2 | 0 .../backend/README.md | 0 .../__APP_NAME__/controllers/__init__.py | 0 .../backend/__APP_NAME__/controllers/test.py | 0 .../backend/django_baseapp/settings.py | 0 .../backend/django_baseapp/urls.py | 0 .../backend/requirements.txt | 0 .../dev-setup.sh | 0 application-templates/django-ninja/Dockerfile | 35 ++++ application-templates/django-ninja/README.md | 86 +++++++++ .../backend/__APP_NAME__/api/__init__.py | 40 +++++ .../backend/__APP_NAME__/exceptions.py | 6 + .../backend/__APP_NAME__/schema.py | 3 + .../backend/django_baseapp/settings.py | 167 ++++++++++++++++++ .../backend/django_baseapp/urls.py | 31 ++++ .../django-ninja/backend/requirements.txt | 2 + .../django-ninja/dev-setup.sh | 50 ++++++ .../ch_cli_tools/application_builders.py | 75 +++++--- .../ch_cli_tools/common_types.py | 5 + .../deployment-cli-tools/harness-application | 15 +- tools/deployment-cli-tools/harness-generate | 15 +- 51 files changed, 496 insertions(+), 34 deletions(-) rename application-templates/{django-app => django-base}/.dockerignore (100%) rename application-templates/{django-app => django-base}/api/test_st.py (100%) rename application-templates/{django-app => django-base}/backend/__APP_NAME__/__init__.py (100%) rename application-templates/{django-app => django-base}/backend/__APP_NAME__/admin.py (100%) rename application-templates/{django-app => django-base}/backend/__APP_NAME__/apps.py (100%) rename application-templates/{django-app => django-base}/backend/__APP_NAME__/migrations/0001_initial.py (100%) rename application-templates/{django-app => django-base}/backend/__APP_NAME__/migrations/__init__.py (100%) rename application-templates/{django-app => django-base}/backend/__APP_NAME__/models.py (100%) rename application-templates/{django-app => django-base}/backend/__APP_NAME__/tests.py (100%) rename application-templates/{django-app => django-base}/backend/__APP_NAME__/views.py (100%) rename application-templates/{django-app => django-base}/backend/django_baseapp/__init__.py (100%) rename application-templates/{django-app => django-base}/backend/django_baseapp/asgi.py (100%) rename application-templates/{django-app => django-base}/backend/django_baseapp/migrations/0001_initial.py (100%) rename application-templates/{django-app => django-base}/backend/django_baseapp/migrations/__init__.py (100%) rename application-templates/{django-app => django-base}/backend/django_baseapp/models.py (100%) rename application-templates/{django-app => django-base}/backend/django_baseapp/static/www/index.html (100%) rename application-templates/{django-app => django-base}/backend/django_baseapp/templates/__APP_NAME__/index.html (100%) rename application-templates/{django-app => django-base}/backend/django_baseapp/templates/__APP_NAME__/swagger-ui.html (100%) rename application-templates/{django-app => django-base}/backend/django_baseapp/views.py (100%) rename application-templates/{django-app => django-base}/backend/django_baseapp/wsgi.py (100%) rename application-templates/{django-app => django-base}/backend/manage.py (100%) rename application-templates/{django-app => django-base}/backend/openapi/.gitkeep (100%) rename application-templates/{django-app => django-base}/backend/persistent/.gitkeep (100%) rename application-templates/{django-app => django-base}/backend/setup.py (100%) rename application-templates/{django-app => django-base}/backend/static/www/.gitkeep (100%) rename application-templates/{django-app => django-base}/deploy/values.yaml (100%) rename application-templates/{django-app => django-fastapi}/Dockerfile (100%) rename application-templates/{django-app => django-fastapi}/README.md (100%) rename application-templates/{django-app => django-fastapi}/api/genapi.sh (100%) rename application-templates/{django-app => django-fastapi}/api/openapi.yaml (100%) rename application-templates/{django-app => django-fastapi}/api/templates/main.jinja2 (100%) rename application-templates/{django-app => django-fastapi}/backend/README.md (100%) rename application-templates/{django-app => django-fastapi}/backend/__APP_NAME__/controllers/__init__.py (100%) rename application-templates/{django-app => django-fastapi}/backend/__APP_NAME__/controllers/test.py (100%) rename application-templates/{django-app => django-fastapi}/backend/django_baseapp/settings.py (100%) rename application-templates/{django-app => django-fastapi}/backend/django_baseapp/urls.py (100%) rename application-templates/{django-app => django-fastapi}/backend/requirements.txt (100%) rename application-templates/{django-app => django-fastapi}/dev-setup.sh (100%) create mode 100644 application-templates/django-ninja/Dockerfile create mode 100644 application-templates/django-ninja/README.md create mode 100644 application-templates/django-ninja/backend/__APP_NAME__/api/__init__.py create mode 100644 application-templates/django-ninja/backend/__APP_NAME__/exceptions.py create mode 100644 application-templates/django-ninja/backend/__APP_NAME__/schema.py create mode 100644 application-templates/django-ninja/backend/django_baseapp/settings.py create mode 100644 application-templates/django-ninja/backend/django_baseapp/urls.py create mode 100644 application-templates/django-ninja/backend/requirements.txt create mode 100644 application-templates/django-ninja/dev-setup.sh diff --git a/application-templates/django-app/.dockerignore b/application-templates/django-base/.dockerignore similarity index 100% rename from application-templates/django-app/.dockerignore rename to application-templates/django-base/.dockerignore diff --git a/application-templates/django-app/api/test_st.py b/application-templates/django-base/api/test_st.py similarity index 100% rename from application-templates/django-app/api/test_st.py rename to application-templates/django-base/api/test_st.py diff --git a/application-templates/django-app/backend/__APP_NAME__/__init__.py b/application-templates/django-base/backend/__APP_NAME__/__init__.py similarity index 100% rename from application-templates/django-app/backend/__APP_NAME__/__init__.py rename to application-templates/django-base/backend/__APP_NAME__/__init__.py diff --git a/application-templates/django-app/backend/__APP_NAME__/admin.py b/application-templates/django-base/backend/__APP_NAME__/admin.py similarity index 100% rename from application-templates/django-app/backend/__APP_NAME__/admin.py rename to application-templates/django-base/backend/__APP_NAME__/admin.py diff --git a/application-templates/django-app/backend/__APP_NAME__/apps.py b/application-templates/django-base/backend/__APP_NAME__/apps.py similarity index 100% rename from application-templates/django-app/backend/__APP_NAME__/apps.py rename to application-templates/django-base/backend/__APP_NAME__/apps.py diff --git a/application-templates/django-app/backend/__APP_NAME__/migrations/0001_initial.py b/application-templates/django-base/backend/__APP_NAME__/migrations/0001_initial.py similarity index 100% rename from application-templates/django-app/backend/__APP_NAME__/migrations/0001_initial.py rename to application-templates/django-base/backend/__APP_NAME__/migrations/0001_initial.py diff --git a/application-templates/django-app/backend/__APP_NAME__/migrations/__init__.py b/application-templates/django-base/backend/__APP_NAME__/migrations/__init__.py similarity index 100% rename from application-templates/django-app/backend/__APP_NAME__/migrations/__init__.py rename to application-templates/django-base/backend/__APP_NAME__/migrations/__init__.py diff --git a/application-templates/django-app/backend/__APP_NAME__/models.py b/application-templates/django-base/backend/__APP_NAME__/models.py similarity index 100% rename from application-templates/django-app/backend/__APP_NAME__/models.py rename to application-templates/django-base/backend/__APP_NAME__/models.py diff --git a/application-templates/django-app/backend/__APP_NAME__/tests.py b/application-templates/django-base/backend/__APP_NAME__/tests.py similarity index 100% rename from application-templates/django-app/backend/__APP_NAME__/tests.py rename to application-templates/django-base/backend/__APP_NAME__/tests.py diff --git a/application-templates/django-app/backend/__APP_NAME__/views.py b/application-templates/django-base/backend/__APP_NAME__/views.py similarity index 100% rename from application-templates/django-app/backend/__APP_NAME__/views.py rename to application-templates/django-base/backend/__APP_NAME__/views.py diff --git a/application-templates/django-app/backend/django_baseapp/__init__.py b/application-templates/django-base/backend/django_baseapp/__init__.py similarity index 100% rename from application-templates/django-app/backend/django_baseapp/__init__.py rename to application-templates/django-base/backend/django_baseapp/__init__.py diff --git a/application-templates/django-app/backend/django_baseapp/asgi.py b/application-templates/django-base/backend/django_baseapp/asgi.py similarity index 100% rename from application-templates/django-app/backend/django_baseapp/asgi.py rename to application-templates/django-base/backend/django_baseapp/asgi.py diff --git a/application-templates/django-app/backend/django_baseapp/migrations/0001_initial.py b/application-templates/django-base/backend/django_baseapp/migrations/0001_initial.py similarity index 100% rename from application-templates/django-app/backend/django_baseapp/migrations/0001_initial.py rename to application-templates/django-base/backend/django_baseapp/migrations/0001_initial.py diff --git a/application-templates/django-app/backend/django_baseapp/migrations/__init__.py b/application-templates/django-base/backend/django_baseapp/migrations/__init__.py similarity index 100% rename from application-templates/django-app/backend/django_baseapp/migrations/__init__.py rename to application-templates/django-base/backend/django_baseapp/migrations/__init__.py diff --git a/application-templates/django-app/backend/django_baseapp/models.py b/application-templates/django-base/backend/django_baseapp/models.py similarity index 100% rename from application-templates/django-app/backend/django_baseapp/models.py rename to application-templates/django-base/backend/django_baseapp/models.py diff --git a/application-templates/django-app/backend/django_baseapp/static/www/index.html b/application-templates/django-base/backend/django_baseapp/static/www/index.html similarity index 100% rename from application-templates/django-app/backend/django_baseapp/static/www/index.html rename to application-templates/django-base/backend/django_baseapp/static/www/index.html diff --git a/application-templates/django-app/backend/django_baseapp/templates/__APP_NAME__/index.html b/application-templates/django-base/backend/django_baseapp/templates/__APP_NAME__/index.html similarity index 100% rename from application-templates/django-app/backend/django_baseapp/templates/__APP_NAME__/index.html rename to application-templates/django-base/backend/django_baseapp/templates/__APP_NAME__/index.html diff --git a/application-templates/django-app/backend/django_baseapp/templates/__APP_NAME__/swagger-ui.html b/application-templates/django-base/backend/django_baseapp/templates/__APP_NAME__/swagger-ui.html similarity index 100% rename from application-templates/django-app/backend/django_baseapp/templates/__APP_NAME__/swagger-ui.html rename to application-templates/django-base/backend/django_baseapp/templates/__APP_NAME__/swagger-ui.html diff --git a/application-templates/django-app/backend/django_baseapp/views.py b/application-templates/django-base/backend/django_baseapp/views.py similarity index 100% rename from application-templates/django-app/backend/django_baseapp/views.py rename to application-templates/django-base/backend/django_baseapp/views.py diff --git a/application-templates/django-app/backend/django_baseapp/wsgi.py b/application-templates/django-base/backend/django_baseapp/wsgi.py similarity index 100% rename from application-templates/django-app/backend/django_baseapp/wsgi.py rename to application-templates/django-base/backend/django_baseapp/wsgi.py diff --git a/application-templates/django-app/backend/manage.py b/application-templates/django-base/backend/manage.py similarity index 100% rename from application-templates/django-app/backend/manage.py rename to application-templates/django-base/backend/manage.py diff --git a/application-templates/django-app/backend/openapi/.gitkeep b/application-templates/django-base/backend/openapi/.gitkeep similarity index 100% rename from application-templates/django-app/backend/openapi/.gitkeep rename to application-templates/django-base/backend/openapi/.gitkeep diff --git a/application-templates/django-app/backend/persistent/.gitkeep b/application-templates/django-base/backend/persistent/.gitkeep similarity index 100% rename from application-templates/django-app/backend/persistent/.gitkeep rename to application-templates/django-base/backend/persistent/.gitkeep diff --git a/application-templates/django-app/backend/setup.py b/application-templates/django-base/backend/setup.py similarity index 100% rename from application-templates/django-app/backend/setup.py rename to application-templates/django-base/backend/setup.py diff --git a/application-templates/django-app/backend/static/www/.gitkeep b/application-templates/django-base/backend/static/www/.gitkeep similarity index 100% rename from application-templates/django-app/backend/static/www/.gitkeep rename to application-templates/django-base/backend/static/www/.gitkeep diff --git a/application-templates/django-app/deploy/values.yaml b/application-templates/django-base/deploy/values.yaml similarity index 100% rename from application-templates/django-app/deploy/values.yaml rename to application-templates/django-base/deploy/values.yaml diff --git a/application-templates/django-app/Dockerfile b/application-templates/django-fastapi/Dockerfile similarity index 100% rename from application-templates/django-app/Dockerfile rename to application-templates/django-fastapi/Dockerfile diff --git a/application-templates/django-app/README.md b/application-templates/django-fastapi/README.md similarity index 100% rename from application-templates/django-app/README.md rename to application-templates/django-fastapi/README.md diff --git a/application-templates/django-app/api/genapi.sh b/application-templates/django-fastapi/api/genapi.sh similarity index 100% rename from application-templates/django-app/api/genapi.sh rename to application-templates/django-fastapi/api/genapi.sh diff --git a/application-templates/django-app/api/openapi.yaml b/application-templates/django-fastapi/api/openapi.yaml similarity index 100% rename from application-templates/django-app/api/openapi.yaml rename to application-templates/django-fastapi/api/openapi.yaml diff --git a/application-templates/django-app/api/templates/main.jinja2 b/application-templates/django-fastapi/api/templates/main.jinja2 similarity index 100% rename from application-templates/django-app/api/templates/main.jinja2 rename to application-templates/django-fastapi/api/templates/main.jinja2 diff --git a/application-templates/django-app/backend/README.md b/application-templates/django-fastapi/backend/README.md similarity index 100% rename from application-templates/django-app/backend/README.md rename to application-templates/django-fastapi/backend/README.md diff --git a/application-templates/django-app/backend/__APP_NAME__/controllers/__init__.py b/application-templates/django-fastapi/backend/__APP_NAME__/controllers/__init__.py similarity index 100% rename from application-templates/django-app/backend/__APP_NAME__/controllers/__init__.py rename to application-templates/django-fastapi/backend/__APP_NAME__/controllers/__init__.py diff --git a/application-templates/django-app/backend/__APP_NAME__/controllers/test.py b/application-templates/django-fastapi/backend/__APP_NAME__/controllers/test.py similarity index 100% rename from application-templates/django-app/backend/__APP_NAME__/controllers/test.py rename to application-templates/django-fastapi/backend/__APP_NAME__/controllers/test.py diff --git a/application-templates/django-app/backend/django_baseapp/settings.py b/application-templates/django-fastapi/backend/django_baseapp/settings.py similarity index 100% rename from application-templates/django-app/backend/django_baseapp/settings.py rename to application-templates/django-fastapi/backend/django_baseapp/settings.py diff --git a/application-templates/django-app/backend/django_baseapp/urls.py b/application-templates/django-fastapi/backend/django_baseapp/urls.py similarity index 100% rename from application-templates/django-app/backend/django_baseapp/urls.py rename to application-templates/django-fastapi/backend/django_baseapp/urls.py diff --git a/application-templates/django-app/backend/requirements.txt b/application-templates/django-fastapi/backend/requirements.txt similarity index 100% rename from application-templates/django-app/backend/requirements.txt rename to application-templates/django-fastapi/backend/requirements.txt diff --git a/application-templates/django-app/dev-setup.sh b/application-templates/django-fastapi/dev-setup.sh similarity index 100% rename from application-templates/django-app/dev-setup.sh rename to application-templates/django-fastapi/dev-setup.sh diff --git a/application-templates/django-ninja/Dockerfile b/application-templates/django-ninja/Dockerfile new file mode 100644 index 00000000..f13bfaac --- /dev/null +++ b/application-templates/django-ninja/Dockerfile @@ -0,0 +1,35 @@ +ARG MNP_UI +ARG CLOUDHARNESS_DJANGO + +FROM $MNP_UI AS frontend + +ARG APP_DIR=/app + +WORKDIR ${APP_DIR} +COPY frontend/package.json . +COPY frontend/yarn.lock . +RUN yarn install --timeout 60000 + +COPY frontend . +RUN yarn build + +##### + +FROM $CLOUDHARNESS_DJANGO + +WORKDIR ${APP_DIR} +RUN mkdir -p ${APP_DIR}/static/www + +COPY backend/requirements.txt ${APP_DIR} +RUN --mount=type=cache,target=/root/.cache python -m pip install --upgrade pip &&\ + pip3 install --no-cache-dir -r requirements.txt --prefer-binary + +COPY backend/requirements.txt backend/setup.py ${APP_DIR} +RUN python3 -m pip install -e . + +COPY backend ${APP_DIR} +RUN python3 manage.py collectstatic --noinput + +COPY --from=frontend /app/dist ${APP_DIR}/static/www + +ENTRYPOINT uvicorn --workers ${WORKERS} --host 0.0.0.0 --port ${PORT} django_baseapp.asgi:application diff --git a/application-templates/django-ninja/README.md b/application-templates/django-ninja/README.md new file mode 100644 index 00000000..cc4b4c09 --- /dev/null +++ b/application-templates/django-ninja/README.md @@ -0,0 +1,86 @@ +# __APP_NAME__ + +Django-Ninja/React-based web application. +This application is constructed to be deployed inside a cloud-harness Kubernetes. +It can be also run locally for development and test purpose. + +The code is generated with the script `harness-application`. + +## Configuration + +### Accounts + +The CloudHarness Django application template comes with a configuration that can retrieve user account updates from Keycloak (accounts) +To enable this feature: +* log in into the accounts admin interface +* select in the left sidebar Events +* select the `Config` tab +* enable "metacell-admin-event-listener" under the `Events Config` - `Event Listeners` + +An other option is to enable the "metacell-admin-event-listener" through customizing the Keycloak realm.json from the CloudHarness repository. + +## Develop + +This application is composed of a Django-Ninja backend and a React frontend. + +### Backend + +Backend code is inside the *backend* directory. +See [backend/README.md#Develop] + +### Frontend + +Backend code is inside the *frontend* directory. + +Frontend is by default generated as a React web application, but no constraint about this specific technology. + +#### Call the backend apis +All the api stubs are automatically generated in the [frontend/rest](frontend/rest) directory by `harness-application` +and `harness-generate`. + +## Local build & run + +### Install dependencies +1 - Clone cloud-harness into your project root folder + +2 - Run the dev setup script +``` +cd applications/__APP_NAME__ +bash dev-setup.sh +``` + +### Prepare backend + +Create a Django local superuser account, this you only need to do on initial setup +```bash +cd backend +python3 manage.py migrate # to sync the database with the Django models +python3 manage.py collectstatic --noinput # to copy all assets to the static folder +python3 manage.py createsuperuser +# link the frontend dist to the django static folder, this is only needed once, frontend updates will automatically be applied +cd static/www +ln -s ../../../frontend/dist dist +``` + +### Build frontend + +Compile the frontend +```bash +cd frontend +npm install +npm run build +``` + +### Run backend application + +start the Django server +```bash +uvicorn --workers 2 --host 0.0.0.0 --port 8000 django_baseapp.asgi:application +``` + + +### Running local with port forwardings to a kubernetes cluster +When you create port forwards to microservices in your k8s cluster you want to forced your local backend server to initialize +the AuthService and EventService services. +This can be done by setting the `KUBERNETES_SERVICE_HOST` environment variable to a dummy or correct k8s service host. +The `KUBERNETES_SERVICE_HOST` switch will activate the creation of the keycloak client and client roles of this microservice. diff --git a/application-templates/django-ninja/backend/__APP_NAME__/api/__init__.py b/application-templates/django-ninja/backend/__APP_NAME__/api/__init__.py new file mode 100644 index 00000000..647697a0 --- /dev/null +++ b/application-templates/django-ninja/backend/__APP_NAME__/api/__init__.py @@ -0,0 +1,40 @@ +import time +from django.http import HttpRequest +from ninja import NinjaAPI +from ..exceptions import Http401, Http403 + + +api = NinjaAPI(title='__APP_NAME__ API', version='0.1.0') + + +@api.exception_handler(Http401) +def unauthorized(request, exc): + return api.create_response( + request, + {'message': 'Unauthorized'}, + status=401, + ) + + +@api.exception_handler(Http403) +def forbidden(request, exc): + return api.create_response( + request, + {'message': 'Forbidden'}, + status=403, + ) + + +@api.get('/ping', response={200: float}) +def ping(request: HttpRequest): + return time.time() + + +@api.get('/live', response={200: str}) +def live(request: HttpRequest): + return 'OK' + + +@api.get('/ready', response={200: str}) +def ready(request: HttpRequest): + return 'OK' diff --git a/application-templates/django-ninja/backend/__APP_NAME__/exceptions.py b/application-templates/django-ninja/backend/__APP_NAME__/exceptions.py new file mode 100644 index 00000000..b5b053c1 --- /dev/null +++ b/application-templates/django-ninja/backend/__APP_NAME__/exceptions.py @@ -0,0 +1,6 @@ +class Http401(Exception): + pass + + +class Http403(Exception): + pass diff --git a/application-templates/django-ninja/backend/__APP_NAME__/schema.py b/application-templates/django-ninja/backend/__APP_NAME__/schema.py new file mode 100644 index 00000000..b3d9c8b3 --- /dev/null +++ b/application-templates/django-ninja/backend/__APP_NAME__/schema.py @@ -0,0 +1,3 @@ +from ninja import Schema + +# Create your schema here diff --git a/application-templates/django-ninja/backend/django_baseapp/settings.py b/application-templates/django-ninja/backend/django_baseapp/settings.py new file mode 100644 index 00000000..e54f20f0 --- /dev/null +++ b/application-templates/django-ninja/backend/django_baseapp/settings.py @@ -0,0 +1,167 @@ +""" +Django settings for the MNP Checkout project. + +Generated by 'django-admin startproject' using Django 3.2.12. + +For more information on this file, see +https://docs.djangoproject.com/en/3.2/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/3.2/ref/settings/ +""" + +import os +from pathlib import Path + +# Build paths inside the project like this: BASE_DIR / 'subdir'. +BASE_DIR = Path(__file__).resolve().parent.parent + + +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/3.2/howto/deployment/checklist/ + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = "django-insecure-81kv$0=07xac7r(pgz6ndb5t0at4-z@ae6&f@u6_3jo&9d#4kl" + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = False if os.environ.get("PRODUCTION", None) else True + +ALLOWED_HOSTS = [ + "*", +] + +# Application definition + +INSTALLED_APPS = [ + "django.contrib.admin", + "django.contrib.auth", + "django.contrib.contenttypes", + "django.contrib.sessions", + "django.contrib.messages", + "django.contrib.staticfiles", +] + +MIDDLEWARE = [ + "django.middleware.security.SecurityMiddleware", + "django.contrib.sessions.middleware.SessionMiddleware", + "django.middleware.common.CommonMiddleware", + 'django.middleware.csrf.CsrfViewMiddleware', + "django.contrib.auth.middleware.AuthenticationMiddleware", + "django.contrib.messages.middleware.MessageMiddleware", + "django.middleware.clickjacking.XFrameOptionsMiddleware", + "cloudharness.middleware.django.CloudharnessMiddleware", +] + + +ROOT_URLCONF = "django_baseapp.urls" + +TEMPLATES = [ + { + "BACKEND": "django.template.backends.django.DjangoTemplates", + "DIRS": [BASE_DIR / "templates"], + "APP_DIRS": True, + "OPTIONS": { + "context_processors": [ + "django.template.context_processors.debug", + "django.template.context_processors.request", + "django.contrib.auth.context_processors.auth", + "django.contrib.messages.context_processors.messages", + ], + }, + }, +] + +WSGI_APPLICATION = "django_baseapp.wsgi.application" + + +# Password validation +# https://docs.djangoproject.com/en/3.2/ref/settings/#auth-password-validators + +AUTH_PASSWORD_VALIDATORS = [ + { + "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator", + }, + { + "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator", + }, + { + "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator", + }, + { + "NAME": "django.contrib.auth.password_validation.NumericPasswordValidator", + }, +] + + +# Internationalization +# https://docs.djangoproject.com/en/3.2/topics/i18n/ + +LANGUAGE_CODE = "en-us" + +TIME_ZONE = "UTC" + +USE_I18N = True + +USE_L10N = True + +USE_TZ = True + +# Default primary key field type +# https://docs.djangoproject.com/en/3.2/ref/settings/#default-auto-field + +DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" + + +PROJECT_NAME = "__APP_NAME__".upper() + +# Persistent storage +PERSISTENT_ROOT = os.path.join(BASE_DIR, "persistent") + +# *********************************************************************** +# * __APP_NAME__ settings +# *********************************************************************** +from cloudharness.applications import get_configuration # noqa E402 +from cloudharness.utils.config import ALLVALUES_PATH, CloudharnessConfig # noqa E402 + +# *********************************************************************** +# * import base CloudHarness Django settings +# *********************************************************************** +from cloudharness_django.settings import * # noqa E402 + +# add the local apps +INSTALLED_APPS += [ + "__APP_NAME__", + "django_baseapp", + "ninja", +] + +# override django admin base template with a local template +# to add some custom styling +TEMPLATES[0]["DIRS"] = [BASE_DIR / "templates"] + +# Static files (CSS, JavaScript, Images) +MEDIA_ROOT = PERSISTENT_ROOT +STATIC_ROOT = os.path.join(BASE_DIR, "static") +MEDIA_URL = "/media/" +STATIC_URL = "/static/" + +# KC Client & roles +KC_CLIENT_NAME = PROJECT_NAME.lower() + +# __APP_NAME__ specific roles + +# Default KC roles +KC_ADMIN_ROLE = f"{KC_CLIENT_NAME}-administrator" # admin user +KC_MANAGER_ROLE = f"{KC_CLIENT_NAME}-manager" # manager user +KC_USER_ROLE = f"{KC_CLIENT_NAME}-user" # customer user +KC_ALL_ROLES = [ + KC_ADMIN_ROLE, + KC_MANAGER_ROLE, + KC_USER_ROLE, +] +KC_PRIVILEGED_ROLES = [ + KC_ADMIN_ROLE, + KC_MANAGER_ROLE, +] + +KC_DEFAULT_USER_ROLE = None # don't add the user role to the realm default role diff --git a/application-templates/django-ninja/backend/django_baseapp/urls.py b/application-templates/django-ninja/backend/django_baseapp/urls.py new file mode 100644 index 00000000..aae3d635 --- /dev/null +++ b/application-templates/django-ninja/backend/django_baseapp/urls.py @@ -0,0 +1,31 @@ +"""MNP Checkout URL Configuration + +The `urlpatterns` list routes URLs to views. For more information please see: + https://docs.djangoproject.com/en/3.2/topics/http/urls/ +Examples: +Function views + 1. Add an import: from my_app import views + 2. Add a URL to urlpatterns: path('', views.home, name='home') +Class-based views + 1. Add an import: from other_app.views import Home + 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') +Including another URLconf + 1. Import the include() function: from django.urls import include, path + 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) +""" +import re +from django.conf import settings +from django.views.static import serve +from django.contrib import admin +from django.urls import path, re_path +from __APP_NAME__.api import api +from django_baseapp.views import index + + +urlpatterns = [ + path("admin/", admin.site.urls), + path("api/", api.urls), + re_path(r"^%s(?P.*)$" % re.escape(settings.MEDIA_URL.lstrip("/")), serve, kwargs=dict(document_root=settings.MEDIA_ROOT)), + re_path(r"^%s(?P.*)$" % re.escape(settings.STATIC_URL.lstrip("/")), serve, kwargs=dict(document_root=settings.STATIC_ROOT)), + re_path(r"^(?P.*)$", index, name="index"), +] diff --git a/application-templates/django-ninja/backend/requirements.txt b/application-templates/django-ninja/backend/requirements.txt new file mode 100644 index 00000000..97c58008 --- /dev/null +++ b/application-templates/django-ninja/backend/requirements.txt @@ -0,0 +1,2 @@ +pydantic==2.9.2 +ninja \ No newline at end of file diff --git a/application-templates/django-ninja/dev-setup.sh b/application-templates/django-ninja/dev-setup.sh new file mode 100644 index 00000000..24970bf3 --- /dev/null +++ b/application-templates/django-ninja/dev-setup.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash + +CURRENT_PATH=$(pwd) +CH_DIRECTORY="../../cloud-harness" +INSTALL_PYTEST=false +CURRENT_DIRECTORY="$(pwd)" +APP_NAME="__APP_NAME__" + +pip_upgrade_error() { + echo "Unable to upgrade pip" + exit 1 +} + +install_error () { + echo "Unable to install $1" 1>&2 + exit 1 +} + +while getopts ch_directory:pytest arg; +do + case "$arg" in + ch_directory) CH_DIRECTORY=${OPTARG};; + pytest) INSTALL_PYTEST=true;; + esac +done + +pip install --upgrade pip || pip_upgrade_error + +# Install pip dependencies from cloudharness-base-debian image + +if $INSTALL_PYTEST; then + pip install pytest || install_error pytest +fi + +pip install -r "$CH_DIRECTORY/libraries/models/requirements.txt" || install_error "models requirements" +pip install -r "$CH_DIRECTORY/libraries/cloudharness-common/requirements.txt" || install_error "cloudharness-common requirements" +pip install -r "$CH_DIRECTORY/libraries/client/cloudharness_cli/requirements.txt" || install_error "cloudharness_cli requirements" + +pip install -e "$CH_DIRECTORY/libraries/models" || install_error models +pip install -e "$CH_DIRECTORY/libraries/cloudharness-common" || install_error cloudharness-common +pip install -e "$CH_DIRECTORY/libraries/client/cloudharness_cli" || install_error cloudharness_cli + +# Install pip dependencies from cloudharness-django image + +pip install -e "$CH_DIRECTORY/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django" || install_error cloudharness-django + +# Install application + +pip install -r "$CURRENT_DIRECTORY/backend/requirements.txt" || install_error "$APP_NAME dependencies" +pip install -e "$CURRENT_DIRECTORY/backend" || install_error "$APP_NAME" \ No newline at end of file diff --git a/tools/deployment-cli-tools/ch_cli_tools/application_builders.py b/tools/deployment-cli-tools/ch_cli_tools/application_builders.py index 0f5bca0e..611955b5 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/application_builders.py +++ b/tools/deployment-cli-tools/ch_cli_tools/application_builders.py @@ -7,7 +7,7 @@ import tempfile from .common_types import TemplateType from .openapi import generate_fastapi_server, generate_server, generate_ts_client -from .utils import copymergedir, get_json_template, merge_configuration_directories, replace_in_dict, replace_in_file, to_python_module +from .utils import copymergedir, get_json_template, merge_configuration_directories, replace_in_dict, replace_in_file, replaceindir, to_python_module from . import CH_ROOT from cloudharness_utils.constants import APPLICATION_TEMPLATE_PATH @@ -134,25 +134,13 @@ def handle_post_merge(self): generate_server(self.app_path) -class DjangoFastApiBuilder(ApplicationBuilder): - def handles(self, templates): - return TemplateType.DJANGO_FASTAPI in templates - - def handle_pre_merge(self): - pass - +class BaseDjangoAppBuilder(ApplicationBuilder): + @abc.abstractmethod def handle_merge(self): - self.merge_template_directories(TemplateType.DJANGO_FASTAPI) + self.merge_template_directories('django-base') + @abc.abstractmethod def handle_post_merge(self): - replace_in_file( - self.api_path / 'templates' / 'main.jinja2', - self.APP_NAME_PLACEHOLDER, - self.python_app_name, - ) - replace_in_file(self.api_path / 'genapi.sh', self.APP_NAME_PLACEHOLDER, self.app_name) - generate_fastapi_server(self.app_path) - replace_in_file( self.app_path / 'deploy' / 'values.yaml', f'{self.APP_NAME_PLACEHOLDER}:{self.APP_NAME_PLACEHOLDER}', @@ -161,12 +149,6 @@ def handle_post_merge(self): replace_in_file(self.app_path / 'dev-setup.sh', self.APP_NAME_PLACEHOLDER, self.app_name) self.create_django_app_vscode_debug_configuration() - - (self.backend_path / self.APP_NAME_PLACEHOLDER / '__main__.py').unlink(missing_ok=True) - - @property - def python_app_name(self): - return to_python_module(self.app_name) def create_django_app_vscode_debug_configuration(self): vscode_launch_path = pathlib.Path('.vscode/launch.json') @@ -188,6 +170,50 @@ def create_django_app_vscode_debug_configuration(self): with vscode_launch_path.open('w') as f: json.dump(launch_config, f, indent=2, sort_keys=True) + @property + def python_app_name(self): + return to_python_module(self.app_name) + + +class DjangoFastApiBuilder(BaseDjangoAppBuilder): + def handles(self, templates): + return TemplateType.DJANGO_FASTAPI in templates + + def handle_pre_merge(self): + pass + + def handle_merge(self): + super().handle_merge() + self.merge_template_directories(TemplateType.DJANGO_FASTAPI) + + def handle_post_merge(self): + super().handle_post_merge() + + replace_in_file( + self.api_path / 'templates' / 'main.jinja2', + self.APP_NAME_PLACEHOLDER, + self.python_app_name, + ) + replace_in_file(self.api_path / 'genapi.sh', self.APP_NAME_PLACEHOLDER, self.app_name) + generate_fastapi_server(self.app_path) + + (self.backend_path / self.APP_NAME_PLACEHOLDER / '__main__.py').unlink(missing_ok=True) + + +class DjangoNinjaBuilder(BaseDjangoAppBuilder): + def handles(self, templates): + return TemplateType.DJANGO_NINJA in templates + + def handle_pre_merge(self): + pass + + def handle_merge(self): + super().handle_merge() + self.merge_template_directories(TemplateType.DJANGO_NINJA) + + def handle_post_merge(self): + super().handle_post_merge() + class AppBuilderPipeline(ApplicationBuilder): def __init__(self, app_name: str, app_path: pathlib.Path, templates: list[str]): @@ -198,6 +224,7 @@ def __init__(self, app_name: str, app_path: pathlib.Path, templates: list[str]): TemplateType.SERVER: ServerAppBuilder(app_name, app_path), TemplateType.FLASK_SERVER: FlaskServerAppBuilder(app_name, app_path), TemplateType.DJANGO_FASTAPI: DjangoFastApiBuilder(app_name, app_path), + TemplateType.DJANGO_NINJA: DjangoNinjaBuilder(app_name, app_path), } def handles(self, templates): @@ -207,6 +234,7 @@ def handle_pre_merge(self): pre_merge_template_order = [ TemplateType.FLASK_SERVER, TemplateType.DJANGO_FASTAPI, + TemplateType.DJANGO_NINJA, TemplateType.WEBAPP, TemplateType.SERVER, ] @@ -232,6 +260,7 @@ def handle_post_merge(self): post_merge_template_order = [ TemplateType.FLASK_SERVER, TemplateType.DJANGO_FASTAPI, + TemplateType.DJANGO_NINJA, TemplateType.WEBAPP, TemplateType.SERVER, ] diff --git a/tools/deployment-cli-tools/ch_cli_tools/common_types.py b/tools/deployment-cli-tools/ch_cli_tools/common_types.py index 99b9d41c..b88a19c9 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/common_types.py +++ b/tools/deployment-cli-tools/ch_cli_tools/common_types.py @@ -17,12 +17,17 @@ class TemplateType(StrEnum): DB_NEO4J = 'db-neo4j' DB_MONGO = 'db-mongo' DJANGO_FASTAPI = 'django-fastapi' + DJANGO_NINJA = 'django-ninja' SERVER = 'server' @classmethod def database_templates(cls): return [cls.DB_POSTGRES, cls.DB_NEO4J, cls.DB_MONGO] + @classmethod + def django_templates(cls) -> list[str]: + return [cls.DJANGO_FASTAPI, cls.DJANGO_FASTAPI] + @dataclass class CloudHarnessManifest(): diff --git a/tools/deployment-cli-tools/harness-application b/tools/deployment-cli-tools/harness-application index 2a65fe37..7484f754 100644 --- a/tools/deployment-cli-tools/harness-application +++ b/tools/deployment-cli-tools/harness-application @@ -66,6 +66,7 @@ def get_command_line_arguments() -> tuple[str, list[str]]: - db-neo4j - db-mongo - django-fastapi (fastapi django backend based on openapi) + - django-ninja (django ninja backend) """) args, unknown = parser.parse_known_args(sys.argv[1:]) @@ -91,14 +92,16 @@ def get_command_line_arguments() -> tuple[str, list[str]]: def normalize_templates(templates: list[str]) -> list[str]: templates = list(templates) - if TemplateType.DJANGO_FASTAPI in templates and TemplateType.WEBAPP not in templates: - django_app_index = templates.index(TemplateType.DJANGO_FASTAPI) - templates.insert(django_app_index, TemplateType.WEBAPP) + def django_template_index(): + return next(index for index, template in enumerate(templates) if template in TemplateType.django_templates()) + + has_django_template = any(template in TemplateType.django_templates() for template in templates) + if has_django_template and TemplateType.WEBAPP not in templates: + templates.insert(django_template_index(), TemplateType.WEBAPP) has_database_template = any(template in TemplateType.database_templates() for template in templates) - if TemplateType.DJANGO_FASTAPI in templates and not has_database_template: - django_app_index = templates.index(TemplateType.DJANGO_FASTAPI) - templates.insert(django_app_index, TemplateType.DB_POSTGRES) + if has_django_template and not has_database_template: + templates.insert(django_template_index(), TemplateType.DB_POSTGRES) return templates diff --git a/tools/deployment-cli-tools/harness-generate b/tools/deployment-cli-tools/harness-generate index 00b80019..8cf6e466 100644 --- a/tools/deployment-cli-tools/harness-generate +++ b/tools/deployment-cli-tools/harness-generate @@ -312,14 +312,19 @@ def infer_webapp_template(app_path: pathlib.Path, templates: list[str]) -> None: def infer_server_template(app_path: pathlib.Path, templates: list[str]) -> None: - genapi_path = app_path / 'api' / 'genapi.sh' - - if genapi_path.exists(): - templates.append(TemplateType.DJANGO_FASTAPI) + backend_path = app_path / 'backend' + manage_path = backend_path / 'manage.py' + + if manage_path.exists(): + requirements_path = backend_path / 'requirements.txt' + requirements = requirements_path.read_text() + if 'ninja' in requirements: + templates.append(TemplateType.DJANGO_NINJA) + else: + templates.append(TemplateType.DJANGO_FASTAPI) return server_path = app_path / 'server' - backend_path = app_path / 'backend' if server_path.exists() or backend_path.exists(): templates.append(TemplateType.FLASK_SERVER) From 5b790444ebc4b84bd8b6699186e80d467ee405e4 Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Fri, 1 Nov 2024 16:18:15 +0000 Subject: [PATCH 33/93] CH-157 Add specific debug templates for the two different djano templates --- ...vscode-django-fastapi-debug-template.json} | 0 .../vscode-django-ninja-debug-template.json | 24 +++++++++++++++++++ .../ch_cli_tools/application_builders.py | 11 ++++++++- 3 files changed, 34 insertions(+), 1 deletion(-) rename deployment-configuration/{vscode-django-app-debug-template.json => vscode-django-fastapi-debug-template.json} (100%) create mode 100644 deployment-configuration/vscode-django-ninja-debug-template.json diff --git a/deployment-configuration/vscode-django-app-debug-template.json b/deployment-configuration/vscode-django-fastapi-debug-template.json similarity index 100% rename from deployment-configuration/vscode-django-app-debug-template.json rename to deployment-configuration/vscode-django-fastapi-debug-template.json diff --git a/deployment-configuration/vscode-django-ninja-debug-template.json b/deployment-configuration/vscode-django-ninja-debug-template.json new file mode 100644 index 00000000..198d465d --- /dev/null +++ b/deployment-configuration/vscode-django-ninja-debug-template.json @@ -0,0 +1,24 @@ +{ + "args": [ + "--host", + "0.0.0.0", + "--port", + "8000", + "django_baseapp.asgi:application" + ], + "console": "integratedTerminal", + "cwd": "${workspaceFolder}/applications/__APP_NAME__/backend", + "env": { + "ACCOUNTS_ADMIN_PASSWORD": "metacell", + "ACCOUNTS_ADMIN_USERNAME": "admin", + "CH_CURRENT_APP_NAME": "__APP_NAME__", + "CH_VALUES_PATH": "${workspaceFolder}/deployment/helm/values.yaml", + "DJANGO_SETTINGS_MODULE": "django_baseapp.settings", + "KUBERNETES_SERVICE_HOST": "ssdds" + }, + "justMyCode": false, + "module": "uvicorn", + "name": "__APP_NAME__ backend", + "request": "launch", + "type": "debugpy" + } \ No newline at end of file diff --git a/tools/deployment-cli-tools/ch_cli_tools/application_builders.py b/tools/deployment-cli-tools/ch_cli_tools/application_builders.py index 611955b5..e58ff681 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/application_builders.py +++ b/tools/deployment-cli-tools/ch_cli_tools/application_builders.py @@ -161,7 +161,7 @@ def create_django_app_vscode_debug_configuration(self): if configuration['name'] != configuration_name ] - debug_config = get_json_template('vscode-django-app-debug-template.json', True) + debug_config = get_json_template(self.debug_template_file, True) debug_config = replace_in_dict(debug_config, self.APP_NAME_PLACEHOLDER, self.app_name) launch_config['configurations'].append(debug_config) @@ -173,9 +173,16 @@ def create_django_app_vscode_debug_configuration(self): @property def python_app_name(self): return to_python_module(self.app_name) + + @property + @abc.abstractmethod + def debug_template_file(self) -> str: + raise NotImplementedError() class DjangoFastApiBuilder(BaseDjangoAppBuilder): + debug_template_name = 'vscode-django-fastapi-debug-template.json' + def handles(self, templates): return TemplateType.DJANGO_FASTAPI in templates @@ -201,6 +208,8 @@ def handle_post_merge(self): class DjangoNinjaBuilder(BaseDjangoAppBuilder): + debug_template_name = 'vscode-django-ninja-debug-template.json' + def handles(self, templates): return TemplateType.DJANGO_NINJA in templates From 025d5c80169628cdb5c091be9fa8a3e8ea882da6 Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Fri, 1 Nov 2024 17:56:40 +0000 Subject: [PATCH 34/93] CH-157 Fix errors in django-ninja template --- .../django-ninja/backend/__APP_NAME__/api/__init__.py | 6 +++--- application-templates/django-ninja/backend/requirements.txt | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/application-templates/django-ninja/backend/__APP_NAME__/api/__init__.py b/application-templates/django-ninja/backend/__APP_NAME__/api/__init__.py index 647697a0..ab1fdac8 100644 --- a/application-templates/django-ninja/backend/__APP_NAME__/api/__init__.py +++ b/application-templates/django-ninja/backend/__APP_NAME__/api/__init__.py @@ -25,16 +25,16 @@ def forbidden(request, exc): ) -@api.get('/ping', response={200: float}) +@api.get('/ping', response={200: float}, tags=['test']) def ping(request: HttpRequest): return time.time() -@api.get('/live', response={200: str}) +@api.get('/live', response={200: str}, tags=['test']) def live(request: HttpRequest): return 'OK' -@api.get('/ready', response={200: str}) +@api.get('/ready', response={200: str}, tags=['test']) def ready(request: HttpRequest): return 'OK' diff --git a/application-templates/django-ninja/backend/requirements.txt b/application-templates/django-ninja/backend/requirements.txt index 97c58008..b3575416 100644 --- a/application-templates/django-ninja/backend/requirements.txt +++ b/application-templates/django-ninja/backend/requirements.txt @@ -1,2 +1,2 @@ pydantic==2.9.2 -ninja \ No newline at end of file +django-ninja \ No newline at end of file From 46488238f381ce6265d67a6f0f7016808055ade7 Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Fri, 1 Nov 2024 17:57:20 +0000 Subject: [PATCH 35/93] CH-157 Add utility method to run djano-ninja openapi export --- tools/deployment-cli-tools/ch_cli_tools/openapi.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tools/deployment-cli-tools/ch_cli_tools/openapi.py b/tools/deployment-cli-tools/ch_cli_tools/openapi.py index 7de31d8c..fcc91974 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/openapi.py +++ b/tools/deployment-cli-tools/ch_cli_tools/openapi.py @@ -124,6 +124,20 @@ def generate_ts_client(openapi_file): replaceindir(out_dir, "http://localhost", '') +def generate_openapi_from_ninja_schema(app_name: str, app_path: pathlib.Path) -> None: + out_path = app_path / 'api' / 'openapi.yaml' + manage_path = app_path / 'backend' / 'manage.py' + command = [ + 'python', manage_path, 'export_openapi_schema', + '--settings', 'django_baseapp.settings', + '--api', f'{to_python_module(app_name)}.api.api', + '--output', out_path, + '--indent', '2', + ] + + subprocess.run(command) + + def get_dependencies(): """ Checks if java is installed From 9febcd62a970206f1d10b0cf03d44c3298c6cb0c Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Fri, 1 Nov 2024 17:57:58 +0000 Subject: [PATCH 36/93] CH-157 Fix bug with django ninja not considered a django template --- tools/deployment-cli-tools/ch_cli_tools/common_types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment-cli-tools/ch_cli_tools/common_types.py b/tools/deployment-cli-tools/ch_cli_tools/common_types.py index b88a19c9..6c8e2c4c 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/common_types.py +++ b/tools/deployment-cli-tools/ch_cli_tools/common_types.py @@ -26,7 +26,7 @@ def database_templates(cls): @classmethod def django_templates(cls) -> list[str]: - return [cls.DJANGO_FASTAPI, cls.DJANGO_FASTAPI] + return [cls.DJANGO_FASTAPI, cls.DJANGO_NINJA] @dataclass From 33d9803df9c45db9525ccf50bbe266df1bbdff34 Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Fri, 1 Nov 2024 17:58:42 +0000 Subject: [PATCH 37/93] CH-157 Fix bug with incorrectly named field in derived app builders --- .../deployment-cli-tools/ch_cli_tools/application_builders.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/deployment-cli-tools/ch_cli_tools/application_builders.py b/tools/deployment-cli-tools/ch_cli_tools/application_builders.py index e58ff681..ef020633 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/application_builders.py +++ b/tools/deployment-cli-tools/ch_cli_tools/application_builders.py @@ -181,7 +181,7 @@ def debug_template_file(self) -> str: class DjangoFastApiBuilder(BaseDjangoAppBuilder): - debug_template_name = 'vscode-django-fastapi-debug-template.json' + debug_template_file = 'vscode-django-fastapi-debug-template.json' def handles(self, templates): return TemplateType.DJANGO_FASTAPI in templates @@ -208,7 +208,7 @@ def handle_post_merge(self): class DjangoNinjaBuilder(BaseDjangoAppBuilder): - debug_template_name = 'vscode-django-ninja-debug-template.json' + debug_template_file = 'vscode-django-ninja-debug-template.json' def handles(self, templates): return TemplateType.DJANGO_NINJA in templates From dca0eae2c16a52f0990b3eb2f0ad66e9acae6d2d Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Fri, 1 Nov 2024 17:59:42 +0000 Subject: [PATCH 38/93] CH-157 Update harness-generate to leverage django-ninja schema generation --- tools/deployment-cli-tools/harness-generate | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/tools/deployment-cli-tools/harness-generate b/tools/deployment-cli-tools/harness-generate index 8cf6e466..545e6612 100644 --- a/tools/deployment-cli-tools/harness-generate +++ b/tools/deployment-cli-tools/harness-generate @@ -11,7 +11,7 @@ import logging from typing import Callable, Optional from ruamel.yaml.error import YAMLError -from ch_cli_tools.openapi import LIB_NAME, generate_python_client, generate_server, generate_fastapi_server, \ +from ch_cli_tools.openapi import LIB_NAME, generate_openapi_from_ninja_schema, generate_python_client, generate_server, generate_fastapi_server, \ get_dependencies, generate_ts_client, generate_model from ch_cli_tools.utils import copymergedir, load_yaml, save_yaml from ch_cli_tools.common_types import CloudHarnessManifest, TemplateType @@ -184,21 +184,25 @@ def generate_clients( if not should_generate('client libraries'): return - openapi_files = get_openapi_file_paths(root_path) client_src_path = root_path / 'libraries' / 'client' / client_lib_name + apps_path = root_path / 'applications' + apps = (app for app in apps_path.iterdir() if app.is_dir()) - for openapi_file in openapi_files: - app_path = openapi_file.parent.parent + for app_path in apps: manifest = get_manifest(app_path) if app_name and manifest.app_name != app_name: continue - if ClientType.PYTHON_CLIENT in client_types: - generate_python_client(manifest.app_name, openapi_file, client_src_path, lib_name=client_lib_name) + if TemplateType.DJANGO_NINJA in manifest.templates: + generate_openapi_from_ninja_schema(manifest.app_name, app_path) + + for openapi_file in app_path.glob('api/*.yaml'): + if ClientType.PYTHON_CLIENT in client_types: + generate_python_client(manifest.app_name, openapi_file, client_src_path, lib_name=client_lib_name) - if TemplateType.WEBAPP in manifest.templates and ClientType.TS_CLIENT in client_types: - generate_ts_client(openapi_file) + if TemplateType.WEBAPP in manifest.templates and ClientType.TS_CLIENT in client_types: + generate_ts_client(openapi_file) aggregate_packages(client_src_path, client_lib_name) From 4b768ef829497428ee3930f37585f36346fa4e15 Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Fri, 1 Nov 2024 18:27:25 +0000 Subject: [PATCH 39/93] CH-157 Move manifest management code into its own file --- .../ch_cli_tools/common_types.py | 16 +-- .../ch_cli_tools/manifest.py | 135 ++++++++++++++++++ tools/deployment-cli-tools/harness-generate | 87 +---------- 3 files changed, 140 insertions(+), 98 deletions(-) create mode 100644 tools/deployment-cli-tools/ch_cli_tools/manifest.py diff --git a/tools/deployment-cli-tools/ch_cli_tools/common_types.py b/tools/deployment-cli-tools/ch_cli_tools/common_types.py index 6c8e2c4c..dbff707a 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/common_types.py +++ b/tools/deployment-cli-tools/ch_cli_tools/common_types.py @@ -51,18 +51,4 @@ def to_dict(self) -> dict: 'version': self.version, 'inferred': self.inferred, 'templates': [str(template) for template in self.templates], - } - - @classmethod - def migrate(cls, data: dict) -> tuple[dict, bool]: - data_copy = copy.deepcopy(data) - update_manifest = False - - if data_copy['version'] < '2': - update_manifest = True - data_copy['templates'] = [ - template if template != 'django-app' else 'django-fastapi' - for template in data_copy['templates'] - ] - - return data_copy, update_manifest \ No newline at end of file + } \ No newline at end of file diff --git a/tools/deployment-cli-tools/ch_cli_tools/manifest.py b/tools/deployment-cli-tools/ch_cli_tools/manifest.py new file mode 100644 index 00000000..155dc750 --- /dev/null +++ b/tools/deployment-cli-tools/ch_cli_tools/manifest.py @@ -0,0 +1,135 @@ +import abc +import copy +import logging +import pathlib +from typing import Iterable +from ruamel.yaml.error import YAMLError +from .common_types import CloudHarnessManifest, TemplateType +from .utils import load_yaml, save_yaml + + +def get_manifest(app_path: pathlib.Path) -> CloudHarnessManifest: + manifest_file = app_path / '.ch-manifest' + + try: + manifest_data = load_yaml(manifest_file) + return CloudHarnessManifest.from_dict(manifest_data) + except (FileNotFoundError, YAMLError): + logging.info(f'Could not load manifest file {manifest_file}, inferring manifest from app structure...') + manifest = CloudHarnessManifest( + app_name=app_path.name, + inferred=True, + templates=infer_templates(app_path), + ) + save_yaml(manifest_file, manifest.to_dict()) + return manifest + + +def load_manifest(manifest_file: pathlib.Path) -> dict: + manifest_data = load_yaml(manifest_file) + migrated_data = migrate_manifest_data(manifest_data) + + if manifest_data != migrated_data: + save_yaml(manifest_file, migrated_data) + + return migrated_data + + +def migrate_manifest_data(data: dict) -> dict: + data = copy.deepcopy(data) + data_version = data['version'] + migrations = [ + migration for migration in _MIGRATIONS_LIST + if data_version < migration.change_version + ] + + for migration in migrations: + migration.migrate(data) + + return data + + +def infer_templates(app_path: pathlib.Path) -> list[str]: + return [ + TemplateType.BASE, + *infer_webapp_template(app_path), + *infer_server_template(app_path), + *infer_database_template(app_path), + ] + + +def infer_webapp_template(app_path: pathlib.Path) -> Iterable[str]: + frontend_path = app_path / 'frontend' + if frontend_path.exists(): + yield TemplateType.WEBAPP + + +def infer_server_template(app_path: pathlib.Path) -> Iterable[str]: + backend_path = app_path / 'backend' + manage_path = backend_path / 'manage.py' + + if manage_path.exists(): + yield from infer_django_template(backend_path) + return + + server_path = app_path / 'server' + if server_path.exists() or backend_path.exists(): + yield TemplateType.FLASK_SERVER + + +def infer_django_template(backend_path: pathlib.Path) -> Iterable[str]: + requirements_path = backend_path / 'requirements.txt' + requirements = requirements_path.read_text() + + if 'django-ninja' in requirements: + yield TemplateType.DJANGO_NINJA + else: + yield TemplateType.DJANGO_FASTAPI + + +def infer_database_template(app_path: pathlib.Path) -> Iterable[str]: + values_file = app_path / 'deploy' / 'values.yaml' + + try: + values_data = load_yaml(values_file) + database_config = values_data['harness']['database'] + if not database_config['auto']: + return + + database_type = database_config['type'] + database_type_to_template_map = { + 'mongo': TemplateType.DB_MONGO, + 'neo4j': TemplateType.DB_NEO4J, + 'postgres': TemplateType.DB_POSTGRES, + } + + if database_type in database_type_to_template_map: + yield database_type_to_template_map[database_type] + + except(FileNotFoundError, YAMLError, KeyError): + pass + + +class ManifestMigration(abc.ABC): + @property + @abc.abstractmethod + def change_version(self) -> str: + ... + + @abc.abstractmethod + def migrate(data: dict) -> None: + ... + + +class NameChangeFromDjangoAppToDjangoFastapi(ManifestMigration): + change_version = '2' + + def migrate(data): + data['templates'] = [ + template if template != 'django-app' else 'django-fastapi' + for template in data['templates'] + ] + +_MIGRATIONS_LIST: list[ManifestMigration] = [ + NameChangeFromDjangoAppToDjangoFastapi(), +] \ No newline at end of file diff --git a/tools/deployment-cli-tools/harness-generate b/tools/deployment-cli-tools/harness-generate index 545e6612..705a40c6 100644 --- a/tools/deployment-cli-tools/harness-generate +++ b/tools/deployment-cli-tools/harness-generate @@ -9,12 +9,12 @@ import pathlib import shutil import logging from typing import Callable, Optional -from ruamel.yaml.error import YAMLError from ch_cli_tools.openapi import LIB_NAME, generate_openapi_from_ninja_schema, generate_python_client, generate_server, generate_fastapi_server, \ get_dependencies, generate_ts_client, generate_model -from ch_cli_tools.utils import copymergedir, load_yaml, save_yaml -from ch_cli_tools.common_types import CloudHarnessManifest, TemplateType +from ch_cli_tools.utils import copymergedir +from ch_cli_tools.common_types import TemplateType +from ch_cli_tools.manifest import get_manifest def main(): @@ -152,7 +152,7 @@ def generate_servers( """ Generates server stubs """ - openapi_files = get_openapi_file_paths(root_path) + openapi_files = [path for path in root_path.glob('applications/*/api/*.yaml')] for openapi_file in openapi_files: app_path = openapi_file.parent.parent @@ -207,10 +207,6 @@ def generate_clients( aggregate_packages(client_src_path, client_lib_name) -def get_openapi_file_paths(root_path: pathlib.Path) -> list[pathlib.Path]: - return [path for path in root_path.glob('applications/*/api/*.yaml')] - - def aggregate_packages(client_source_path: pathlib.Path, lib_name=LIB_NAME): client_source_path.mkdir(parents=True, exist_ok=True) @@ -278,80 +274,5 @@ def aggregate_packages(client_source_path: pathlib.Path, lib_name=LIB_NAME): shutil.rmtree(temp_module_path) -def get_manifest(app_path: pathlib.Path) -> CloudHarnessManifest: - manifest_file = app_path / '.ch-manifest' - - try: - manifest_data = load_yaml(manifest_file) - manifest_data, update_manifest = CloudHarnessManifest.migrate(manifest_data) - if update_manifest: - save_yaml(manifest_file, manifest_data) - manifest = CloudHarnessManifest.from_dict(manifest_data) - except (FileNotFoundError, YAMLError): - logging.info(f'Could not find manifest file {manifest_file}, inferring manifest from app structure...') - manifest = CloudHarnessManifest( - app_name=app_path.name, - inferred=True, - templates=infer_templates(app_path), - ) - save_yaml(manifest_file, manifest.to_dict()) - - return manifest - - -def infer_templates(app_path: pathlib.Path) -> list[str]: - templates = [TemplateType.BASE] - - infer_webapp_template(app_path, templates) - infer_server_template(app_path, templates) - infer_database_template(app_path, templates) - - return templates - - -def infer_webapp_template(app_path: pathlib.Path, templates: list[str]) -> None: - frontend_path = app_path / 'frontend' - if frontend_path.exists(): - templates.append(TemplateType.WEBAPP) - - -def infer_server_template(app_path: pathlib.Path, templates: list[str]) -> None: - backend_path = app_path / 'backend' - manage_path = backend_path / 'manage.py' - - if manage_path.exists(): - requirements_path = backend_path / 'requirements.txt' - requirements = requirements_path.read_text() - if 'ninja' in requirements: - templates.append(TemplateType.DJANGO_NINJA) - else: - templates.append(TemplateType.DJANGO_FASTAPI) - return - - server_path = app_path / 'server' - if server_path.exists() or backend_path.exists(): - templates.append(TemplateType.FLASK_SERVER) - - -def infer_database_template(app_path: pathlib.Path, templates: list[str]) -> None: - values_file = app_path / 'deploy' / 'values.yaml' - - try: - values_data = load_yaml(values_file) - database_config = values_data['harness']['database'] - if not database_config['auto']: - return - - database_type = database_config['type'] - if database_type == 'mongo': - templates.append(TemplateType.DB_MONGO) - if database_type == 'neo4j': - templates.append(TemplateType.DB_NEO4J) - if database_type == 'postgres': - templates.append(TemplateType.DB_POSTGRES) - except (FileNotFoundError, YAMLError, KeyError): - pass - - if __name__ == "__main__": main() From 11590babcd48b844db282c32a0c1fbf2afa28e32 Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Mon, 4 Nov 2024 17:20:39 +0000 Subject: [PATCH 40/93] CH-157 Fix linting issues --- .../ch_cli_tools/application_builders.py | 18 +++++++++--------- .../ch_cli_tools/common_types.py | 2 +- .../ch_cli_tools/manifest.py | 11 ++++++----- 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/tools/deployment-cli-tools/ch_cli_tools/application_builders.py b/tools/deployment-cli-tools/ch_cli_tools/application_builders.py index ef020633..d7701445 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/application_builders.py +++ b/tools/deployment-cli-tools/ch_cli_tools/application_builders.py @@ -18,7 +18,7 @@ class ApplicationBuilder(abc.ABC): def __init__(self, app_name: str, app_path: pathlib.Path): self.app_name = app_name self.app_path = app_path - + @abc.abstractmethod def handles(self, templates: list[str]) -> bool: pass @@ -51,19 +51,19 @@ def merge_template_directories(self, template_name: str) -> None: @property def frontend_path(self): return self.app_path / 'frontend' - + @property def backend_path(self): return self.app_path / 'backend' - + @property def api_path(self): return self.app_path / 'api' - + @property def ch_root(self): return pathlib.Path(CH_ROOT) - + @property def app_template_path(self): return self.ch_root / APPLICATION_TEMPLATE_PATH @@ -103,7 +103,7 @@ def install_frontend_dependencies(self) -> None: class ServerAppBuilder(ApplicationBuilder): def handles(self, templates): return TemplateType.SERVER in templates - + def handle_pre_merge(self): with tempfile.TemporaryDirectory() as tmp_dirname: tmp_path = pathlib.Path(tmp_dirname) @@ -123,7 +123,7 @@ def handle_post_merge(self): class FlaskServerAppBuilder(ApplicationBuilder): def handles(self, templates): return TemplateType.FLASK_SERVER in templates - + def handle_pre_merge(self): pass @@ -149,7 +149,7 @@ def handle_post_merge(self): replace_in_file(self.app_path / 'dev-setup.sh', self.APP_NAME_PLACEHOLDER, self.app_name) self.create_django_app_vscode_debug_configuration() - + def create_django_app_vscode_debug_configuration(self): vscode_launch_path = pathlib.Path('.vscode/launch.json') configuration_name = f'{self.app_name} backend' @@ -173,7 +173,7 @@ def create_django_app_vscode_debug_configuration(self): @property def python_app_name(self): return to_python_module(self.app_name) - + @property @abc.abstractmethod def debug_template_file(self) -> str: diff --git a/tools/deployment-cli-tools/ch_cli_tools/common_types.py b/tools/deployment-cli-tools/ch_cli_tools/common_types.py index dbff707a..eb1e5feb 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/common_types.py +++ b/tools/deployment-cli-tools/ch_cli_tools/common_types.py @@ -51,4 +51,4 @@ def to_dict(self) -> dict: 'version': self.version, 'inferred': self.inferred, 'templates': [str(template) for template in self.templates], - } \ No newline at end of file + } diff --git a/tools/deployment-cli-tools/ch_cli_tools/manifest.py b/tools/deployment-cli-tools/ch_cli_tools/manifest.py index 155dc750..0425d806 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/manifest.py +++ b/tools/deployment-cli-tools/ch_cli_tools/manifest.py @@ -24,14 +24,14 @@ def get_manifest(app_path: pathlib.Path) -> CloudHarnessManifest: save_yaml(manifest_file, manifest.to_dict()) return manifest - + def load_manifest(manifest_file: pathlib.Path) -> dict: manifest_data = load_yaml(manifest_file) migrated_data = migrate_manifest_data(manifest_data) if manifest_data != migrated_data: save_yaml(manifest_file, migrated_data) - + return migrated_data @@ -95,7 +95,7 @@ def infer_database_template(app_path: pathlib.Path) -> Iterable[str]: database_config = values_data['harness']['database'] if not database_config['auto']: return - + database_type = database_config['type'] database_type_to_template_map = { 'mongo': TemplateType.DB_MONGO, @@ -106,7 +106,7 @@ def infer_database_template(app_path: pathlib.Path) -> Iterable[str]: if database_type in database_type_to_template_map: yield database_type_to_template_map[database_type] - except(FileNotFoundError, YAMLError, KeyError): + except (FileNotFoundError, YAMLError, KeyError): pass @@ -130,6 +130,7 @@ def migrate(data): for template in data['templates'] ] + _MIGRATIONS_LIST: list[ManifestMigration] = [ NameChangeFromDjangoAppToDjangoFastapi(), -] \ No newline at end of file +] From 6c856568ea0cfc3bcf47bdd6b5899b7d52f542fe Mon Sep 17 00:00:00 2001 From: Filippo Ledda Date: Tue, 5 Nov 2024 11:10:42 +0100 Subject: [PATCH 41/93] CH-158 option to edit user on django --- .../cloudharness-django/cloudharness_django/admin.py | 6 +++--- .../cloudharness-django/cloudharness_django/settings.py | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/admin.py b/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/admin.py index 493384e0..2f3640f4 100644 --- a/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/admin.py +++ b/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/admin.py @@ -16,13 +16,13 @@ class CHUserAdmin(ExtraButtonsMixin, UserAdmin): def has_add_permission(self, request): - return settings.DEBUG + return settings.DEBUG or settings.USER_CHANGE_ENABLED def has_change_permission(self, request, obj=None): - return settings.DEBUG + return settings.DEBUG or settings.USER_CHANGE_ENABLED def has_delete_permission(self, request, obj=None): - return settings.DEBUG + return settings.DEBUG or settings.USER_CHANGE_ENABLED @button() def sync_keycloak(self, request): diff --git a/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/settings.py b/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/settings.py index b9efd572..b5c71a1f 100644 --- a/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/settings.py +++ b/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/settings.py @@ -26,6 +26,8 @@ 'cloudharness_django.middleware.BearerTokenMiddleware', ] +USER_CHANGE_ENABLED = False + # test if the kubernetes CH all values exists, if so then set up specific k8s stuff # IMPROTANT NOTE: # when testing/debugging with Kafka then copy the deployment/helm/values.yaml to the ALLVALUES_PATH From b27916fa3e589ae163ba3fed41fd1919d5485103 Mon Sep 17 00:00:00 2001 From: Filippo Ledda Date: Wed, 6 Nov 2024 11:58:42 +0100 Subject: [PATCH 42/93] CH-162 fix random values generation --- .../helm/templates/auto-secrets.yaml | 27 +++++++++---------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/deployment-configuration/helm/templates/auto-secrets.yaml b/deployment-configuration/helm/templates/auto-secrets.yaml index a0a37a2f..27851ad9 100644 --- a/deployment-configuration/helm/templates/auto-secrets.yaml +++ b/deployment-configuration/helm/templates/auto-secrets.yaml @@ -9,23 +9,20 @@ metadata: labels: app: {{ .app.harness.deployment.name }} type: Opaque - {{- $secret := (lookup "v1" "Secret" .root.Values.namespace $secret_name) }} - {{- if $secret }} +{{- $secret := (lookup "v1" "Secret" .root.Values.namespace $secret_name) }} +{{- if $secret }} # secret already exists - {{- if not (compact (values .app.harness.secrets)) }} -# secret values are null, copy from the existing secret -data: - {{- range $k, $v := $secret.data }} - {{ $k }}: {{ $v }} - {{- end }} - {{- else }} -# there are non default values in values.yaml, use these stringData: - {{- range $k, $v := .app.harness.secrets }} - {{ $k }}: {{ $v | default (randAlphaNum 20) }} - {{- end }} - {{- end }} - {{- else }} + {{- range $k, $v := .app.harness.secrets }} + {{- if $v }} + # Set secret value to value in values.yaml if specified + {{ $k }}: {{ $v }} + {{- else if not (hasKey $secret.data $k) }} + # Create a random secret value if not specified in values.yaml and is not already in the secret + {{ $k }}: {{ randAlphaNum 20 }} + {{- end}} + {{- end }} +{{- else }} # secret doesn't exist stringData: {{- range $k, $v := .app.harness.secrets }} From d938c9779450b689df03306ea22f8e15b618cfff Mon Sep 17 00:00:00 2001 From: Filippo Ledda Date: Wed, 6 Nov 2024 12:09:04 +0100 Subject: [PATCH 43/93] CH-162 support for dynamic random secrets --- .../helm/templates/auto-secrets.yaml | 9 ++++++--- docs/applications/secrets.md | 7 +++++-- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/deployment-configuration/helm/templates/auto-secrets.yaml b/deployment-configuration/helm/templates/auto-secrets.yaml index 27851ad9..3dd6427f 100644 --- a/deployment-configuration/helm/templates/auto-secrets.yaml +++ b/deployment-configuration/helm/templates/auto-secrets.yaml @@ -12,13 +12,16 @@ type: Opaque {{- $secret := (lookup "v1" "Secret" .root.Values.namespace $secret_name) }} {{- if $secret }} # secret already exists + stringData: {{- range $k, $v := .app.harness.secrets }} - {{- if $v }} + {{- if and $v (ne $v "?") }} # Set secret value to value in values.yaml if specified {{ $k }}: {{ $v }} - {{- else if not (hasKey $secret.data $k) }} - # Create a random secret value if not specified in values.yaml and is not already in the secret + {{- else if or (eq $v "?") (not (hasKey $secret.data $k)) }} + # Create a random secret value if not specified in values.yaml if: + # 1. it is not set and it is not already in the deployed secret (static random secret) + # 2. its value is ? (dynamic random secret) {{ $k }}: {{ randAlphaNum 20 }} {{- end}} {{- end }} diff --git a/docs/applications/secrets.md b/docs/applications/secrets.md index dfd72256..5cac227a 100644 --- a/docs/applications/secrets.md +++ b/docs/applications/secrets.md @@ -20,13 +20,16 @@ harness: secrets: unsecureSecret: secureSecret: - random-secret: "" + random-static-secret: "" + random-dynamic-secret: ? ``` Secret values are initialized in three different ways: * Set the secret's value (as in `unsecureSecret`). Do that only if you aware of what you are doing as the value may be pushed in the git(hub) repository. * Leave the secret's value `null` (as in `secureSecret`) to configure manually later in the ci/cd pipeline. -* Use the "" (empty string) value (as in `random-secret`) to let cloudharness generate a random value for you. +* Use the "" (empty string) value (as in `random-static-secret`) to let CloudHarness generate a random value for you. + This secret won't be updated after being set by any of the CloudHarness automations, so has to be managed through `kubectl` directly. +* Use the `?` value (as in `random-dynamic-secret`) to get a new random value for every deployment upgrade Secret editing/maintenance alternatives: * CI/CD Codefresh support: all `null` and `` secrets will be added to the codefresh deployment file(s) and can be set/overwritten through the codefresh variable configuration From fa4c3fee29109336332b3009b7eb805369a2bded Mon Sep 17 00:00:00 2001 From: Filippo Ledda Date: Wed, 6 Nov 2024 13:02:01 +0100 Subject: [PATCH 44/93] Django: Improve db coherence on user sync issues --- .../cloudharness-django/cloudharness_django/admin.py | 8 +++++++- .../cloudharness_django/services/user.py | 4 +++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/admin.py b/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/admin.py index 2f3640f4..ea3d423d 100644 --- a/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/admin.py +++ b/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/admin.py @@ -4,7 +4,7 @@ from django.contrib.auth.models import User, Group from admin_extra_buttons.api import ExtraButtonsMixin, button - +from .models import Member from cloudharness_django.services import get_user_service # Register your models here. @@ -13,8 +13,14 @@ admin.site.unregister(Group) +class MemberAdmin(admin.StackedInline): + model = Member + + class CHUserAdmin(ExtraButtonsMixin, UserAdmin): + inlines = [MemberAdmin] + def has_add_permission(self, request): return settings.DEBUG or settings.USER_CHANGE_ENABLED diff --git a/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/services/user.py b/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/services/user.py index fc49ba36..4aa0677c 100644 --- a/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/services/user.py +++ b/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/services/user.py @@ -100,7 +100,9 @@ def sync_kc_user(self, kc_user, is_superuser=False, delete=False): user, created = User.objects.get_or_create(username=kc_user["username"]) - Member.objects.get_or_create(user=user, kc_id=kc_user["id"]) + member = Member.objects.get_or_create(user=user) + member.kc_id = kc_user["id"] + member.save() user = self._map_kc_user(user, kc_user, is_superuser, delete) user.save() return user From 9be9c963266c2649597b0c9e7ce5f29958fa3341 Mon Sep 17 00:00:00 2001 From: Filippo Ledda Date: Wed, 6 Nov 2024 17:08:05 +0100 Subject: [PATCH 45/93] Django: Fix: Improve db coherence on user sync issues --- .../cloudharness-django/cloudharness_django/services/user.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/services/user.py b/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/services/user.py index 4aa0677c..189e6516 100644 --- a/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/services/user.py +++ b/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/services/user.py @@ -100,7 +100,7 @@ def sync_kc_user(self, kc_user, is_superuser=False, delete=False): user, created = User.objects.get_or_create(username=kc_user["username"]) - member = Member.objects.get_or_create(user=user) + member, created = Member.objects.get_or_create(user=user) member.kc_id = kc_user["id"] member.save() user = self._map_kc_user(user, kc_user, is_superuser, delete) From b4e4102c8fbfb88b3e3c208ede71cbeaa893a7cd Mon Sep 17 00:00:00 2001 From: Filippo Ledda Date: Wed, 6 Nov 2024 18:40:25 +0100 Subject: [PATCH 46/93] CH-162 fix for non string secrets --- deployment-configuration/helm/templates/auto-secrets.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deployment-configuration/helm/templates/auto-secrets.yaml b/deployment-configuration/helm/templates/auto-secrets.yaml index 3dd6427f..df92258a 100644 --- a/deployment-configuration/helm/templates/auto-secrets.yaml +++ b/deployment-configuration/helm/templates/auto-secrets.yaml @@ -15,10 +15,10 @@ type: Opaque stringData: {{- range $k, $v := .app.harness.secrets }} - {{- if and $v (ne $v "?") }} + {{- if and $v (and (eq (typeOf $v) "string") (ne $v "?")) }} # Set secret value to value in values.yaml if specified {{ $k }}: {{ $v }} - {{- else if or (eq $v "?") (not (hasKey $secret.data $k)) }} + {{- else if or (and (eq (typeOf $v) "string") (eq $v "?")) (not (hasKey $secret.data $k)) }} # Create a random secret value if not specified in values.yaml if: # 1. it is not set and it is not already in the deployed secret (static random secret) # 2. its value is ? (dynamic random secret) From 49bc099153340e2172c02fec6512819464cdc550 Mon Sep 17 00:00:00 2001 From: Filippo Ledda Date: Fri, 8 Nov 2024 16:05:56 +0100 Subject: [PATCH 47/93] CH-162 fix for non string secrets +1 --- deployment-configuration/helm/templates/auto-secrets.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment-configuration/helm/templates/auto-secrets.yaml b/deployment-configuration/helm/templates/auto-secrets.yaml index df92258a..9ab6783a 100644 --- a/deployment-configuration/helm/templates/auto-secrets.yaml +++ b/deployment-configuration/helm/templates/auto-secrets.yaml @@ -18,7 +18,7 @@ stringData: {{- if and $v (and (eq (typeOf $v) "string") (ne $v "?")) }} # Set secret value to value in values.yaml if specified {{ $k }}: {{ $v }} - {{- else if or (and (eq (typeOf $v) "string") (eq $v "?")) (not (hasKey $secret.data $k)) }} + {{- else if and (eq (typeOf $v) "string") (or (eq $v "?") (not (hasKey $secret.data $k))) }} # Create a random secret value if not specified in values.yaml if: # 1. it is not set and it is not already in the deployed secret (static random secret) # 2. its value is ? (dynamic random secret) From 82b44dad1888489854db7b6ec0e1c8d9de307330 Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Tue, 12 Nov 2024 10:40:26 +0000 Subject: [PATCH 48/93] CH-157 Fix typo in django template readme files --- application-templates/django-fastapi/README.md | 2 +- application-templates/django-ninja/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/application-templates/django-fastapi/README.md b/application-templates/django-fastapi/README.md index d77502b9..b237db70 100644 --- a/application-templates/django-fastapi/README.md +++ b/application-templates/django-fastapi/README.md @@ -31,7 +31,7 @@ See [backend/README.md#Develop] ### Frontend -Backend code is inside the *frontend* directory. +Frontend code is inside the *frontend* directory. Frontend is by default generated as a React web application, but no constraint about this specific technology. diff --git a/application-templates/django-ninja/README.md b/application-templates/django-ninja/README.md index cc4b4c09..423842d6 100644 --- a/application-templates/django-ninja/README.md +++ b/application-templates/django-ninja/README.md @@ -30,7 +30,7 @@ See [backend/README.md#Develop] ### Frontend -Backend code is inside the *frontend* directory. +Frontend code is inside the *frontend* directory. Frontend is by default generated as a React web application, but no constraint about this specific technology. From de3f097088c5c2842b0c26650ee4b85131734447 Mon Sep 17 00:00:00 2001 From: Filippo Ledda Date: Tue, 12 Nov 2024 17:01:17 +0100 Subject: [PATCH 49/93] CH-163 Gatekeeper version update --- .../helm/templates/auto-gatekeepers.yaml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/deployment-configuration/helm/templates/auto-gatekeepers.yaml b/deployment-configuration/helm/templates/auto-gatekeepers.yaml index 898995cd..b0a7914e 100644 --- a/deployment-configuration/helm/templates/auto-gatekeepers.yaml +++ b/deployment-configuration/helm/templates/auto-gatekeepers.yaml @@ -1,6 +1,18 @@ {{/* Secured Services/Deployments */}} + +{{- define "check_no_wildcard_uri" -}} +{{- $check := true -}} +{{- range .uri_role_mapping -}} + {{- if eq .uri "/*" -}} + {{- $check = false -}} + {{- end -}} +{{- end -}} +{{- $check -}} +{{- end -}} + {{- define "deploy_utils.securedservice" }} {{- $tls := not (not .root.Values.tls) }} +{{- $noWildcards := include "check_no_wildcard_uri" (dict "uri_role_mapping" .app.harness.uri_role_mapping) -}} apiVersion: v1 kind: ConfigMap metadata: @@ -15,7 +27,7 @@ data: client-secret: {{ .root.Values.apps.accounts.webclient.secret }} secure-cookie: {{ $tls }} forbidden-page: /templates/access-denied.html.tmpl - enable-default-deny: {{ eq (.app.harness.secured | toString) "true" }} + enable-default-deny: {{ $noWildcards }} listen: 0.0.0.0:8080 enable-refresh-tokens: true server-write-timeout: {{ .app.harness.proxy.timeout.send | default .root.Values.proxy.timeout.send | default 180 }}s @@ -115,7 +127,7 @@ spec: {{ include "deploy_utils.etcHosts" .root | indent 6 }} containers: - name: {{ .app.harness.service.name | quote }} - image: "quay.io/gogatekeeper/gatekeeper:1.3.8" + image: "quay.io/gogatekeeper/gatekeeper:2.14.3" imagePullPolicy: IfNotPresent {{ if .root.Values.local }} securityContext: From a0302fcc8d2763a9f5f69afac977c3bbaf3fd61a Mon Sep 17 00:00:00 2001 From: Filippo Ledda Date: Mon, 18 Nov 2024 13:04:47 +0100 Subject: [PATCH 50/93] Fix secrets update --- .../helm/templates/auto-secrets.yaml | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/deployment-configuration/helm/templates/auto-secrets.yaml b/deployment-configuration/helm/templates/auto-secrets.yaml index 9ab6783a..b1982c38 100644 --- a/deployment-configuration/helm/templates/auto-secrets.yaml +++ b/deployment-configuration/helm/templates/auto-secrets.yaml @@ -12,16 +12,19 @@ type: Opaque {{- $secret := (lookup "v1" "Secret" .root.Values.namespace $secret_name) }} {{- if $secret }} # secret already exists - stringData: {{- range $k, $v := .app.harness.secrets }} - {{- if and $v (and (eq (typeOf $v) "string") (ne $v "?")) }} + {{- if and $v (eq (typeOf $v) "string") }} + {{- if (ne $v "?")}} # Set secret value to value in values.yaml if specified {{ $k }}: {{ $v }} - {{- else if and (eq (typeOf $v) "string") (or (eq $v "?") (not (hasKey $secret.data $k))) }} - # Create a random secret value if not specified in values.yaml if: - # 1. it is not set and it is not already in the deployed secret (static random secret) - # 2. its value is ? (dynamic random secret) + {{- else }} + # Refresh at any deployment for ? (pure random) value + {{ $k }}: {{ randAlphaNum 20 }} + {{- end }} + {{- else if not (hasKey $secret.data $k) }} + # Create a random secret value if not specified in values.yaml if + # it is not set and it is not already in the deployed secret (static random secret) {{ $k }}: {{ randAlphaNum 20 }} {{- end}} {{- end }} From ecca6982faba2c90c2a466b80c2213d1a0c94008 Mon Sep 17 00:00:00 2001 From: Filippo Ledda Date: Fri, 22 Nov 2024 10:07:22 +0100 Subject: [PATCH 51/93] CH-162 fix for secrets controls --- deployment-configuration/helm/templates/auto-secrets.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/deployment-configuration/helm/templates/auto-secrets.yaml b/deployment-configuration/helm/templates/auto-secrets.yaml index b1982c38..f1104782 100644 --- a/deployment-configuration/helm/templates/auto-secrets.yaml +++ b/deployment-configuration/helm/templates/auto-secrets.yaml @@ -22,10 +22,13 @@ stringData: # Refresh at any deployment for ? (pure random) value {{ $k }}: {{ randAlphaNum 20 }} {{- end }} - {{- else if not (hasKey $secret.data $k) }} + + {{- else if eq (typeOf $secret.data) "map" }} + {{- if not (hasKey $secret.data $k) }} # Create a random secret value if not specified in values.yaml if # it is not set and it is not already in the deployed secret (static random secret) {{ $k }}: {{ randAlphaNum 20 }} + {{- end}} {{- end}} {{- end }} {{- else }} From f8cf3bae5af5b53e0cd91e37a5d36099024b132a Mon Sep 17 00:00:00 2001 From: Filippo Ledda Date: Fri, 22 Nov 2024 10:35:35 +0100 Subject: [PATCH 52/93] CH-162 fix empty secrets gen --- deployment-configuration/helm/templates/auto-secrets.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/deployment-configuration/helm/templates/auto-secrets.yaml b/deployment-configuration/helm/templates/auto-secrets.yaml index f1104782..7d575074 100644 --- a/deployment-configuration/helm/templates/auto-secrets.yaml +++ b/deployment-configuration/helm/templates/auto-secrets.yaml @@ -41,7 +41,6 @@ stringData: {{- end }} --- {{- end }} ---- {{- range $app := .Values.apps }} --- {{- include "deploy_utils.secret" (dict "root" $ "app" $app) }} From 3461199ffd93d53918f39697ce859accc00cba31 Mon Sep 17 00:00:00 2001 From: Filippo Ledda Date: Fri, 22 Nov 2024 10:44:02 +0100 Subject: [PATCH 53/93] CH-162 fix empty secrets gen --- deployment-configuration/helm/templates/auto-secrets.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/deployment-configuration/helm/templates/auto-secrets.yaml b/deployment-configuration/helm/templates/auto-secrets.yaml index 7d575074..df91c466 100644 --- a/deployment-configuration/helm/templates/auto-secrets.yaml +++ b/deployment-configuration/helm/templates/auto-secrets.yaml @@ -42,7 +42,6 @@ stringData: --- {{- end }} {{- range $app := .Values.apps }} ---- {{- include "deploy_utils.secret" (dict "root" $ "app" $app) }} {{- range $subapp := $app }} {{- if contains "map" (typeOf $subapp) }} From 663a7e8c92492b5822f26625b9921feb73396ef5 Mon Sep 17 00:00:00 2001 From: Filippo Ledda Date: Fri, 22 Nov 2024 11:19:34 +0100 Subject: [PATCH 54/93] CH-162 fix empty secrets gen --- deployment-configuration/helm/templates/auto-secrets.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/deployment-configuration/helm/templates/auto-secrets.yaml b/deployment-configuration/helm/templates/auto-secrets.yaml index df91c466..ef6da926 100644 --- a/deployment-configuration/helm/templates/auto-secrets.yaml +++ b/deployment-configuration/helm/templates/auto-secrets.yaml @@ -1,5 +1,4 @@ {{- define "deploy_utils.secret" }} -{{- if .app.harness.secrets }} {{- $secret_name := printf "%s" .app.harness.deployment.name }} apiVersion: v1 kind: Secret @@ -38,15 +37,15 @@ stringData: {{ $k }}: {{ $v | default (randAlphaNum 20) }} {{- end }} {{- end }} -{{- end }} --- {{- end }} {{- range $app := .Values.apps }} + {{- if $app.harness.secrets }}{{- if ne (len $app.harness.secrets) 0 }} {{- include "deploy_utils.secret" (dict "root" $ "app" $app) }} + {{- end }}{{- end }} {{- range $subapp := $app }} {{- if contains "map" (typeOf $subapp) }} {{- if hasKey $subapp "harness" }} ---- {{- include "deploy_utils.secret" (dict "root" $ "app" $subapp) }} {{- end }} {{- end }} From 7ce854d026e151f140105e511f180eedd5518bfc Mon Sep 17 00:00:00 2001 From: Filippo Ledda Date: Fri, 22 Nov 2024 11:35:02 +0100 Subject: [PATCH 55/93] attempt fix codefresh deployment issues --- deployment-configuration/helm/templates/auto-secrets.yaml | 4 ++-- deployment/codefresh-test.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/deployment-configuration/helm/templates/auto-secrets.yaml b/deployment-configuration/helm/templates/auto-secrets.yaml index ef6da926..b59cd03f 100644 --- a/deployment-configuration/helm/templates/auto-secrets.yaml +++ b/deployment-configuration/helm/templates/auto-secrets.yaml @@ -45,9 +45,9 @@ stringData: {{- end }}{{- end }} {{- range $subapp := $app }} {{- if contains "map" (typeOf $subapp) }} - {{- if hasKey $subapp "harness" }} + {{- if hasKey $subapp "harness" }}{{- if $app.harness.secrets }}{{- if ne (len $app.harness.secrets) 0 }} {{- include "deploy_utils.secret" (dict "root" $ "app" $subapp) }} - {{- end }} + {{- end }}{{- end }}{{- end }} {{- end }} {{- end }} {{- end }} \ No newline at end of file diff --git a/deployment/codefresh-test.yaml b/deployment/codefresh-test.yaml index 3a144406..b6a6e883 100644 --- a/deployment/codefresh-test.yaml +++ b/deployment/codefresh-test.yaml @@ -368,7 +368,7 @@ steps: working_directory: ./${{CF_REPO_NAME}} title: Installing chart arguments: - helm_version: 3.6.2 + helm_version: 3.11.0 chart_name: deployment/helm release_name: test-${{NAMESPACE_BASENAME}} kube_context: '${{CLUSTER_NAME}}' From 4466edde317e3fb73c83515975d73f5dd6ff1dae Mon Sep 17 00:00:00 2001 From: Filippo Ledda Date: Fri, 22 Nov 2024 11:52:55 +0100 Subject: [PATCH 56/93] CH-162 add failsafe value --- .../helm/templates/auto-secrets.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/deployment-configuration/helm/templates/auto-secrets.yaml b/deployment-configuration/helm/templates/auto-secrets.yaml index b59cd03f..a908069c 100644 --- a/deployment-configuration/helm/templates/auto-secrets.yaml +++ b/deployment-configuration/helm/templates/auto-secrets.yaml @@ -20,15 +20,15 @@ stringData: {{- else }} # Refresh at any deployment for ? (pure random) value {{ $k }}: {{ randAlphaNum 20 }} - {{- end }} - - {{- else if eq (typeOf $secret.data) "map" }} - {{- if not (hasKey $secret.data $k) }} + {{- end }} + {{- else if eq (typeOf $secret.data) "map" }}{{- if not (hasKey $secret.data $k) }} # Create a random secret value if not specified in values.yaml if # it is not set and it is not already in the deployed secret (static random secret) - {{ $k }}: {{ randAlphaNum 20 }} - {{- end}} - {{- end}} + {{ $k }}: {{ randAlphaNum 20 }} + {{- end}}{{- end}} + {{- else }} + failsafe: check your values file + {{- end }} {{- end }} {{- else }} # secret doesn't exist From 8ea73a849223bc2355191c8ee86d65772dd5eb28 Mon Sep 17 00:00:00 2001 From: Filippo Ledda Date: Fri, 22 Nov 2024 12:00:31 +0100 Subject: [PATCH 57/93] CH-162 add failsafe value fix --- deployment-configuration/helm/templates/auto-secrets.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/deployment-configuration/helm/templates/auto-secrets.yaml b/deployment-configuration/helm/templates/auto-secrets.yaml index a908069c..a7d3b0b8 100644 --- a/deployment-configuration/helm/templates/auto-secrets.yaml +++ b/deployment-configuration/helm/templates/auto-secrets.yaml @@ -21,11 +21,12 @@ stringData: # Refresh at any deployment for ? (pure random) value {{ $k }}: {{ randAlphaNum 20 }} {{- end }} - {{- else if eq (typeOf $secret.data) "map" }}{{- if not (hasKey $secret.data $k) }} + {{- else if eq (typeOf $secret.data) "map" }} + {{- if not (hasKey $secret.data $k) }} # Create a random secret value if not specified in values.yaml if # it is not set and it is not already in the deployed secret (static random secret) {{ $k }}: {{ randAlphaNum 20 }} - {{- end}}{{- end}} + {{- end}} {{- else }} failsafe: check your values file {{- end }} From c4989d41e886f6bfac9240d5311355957c0707c0 Mon Sep 17 00:00:00 2001 From: Filippo Ledda Date: Fri, 22 Nov 2024 15:05:47 +0100 Subject: [PATCH 58/93] CH-162 fix comments --- .../helm/templates/auto-secrets.yaml | 41 +++++++++---------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/deployment-configuration/helm/templates/auto-secrets.yaml b/deployment-configuration/helm/templates/auto-secrets.yaml index a7d3b0b8..280e1e27 100644 --- a/deployment-configuration/helm/templates/auto-secrets.yaml +++ b/deployment-configuration/helm/templates/auto-secrets.yaml @@ -8,36 +8,35 @@ metadata: labels: app: {{ .app.harness.deployment.name }} type: Opaque -{{- $secret := (lookup "v1" "Secret" .root.Values.namespace $secret_name) }} -{{- if $secret }} -# secret already exists +{{ $secret := (lookup "v1" "Secret" .root.Values.namespace $secret_name) }} +{{/*- $secret := dict "data" (dict "test" "test") */}} stringData: +{{- if $secret }} {{- range $k, $v := .app.harness.secrets }} - {{- if and $v (eq (typeOf $v) "string") }} - {{- if (ne $v "?")}} - # Set secret value to value in values.yaml if specified - {{ $k }}: {{ $v }} - {{- else }} - # Refresh at any deployment for ? (pure random) value - {{ $k }}: {{ randAlphaNum 20 }} - {{- end }} + {{- if $v }} + {{- if eq (typeOf $v) "string" }} + {{- if ne $v "?" }} + {{/* Set secret value to value in values.yaml if specified */}} + {{ $k }}: {{ $v | quote }} + {{- else }} + {{/* Refresh at any deployment for ? (pure random) value */}} + {{ $k }}: {{ randAlphaNum 20 | quote }} + {{- end }} + {{- end }} {{- else if eq (typeOf $secret.data) "map" }} {{- if not (hasKey $secret.data $k) }} - # Create a random secret value if not specified in values.yaml if - # it is not set and it is not already in the deployed secret (static random secret) - {{ $k }}: {{ randAlphaNum 20 }} + {{/* Create a random secret value if not specified in values.yaml if + it is not set and it is not already in the deployed secret (static random secret) */}} + {{ $k }}: {{ randAlphaNum 20 | quote }} {{- end}} - {{- else }} - failsafe: check your values file + {{/* if it's not set and it's not ?, ignore */}} {{- end }} {{- end }} {{- else }} -# secret doesn't exist -stringData: - {{- range $k, $v := .app.harness.secrets }} - {{ $k }}: {{ $v | default (randAlphaNum 20) }} - {{- end }} + {{- range $k, $v := .app.harness.secrets }} + {{ $k }}: {{ $v | default (randAlphaNum 20) | quote }} {{- end }} +{{- end }} --- {{- end }} {{- range $app := .Values.apps }} From 1447db6bbb0f8d2a14ba86c9bfc596903a0a501a Mon Sep 17 00:00:00 2001 From: Filippo Ledda Date: Fri, 22 Nov 2024 15:22:42 +0100 Subject: [PATCH 59/93] CH-162 another helm fix attempt --- .../helm/templates/auto-secrets.yaml | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/deployment-configuration/helm/templates/auto-secrets.yaml b/deployment-configuration/helm/templates/auto-secrets.yaml index 280e1e27..f43ba160 100644 --- a/deployment-configuration/helm/templates/auto-secrets.yaml +++ b/deployment-configuration/helm/templates/auto-secrets.yaml @@ -6,9 +6,10 @@ metadata: name: {{ $secret_name }} namespace: {{ .root.Values.namespace }} labels: + gianni: z app: {{ .app.harness.deployment.name }} type: Opaque -{{ $secret := (lookup "v1" "Secret" .root.Values.namespace $secret_name) }} +{{- $secret := (lookup "v1" "Secret" .root.Values.namespace $secret_name) }} {{/*- $secret := dict "data" (dict "test" "test") */}} stringData: {{- if $secret }} @@ -16,21 +17,23 @@ stringData: {{- if $v }} {{- if eq (typeOf $v) "string" }} {{- if ne $v "?" }} - {{/* Set secret value to value in values.yaml if specified */}} + # Set secret value to value in values.yaml if specified {{ $k }}: {{ $v | quote }} {{- else }} - {{/* Refresh at any deployment for ? (pure random) value */}} + # Refresh at any deployment for ? (pure random) value {{ $k }}: {{ randAlphaNum 20 | quote }} {{- end }} {{- end }} {{- else if eq (typeOf $secret.data) "map" }} {{- if not (hasKey $secret.data $k) }} - {{/* Create a random secret value if not specified in values.yaml if - it is not set and it is not already in the deployed secret (static random secret) */}} + # Create a random secret value if not specified in values.yaml if it is not set and it is not already in the deployed secret (static random secret) */}} {{ $k }}: {{ randAlphaNum 20 | quote }} {{- end}} - {{/* if it's not set and it's not ?, ignore */}} + {{- else }} + # This value should have been set in the values files or in the CI/CD pipeline" + {{ $k }}: "" {{- end }} + {{- end }} {{- else }} {{- range $k, $v := .app.harness.secrets }} From cd42d55a6ab6ba38bd6c230e3d879e77cf4a7b15 Mon Sep 17 00:00:00 2001 From: Filippo Ledda Date: Fri, 22 Nov 2024 15:36:24 +0100 Subject: [PATCH 60/93] CH-163 fix template --- .../helm/templates/auto-gatekeepers.yaml | 23 ++++++++----------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/deployment-configuration/helm/templates/auto-gatekeepers.yaml b/deployment-configuration/helm/templates/auto-gatekeepers.yaml index b0a7914e..8e1c3ac0 100644 --- a/deployment-configuration/helm/templates/auto-gatekeepers.yaml +++ b/deployment-configuration/helm/templates/auto-gatekeepers.yaml @@ -1,18 +1,16 @@ {{/* Secured Services/Deployments */}} - -{{- define "check_no_wildcard_uri" -}} -{{- $check := true -}} -{{- range .uri_role_mapping -}} - {{- if eq .uri "/*" -}} - {{- $check = false -}} - {{- end -}} -{{- end -}} -{{- $check -}} -{{- end -}} - +{{- define "check_no_wildcard_uri" }} +{{- $check := true }} +{{- range .uri_role_mapping }} + {{- if eq .uri "/*" }} + {{- $check = false }} + {{- end }} +{{- end }} +{{- $check }} +{{- end }} {{- define "deploy_utils.securedservice" }} {{- $tls := not (not .root.Values.tls) }} -{{- $noWildcards := include "check_no_wildcard_uri" (dict "uri_role_mapping" .app.harness.uri_role_mapping) -}} +{{- $noWildcards := include "check_no_wildcard_uri" (dict "uri_role_mapping" .app.harness.uri_role_mapping) }} apiVersion: v1 kind: ConfigMap metadata: @@ -111,7 +109,6 @@ metadata: name: "{{ .app.harness.service.name }}-gk" labels: app: "{{ .app.harness.service.name }}-gk" - spec: replicas: 1 selector: From 81f5dfb8fb38f4874c8e7003f907fbc7ef254c89 Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Mon, 9 Dec 2024 11:23:59 +0000 Subject: [PATCH 61/93] CH-157 Fixed typo in asgi and wsgi --- .../django-base/backend/django_baseapp/asgi.py | 4 ++-- .../django-base/backend/django_baseapp/wsgi.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/application-templates/django-base/backend/django_baseapp/asgi.py b/application-templates/django-base/backend/django_baseapp/asgi.py index 049df40e..76fc7817 100644 --- a/application-templates/django-base/backend/django_baseapp/asgi.py +++ b/application-templates/django-base/backend/django_baseapp/asgi.py @@ -21,6 +21,6 @@ init_services() # start the kafka event listener -from cloudharness_django.services.events import init_listner # noqa E402 +from cloudharness_django.services.events import init_listener # noqa E402 -init_listner() +init_listener() diff --git a/application-templates/django-base/backend/django_baseapp/wsgi.py b/application-templates/django-base/backend/django_baseapp/wsgi.py index e2d6fc09..fccfadef 100644 --- a/application-templates/django-base/backend/django_baseapp/wsgi.py +++ b/application-templates/django-base/backend/django_baseapp/wsgi.py @@ -21,6 +21,6 @@ init_services() # start the kafka event listener -from cloudharness_django.services.events import init_listner # noqa E402 +from cloudharness_django.services.events import init_listener # noqa E402 -init_listner() +init_listener() From a441b95d1d5c1aa8b0ef80eb8c12ea73225da287 Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Mon, 9 Dec 2024 11:47:18 +0000 Subject: [PATCH 62/93] CH-157 Fixed issue with incorrect django settings module for asgi --- .../django-base/backend/django_baseapp/asgi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application-templates/django-base/backend/django_baseapp/asgi.py b/application-templates/django-base/backend/django_baseapp/asgi.py index 76fc7817..4ec66b7d 100644 --- a/application-templates/django-base/backend/django_baseapp/asgi.py +++ b/application-templates/django-base/backend/django_baseapp/asgi.py @@ -11,7 +11,7 @@ from django.core.asgi import get_asgi_application -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "__APP_NAME__.settings") +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "django_baseapp.settings") application = get_asgi_application() From 23a6fed7514118c966771b44b13318bc6bbfa151 Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Mon, 9 Dec 2024 11:47:48 +0000 Subject: [PATCH 63/93] CH-157 Change name of frontend build stage to be more generic --- application-templates/django-ninja/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/application-templates/django-ninja/Dockerfile b/application-templates/django-ninja/Dockerfile index f13bfaac..dc8e2223 100644 --- a/application-templates/django-ninja/Dockerfile +++ b/application-templates/django-ninja/Dockerfile @@ -1,7 +1,7 @@ -ARG MNP_UI +ARG CLOUDHARNESS_FRONTEND_BUILD ARG CLOUDHARNESS_DJANGO -FROM $MNP_UI AS frontend +FROM $CLOUDHARNESS_FRONTEND_BUILD AS frontend ARG APP_DIR=/app From 2a4bd6e1500c4e0c1932f425f4786f92435230e3 Mon Sep 17 00:00:00 2001 From: Filippo Ledda Date: Tue, 7 Jan 2025 11:43:29 +0100 Subject: [PATCH 64/93] CH-166 fix syntax issue --- tools/deployment-cli-tools/ch_cli_tools/codefresh.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/deployment-cli-tools/ch_cli_tools/codefresh.py b/tools/deployment-cli-tools/ch_cli_tools/codefresh.py index 3ea43e31..8bcf2b79 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/codefresh.py +++ b/tools/deployment-cli-tools/ch_cli_tools/codefresh.py @@ -175,7 +175,8 @@ def codefresh_steps_from_base_path(base_path, build_step, fixed_context=None, in if app_config and app_config.dependencies and app_config.dependencies.git: for dep in app_config.dependencies.git: - steps[CD_BUILD_STEP_DEPENDENCIES]['steps'][f"clone_{basename(dep.url).replace(".", "_")}_{basename(dockerfile_relative_to_root).replace(".", "_")}"] = clone_step_spec(dep, dockerfile_relative_to_root) + step_name = f"clone_{basename(dep.url).replace('.', '_')}_{basename(dockerfile_relative_to_root).replace('.', '_')}" + steps[CD_BUILD_STEP_DEPENDENCIES]['steps'][step_name] = clone_step_spec(dep, dockerfile_relative_to_root) build = None if build_step in steps: From 6e1dee33f4a8e44e4b8375571d82ec6d3e838df1 Mon Sep 17 00:00:00 2001 From: Filippo Ledda Date: Tue, 14 Jan 2025 18:23:48 +0100 Subject: [PATCH 65/93] CH-170 fix secrets upgrade --- .../helm/templates/auto-secrets.yaml | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/deployment-configuration/helm/templates/auto-secrets.yaml b/deployment-configuration/helm/templates/auto-secrets.yaml index f43ba160..345ea984 100644 --- a/deployment-configuration/helm/templates/auto-secrets.yaml +++ b/deployment-configuration/helm/templates/auto-secrets.yaml @@ -14,11 +14,76 @@ type: Opaque stringData: {{- if $secret }} {{- range $k, $v := .app.harness.secrets }} + {{- define "deploy_utils.secret" }} +{{- $secret_name := printf "%s" .app.harness.deployment.name }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ $secret_name }} + namespace: {{ .root.Values.namespace }} + labels: + app: {{ .app.harness.deployment.name }} +type: Opaque +{{- $secret := (lookup "v1" "Secret" .root.Values.namespace $secret_name) }} +{{/*- $secret := dict "data" (dict "test" "test") */}} +stringData: +{{- if $secret }} + {{- range $k, $v := .app.harness.secrets }} + {{- if $v }} + {{- if eq (typeOf $v) "string" }} + {{- if ne $v "?" }} + # Update/set value to value in values.yaml if specified + {{ $k }}: {{ $v | quote }} +# {{ $k }}-updated: "it's a non empty string" + {{- else }} + # Refresh at any deployment for ? (pure random) value +# {{ $k }}-random: "set to a random string" + {{ $k }}: {{ randAlphaNum 20 | quote }} + {{- end }} + {{- else }} + # Type not recognized: setting to a empty string" + {{ $k }}: "" + # {{ $k }}-formatnotrecognized: {{ typeOf $v }} + {{- end }} + {{- else if eq (typeOf $secret.data) (typeOf dict) }} + # Value empty or null in the values.yaml + {{- if not (hasKey $secret.data $k) }} + # Create a random secret value if not specified in values.yaml if it is not set and it is not already in the deployed secret (static random secret) */}} +# {{ $k }}-random-empty: new-random + {{ $k }}: {{ randAlphaNum 20 | quote }} + {{- else }} + # confirm previous value from the secret (static random secret already set, do nothing) +# {{ $k }}-confirm: {{ index $secret.data $k | quote }} + {{- end}} + {{- end }} + {{- end }} # range end +{{- else }} +# New secret + {{- range $k, $v := .app.harness.secrets }} + {{ $k }}-new: {{ $v | default (randAlphaNum 20) | quote }} + {{ $k }}: {{ $v | default (randAlphaNum 20) | quote }} + {{- end }} +{{- end }} +--- +{{- end }} +{{- range $app := .Values.apps }} + {{- if $app.harness.secrets }}{{- if ne (len $app.harness.secrets) 0 }} + {{- include "deploy_utils.secret" (dict "root" $ "app" $app) }} + {{- end }}{{- end }} + {{- range $subapp := $app }} + {{- if contains "map" (typeOf $subapp) }} + {{- if hasKey $subapp "harness" }}{{- if $app.harness.secrets }}{{- if ne (len $app.harness.secrets) 0 }} + {{- include "deploy_utils.secret" (dict "root" $ "app" $subapp) }} + {{- end }}{{- end }}{{- end }} + {{- end }} + {{- end }} +{{- end }} {{- if $v }} {{- if eq (typeOf $v) "string" }} {{- if ne $v "?" }} # Set secret value to value in values.yaml if specified {{ $k }}: {{ $v | quote }} +{{ $k }}-new: {{ "updated" }} {{- else }} # Refresh at any deployment for ? (pure random) value {{ $k }}: {{ randAlphaNum 20 | quote }} From 04aa612885a89b9d8488d30b283b278e089b61ad Mon Sep 17 00:00:00 2001 From: Filippo Ledda Date: Tue, 14 Jan 2025 19:41:36 +0100 Subject: [PATCH 66/93] CH-170 fix secrets upgrade -- bad push --- .../helm/templates/auto-secrets.yaml | 65 ------------------- 1 file changed, 65 deletions(-) diff --git a/deployment-configuration/helm/templates/auto-secrets.yaml b/deployment-configuration/helm/templates/auto-secrets.yaml index 345ea984..cb480796 100644 --- a/deployment-configuration/helm/templates/auto-secrets.yaml +++ b/deployment-configuration/helm/templates/auto-secrets.yaml @@ -6,7 +6,6 @@ metadata: name: {{ $secret_name }} namespace: {{ .root.Values.namespace }} labels: - gianni: z app: {{ .app.harness.deployment.name }} type: Opaque {{- $secret := (lookup "v1" "Secret" .root.Values.namespace $secret_name) }} @@ -14,70 +13,6 @@ type: Opaque stringData: {{- if $secret }} {{- range $k, $v := .app.harness.secrets }} - {{- define "deploy_utils.secret" }} -{{- $secret_name := printf "%s" .app.harness.deployment.name }} -apiVersion: v1 -kind: Secret -metadata: - name: {{ $secret_name }} - namespace: {{ .root.Values.namespace }} - labels: - app: {{ .app.harness.deployment.name }} -type: Opaque -{{- $secret := (lookup "v1" "Secret" .root.Values.namespace $secret_name) }} -{{/*- $secret := dict "data" (dict "test" "test") */}} -stringData: -{{- if $secret }} - {{- range $k, $v := .app.harness.secrets }} - {{- if $v }} - {{- if eq (typeOf $v) "string" }} - {{- if ne $v "?" }} - # Update/set value to value in values.yaml if specified - {{ $k }}: {{ $v | quote }} -# {{ $k }}-updated: "it's a non empty string" - {{- else }} - # Refresh at any deployment for ? (pure random) value -# {{ $k }}-random: "set to a random string" - {{ $k }}: {{ randAlphaNum 20 | quote }} - {{- end }} - {{- else }} - # Type not recognized: setting to a empty string" - {{ $k }}: "" - # {{ $k }}-formatnotrecognized: {{ typeOf $v }} - {{- end }} - {{- else if eq (typeOf $secret.data) (typeOf dict) }} - # Value empty or null in the values.yaml - {{- if not (hasKey $secret.data $k) }} - # Create a random secret value if not specified in values.yaml if it is not set and it is not already in the deployed secret (static random secret) */}} -# {{ $k }}-random-empty: new-random - {{ $k }}: {{ randAlphaNum 20 | quote }} - {{- else }} - # confirm previous value from the secret (static random secret already set, do nothing) -# {{ $k }}-confirm: {{ index $secret.data $k | quote }} - {{- end}} - {{- end }} - {{- end }} # range end -{{- else }} -# New secret - {{- range $k, $v := .app.harness.secrets }} - {{ $k }}-new: {{ $v | default (randAlphaNum 20) | quote }} - {{ $k }}: {{ $v | default (randAlphaNum 20) | quote }} - {{- end }} -{{- end }} ---- -{{- end }} -{{- range $app := .Values.apps }} - {{- if $app.harness.secrets }}{{- if ne (len $app.harness.secrets) 0 }} - {{- include "deploy_utils.secret" (dict "root" $ "app" $app) }} - {{- end }}{{- end }} - {{- range $subapp := $app }} - {{- if contains "map" (typeOf $subapp) }} - {{- if hasKey $subapp "harness" }}{{- if $app.harness.secrets }}{{- if ne (len $app.harness.secrets) 0 }} - {{- include "deploy_utils.secret" (dict "root" $ "app" $subapp) }} - {{- end }}{{- end }}{{- end }} - {{- end }} - {{- end }} -{{- end }} {{- if $v }} {{- if eq (typeOf $v) "string" }} {{- if ne $v "?" }} From b101e44578781bee4dd8ac22bc08b64b3e86c4f9 Mon Sep 17 00:00:00 2001 From: Filippo Ledda Date: Wed, 15 Jan 2025 10:33:32 +0100 Subject: [PATCH 67/93] CH-170 fix secrets upgrade -- bad push --- .../helm/templates/auto-secrets.yaml | 32 ++++++++++++------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/deployment-configuration/helm/templates/auto-secrets.yaml b/deployment-configuration/helm/templates/auto-secrets.yaml index cb480796..9519882b 100644 --- a/deployment-configuration/helm/templates/auto-secrets.yaml +++ b/deployment-configuration/helm/templates/auto-secrets.yaml @@ -16,27 +16,35 @@ stringData: {{- if $v }} {{- if eq (typeOf $v) "string" }} {{- if ne $v "?" }} - # Set secret value to value in values.yaml if specified + # Update/set value to value in values.yaml if specified {{ $k }}: {{ $v | quote }} -{{ $k }}-new: {{ "updated" }} +# {{ $k }}-updated: "it's a non empty string" {{- else }} - # Refresh at any deployment for ? (pure random) value + # Refresh at any deployment for ? (pure random) value +# {{ $k }}-random: "set to a random string" {{ $k }}: {{ randAlphaNum 20 | quote }} - {{- end }} - {{- end }} - {{- else if eq (typeOf $secret.data) "map" }} + {{- end }} + {{- else }} + # Type not recognized: setting to a empty string" + {{ $k }}: "" + # {{ $k }}-formatnotrecognized: {{ typeOf $v }} + {{- end }} + {{- else if eq (typeOf $secret.data) (typeOf dict) }} + # Value empty or null in the values.yaml {{- if not (hasKey $secret.data $k) }} - # Create a random secret value if not specified in values.yaml if it is not set and it is not already in the deployed secret (static random secret) */}} + # Create a random secret value if not specified in values.yaml if it is not set and it is not already in the deployed secret (static random secret) */}} +# {{ $k }}-random-empty: new-random {{ $k }}: {{ randAlphaNum 20 | quote }} + {{- else }} + # confirm previous value from the secret (static random secret already set, do nothing) +# {{ $k }}-confirm: {{ index $secret.data $k | quote }} {{- end}} - {{- else }} - # This value should have been set in the values files or in the CI/CD pipeline" - {{ $k }}: "" {{- end }} - - {{- end }} + {{- end }} # range end {{- else }} +# New secret {{- range $k, $v := .app.harness.secrets }} + {{ $k }}-new: {{ $v | default (randAlphaNum 20) | quote }} {{ $k }}: {{ $v | default (randAlphaNum 20) | quote }} {{- end }} {{- end }} From 53d68ee337be95f828f4a3d94e1a3a49dd2dcf5a Mon Sep 17 00:00:00 2001 From: Filippo Ledda Date: Wed, 15 Jan 2025 18:10:10 +0100 Subject: [PATCH 68/93] CH-170 fix secrets upgrade possible reset --- deployment/secret.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 deployment/secret.yaml diff --git a/deployment/secret.yaml b/deployment/secret.yaml new file mode 100644 index 00000000..ff1ad4f3 --- /dev/null +++ b/deployment/secret.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Secret +metadata: + annotations: + meta.helm.sh/release-name: mnp + meta.helm.sh/release-namespace: mnp + labels: + app: accounts + app.kubernetes.io/managed-by: Helm + name: accounts + namespace: mnp +stringData: + pino: gianni +data: + api_user_password: ZGRkYTNjMTYxOTRhZTc2MjE5YzNiYjdjYjRiN2FlMDMzYThiN2ZlNQ== +type: Opaque From 77852b78ead2f722b8f9445a9ac00432a0be6f36 Mon Sep 17 00:00:00 2001 From: Filippo Ledda Date: Thu, 16 Jan 2025 10:41:36 +0100 Subject: [PATCH 69/93] CH-170 fix for empty secrets reset --- deployment-configuration/helm/templates/auto-secrets.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/deployment-configuration/helm/templates/auto-secrets.yaml b/deployment-configuration/helm/templates/auto-secrets.yaml index 9519882b..e4f0b7a4 100644 --- a/deployment-configuration/helm/templates/auto-secrets.yaml +++ b/deployment-configuration/helm/templates/auto-secrets.yaml @@ -11,6 +11,7 @@ type: Opaque {{- $secret := (lookup "v1" "Secret" .root.Values.namespace $secret_name) }} {{/*- $secret := dict "data" (dict "test" "test") */}} stringData: + updated: {{ now | quote }} # Added because in case of update, if no field is updated, alla data is erased {{- if $secret }} {{- range $k, $v := .app.harness.secrets }} {{- if $v }} From 80c72368f48ea4b9a9f6421377bded79124026f4 Mon Sep 17 00:00:00 2001 From: Filippo Ledda Date: Thu, 16 Jan 2025 10:42:15 +0100 Subject: [PATCH 70/93] Fix e2e tests run --- ch-166.patch | 14 ++++++++++++++ .../codefresh-template-test.yaml | 1 + deployment/codefresh-test.yaml | 1 + 3 files changed, 16 insertions(+) create mode 100644 ch-166.patch diff --git a/ch-166.patch b/ch-166.patch new file mode 100644 index 00000000..1603e0a0 --- /dev/null +++ b/ch-166.patch @@ -0,0 +1,14 @@ +diff --git a/tools/deployment-cli-tools/ch_cli_tools/codefresh.py b/tools/deployment-cli-tools/ch_cli_tools/codefresh.py +index 8bcf2b79..3ea43e31 100644 +--- a/tools/deployment-cli-tools/ch_cli_tools/codefresh.py ++++ b/tools/deployment-cli-tools/ch_cli_tools/codefresh.py +@@ -175,8 +175,7 @@ def create_codefresh_deployment_scripts(root_paths, envs=(), include=(), exclude + + if app_config and app_config.dependencies and app_config.dependencies.git: + for dep in app_config.dependencies.git: +- step_name = f"clone_{basename(dep.url).replace('.', '_')}_{basename(dockerfile_relative_to_root).replace('.', '_')}" +- steps[CD_BUILD_STEP_DEPENDENCIES]['steps'][step_name] = clone_step_spec(dep, dockerfile_relative_to_root) ++ steps[CD_BUILD_STEP_DEPENDENCIES]['steps'][f"clone_{basename(dep.url).replace(".", "_")}_{basename(dockerfile_relative_to_root).replace(".", "_")}"] = clone_step_spec(dep, dockerfile_relative_to_root) + + build = None + if build_step in steps: diff --git a/deployment-configuration/codefresh-template-test.yaml b/deployment-configuration/codefresh-template-test.yaml index b727d53c..45febee4 100644 --- a/deployment-configuration/codefresh-template-test.yaml +++ b/deployment-configuration/codefresh-template-test.yaml @@ -112,6 +112,7 @@ steps: image: "${{test-e2e}}" fail_fast: false commands: + - npx puppeteer browsers install chrome - yarn test scale: {} hooks: diff --git a/deployment/codefresh-test.yaml b/deployment/codefresh-test.yaml index 89e03209..144e3f36 100644 --- a/deployment/codefresh-test.yaml +++ b/deployment/codefresh-test.yaml @@ -617,6 +617,7 @@ steps: image: '${{REGISTRY}}/cloudharness/test-e2e:${{TEST_E2E_TAG}}' fail_fast: false commands: + - npx puppeteer browsers install chrome - yarn test scale: jupyterhub_e2e_test: From b8ca1e3d1cb0129f2328b665cdc71f32d3e91947 Mon Sep 17 00:00:00 2001 From: Filippo Ledda Date: Thu, 16 Jan 2025 14:52:35 +0100 Subject: [PATCH 71/93] CH-170 fix secrets upgrade error --- .../helm/templates/auto-secrets.yaml | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/deployment-configuration/helm/templates/auto-secrets.yaml b/deployment-configuration/helm/templates/auto-secrets.yaml index e4f0b7a4..0a1ccc3b 100644 --- a/deployment-configuration/helm/templates/auto-secrets.yaml +++ b/deployment-configuration/helm/templates/auto-secrets.yaml @@ -19,33 +19,28 @@ stringData: {{- if ne $v "?" }} # Update/set value to value in values.yaml if specified {{ $k }}: {{ $v | quote }} -# {{ $k }}-updated: "it's a non empty string" {{- else }} # Refresh at any deployment for ? (pure random) value -# {{ $k }}-random: "set to a random string" {{ $k }}: {{ randAlphaNum 20 | quote }} {{- end }} {{- else }} # Type not recognized: setting to a empty string" - {{ $k }}: "" - # {{ $k }}-formatnotrecognized: {{ typeOf $v }} + {{ $k }}-formatnotrecognized: {{ $v }} + {{ $k }}: "" {{- end }} {{- else if eq (typeOf $secret.data) (typeOf dict) }} # Value empty or null in the values.yaml {{- if not (hasKey $secret.data $k) }} # Create a random secret value if not specified in values.yaml if it is not set and it is not already in the deployed secret (static random secret) */}} -# {{ $k }}-random-empty: new-random {{ $k }}: {{ randAlphaNum 20 | quote }} {{- else }} - # confirm previous value from the secret (static random secret already set, do nothing) -# {{ $k }}-confirm: {{ index $secret.data $k | quote }} + # confirm previous value from the secret (static random secret already set, do nothing)} {{- end}} {{- end }} {{- end }} # range end {{- else }} # New secret {{- range $k, $v := .app.harness.secrets }} - {{ $k }}-new: {{ $v | default (randAlphaNum 20) | quote }} {{ $k }}: {{ $v | default (randAlphaNum 20) | quote }} {{- end }} {{- end }} From 5432c84b7abd197aafd11331cca6fec1e0f97039 Mon Sep 17 00:00:00 2001 From: Filippo Ledda Date: Tue, 21 Jan 2025 14:21:15 +0100 Subject: [PATCH 72/93] CH-157 refactor, small fixes --- .../django-ninja/dev-setup.sh | 2 +- .../ch_cli_tools/application_builders.py | 16 +- .../ch_cli_tools/openapi.py | 173 +++++++++++++++++- .../ch_cli_tools/utils.py | 6 +- .../deployment-cli-tools/harness-application | 18 +- tools/deployment-cli-tools/harness-generate | 160 +--------------- 6 files changed, 192 insertions(+), 183 deletions(-) diff --git a/application-templates/django-ninja/dev-setup.sh b/application-templates/django-ninja/dev-setup.sh index 24970bf3..d7d062d1 100644 --- a/application-templates/django-ninja/dev-setup.sh +++ b/application-templates/django-ninja/dev-setup.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash CURRENT_PATH=$(pwd) -CH_DIRECTORY="../../cloud-harness" +CH_DIRECTORY="__CLOUDHARNESS_PATH__" INSTALL_PYTEST=false CURRENT_DIRECTORY="$(pwd)" APP_NAME="__APP_NAME__" diff --git a/tools/deployment-cli-tools/ch_cli_tools/application_builders.py b/tools/deployment-cli-tools/ch_cli_tools/application_builders.py index d7701445..b10f17e1 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/application_builders.py +++ b/tools/deployment-cli-tools/ch_cli_tools/application_builders.py @@ -6,7 +6,7 @@ import subprocess import tempfile from .common_types import TemplateType -from .openapi import generate_fastapi_server, generate_server, generate_ts_client +from .openapi import generate_fastapi_server, generate_flask_server, generate_openapi_from_ninja_schema, generate_ts_client from .utils import copymergedir, get_json_template, merge_configuration_directories, replace_in_dict, replace_in_file, replaceindir, to_python_module from . import CH_ROOT from cloudharness_utils.constants import APPLICATION_TEMPLATE_PATH @@ -14,6 +14,7 @@ class ApplicationBuilder(abc.ABC): APP_NAME_PLACEHOLDER = '__APP_NAME__' + CLOUDHARNESS_PATH_PLACEHOLDER = '__CLOUDHARNESS_PATH__' def __init__(self, app_name: str, app_path: pathlib.Path): self.app_name = app_name @@ -76,7 +77,7 @@ def handles(self, templates): def handle_pre_merge(self): if self.frontend_path.exists(): shutil.rmtree(self.frontend_path) - + logging.info('Creating vite app') self.create_vite_skaffold(self.frontend_path) def handle_merge(self): @@ -85,8 +86,9 @@ def handle_merge(self): def handle_post_merge(self): backend_dockerfile_path = self.backend_path / 'Dockerfile' backend_dockerfile_path.unlink(missing_ok=True) - + logging.info('Installing frontend dependencies') self.install_frontend_dependencies() + logging.info('Generating ts client') generate_ts_client(self.api_path / 'openapi.yaml') def create_vite_skaffold(self, frontend_path: pathlib.Path) -> None: @@ -111,7 +113,7 @@ def handle_pre_merge(self): copymergedir(server_template_path, tmp_path) merge_configuration_directories(self.app_path, tmp_path) - generate_server(self.app_name, tmp_path) + generate_flask_server(self.app_name, tmp_path) def handle_merge(self): self.merge_template_directories(TemplateType.SERVER) @@ -131,7 +133,7 @@ def handle_merge(self): self.merge_template_directories(TemplateType.FLASK_SERVER) def handle_post_merge(self): - generate_server(self.app_path) + generate_flask_server(self.app_path) class BaseDjangoAppBuilder(ApplicationBuilder): @@ -147,10 +149,12 @@ def handle_post_merge(self): f'{self.python_app_name}:{self.python_app_name}', ) replace_in_file(self.app_path / 'dev-setup.sh', self.APP_NAME_PLACEHOLDER, self.app_name) + replace_in_file(self.app_path / 'dev-setup.sh', self.CLOUDHARNESS_PATH_PLACEHOLDER, self.ch_root) self.create_django_app_vscode_debug_configuration() def create_django_app_vscode_debug_configuration(self): + vscode_launch_path = pathlib.Path('.vscode/launch.json') configuration_name = f'{self.app_name} backend' @@ -222,6 +226,8 @@ def handle_merge(self): def handle_post_merge(self): super().handle_post_merge() + logging.info('Generating openapi from ninja schema') + generate_openapi_from_ninja_schema(self.app_name, self.app_path) class AppBuilderPipeline(ApplicationBuilder): diff --git a/tools/deployment-cli-tools/ch_cli_tools/openapi.py b/tools/deployment-cli-tools/ch_cli_tools/openapi.py index fcc91974..ec2267e2 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/openapi.py +++ b/tools/deployment-cli-tools/ch_cli_tools/openapi.py @@ -1,17 +1,23 @@ +import enum +import functools import glob import json import logging +import operator import os import pathlib import shutil import subprocess import sys -from typing import Optional +from typing import Callable, Optional import urllib.request from os.path import dirname as dn, join +from ch_cli_tools.common_types import TemplateType +from ch_cli_tools.manifest import get_manifest + from . import HERE -from .utils import replaceindir, to_python_module +from .utils import copymergedir, replaceindir, to_python_module CODEGEN = os.path.join(HERE, 'bin', 'openapi-generator-cli.jar') APPLICATIONS_SRC_PATH = os.path.join('applications') @@ -21,7 +27,16 @@ OPENAPI_GEN_URL = 'https://repo1.maven.org/maven2/org/openapitools/openapi-generator-cli/7.7.0/openapi-generator-cli-7.7.0.jar' -def generate_server(app_path: pathlib.Path, overrides_folder: Optional[pathlib.Path] = None) -> None: +class ClientType(enum.Flag): + TS_CLIENT = enum.auto() + PYTHON_CLIENT = enum.auto() + + @classmethod + def all(cls): + return functools.reduce(operator.or_, cls) + + +def generate_flask_server(app_path: pathlib.Path, overrides_folder: Optional[pathlib.Path] = None) -> None: get_dependencies() openapi_directory = app_path / 'api' @@ -45,6 +60,8 @@ def generate_server(app_path: pathlib.Path, overrides_folder: Optional[pathlib.P def generate_fastapi_server(app_path: pathlib.Path) -> None: + # Install the fastapi code generator here as it comes with potential problematic dependencies + subprocess.check_call([sys.executable, "-m", "pip", "install", "fastapi-code-generator"]) api_directory = app_path / 'api' backend_directory = app_path / 'backend' temp_directory = api_directory / 'app' @@ -125,7 +142,9 @@ def generate_ts_client(openapi_file): def generate_openapi_from_ninja_schema(app_name: str, app_path: pathlib.Path) -> None: + subprocess.check_call(["sh", "dev-setup.sh"], cwd=app_path) out_path = app_path / 'api' / 'openapi.yaml' + manage_path = app_path / 'backend' / 'manage.py' command = [ 'python', manage_path, 'export_openapi_schema', @@ -156,3 +175,151 @@ def get_dependencies(): if not os.path.exists(cdir): os.makedirs(cdir) urllib.request.urlretrieve(OPENAPI_GEN_URL, CODEGEN) + + +def generate_models( + root_path: pathlib.Path, + should_generate: Callable[[str], bool], +) -> None: + """ + Generates the main model + """ + library_models_path = root_path / 'libraries' / 'models' + + if not library_models_path.exists(): + return + + if not should_generate('the main model'): + return + + generate_model() + + +def generate_servers( + root_path: pathlib.Path, + should_generate: Callable[[str], bool], + app_name: Optional[str], +) -> None: + """ + Generates server stubs + """ + openapi_files = [path for path in root_path.glob('applications/*/api/*.yaml')] + + for openapi_file in openapi_files: + app_path = openapi_file.parent.parent + manifest = get_manifest(app_path) + + if app_name and manifest.app_name != app_name: + continue + + if not should_generate(f'server stubs for {openapi_file}'): + continue + + if TemplateType.DJANGO_FASTAPI in manifest.templates: + generate_fastapi_server(app_path) + + if TemplateType.FLASK_SERVER in manifest.templates: + generate_flask_server(app_path) + + +def generate_clients( + root_path: pathlib.Path, + should_generate: Callable[[str], bool], + app_name: Optional[str], + client_lib_name: str, + client_types: ClientType, +) -> None: + """ + Generates client stubs + """ + if not should_generate('client libraries'): + return + + client_src_path = root_path / 'libraries' / 'client' / client_lib_name + apps_path = root_path / 'applications' + apps = (app for app in apps_path.iterdir() if app.is_dir()) + + for app_path in apps: + manifest = get_manifest(app_path) + + if app_name and manifest.app_name != app_name: + continue + + if TemplateType.DJANGO_NINJA in manifest.templates: + generate_openapi_from_ninja_schema(manifest.app_name, app_path) + + for openapi_file in app_path.glob('api/*.yaml'): + if ClientType.PYTHON_CLIENT in client_types: + generate_python_client(manifest.app_name, openapi_file, client_src_path, lib_name=client_lib_name) + + if TemplateType.WEBAPP in manifest.templates and ClientType.TS_CLIENT in client_types: + generate_ts_client(openapi_file) + + aggregate_packages(client_src_path, client_lib_name) + + +def aggregate_packages(client_source_path: pathlib.Path, lib_name=LIB_NAME): + client_source_path.mkdir(parents=True, exist_ok=True) + + client_docs_path = client_source_path / 'docs' + client_docs_path.mkdir(exist_ok=True) + + client_test_path = client_source_path / 'test' + client_test_path.mkdir(exist_ok=True) + + client_readme_file = client_source_path / 'README.md' + client_readme_file.unlink(missing_ok=True) + + client_requirements_file = client_source_path / 'requirements.txt' + client_requirements_file.unlink(missing_ok=True) + + client_test_requirements_file = client_source_path / 'test-requirements.txt' + client_test_requirements_file.unlink(missing_ok=True) + + requirements_lines_seen = set() + test_requirements_lines_seen = set() + + for temp_module_path in client_source_path.glob('tmp-*/'): + module = ( + temp_module_path + .name + .removeprefix('tmp-') + .replace('-', '_') + ) + + code_destination_directory = client_source_path / lib_name / module + copymergedir(temp_module_path / lib_name / module, code_destination_directory) + copymergedir(temp_module_path / f'{lib_name}.{module}', code_destination_directory) # Fixes a bug with nested packages + + module_docs_path = client_docs_path / module + module_docs_path.mkdir(parents=True, exist_ok=True) + copymergedir(client_source_path / temp_module_path.name / 'docs', module_docs_path) + + module_tests_path = client_source_path / 'test' / module + copymergedir(temp_module_path / 'test', module_tests_path) + + readme_file = temp_module_path / 'README.md' + if not readme_file.exists(): + logging.warning(f'Readme file not found: {readme_file}.') + continue + + with client_readme_file.open('+a') as out_file, readme_file.open('r') as in_file: + file_data = in_file.read() + updated_file_data = file_data.replace('docs/', f'docs/{module}/') + out_file.write(updated_file_data) + + # FIXME: Different package versions will remain in the output file + requirements_file = temp_module_path / 'requirements.txt' + with requirements_file.open('r') as in_file, client_requirements_file.open('+a') as out_file: + unseen_lines = [line for line in in_file if line not in requirements_lines_seen] + requirements_lines_seen.update(unseen_lines) + out_file.writelines(unseen_lines) + + # FIXME: Different package versions will remain in the output file + test_requirements_file = temp_module_path / 'test-requirements.txt' + with test_requirements_file.open('r') as in_file, client_test_requirements_file.open('+a') as out_file: + unseen_lines = [line for line in in_file if line not in test_requirements_lines_seen] + test_requirements_lines_seen.update(unseen_lines) + out_file.writelines(unseen_lines) + + shutil.rmtree(temp_module_path) diff --git a/tools/deployment-cli-tools/ch_cli_tools/utils.py b/tools/deployment-cli-tools/ch_cli_tools/utils.py index 53f36953..4468420e 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/utils.py +++ b/tools/deployment-cli-tools/ch_cli_tools/utils.py @@ -180,14 +180,14 @@ def replaceindir(root_src_dir, source, replace): replace_in_file(src_file, source, replace) -def replace_in_file(src_file: pathlib.Path, source: str, replacement: str) -> None: +def replace_in_file(src_file: pathlib.Path, source: str, replacement) -> None: if src_file.name.endswith('.py') or src_file.name == 'Dockerfile': - replacement = to_python_module(replacement) + replacement = to_python_module(str(replacement)) with fileinput.input(src_file, inplace=True) as file: try: for line in file: - print(line.replace(source, replacement), end='') + print(line.replace(source, str(replacement)), end='') except UnicodeDecodeError: pass diff --git a/tools/deployment-cli-tools/harness-application b/tools/deployment-cli-tools/harness-application index 7484f754..8aadbf46 100644 --- a/tools/deployment-cli-tools/harness-application +++ b/tools/deployment-cli-tools/harness-application @@ -13,7 +13,7 @@ from typing import Union from ch_cli_tools import CH_ROOT from cloudharness_utils.constants import APPLICATION_TEMPLATE_PATH -from ch_cli_tools.openapi import generate_server, generate_fastapi_server, APPLICATIONS_SRC_PATH, generate_ts_client +from ch_cli_tools.openapi import generate_flask_server, generate_fastapi_server, APPLICATIONS_SRC_PATH, generate_openapi_from_ninja_schema, generate_ts_client from ch_cli_tools.utils import merge_configuration_directories, replaceindir, replace_in_file, save_yaml, \ to_python_module, copymergedir, get_json_template, replace_in_dict from ch_cli_tools.common_types import CloudHarnessManifest, TemplateType @@ -44,7 +44,6 @@ def main() -> None: pipeline.handle_post_merge() create_manifest_file(app_path, app_name, templates) - call_harness_generate(app_path, app_name) def get_command_line_arguments() -> tuple[str, list[str]]: @@ -96,12 +95,14 @@ def normalize_templates(templates: list[str]) -> list[str]: return next(index for index, template in enumerate(templates) if template in TemplateType.django_templates()) has_django_template = any(template in TemplateType.django_templates() for template in templates) - if has_django_template and TemplateType.WEBAPP not in templates: - templates.insert(django_template_index(), TemplateType.WEBAPP) + if TemplateType.WEBAPP not in templates: + if (input(f'Do you want to generate Vite frontend application? [Y/n] ').casefold() == 'y'): + templates.insert(django_template_index(), TemplateType.WEBAPP) has_database_template = any(template in TemplateType.database_templates() for template in templates) if has_django_template and not has_database_template: - templates.insert(django_template_index(), TemplateType.DB_POSTGRES) + if (input(f'Do you want to use a postgres database? [Y/n] ').casefold() == 'y'): + templates.insert(django_template_index(), TemplateType.DB_POSTGRES) return templates @@ -119,12 +120,5 @@ def create_manifest_file(app_path: pathlib.Path, app_name: str, templates: list[ save_yaml(manifest_file, manifest.to_dict()) -def call_harness_generate(app_path: pathlib.Path, app_name: str): - logging.info('Running initial harness generate...') - root_path = app_path.parent.parent - command = ['harness-generate', 'all', '--ts-only', '--app-name', app_name, root_path] - subprocess.run(command) - - if __name__ == "__main__": main() diff --git a/tools/deployment-cli-tools/harness-generate b/tools/deployment-cli-tools/harness-generate index 705a40c6..ae7819ed 100644 --- a/tools/deployment-cli-tools/harness-generate +++ b/tools/deployment-cli-tools/harness-generate @@ -10,8 +10,7 @@ import shutil import logging from typing import Callable, Optional -from ch_cli_tools.openapi import LIB_NAME, generate_openapi_from_ninja_schema, generate_python_client, generate_server, generate_fastapi_server, \ - get_dependencies, generate_ts_client, generate_model +from ch_cli_tools.openapi import LIB_NAME, ClientType, generate_clients, generate_models, generate_servers, get_dependencies from ch_cli_tools.utils import copymergedir from ch_cli_tools.common_types import TemplateType from ch_cli_tools.manifest import get_manifest @@ -35,15 +34,6 @@ def main(): generate_clients(root_path, should_generate, args.app_name, args.client_name, args.client_types) -class ClientType(enum.Flag): - TS_CLIENT = enum.auto() - PYTHON_CLIENT = enum.auto() - - @classmethod - def all(cls): - return functools.reduce(operator.or_, cls) - - class GenerationMode(enum.Flag): CLIENTS = enum.auto() MODELS = enum.auto() @@ -126,153 +116,5 @@ def should_generate_interactive(resource: str) -> bool: return user_input == 'y' -def generate_models( - root_path: pathlib.Path, - should_generate: Callable[[str], bool], -) -> None: - """ - Generates the main model - """ - library_models_path = root_path / 'libraries' / 'models' - - if not library_models_path.exists(): - return - - if not should_generate('the main model'): - return - - generate_model() - - -def generate_servers( - root_path: pathlib.Path, - should_generate: Callable[[str], bool], - app_name: Optional[str], -) -> None: - """ - Generates server stubs - """ - openapi_files = [path for path in root_path.glob('applications/*/api/*.yaml')] - - for openapi_file in openapi_files: - app_path = openapi_file.parent.parent - manifest = get_manifest(app_path) - - if app_name and manifest.app_name != app_name: - continue - - if not should_generate(f'server stubs for {openapi_file}'): - continue - - if TemplateType.DJANGO_FASTAPI in manifest.templates: - generate_fastapi_server(app_path) - - if TemplateType.FLASK_SERVER in manifest.templates: - generate_server(app_path) - - -def generate_clients( - root_path: pathlib.Path, - should_generate: Callable[[str], bool], - app_name: Optional[str], - client_lib_name: str, - client_types: ClientType, -) -> None: - """ - Generates client stubs - """ - if not should_generate('client libraries'): - return - - client_src_path = root_path / 'libraries' / 'client' / client_lib_name - apps_path = root_path / 'applications' - apps = (app for app in apps_path.iterdir() if app.is_dir()) - - for app_path in apps: - manifest = get_manifest(app_path) - - if app_name and manifest.app_name != app_name: - continue - - if TemplateType.DJANGO_NINJA in manifest.templates: - generate_openapi_from_ninja_schema(manifest.app_name, app_path) - - for openapi_file in app_path.glob('api/*.yaml'): - if ClientType.PYTHON_CLIENT in client_types: - generate_python_client(manifest.app_name, openapi_file, client_src_path, lib_name=client_lib_name) - - if TemplateType.WEBAPP in manifest.templates and ClientType.TS_CLIENT in client_types: - generate_ts_client(openapi_file) - - aggregate_packages(client_src_path, client_lib_name) - - -def aggregate_packages(client_source_path: pathlib.Path, lib_name=LIB_NAME): - client_source_path.mkdir(parents=True, exist_ok=True) - - client_docs_path = client_source_path / 'docs' - client_docs_path.mkdir(exist_ok=True) - - client_test_path = client_source_path / 'test' - client_test_path.mkdir(exist_ok=True) - - client_readme_file = client_source_path / 'README.md' - client_readme_file.unlink(missing_ok=True) - - client_requirements_file = client_source_path / 'requirements.txt' - client_requirements_file.unlink(missing_ok=True) - - client_test_requirements_file = client_source_path / 'test-requirements.txt' - client_test_requirements_file.unlink(missing_ok=True) - - requirements_lines_seen = set() - test_requirements_lines_seen = set() - - for temp_module_path in client_source_path.glob('tmp-*/'): - module = ( - temp_module_path - .name - .removeprefix('tmp-') - .replace('-', '_') - ) - - code_destination_directory = client_source_path / lib_name / module - copymergedir(temp_module_path / lib_name / module, code_destination_directory) - copymergedir(temp_module_path / f'{lib_name}.{module}', code_destination_directory) # Fixes a bug with nested packages - - module_docs_path = client_docs_path / module - module_docs_path.mkdir(parents=True, exist_ok=True) - copymergedir(client_source_path / temp_module_path.name / 'docs', module_docs_path) - - module_tests_path = client_source_path / 'test' / module - copymergedir(temp_module_path / 'test', module_tests_path) - - readme_file = temp_module_path / 'README.md' - if not readme_file.exists(): - logging.warning(f'Readme file not found: {readme_file}.') - continue - - with client_readme_file.open('+a') as out_file, readme_file.open('r') as in_file: - file_data = in_file.read() - updated_file_data = file_data.replace('docs/', f'docs/{module}/') - out_file.write(updated_file_data) - - # FIXME: Different package versions will remain in the output file - requirements_file = temp_module_path / 'requirements.txt' - with requirements_file.open('r') as in_file, client_requirements_file.open('+a') as out_file: - unseen_lines = [line for line in in_file if line not in requirements_lines_seen] - requirements_lines_seen.update(unseen_lines) - out_file.writelines(unseen_lines) - - # FIXME: Different package versions will remain in the output file - test_requirements_file = temp_module_path / 'test-requirements.txt' - with test_requirements_file.open('r') as in_file, client_test_requirements_file.open('+a') as out_file: - unseen_lines = [line for line in in_file if line not in test_requirements_lines_seen] - test_requirements_lines_seen.update(unseen_lines) - out_file.writelines(unseen_lines) - - shutil.rmtree(temp_module_path) - - if __name__ == "__main__": main() From 8176368fdb5ec347af71856683ab6ad6c773c715 Mon Sep 17 00:00:00 2001 From: Filippo Ledda Date: Tue, 21 Jan 2025 14:21:47 +0100 Subject: [PATCH 73/93] Update version numbers --- deployment-configuration/compose/Chart.yaml | 2 +- deployment-configuration/compose/values.yaml | 2 +- deployment-configuration/helm/Chart.yaml | 2 +- deployment-configuration/helm/values.yaml | 2 +- libraries/cloudharness-common/setup.py | 2 +- libraries/cloudharness-utils/setup.py | 2 +- tools/cloudharness-test/setup.py | 2 +- tools/deployment-cli-tools/PKG-INFO | 18 ++++++++++++++++++ tools/deployment-cli-tools/setup.py | 3 +-- 9 files changed, 26 insertions(+), 9 deletions(-) create mode 100644 tools/deployment-cli-tools/PKG-INFO diff --git a/deployment-configuration/compose/Chart.yaml b/deployment-configuration/compose/Chart.yaml index 1b396ffe..e71aca49 100644 --- a/deployment-configuration/compose/Chart.yaml +++ b/deployment-configuration/compose/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v1 -appVersion: 2.4.0-compose +appVersion: 2.5.0-compose description: CloudHarness Docker compose Helm Chart maintainers: - {email: filippo@metacell.us, name: Filippo Ledda} diff --git a/deployment-configuration/compose/values.yaml b/deployment-configuration/compose/values.yaml index 0fa87e29..023ab9f5 100644 --- a/deployment-configuration/compose/values.yaml +++ b/deployment-configuration/compose/values.yaml @@ -21,7 +21,7 @@ apps: {} env: # -- Cloud Harness version - name: CH_VERSION - value: 2.4.0 + value: 2.5.0 proxy: timeout: # -- Timeout for proxy connections in seconds. diff --git a/deployment-configuration/helm/Chart.yaml b/deployment-configuration/helm/Chart.yaml index 31dbc5ff..09aa97e5 100644 --- a/deployment-configuration/helm/Chart.yaml +++ b/deployment-configuration/helm/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v1 -appVersion: 2.4.0 +appVersion: 2.5.0 description: CloudHarness Helm Chart maintainers: - {email: filippo@metacell.us, name: Filippo Ledda} diff --git a/deployment-configuration/helm/values.yaml b/deployment-configuration/helm/values.yaml index 86ada06f..75a14f3a 100644 --- a/deployment-configuration/helm/values.yaml +++ b/deployment-configuration/helm/values.yaml @@ -21,7 +21,7 @@ apps: {} env: # -- Cloud Harness version - name: CH_VERSION - value: 2.4.0 + value: 2.5.0 privenv: # -- Defines a secret as private environment variable that is injected in containers. - name: CH_SECRET diff --git a/libraries/cloudharness-common/setup.py b/libraries/cloudharness-common/setup.py index 07184507..d6aa8433 100644 --- a/libraries/cloudharness-common/setup.py +++ b/libraries/cloudharness-common/setup.py @@ -3,7 +3,7 @@ NAME = "cloudharness" -VERSION = "2.4.0" +VERSION = "2.5.0" # To install the library, run the following # # python setup.py install diff --git a/libraries/cloudharness-utils/setup.py b/libraries/cloudharness-utils/setup.py index 6a2b4b48..91d3e31d 100644 --- a/libraries/cloudharness-utils/setup.py +++ b/libraries/cloudharness-utils/setup.py @@ -11,7 +11,7 @@ NAME = "cloudharness_utils" -VERSION = "2.4.0" +VERSION = "2.5.0" # To install the library, run the following # # python setup.py install diff --git a/tools/cloudharness-test/setup.py b/tools/cloudharness-test/setup.py index 5f07dfd6..c82959f0 100644 --- a/tools/cloudharness-test/setup.py +++ b/tools/cloudharness-test/setup.py @@ -12,7 +12,7 @@ NAME = "cloudharness-test" -VERSION = "2.3.0" +VERSION = "2.5.0" # To install the library, run the following # # python setup.py install diff --git a/tools/deployment-cli-tools/PKG-INFO b/tools/deployment-cli-tools/PKG-INFO new file mode 100644 index 00000000..ddcea8a6 --- /dev/null +++ b/tools/deployment-cli-tools/PKG-INFO @@ -0,0 +1,18 @@ +Metadata-Version: 2.1 +Name: cloudharness-cli-tools +Version: 2.5.0 +Summary: CloudHarness deploy and code generation tools +Home-page: +Author-email: cloudharness@metacell.us +Keywords: Cloud,Kubernetes,Helm,Deploy +Requires-Dist: ruamel.yaml +Requires-Dist: oyaml +Requires-Dist: docker +Requires-Dist: six +Requires-Dist: cloudharness_model +Requires-Dist: cloudharness_utils +Requires-Dist: dirhash +Requires-Dist: StrEnum; python_version < "3.11" + + CloudHarness deploy library + diff --git a/tools/deployment-cli-tools/setup.py b/tools/deployment-cli-tools/setup.py index 280cd716..1b9b1bf2 100644 --- a/tools/deployment-cli-tools/setup.py +++ b/tools/deployment-cli-tools/setup.py @@ -12,7 +12,7 @@ NAME = "cloudharness-cli-tools" -VERSION = "2.3.0" +VERSION = "2.5.0" # To install the library, run the following # # python setup.py install @@ -27,7 +27,6 @@ 'six', 'cloudharness_model', 'cloudharness_utils', - 'fastapi-code-generator', 'dirhash', "StrEnum ; python_version < '3.11'", ] From 6a6b5f32a69551f77c5914282c110ffcc065af9d Mon Sep 17 00:00:00 2001 From: Filippo Ledda Date: Tue, 21 Jan 2025 14:22:26 +0100 Subject: [PATCH 74/93] Add app manifests, update versions --- applications/accounts/.ch-manifest | 4 ++++ applications/argo/.ch-manifest | 4 ++++ applications/common/.ch-manifest | 4 ++++ applications/common/server/setup.py | 2 +- applications/events/.ch-manifest | 4 ++++ applications/jupyterhub/.ch-manifest | 4 ++++ applications/neo4j/.ch-manifest | 4 ++++ applications/nfsserver/.ch-manifest | 4 ++++ 8 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 applications/accounts/.ch-manifest create mode 100644 applications/argo/.ch-manifest create mode 100644 applications/common/.ch-manifest create mode 100644 applications/events/.ch-manifest create mode 100644 applications/jupyterhub/.ch-manifest create mode 100644 applications/neo4j/.ch-manifest create mode 100644 applications/nfsserver/.ch-manifest diff --git a/applications/accounts/.ch-manifest b/applications/accounts/.ch-manifest new file mode 100644 index 00000000..8862ec5b --- /dev/null +++ b/applications/accounts/.ch-manifest @@ -0,0 +1,4 @@ +app-name: accounts +inferred: true +templates: [base, db-postgres] +version: '2' diff --git a/applications/argo/.ch-manifest b/applications/argo/.ch-manifest new file mode 100644 index 00000000..6271cbc0 --- /dev/null +++ b/applications/argo/.ch-manifest @@ -0,0 +1,4 @@ +app-name: argo +inferred: true +templates: [base] +version: '2' diff --git a/applications/common/.ch-manifest b/applications/common/.ch-manifest new file mode 100644 index 00000000..10271ee5 --- /dev/null +++ b/applications/common/.ch-manifest @@ -0,0 +1,4 @@ +app-name: common +inferred: true +templates: [base, flask-server] +version: '2' diff --git a/applications/common/server/setup.py b/applications/common/server/setup.py index 1935137a..fdfc3ea8 100644 --- a/applications/common/server/setup.py +++ b/applications/common/server/setup.py @@ -4,7 +4,7 @@ from setuptools import setup, find_packages NAME = "openapi_server" -VERSION = "2.4.0" +VERSION = "2.5.0" # To install the library, run the following # diff --git a/applications/events/.ch-manifest b/applications/events/.ch-manifest new file mode 100644 index 00000000..5cdb28f0 --- /dev/null +++ b/applications/events/.ch-manifest @@ -0,0 +1,4 @@ +app-name: events +inferred: true +templates: [base] +version: '2' diff --git a/applications/jupyterhub/.ch-manifest b/applications/jupyterhub/.ch-manifest new file mode 100644 index 00000000..4f42b848 --- /dev/null +++ b/applications/jupyterhub/.ch-manifest @@ -0,0 +1,4 @@ +app-name: jupyterhub +inferred: true +templates: [base] +version: '2' diff --git a/applications/neo4j/.ch-manifest b/applications/neo4j/.ch-manifest new file mode 100644 index 00000000..63ef59d4 --- /dev/null +++ b/applications/neo4j/.ch-manifest @@ -0,0 +1,4 @@ +app-name: neo4j +inferred: true +templates: [base, db-neo4j] +version: '2' diff --git a/applications/nfsserver/.ch-manifest b/applications/nfsserver/.ch-manifest new file mode 100644 index 00000000..b982761e --- /dev/null +++ b/applications/nfsserver/.ch-manifest @@ -0,0 +1,4 @@ +app-name: nfsserver +inferred: true +templates: [base] +version: '2' From c5a64e978b6a80a732a6ba3b2b9e73ac9b3e7a04 Mon Sep 17 00:00:00 2001 From: Filippo Ledda Date: Tue, 21 Jan 2025 14:22:51 +0100 Subject: [PATCH 75/93] Add app manifests, update versions --- applications/notifications/.ch-manifest | 4 ++++ applications/notifications/server/setup.py | 2 +- applications/samples/.ch-manifest | 4 ++++ applications/sentry/.ch-manifest | 4 ++++ applications/volumemanager/.ch-manifest | 4 ++++ applications/volumemanager/server/setup.py | 2 +- applications/workflows/.ch-manifest | 4 ++++ applications/workflows/server/setup.py | 2 +- 8 files changed, 23 insertions(+), 3 deletions(-) create mode 100644 applications/notifications/.ch-manifest create mode 100644 applications/samples/.ch-manifest create mode 100644 applications/sentry/.ch-manifest create mode 100644 applications/volumemanager/.ch-manifest create mode 100644 applications/workflows/.ch-manifest diff --git a/applications/notifications/.ch-manifest b/applications/notifications/.ch-manifest new file mode 100644 index 00000000..cf24fc50 --- /dev/null +++ b/applications/notifications/.ch-manifest @@ -0,0 +1,4 @@ +app-name: notifications +inferred: true +templates: [base, flask-server] +version: '2' diff --git a/applications/notifications/server/setup.py b/applications/notifications/server/setup.py index f625329c..590a1fd2 100644 --- a/applications/notifications/server/setup.py +++ b/applications/notifications/server/setup.py @@ -4,7 +4,7 @@ from setuptools import setup, find_packages NAME = "notifications" -VERSION = "2.4.0" +VERSION = "2.5.0" # To install the library, run the following # diff --git a/applications/samples/.ch-manifest b/applications/samples/.ch-manifest new file mode 100644 index 00000000..cd34091e --- /dev/null +++ b/applications/samples/.ch-manifest @@ -0,0 +1,4 @@ +app-name: samples +inferred: true +templates: [base, webapp, flask-server] +version: '2' diff --git a/applications/sentry/.ch-manifest b/applications/sentry/.ch-manifest new file mode 100644 index 00000000..4acc26d5 --- /dev/null +++ b/applications/sentry/.ch-manifest @@ -0,0 +1,4 @@ +app-name: sentry +inferred: true +templates: [base, db-postgres] +version: '2' diff --git a/applications/volumemanager/.ch-manifest b/applications/volumemanager/.ch-manifest new file mode 100644 index 00000000..f994722e --- /dev/null +++ b/applications/volumemanager/.ch-manifest @@ -0,0 +1,4 @@ +app-name: volumemanager +inferred: true +templates: [base, flask-server] +version: '2' diff --git a/applications/volumemanager/server/setup.py b/applications/volumemanager/server/setup.py index ea5b3b37..21b33fc1 100644 --- a/applications/volumemanager/server/setup.py +++ b/applications/volumemanager/server/setup.py @@ -4,7 +4,7 @@ from setuptools import setup, find_packages NAME = "volumemanager" -VERSION = "2.4.0" +VERSION = "2.5.0" # To install the library, run the following # diff --git a/applications/workflows/.ch-manifest b/applications/workflows/.ch-manifest new file mode 100644 index 00000000..16cb6bbd --- /dev/null +++ b/applications/workflows/.ch-manifest @@ -0,0 +1,4 @@ +app-name: workflows +inferred: true +templates: [base, flask-server] +version: '2' diff --git a/applications/workflows/server/setup.py b/applications/workflows/server/setup.py index 230cdd9d..93f8036b 100644 --- a/applications/workflows/server/setup.py +++ b/applications/workflows/server/setup.py @@ -4,7 +4,7 @@ from setuptools import setup, find_packages NAME = "workflows_api" -VERSION = "2.4.0" +VERSION = "2.5.0" # To install the library, run the following # From 2af22175d5741c7b6d6e0116b1a93571b09a624c Mon Sep 17 00:00:00 2001 From: Filippo Ledda Date: Tue, 21 Jan 2025 18:05:59 +0100 Subject: [PATCH 76/93] CH-157 refactor, docs, enhancements --- application-templates/django-ninja/README.md | 50 ++++++++++++++---- .../django-ninja/port-forward.sh | 6 +++ .../webapp/frontend/package.json | 4 +- .../frontend/src/components/RestTest.tsx | 8 +-- .../cloudharness_django/settings.py | 2 +- .../cloudharness_cli/__init__.py | 0 .../test/ninjatest/__init__.py | 0 .../test/ninjatest/test_test_api.py | 52 +++++++++++++++++++ .../ch_cli_tools/application_builders.py | 2 +- .../ch_cli_tools/openapi.py | 45 +++++++++++++--- .../ch_cli_tools/utils.py | 5 ++ .../deployment-cli-tools/harness-application | 16 ++---- tools/deployment-cli-tools/harness-generate | 29 +++++++---- 13 files changed, 170 insertions(+), 49 deletions(-) create mode 100755 application-templates/django-ninja/port-forward.sh create mode 100644 libraries/client/cloudharness_cli/cloudharness_cli/__init__.py create mode 100644 libraries/client/cloudharness_cli/test/ninjatest/__init__.py create mode 100644 libraries/client/cloudharness_cli/test/ninjatest/test_test_api.py diff --git a/application-templates/django-ninja/README.md b/application-templates/django-ninja/README.md index 423842d6..450c1da8 100644 --- a/application-templates/django-ninja/README.md +++ b/application-templates/django-ninja/README.md @@ -34,19 +34,29 @@ Frontend code is inside the *frontend* directory. Frontend is by default generated as a React web application, but no constraint about this specific technology. -#### Call the backend apis +See also [frontend/README.md] + +#### Generate API client stubs All the api stubs are automatically generated in the [frontend/rest](frontend/rest) directory by `harness-application` and `harness-generate`. +To update frontend client stubs, run + +``` +harness-generate clients __APP_NAME__ -t +``` + +Stubs can also be updated using the `genapi.sh` from the api folder. + ## Local build & run -### Install dependencies +### Install Python dependencies 1 - Clone cloud-harness into your project root folder 2 - Run the dev setup script ``` cd applications/__APP_NAME__ -bash dev-setup.sh +source dev-setup.sh ``` ### Prepare backend @@ -62,25 +72,43 @@ cd static/www ln -s ../../../frontend/dist dist ``` -### Build frontend +### Run frontend -Compile the frontend -```bash -cd frontend -npm install -npm run build -``` +- `yarn dev` Local dev with no backend (no or mock data, cookie required) +- `yarn start` Local dev with backend on localhost:8000 -- see next paragraph (cookie required) +- `yarn start:dev` Local dev with backend on the remote dev deployment (cookie required) +- `yarn start:local` Local dev with backend on the local dev deployment on mnp.local (cookie required) + +To obtain the login cookie, login in the application with the forwarded backend, copy the `kc-access` cookie and set it into localhost:9000 ### Run backend application start the Django server + ```bash -uvicorn --workers 2 --host 0.0.0.0 --port 8000 django_baseapp.asgi:application +ACCOUNTS_ADMIN_PASSWORD=metacell ACCOUNTS_ADMIN_USERNAME=admin CH_CURRENT_APP_NAME=__APP_NAME__ CH_VALUES_PATH=../../../deployment/helm/values.yaml DJANGO_SETTINGS_MODULE=django_baseapp.settings KUBERNETES_SERVICE_HOST=a uvicorn --host 0.0.0.0 --port 8000 django_baseapp.asgi:application ``` +Before running this backend, have to: +- Run `harness-deployment ... -n [NAMESPACE] -i __APP_NAME__` with the setup +- port-forward keycloak and the database (see below) ### Running local with port forwardings to a kubernetes cluster When you create port forwards to microservices in your k8s cluster you want to forced your local backend server to initialize the AuthService and EventService services. This can be done by setting the `KUBERNETES_SERVICE_HOST` environment variable to a dummy or correct k8s service host. The `KUBERNETES_SERVICE_HOST` switch will activate the creation of the keycloak client and client roles of this microservice. + +Run `port-forward.sh` to get the keycloak and database running. + +To access those have to map to the hosts file: + +``` +127.0.0.1 accounts.[NAMESPACE] __APP_NAME__-db +``` + +After running the backend on port 8000, run `yarn start` to get a frontend to it + +#### Vs code run configuration + +Run configuration is automatically generated for VS code (__APP_NAME__ backend) \ No newline at end of file diff --git a/application-templates/django-ninja/port-forward.sh b/application-templates/django-ninja/port-forward.sh new file mode 100755 index 00000000..63a06be8 --- /dev/null +++ b/application-templates/django-ninja/port-forward.sh @@ -0,0 +1,6 @@ +fuser -k 5432/tcp +fuser -k 8080/tcp +namespace=${1:-ch} +echo "Port forwarding for $namespace" +kubectl port-forward --namespace $namespace deployment/accounts 8080:8080 & +kubectl port-forward --namespace $namespace deployment/__APP_NAME__-db 5432:5432 & diff --git a/application-templates/webapp/frontend/package.json b/application-templates/webapp/frontend/package.json index 4d852c34..bd64303d 100644 --- a/application-templates/webapp/frontend/package.json +++ b/application-templates/webapp/frontend/package.json @@ -3,8 +3,8 @@ "scripts": { "dev": "vite", "start": "DOMAIN=http://localhost:5000 vite", - "start:dev": "DOMAIN=https://test.ch.metacell.us vite", - "start:local": "DOMAIN=http://samples.ch vite", + "start:dev": "DOMAIN=https://__APP_NAME__.ch.metacell.us vite", + "start:local": "DOMAIN=http://__APP_NAME__.ch vite", "prebuild": "eslint .", "build": "vite build", "lint": "eslint src --report-unused-disable-directives --fix" diff --git a/application-templates/webapp/frontend/src/components/RestTest.tsx b/application-templates/webapp/frontend/src/components/RestTest.tsx index c1dc7c00..0123e48b 100644 --- a/application-templates/webapp/frontend/src/components/RestTest.tsx +++ b/application-templates/webapp/frontend/src/components/RestTest.tsx @@ -1,7 +1,7 @@ import { useState, useEffect } from 'react'; -import { TestApi } from '../rest/apis/TestApi' +import { TestApi } from '../rest/__APP_NAME__/apis/TestApi' const test = new TestApi(); @@ -9,11 +9,11 @@ const test = new TestApi(); const RestTest = () => { const [result, setResult] = useState(null); useEffect(() => { - test.ping().then((r) => setResult(r), () => setResult( "API error")); + test.ping().then((r) => setResult(r), () => setResult("API error")); }, []); - - return result ?

Backend answered: { result }

:

Backend did not answer

+ + return result ?

Backend answered: {result}

:

Backend did not answer

} export default RestTest; \ No newline at end of file diff --git a/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/settings.py b/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/settings.py index b5c71a1f..d5df3516 100644 --- a/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/settings.py +++ b/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/settings.py @@ -50,7 +50,7 @@ except: # no current app found, fall back to the default settings, there is a god change that # we are running on a developers local machine - log.warning("Error setting current app configuration, continuing...") + log.warning("Error setting current app configuration, was `harness-deployment` executed? Continuing...") current_app = applications.ApplicationConfiguration({ "name": app_name, diff --git a/libraries/client/cloudharness_cli/cloudharness_cli/__init__.py b/libraries/client/cloudharness_cli/cloudharness_cli/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/libraries/client/cloudharness_cli/test/ninjatest/__init__.py b/libraries/client/cloudharness_cli/test/ninjatest/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/libraries/client/cloudharness_cli/test/ninjatest/test_test_api.py b/libraries/client/cloudharness_cli/test/ninjatest/test_test_api.py new file mode 100644 index 00000000..6d93d9a4 --- /dev/null +++ b/libraries/client/cloudharness_cli/test/ninjatest/test_test_api.py @@ -0,0 +1,52 @@ +# coding: utf-8 + +""" + ninjatest API + + No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + + The version of the OpenAPI document: 0.1.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +import unittest + +from cloudharness_cli.ninjatest.api.test_api import TestApi + + +class TestTestApi(unittest.TestCase): + """TestApi unit test stubs""" + + def setUp(self) -> None: + self.api = TestApi() + + def tearDown(self) -> None: + pass + + def test_ninjatest_api_live(self) -> None: + """Test case for ninjatest_api_live + + Live + """ + pass + + def test_ninjatest_api_ping(self) -> None: + """Test case for ninjatest_api_ping + + Ping + """ + pass + + def test_ninjatest_api_ready(self) -> None: + """Test case for ninjatest_api_ready + + Ready + """ + pass + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/deployment-cli-tools/ch_cli_tools/application_builders.py b/tools/deployment-cli-tools/ch_cli_tools/application_builders.py index b10f17e1..329ab580 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/application_builders.py +++ b/tools/deployment-cli-tools/ch_cli_tools/application_builders.py @@ -89,7 +89,7 @@ def handle_post_merge(self): logging.info('Installing frontend dependencies') self.install_frontend_dependencies() logging.info('Generating ts client') - generate_ts_client(self.api_path / 'openapi.yaml') + generate_ts_client(self.api_path / 'openapi.yaml', self.app_name) def create_vite_skaffold(self, frontend_path: pathlib.Path) -> None: self.run_command( diff --git a/tools/deployment-cli-tools/ch_cli_tools/openapi.py b/tools/deployment-cli-tools/ch_cli_tools/openapi.py index ec2267e2..3c3ade29 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/openapi.py +++ b/tools/deployment-cli-tools/ch_cli_tools/openapi.py @@ -17,7 +17,7 @@ from ch_cli_tools.manifest import get_manifest from . import HERE -from .utils import copymergedir, replaceindir, to_python_module +from .utils import confirm, copymergedir, replace_in_file, replaceindir, to_python_module CODEGEN = os.path.join(HERE, 'bin', 'openapi-generator-cli.jar') APPLICATIONS_SRC_PATH = os.path.join('applications') @@ -129,33 +129,60 @@ def generate_python_client(module, openapi_file, client_src_path, lib_name=LIB_N os.system(command) -def generate_ts_client(openapi_file): +def generate_ts_client(openapi_file, app_name=""): get_dependencies() - out_dir = f"{os.path.dirname(os.path.dirname(openapi_file))}/frontend/src/rest" + out_dir = f"{os.path.dirname(os.path.dirname(openapi_file))}/frontend/src/rest/{app_name}" command = f"java -jar {CODEGEN} generate " \ f"-i {openapi_file} " \ f"-g typescript-fetch " \ - f"-o {out_dir}" + f"-o {out_dir} "\ + f"--additional-properties=prefixParameterInterfaces=false" os.system(command) replaceindir(out_dir, "http://localhost", '') +def json2yaml(json_filename, yaml_file=None): + import yaml + if yaml_file is None: + yaml_file = str(json_filename).replace('.json', '.yaml') + with open(json_filename, 'r') as json_filename: + data = json.load(json_filename) + with open(yaml_file, 'w') as yaml_file: + yaml.dump(data, yaml_file) + + def generate_openapi_from_ninja_schema(app_name: str, app_path: pathlib.Path) -> None: - subprocess.check_call(["sh", "dev-setup.sh"], cwd=app_path) - out_path = app_path / 'api' / 'openapi.yaml' + # check if cloudharness_django python library is installed + python_module = to_python_module(app_name) + try: + import cloudharness_django + # dynamicall import python_module + __import__(f'{python_module}') + except ImportError: + if confirm('Runtime env is not installed. Do you want to install it? [Y/n]'): + subprocess.check_call(["sh", "dev-setup.sh"], cwd=app_path) + else: + logging.error('Runtime env is not installed. Cound not generate openapi files for Django Ninja.') + return + logging.info(f"Generating openapi files for Django Ninja for application {app_name}") + out_path = app_path / 'api' / 'openapi.json' manage_path = app_path / 'backend' / 'manage.py' command = [ 'python', manage_path, 'export_openapi_schema', '--settings', 'django_baseapp.settings', - '--api', f'{to_python_module(app_name)}.api.api', + '--api', f'{python_module}.api.api', '--output', out_path, '--indent', '2', ] subprocess.run(command) + replace_in_file(out_path, f'{app_name}_api_', '') + + json2yaml(out_path) + def get_dependencies(): """ @@ -235,6 +262,8 @@ def generate_clients( if not should_generate('client libraries'): return + logging.info('Generating client libraries for %s', str(client_types)) + client_src_path = root_path / 'libraries' / 'client' / client_lib_name apps_path = root_path / 'applications' apps = (app for app in apps_path.iterdir() if app.is_dir()) @@ -253,7 +282,7 @@ def generate_clients( generate_python_client(manifest.app_name, openapi_file, client_src_path, lib_name=client_lib_name) if TemplateType.WEBAPP in manifest.templates and ClientType.TS_CLIENT in client_types: - generate_ts_client(openapi_file) + generate_ts_client(openapi_file, app_name) aggregate_packages(client_src_path, client_lib_name) diff --git a/tools/deployment-cli-tools/ch_cli_tools/utils.py b/tools/deployment-cli-tools/ch_cli_tools/utils.py index 4468420e..85d56eaa 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/utils.py +++ b/tools/deployment-cli-tools/ch_cli_tools/utils.py @@ -180,6 +180,11 @@ def replaceindir(root_src_dir, source, replace): replace_in_file(src_file, source, replace) +def confirm(question, default_true=True): + answer = input(f"{question} (Y/n): ").casefold() + return (default_true and not answer) or answer == "y" + + def replace_in_file(src_file: pathlib.Path, source: str, replacement) -> None: if src_file.name.endswith('.py') or src_file.name == 'Dockerfile': replacement = to_python_module(str(replacement)) diff --git a/tools/deployment-cli-tools/harness-application b/tools/deployment-cli-tools/harness-application index 8aadbf46..4ae81c9b 100644 --- a/tools/deployment-cli-tools/harness-application +++ b/tools/deployment-cli-tools/harness-application @@ -1,21 +1,15 @@ #!/usr/bin/env python -import json import pathlib import sys import re -import shutil -import tempfile -import subprocess import logging import argparse from typing import Union -from ch_cli_tools import CH_ROOT -from cloudharness_utils.constants import APPLICATION_TEMPLATE_PATH -from ch_cli_tools.openapi import generate_flask_server, generate_fastapi_server, APPLICATIONS_SRC_PATH, generate_openapi_from_ninja_schema, generate_ts_client -from ch_cli_tools.utils import merge_configuration_directories, replaceindir, replace_in_file, save_yaml, \ - to_python_module, copymergedir, get_json_template, replace_in_dict +from ch_cli_tools.openapi import APPLICATIONS_SRC_PATH +from ch_cli_tools.utils import confirm, replaceindir, replace_in_file, save_yaml, \ + to_python_module from ch_cli_tools.common_types import CloudHarnessManifest, TemplateType from ch_cli_tools.application_builders import AppBuilderPipeline @@ -96,12 +90,12 @@ def normalize_templates(templates: list[str]) -> list[str]: has_django_template = any(template in TemplateType.django_templates() for template in templates) if TemplateType.WEBAPP not in templates: - if (input(f'Do you want to generate Vite frontend application? [Y/n] ').casefold() == 'y'): + if (confirm(f'Do you want to generate Vite frontend application?')): templates.insert(django_template_index(), TemplateType.WEBAPP) has_database_template = any(template in TemplateType.database_templates() for template in templates) if has_django_template and not has_database_template: - if (input(f'Do you want to use a postgres database? [Y/n] ').casefold() == 'y'): + if (confirm(f'Do you want to use a postgres database?')): templates.insert(django_template_index(), TemplateType.DB_POSTGRES) return templates diff --git a/tools/deployment-cli-tools/harness-generate b/tools/deployment-cli-tools/harness-generate index ae7819ed..6087addc 100644 --- a/tools/deployment-cli-tools/harness-generate +++ b/tools/deployment-cli-tools/harness-generate @@ -6,21 +6,30 @@ import enum import functools import operator import pathlib -import shutil import logging -from typing import Callable, Optional +from typing import Optional from ch_cli_tools.openapi import LIB_NAME, ClientType, generate_clients, generate_models, generate_servers, get_dependencies -from ch_cli_tools.utils import copymergedir -from ch_cli_tools.common_types import TemplateType -from ch_cli_tools.manifest import get_manifest +from ch_cli_tools.utils import confirm def main(): args = get_command_line_arguments() get_dependencies() - root_path = args.path.absolute() + if args.path: + root_path = args.path.absolute() + + else: + root_path = pathlib.Path.cwd() + + # Check if applications folder exists, if not, go up until it's found + while not (root_path / 'applications').exists(): + if root_path == root_path.parent: + logging.error('Could not find applications folder') + return + root_path = root_path.parent + should_generate = should_generate_interactive if args.is_interactive else lambda _: True if args.generate_models: @@ -67,7 +76,7 @@ class CommandLineArguments: def get_command_line_arguments() -> CommandLineArguments: - parser = argparse.ArgumentParser(description='Walk filesystem inside ./applications create application scaffolding.') + parser = argparse.ArgumentParser(description='Walks filesystem inside ./applications to create and update application scaffolding based on API specifications.') common_arguments = argparse.ArgumentParser(add_help=False) common_arguments.add_argument('path', metavar='path', nargs='?', default=pathlib.Path.cwd(), type=pathlib.Path, @@ -79,7 +88,7 @@ def get_command_line_arguments() -> CommandLineArguments: clients_arguments = argparse.ArgumentParser(add_help=False) clients_arguments.add_argument('-cn', '--client-name', dest='client_name', action='store', default=LIB_NAME, - help='specify image registry prefix') + help='specify client prefix name') client_type_group = clients_arguments.add_mutually_exclusive_group(required=False) client_type_group.add_argument('-t', '--ts-only', dest='client_types', action='store_const', const=ClientType.TS_CLIENT, help='Generate only typescript clients') @@ -111,9 +120,7 @@ def get_command_line_arguments() -> CommandLineArguments: def should_generate_interactive(resource: str) -> bool: - user_input = input(f'Do you want to generate {resource}? [Y/n] ').casefold() - - return user_input == 'y' + return confirm(f'Do you want to generate {resource}?') if __name__ == "__main__": From 080178704a6c98890168f626935bb315a8d6a190 Mon Sep 17 00:00:00 2001 From: Filippo Ledda Date: Tue, 21 Jan 2025 18:35:37 +0100 Subject: [PATCH 77/93] Fix puppeteer not working in codefresh --- deployment-configuration/codefresh-template-dev.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/deployment-configuration/codefresh-template-dev.yaml b/deployment-configuration/codefresh-template-dev.yaml index 51620048..a12f30f4 100644 --- a/deployment-configuration/codefresh-template-dev.yaml +++ b/deployment-configuration/codefresh-template-dev.yaml @@ -120,6 +120,7 @@ steps: image: "${{test-e2e}}" fail_fast: false commands: + - npx puppeteer browsers install chrome - yarn test scale: {} when: From 5bd4b737000d10c03d21c4c840df13ddcc0fd312 Mon Sep 17 00:00:00 2001 From: Filippo Ledda Date: Fri, 24 Jan 2025 14:50:07 +0100 Subject: [PATCH 78/93] #792 Small code style tweaks --- tools/deployment-cli-tools/ch_cli_tools/openapi.py | 8 ++++---- tools/deployment-cli-tools/ch_cli_tools/utils.py | 4 ++-- tools/deployment-cli-tools/harness-generate | 8 ++------ 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/tools/deployment-cli-tools/ch_cli_tools/openapi.py b/tools/deployment-cli-tools/ch_cli_tools/openapi.py index 3c3ade29..164bcc5b 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/openapi.py +++ b/tools/deployment-cli-tools/ch_cli_tools/openapi.py @@ -156,11 +156,11 @@ def generate_openapi_from_ninja_schema(app_name: str, app_path: pathlib.Path) -> # check if cloudharness_django python library is installed python_module = to_python_module(app_name) try: - import cloudharness_django - # dynamicall import python_module - __import__(f'{python_module}') + import cloudharness_django # noqa + # dynamically import python_module + __import__(python_module) except ImportError: - if confirm('Runtime env is not installed. Do you want to install it? [Y/n]'): + if confirm('Runtime env is not installed. Do you want to install it?'): subprocess.check_call(["sh", "dev-setup.sh"], cwd=app_path) else: logging.error('Runtime env is not installed. Cound not generate openapi files for Django Ninja.') diff --git a/tools/deployment-cli-tools/ch_cli_tools/utils.py b/tools/deployment-cli-tools/ch_cli_tools/utils.py index 85d56eaa..18b876a7 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/utils.py +++ b/tools/deployment-cli-tools/ch_cli_tools/utils.py @@ -180,9 +180,9 @@ def replaceindir(root_src_dir, source, replace): replace_in_file(src_file, source, replace) -def confirm(question, default_true=True): +def confirm(question): answer = input(f"{question} (Y/n): ").casefold() - return (default_true and not answer) or answer == "y" + return answer == "y" if answer else True def replace_in_file(src_file: pathlib.Path, source: str, replacement) -> None: diff --git a/tools/deployment-cli-tools/harness-generate b/tools/deployment-cli-tools/harness-generate index 6087addc..5bd1b9ac 100644 --- a/tools/deployment-cli-tools/harness-generate +++ b/tools/deployment-cli-tools/harness-generate @@ -17,13 +17,9 @@ def main(): args = get_command_line_arguments() get_dependencies() - if args.path: - root_path = args.path.absolute() + root_path = args.path.absolute() - else: - root_path = pathlib.Path.cwd() - - # Check if applications folder exists, if not, go up until it's found + # Check if applications folder exists, if not, go up until it's found while not (root_path / 'applications').exists(): if root_path == root_path.parent: logging.error('Could not find applications folder') From 08b37169c7ab8d828b4aa8e2b0fd122b79904264 Mon Sep 17 00:00:00 2001 From: Filippo Ledda Date: Fri, 24 Jan 2025 19:09:31 +0100 Subject: [PATCH 79/93] CH-169 implement application build exclusion based on image param --- tools/deployment-cli-tools/ch_cli_tools/codefresh.py | 3 +++ tools/deployment-cli-tools/ch_cli_tools/helm.py | 10 ++++++---- tools/deployment-cli-tools/ch_cli_tools/skaffold.py | 4 ++++ 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/tools/deployment-cli-tools/ch_cli_tools/codefresh.py b/tools/deployment-cli-tools/ch_cli_tools/codefresh.py index 8bcf2b79..203a92c0 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/codefresh.py +++ b/tools/deployment-cli-tools/ch_cli_tools/codefresh.py @@ -173,6 +173,9 @@ def codefresh_steps_from_base_path(base_path, build_step, fixed_context=None, in # Skip excluded apps continue + if app_config and not helm_values.apps[app_key].get('build', True): + continue + if app_config and app_config.dependencies and app_config.dependencies.git: for dep in app_config.dependencies.git: step_name = f"clone_{basename(dep.url).replace('.', '_')}_{basename(dockerfile_relative_to_root).replace('.', '_')}" diff --git a/tools/deployment-cli-tools/ch_cli_tools/helm.py b/tools/deployment-cli-tools/ch_cli_tools/helm.py index b81d0f19..b202a21b 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/helm.py +++ b/tools/deployment-cli-tools/ch_cli_tools/helm.py @@ -190,13 +190,15 @@ def create_app_values_spec(self, app_name, app_path, base_image_name=None, helm_ else: build_dependencies = [] - if len(image_paths) > 0: + deployment_values = values[KEY_HARNESS].get(KEY_DEPLOYMENT, {}) + deployment_image = deployment_values.get('image', None) or values.get('image', None) + values['build'] = not bool(deployment_image) # Used by skaffold and ci/cd to determine if the image should be built + if len(image_paths) > 0 and not deployment_image: image_name = image_name_from_dockerfile_path(os.path.relpath( image_paths[0], os.path.dirname(app_path)), base_image_name) - - values['image'] = self.image_tag( + deployment_values['image'] = self.image_tag( image_name, build_context_path=app_path, dependencies=build_dependencies) - elif KEY_HARNESS in values and not values[KEY_HARNESS].get(KEY_DEPLOYMENT, {}).get('image', None) and values[ + elif KEY_HARNESS in values and not deployment_image and values[ KEY_HARNESS].get(KEY_DEPLOYMENT, {}).get('auto', False): raise Exception(f"At least one Dockerfile must be specified on application {app_name}. " f"Specify harness.deployment.image value if you intend to use a prebuilt image.") diff --git a/tools/deployment-cli-tools/ch_cli_tools/skaffold.py b/tools/deployment-cli-tools/ch_cli_tools/skaffold.py index 9609ad86..b8730971 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/skaffold.py +++ b/tools/deployment-cli-tools/ch_cli_tools/skaffold.py @@ -81,6 +81,8 @@ def process_build_dockerfile( if app_name is None: app_name = app_name_from_path(basename(dockerfile_path)) app_key = app_name.replace("-", "_") + if app_key in helm_values.apps and not helm_values.apps[app_key]['build']: + return if app_name in helm_values[KEY_TASK_IMAGES] or app_key in helm_values.apps: context_path = relpath_if(root_path, output_path) if global_context else relpath_if(dockerfile_path, output_path) @@ -157,6 +159,8 @@ def process_build_dockerfile( # app_image_tag, app_relative_to_skaffold, build_requirements) process_build_dockerfile(dockerfile_path, root_path, requirements=build_requirements, app_name=app_name) app = apps[app_key] + if not app['build']: + continue if app[KEY_HARNESS][KEY_DEPLOYMENT]['image']: release_config['artifactOverrides']['apps'][app_key] = \ { From 4cae093a647732b6a3549a88b9807dca1641a2fb Mon Sep 17 00:00:00 2001 From: Filippo Ledda Date: Fri, 31 Jan 2025 18:02:41 +0100 Subject: [PATCH 80/93] CH-169 tests and fixes --- .../ch_cli_tools/codefresh.py | 6 ++-- .../deployment-cli-tools/ch_cli_tools/helm.py | 4 +-- .../myapp/deploy/values-nobuild.yaml | 5 +++ .../tests/test_codefresh.py | 36 +++++++++++++++++++ tools/deployment-cli-tools/tests/test_helm.py | 24 +++++++++---- .../tests/test_skaffold.py | 34 ++++++++++++++++++ 6 files changed, 98 insertions(+), 11 deletions(-) create mode 100644 tools/deployment-cli-tools/tests/resources/applications/myapp/deploy/values-nobuild.yaml diff --git a/tools/deployment-cli-tools/ch_cli_tools/codefresh.py b/tools/deployment-cli-tools/ch_cli_tools/codefresh.py index 203a92c0..4f9b740d 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/codefresh.py +++ b/tools/deployment-cli-tools/ch_cli_tools/codefresh.py @@ -322,11 +322,13 @@ def add_unit_test_step(app_config: ApplicationHarnessConfig): steps = codefresh["steps"] if CD_E2E_TEST_STEP in steps and not steps[CD_E2E_TEST_STEP]["scale"]: del steps[CD_E2E_TEST_STEP] - del steps[CD_BUILD_STEP_TEST]["steps"]["test-e2e"] + if CD_BUILD_STEP_TEST in steps and 'test-e2e' in steps[CD_BUILD_STEP_TEST]["steps"]: + del steps[CD_BUILD_STEP_TEST]["steps"]["test-e2e"] if CD_API_TEST_STEP in steps and not steps[CD_API_TEST_STEP]["scale"]: del steps[CD_API_TEST_STEP] - del steps[CD_BUILD_STEP_TEST]["steps"]["test-api"] + if CD_BUILD_STEP_TEST in steps and 'test-api' in steps[CD_BUILD_STEP_TEST]["steps"]: + del steps[CD_BUILD_STEP_TEST]["steps"]["test-api"] if CD_BUILD_STEP_TEST in steps and not steps[CD_BUILD_STEP_TEST]["steps"]: del steps[CD_BUILD_STEP_TEST] diff --git a/tools/deployment-cli-tools/ch_cli_tools/helm.py b/tools/deployment-cli-tools/ch_cli_tools/helm.py index b202a21b..0c34df8b 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/helm.py +++ b/tools/deployment-cli-tools/ch_cli_tools/helm.py @@ -190,13 +190,13 @@ def create_app_values_spec(self, app_name, app_path, base_image_name=None, helm_ else: build_dependencies = [] - deployment_values = values[KEY_HARNESS].get(KEY_DEPLOYMENT, {}) + deployment_values = values.get(KEY_HARNESS, {}).get(KEY_DEPLOYMENT, {}) deployment_image = deployment_values.get('image', None) or values.get('image', None) values['build'] = not bool(deployment_image) # Used by skaffold and ci/cd to determine if the image should be built if len(image_paths) > 0 and not deployment_image: image_name = image_name_from_dockerfile_path(os.path.relpath( image_paths[0], os.path.dirname(app_path)), base_image_name) - deployment_values['image'] = self.image_tag( + values['image'] = self.image_tag( image_name, build_context_path=app_path, dependencies=build_dependencies) elif KEY_HARNESS in values and not deployment_image and values[ KEY_HARNESS].get(KEY_DEPLOYMENT, {}).get('auto', False): diff --git a/tools/deployment-cli-tools/tests/resources/applications/myapp/deploy/values-nobuild.yaml b/tools/deployment-cli-tools/tests/resources/applications/myapp/deploy/values-nobuild.yaml new file mode 100644 index 00000000..f6183636 --- /dev/null +++ b/tools/deployment-cli-tools/tests/resources/applications/myapp/deploy/values-nobuild.yaml @@ -0,0 +1,5 @@ +harness: + dependencies: + build: [] + deployment: + image: "custom-image" \ No newline at end of file diff --git a/tools/deployment-cli-tools/tests/test_codefresh.py b/tools/deployment-cli-tools/tests/test_codefresh.py index 0cda8eca..bf1282c9 100644 --- a/tools/deployment-cli-tools/tests/test_codefresh.py +++ b/tools/deployment-cli-tools/tests/test_codefresh.py @@ -282,3 +282,39 @@ def test_create_codefresh_configuration_tests(): finally: shutil.rmtree(BUILD_MERGE_DIR) + + +def test_create_codefresh_configuration_nobuild(): + values = create_helm_chart( + [RESOURCES], + output_path=OUT, + include=['myapp'], + exclude=['events'], + domain="my.local", + namespace='test', + env=['dev', 'nobuild'], + local=False, + tag=1, + registry='reg' + ) + + root_paths = preprocess_build_overrides( + root_paths=[CLOUD_HARNESS_PATH, RESOURCES], + helm_values=values, + merge_build_path=BUILD_MERGE_DIR + ) + + build_included = [app['harness']['name'] + for app in values['apps'].values() if 'harness' in app] + + cf = create_codefresh_deployment_scripts(root_paths, include=build_included, + envs=['dev', 'nobuild'], + base_image_name=values['name'], + helm_values=values, save=False) + l1_steps = cf['steps'] + steps = l1_steps["build_application_images"]["steps"] + assert len(steps) == 1 + assert "myapp" not in steps + assert "myapp-mytask" in steps + assert "publish_myapp" not in l1_steps["publish"]["steps"] + assert "publish_myapp-mytask" in l1_steps["publish"]["steps"] diff --git a/tools/deployment-cli-tools/tests/test_helm.py b/tools/deployment-cli-tools/tests/test_helm.py index 35d903a1..c0a1a148 100644 --- a/tools/deployment-cli-tools/tests/test_helm.py +++ b/tools/deployment-cli-tools/tests/test_helm.py @@ -17,13 +17,6 @@ def test_collect_helm_values(tmp_path): exclude=['events'], domain="my.local", namespace='test', env='dev', local=False, tag=1, registry='reg') - # Auto values - assert values[KEY_APPS]['myapp'][KEY_HARNESS]['deployment']['image'] == 'reg/cloudharness/myapp:1' - assert values.apps['myapp'].harness.deployment.image == 'reg/cloudharness/myapp:1' - assert values[KEY_APPS]['myapp'][KEY_HARNESS]['name'] == 'myapp' - assert values[KEY_APPS]['legacy'][KEY_HARNESS]['name'] == 'legacy' - assert values[KEY_APPS]['accounts'][KEY_HARNESS]['deployment']['image'] == 'reg/cloudharness/accounts:1' - # First level include apps assert 'samples' in values[KEY_APPS] assert 'myapp' in values[KEY_APPS] @@ -41,6 +34,14 @@ def test_collect_helm_values(tmp_path): # Explicit exclude overrides include assert 'events' not in values[KEY_APPS] + # Auto values + assert values[KEY_APPS]['myapp'][KEY_HARNESS]['deployment']['image'] == 'reg/cloudharness/myapp:1' + assert values[KEY_APPS]['myapp']['build'] == True + assert values.apps['myapp'].harness.deployment.image == 'reg/cloudharness/myapp:1' + assert values[KEY_APPS]['myapp'][KEY_HARNESS]['name'] == 'myapp' + assert values[KEY_APPS]['legacy'][KEY_HARNESS]['name'] == 'legacy' + assert values[KEY_APPS]['accounts'][KEY_HARNESS]['deployment']['image'] == 'reg/cloudharness/accounts:1' + # Base values kept assert values[KEY_APPS]['accounts'][KEY_HARNESS]['subdomain'] == 'accounts' @@ -79,6 +80,15 @@ def test_collect_helm_values(tmp_path): assert 'cloudharness-base-debian' not in values[KEY_TASK_IMAGES] +def test_collect_nobuild(tmp_path): + out_folder = tmp_path / 'test_collect_helm_values' + values = create_helm_chart([RESOURCES], output_path=out_folder, include=['myapp'], + exclude=['events'], domain="my.local", + namespace='test', env='nobuild', local=False, tag=1, registry='reg') + assert values[KEY_APPS]['myapp'][KEY_HARNESS]['deployment']['image'] == 'custom-image' + assert values[KEY_APPS]['myapp']['build'] == False + + def test_collect_helm_values_noreg_noinclude(tmp_path): out_path = tmp_path / 'test_collect_helm_values_noreg_noinclude' values = create_helm_chart([CLOUDHARNESS_ROOT, RESOURCES], output_path=out_path, domain="my.local", diff --git a/tools/deployment-cli-tools/tests/test_skaffold.py b/tools/deployment-cli-tools/tests/test_skaffold.py index 3c545d21..232fe7d8 100644 --- a/tools/deployment-cli-tools/tests/test_skaffold.py +++ b/tools/deployment-cli-tools/tests/test_skaffold.py @@ -180,3 +180,37 @@ def test_create_skaffold_configuration_with_conflicting_dependencies_requirement myapp_config = release['overrides']['apps']['myapp2'] assert myapp_config['harness']['deployment']['args'][0] == '/usr/src/app/myapp_code/__main__.py' + + +def test_create_skaffold_configuration_nobuild(): + values = create_helm_chart( + [RESOURCES], + output_path=OUT, + include=['myapp'], + domain="my.local", + namespace='test', + env='nobuild', + local=False, + tag=1, + registry='reg' + ) + + BUILD_DIR = "/tmp/build" + root_paths = preprocess_build_overrides( + root_paths=[CLOUDHARNESS_ROOT, RESOURCES], + helm_values=values, + merge_build_path=BUILD_DIR + ) + + sk = create_skaffold_configuration( + root_paths=root_paths, + helm_values=values, + output_path=OUT + ) + releases = sk['deploy']['helm']['releases'] + + assert len(sk['build']['artifacts']) == 1 + assert len(releases) == 1 # Ensure we only found 1 deployment (for myapp) + + release = releases[0] + assert 'myapp' not in release['overrides']['apps'] From 8e4f088e22f7808cbe39b70df40adad2260cc62c Mon Sep 17 00:00:00 2001 From: Filippo Ledda Date: Fri, 31 Jan 2025 18:09:28 +0100 Subject: [PATCH 81/93] CH-134 remove nfs build --- applications/nfsserver/deploy/values.yaml | 1 + applications/samples/deploy/values-test.yaml | 2 + deployment/codefresh-test.yaml | 54 ++------------------ 3 files changed, 7 insertions(+), 50 deletions(-) diff --git a/applications/nfsserver/deploy/values.yaml b/applications/nfsserver/deploy/values.yaml index 8bf13056..713ee0ec 100644 --- a/applications/nfsserver/deploy/values.yaml +++ b/applications/nfsserver/deploy/values.yaml @@ -4,6 +4,7 @@ harness: auto: false deployment: auto: false + image: gcr.io/metacellllc/cloudharness/nfsserver:1.0 # nfs server pvc disk size (/exports) diff --git a/applications/samples/deploy/values-test.yaml b/applications/samples/deploy/values-test.yaml index ebf25ca3..f21e3a8b 100644 --- a/applications/samples/deploy/values-test.yaml +++ b/applications/samples/deploy/values-test.yaml @@ -5,6 +5,8 @@ harness: - events - common - jupyterhub + - volumemanager + - notifications accounts: roles: - role1 diff --git a/deployment/codefresh-test.yaml b/deployment/codefresh-test.yaml index 144e3f36..db0ca581 100644 --- a/deployment/codefresh-test.yaml +++ b/deployment/codefresh-test.yaml @@ -33,8 +33,8 @@ steps: working_directory: . commands: - bash cloud-harness/install.sh - - 'harness-deployment . -n test-${{NAMESPACE_BASENAME}} -d ${{DOMAIN}} -r ${{REGISTRY}} - -rs ${{REGISTRY_SECRET}} -e test --write-env -N ' + - harness-deployment . -n test-${{NAMESPACE_BASENAME}} -d ${{DOMAIN}} -r ${{REGISTRY}} + -rs ${{REGISTRY_SECRET}} -e test --write-env -N -i samples - cat deployment/.env >> ${{CF_VOLUME_PATH}}/env_vars_to_export - cat ${{CF_VOLUME_PATH}}/env_vars_to_export prepare_deployment_view: @@ -124,27 +124,6 @@ steps: type: parallel stage: build steps: - nfsserver: - type: build - stage: build - dockerfile: Dockerfile - registry: '${{CODEFRESH_REGISTRY}}' - buildkit: true - build_arguments: - - DOMAIN=${{DOMAIN}} - - NOCACHE=${{CF_BUILD_ID}} - - REGISTRY=${{REGISTRY}}/cloudharness/ - image_name: cloudharness/nfsserver - title: Nfsserver - working_directory: ./applications/nfsserver - tag: '${{NFSSERVER_TAG}}' - when: - condition: - any: - buildDoesNotExist: includes('${{NFSSERVER_TAG_EXISTS}}', '{{NFSSERVER_TAG_EXISTS}}') - == true - forceNoCache: includes('${{NFSSERVER_TAG_FORCE_BUILD}}', '{{NFSSERVER_TAG_FORCE_BUILD}}') - == false notifications: type: build stage: build @@ -210,27 +189,6 @@ steps: == true forceNoCache: includes('${{VOLUMEMANAGER_TAG_FORCE_BUILD}}', '{{VOLUMEMANAGER_TAG_FORCE_BUILD}}') == false - sentry: - type: build - stage: build - dockerfile: Dockerfile - registry: '${{CODEFRESH_REGISTRY}}' - buildkit: true - build_arguments: - - DOMAIN=${{DOMAIN}} - - NOCACHE=${{CF_BUILD_ID}} - - REGISTRY=${{REGISTRY}}/cloudharness/ - image_name: cloudharness/sentry - title: Sentry - working_directory: ./applications/sentry - tag: '${{SENTRY_TAG}}' - when: - condition: - any: - buildDoesNotExist: includes('${{SENTRY_TAG_EXISTS}}', '{{SENTRY_TAG_EXISTS}}') - == true - forceNoCache: includes('${{SENTRY_TAG_FORCE_BUILD}}', '{{SENTRY_TAG_FORCE_BUILD}}') - == false jupyterhub: type: build stage: build @@ -466,7 +424,7 @@ steps: working_directory: ./${{CF_REPO_NAME}} title: Installing chart arguments: - helm_version: 3.11.0 + helm_version: 3.6.2 chart_name: deployment/helm release_name: test-${{NAMESPACE_BASENAME}} kube_context: '${{CLUSTER_NAME}}' @@ -478,9 +436,6 @@ steps: custom_values: - apps_notifications_harness_secrets_email-user=${{EMAIL-USER}} - apps_notifications_harness_secrets_email-password=${{EMAIL-PASSWORD}} - - apps_sentry_harness_secrets_email-server=${{EMAIL-SERVER}} - - apps_sentry_harness_secrets_email-user=${{EMAIL-USER}} - - apps_sentry_harness_secrets_email-password=${{EMAIL-PASSWORD}} - apps_samples_harness_secrets_asecret=${{ASECRET}} build_test_images: title: Build test images @@ -540,7 +495,6 @@ steps: - kubectl rollout status deployment/notifications - kubectl rollout status deployment/accounts - kubectl rollout status deployment/volumemanager - - kubectl rollout status deployment/sentry - kubectl rollout status deployment/argo-server-gk - kubectl rollout status deployment/samples - kubectl rollout status deployment/samples-gk @@ -617,7 +571,7 @@ steps: image: '${{REGISTRY}}/cloudharness/test-e2e:${{TEST_E2E_TAG}}' fail_fast: false commands: - - npx puppeteer browsers install chrome + - npx puppeteer browsers install chrome - yarn test scale: jupyterhub_e2e_test: From 116ba731e72c1a7aff5d1ebde0af24a67ab13e3b Mon Sep 17 00:00:00 2001 From: Filippo Ledda Date: Fri, 31 Jan 2025 18:27:36 +0100 Subject: [PATCH 82/93] Test pipeline tweak --- deployment/codefresh-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment/codefresh-test.yaml b/deployment/codefresh-test.yaml index db0ca581..8cc74824 100644 --- a/deployment/codefresh-test.yaml +++ b/deployment/codefresh-test.yaml @@ -601,7 +601,7 @@ steps: description: The pipeline will fail after ${{WAIT_ON_FAIL}} minutes timeout: timeUnit: minutes - duration: '${{WAIT_ON_FAIL}}' + duration: ${{WAIT_ON_FAIL}} finalState: denied when: condition: From 67fdf623bb808b2ef0d939d6d6cdd38b6fe39ced Mon Sep 17 00:00:00 2001 From: Filippo Ledda Date: Fri, 31 Jan 2025 18:46:44 +0100 Subject: [PATCH 83/93] Test pipeline tweak --- applications/samples/deploy/values-test.yaml | 1 - deployment/codefresh-test.yaml | 27 +------------------- 2 files changed, 1 insertion(+), 27 deletions(-) diff --git a/applications/samples/deploy/values-test.yaml b/applications/samples/deploy/values-test.yaml index f21e3a8b..e897ad99 100644 --- a/applications/samples/deploy/values-test.yaml +++ b/applications/samples/deploy/values-test.yaml @@ -6,7 +6,6 @@ harness: - common - jupyterhub - volumemanager - - notifications accounts: roles: - role1 diff --git a/deployment/codefresh-test.yaml b/deployment/codefresh-test.yaml index 8cc74824..f341aaa5 100644 --- a/deployment/codefresh-test.yaml +++ b/deployment/codefresh-test.yaml @@ -124,28 +124,6 @@ steps: type: parallel stage: build steps: - notifications: - type: build - stage: build - dockerfile: Dockerfile - registry: '${{CODEFRESH_REGISTRY}}' - buildkit: true - build_arguments: - - DOMAIN=${{DOMAIN}} - - NOCACHE=${{CF_BUILD_ID}} - - REGISTRY=${{REGISTRY}}/cloudharness/ - - CLOUDHARNESS_BASE=${{REGISTRY}}/cloudharness/cloudharness-base:${{CLOUDHARNESS_BASE_TAG}} - image_name: cloudharness/notifications - title: Notifications - working_directory: ./applications/notifications/server - tag: '${{NOTIFICATIONS_TAG}}' - when: - condition: - any: - buildDoesNotExist: includes('${{NOTIFICATIONS_TAG_EXISTS}}', '{{NOTIFICATIONS_TAG_EXISTS}}') - == true - forceNoCache: includes('${{NOTIFICATIONS_TAG_FORCE_BUILD}}', '{{NOTIFICATIONS_TAG_FORCE_BUILD}}') - == false accounts: type: build stage: build @@ -434,8 +412,6 @@ steps: custom_value_files: - ./deployment/helm/values.yaml custom_values: - - apps_notifications_harness_secrets_email-user=${{EMAIL-USER}} - - apps_notifications_harness_secrets_email-password=${{EMAIL-PASSWORD}} - apps_samples_harness_secrets_asecret=${{ASECRET}} build_test_images: title: Build test images @@ -492,7 +468,6 @@ steps: commands: - kubectl config use-context ${{CLUSTER_NAME}} - kubectl config set-context --current --namespace=test-${{NAMESPACE_BASENAME}} - - kubectl rollout status deployment/notifications - kubectl rollout status deployment/accounts - kubectl rollout status deployment/volumemanager - kubectl rollout status deployment/argo-server-gk @@ -601,7 +576,7 @@ steps: description: The pipeline will fail after ${{WAIT_ON_FAIL}} minutes timeout: timeUnit: minutes - duration: ${{WAIT_ON_FAIL}} + duration: '${{WAIT_ON_FAIL}}' finalState: denied when: condition: From 60cded65d76cc70f6d9ac268480991a820fca97a Mon Sep 17 00:00:00 2001 From: Filippo Ledda Date: Wed, 12 Feb 2025 16:43:03 +0100 Subject: [PATCH 84/93] #792 fix template description --- .../django-fastapi/backend/django_baseapp/urls.py | 2 +- .../django-ninja/backend/django_baseapp/urls.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/application-templates/django-fastapi/backend/django_baseapp/urls.py b/application-templates/django-fastapi/backend/django_baseapp/urls.py index ecfee23e..3032dd29 100644 --- a/application-templates/django-fastapi/backend/django_baseapp/urls.py +++ b/application-templates/django-fastapi/backend/django_baseapp/urls.py @@ -1,4 +1,4 @@ -"""MNP Checkout URL Configuration +"""URL Configuration The `urlpatterns` list routes URLs to views. For more information please see: https://docs.djangoproject.com/en/3.2/topics/http/urls/ diff --git a/application-templates/django-ninja/backend/django_baseapp/urls.py b/application-templates/django-ninja/backend/django_baseapp/urls.py index aae3d635..c95d4926 100644 --- a/application-templates/django-ninja/backend/django_baseapp/urls.py +++ b/application-templates/django-ninja/backend/django_baseapp/urls.py @@ -1,4 +1,4 @@ -"""MNP Checkout URL Configuration +"""URL Configuration The `urlpatterns` list routes URLs to views. For more information please see: https://docs.djangoproject.com/en/3.2/topics/http/urls/ From 33c6e2bc1407e4075f3a496cfa47286906cae7cd Mon Sep 17 00:00:00 2001 From: ddelpiano Date: Wed, 12 Mar 2025 11:18:16 +0100 Subject: [PATCH 85/93] bumping versions up --- libraries/client/cloudharness_cli/setup.py | 2 +- libraries/models/setup.py | 2 +- tools/deployment-cli-tools/harness-generate | 0 3 files changed, 2 insertions(+), 2 deletions(-) mode change 100644 => 100755 tools/deployment-cli-tools/harness-generate diff --git a/libraries/client/cloudharness_cli/setup.py b/libraries/client/cloudharness_cli/setup.py index 07fbb77a..15dd3d4a 100644 --- a/libraries/client/cloudharness_cli/setup.py +++ b/libraries/client/cloudharness_cli/setup.py @@ -12,7 +12,7 @@ from setuptools import setup, find_packages # noqa: H301 NAME = "cloudharness-cli" -VERSION = "2.4.0" +VERSION = "2.5.0" # To install the library, run the following # # python setup.py install diff --git a/libraries/models/setup.py b/libraries/models/setup.py index 7b9ca1f9..e2171e3f 100644 --- a/libraries/models/setup.py +++ b/libraries/models/setup.py @@ -8,7 +8,7 @@ HERE = dn(realpath(__file__)) NAME = "cloudharness_model" -VERSION = "2.4.0" +VERSION = "2.5.0" REQUIREMENTS = [ "Jinja2 >= 3.1.3", "oyaml >= 1.0", diff --git a/tools/deployment-cli-tools/harness-generate b/tools/deployment-cli-tools/harness-generate old mode 100644 new mode 100755 From b073811a020de11404b267e8cd3802ea53d22dbe Mon Sep 17 00:00:00 2001 From: ddelpiano Date: Thu, 13 Mar 2025 13:29:13 +0100 Subject: [PATCH 86/93] small refactoring for harness-generate --- .../ch_cli_tools/openapi.py | 71 +++++++++---------- .../ch_cli_tools/utils.py | 12 ++++ tools/deployment-cli-tools/harness-generate | 34 ++++----- 3 files changed, 65 insertions(+), 52 deletions(-) diff --git a/tools/deployment-cli-tools/ch_cli_tools/openapi.py b/tools/deployment-cli-tools/ch_cli_tools/openapi.py index 164bcc5b..bc3eda81 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/openapi.py +++ b/tools/deployment-cli-tools/ch_cli_tools/openapi.py @@ -17,7 +17,7 @@ from ch_cli_tools.manifest import get_manifest from . import HERE -from .utils import confirm, copymergedir, replace_in_file, replaceindir, to_python_module +from .utils import confirm, copymergedir, replace_in_file, replaceindir, to_python_module, get_apps_paths CODEGEN = os.path.join(HERE, 'bin', 'openapi-generator-cli.jar') APPLICATIONS_SRC_PATH = os.path.join('applications') @@ -39,24 +39,27 @@ def all(cls): def generate_flask_server(app_path: pathlib.Path, overrides_folder: Optional[pathlib.Path] = None) -> None: get_dependencies() - openapi_directory = app_path / 'api' - openapi_file = next(openapi_directory.glob('*.yaml')) - - server_path = app_path / 'server' - backend_path = app_path / 'backend' - out_path = server_path if server_path.exists() else backend_path - - command = [ - 'java', '-jar', CODEGEN, 'generate', - '-i', openapi_file, - '-g', 'python-flask', - '-o', out_path, - '-c', openapi_directory / 'config.json', - ] - if overrides_folder: - command += ['-t', overrides_folder] - - subprocess.run(command) + try: + openapi_directory = app_path / 'api' + openapi_file = next(openapi_directory.glob('*.yaml')) + + server_path = app_path / 'server' + backend_path = app_path / 'backend' + out_path = server_path if server_path.exists() else backend_path + + command = [ + 'java', '-jar', CODEGEN, 'generate', + '-i', openapi_file, + '-g', 'python-flask', + '-o', out_path, + '-c', openapi_directory / 'config.json', + ] + if overrides_folder: + command += ['-t', overrides_folder] + + subprocess.run(command) + except: + logging.error(f'An error occurred while generating the server stubs for {app_path.name}', exc_info=True) def generate_fastapi_server(app_path: pathlib.Path) -> None: @@ -230,16 +233,12 @@ def generate_servers( """ Generates server stubs """ - openapi_files = [path for path in root_path.glob('applications/*/api/*.yaml')] + apps_path = get_apps_paths(root_path, app_name) - for openapi_file in openapi_files: - app_path = openapi_file.parent.parent + for app_path in apps_path: manifest = get_manifest(app_path) - if app_name and manifest.app_name != app_name: - continue - - if not should_generate(f'server stubs for {openapi_file}'): + if not should_generate(f'Should we regenerate the server stubs for {app_path.name} ?'): continue if TemplateType.DJANGO_FASTAPI in manifest.templates: @@ -259,32 +258,32 @@ def generate_clients( """ Generates client stubs """ - if not should_generate('client libraries'): - return + apps_path = get_apps_paths(root_path, app_name) logging.info('Generating client libraries for %s', str(client_types)) - client_src_path = root_path / 'libraries' / 'client' / client_lib_name - apps_path = root_path / 'applications' - apps = (app for app in apps_path.iterdir() if app.is_dir()) + if client_lib_name: + client_src_path = root_path / 'libraries' / 'client' / client_lib_name - for app_path in apps: + for app_path in apps_path: manifest = get_manifest(app_path) - if app_name and manifest.app_name != app_name: + if not should_generate(f'Should we regenerate the client libraries for {app_path.name} ?'): continue if TemplateType.DJANGO_NINJA in manifest.templates: generate_openapi_from_ninja_schema(manifest.app_name, app_path) for openapi_file in app_path.glob('api/*.yaml'): - if ClientType.PYTHON_CLIENT in client_types: + if ClientType.PYTHON_CLIENT in client_types and client_lib_name: generate_python_client(manifest.app_name, openapi_file, client_src_path, lib_name=client_lib_name) if TemplateType.WEBAPP in manifest.templates and ClientType.TS_CLIENT in client_types: - generate_ts_client(openapi_file, app_name) + ts_client_name = app_name if app_name else manifest.app_name + generate_ts_client(openapi_file, ts_client_name) - aggregate_packages(client_src_path, client_lib_name) + if client_lib_name: + aggregate_packages(client_src_path, client_lib_name) def aggregate_packages(client_source_path: pathlib.Path, lib_name=LIB_NAME): diff --git a/tools/deployment-cli-tools/ch_cli_tools/utils.py b/tools/deployment-cli-tools/ch_cli_tools/utils.py index 18b876a7..8f9f5e45 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/utils.py +++ b/tools/deployment-cli-tools/ch_cli_tools/utils.py @@ -490,3 +490,15 @@ def load_yaml(yaml_file: pathlib.Path) -> dict: def save_yaml(yaml_file: pathlib.Path, data: dict) -> None: with yaml_file.open('w') as file: yaml.dump(data, file) + + +def get_apps_paths(root, app_name) -> tuple[str]: + apps_path = [] + + if app_name: + logging.info('### Generating server stubs for %s ###', app_name) + apps_path = [path for path in root.glob(f'applications/{app_name}') if path.is_dir()] + else: + logging.info('### Generating server stubs for all applications ###') + apps_path = [path for path in root.glob('applications/*') if path.is_dir()] + return apps_path diff --git a/tools/deployment-cli-tools/harness-generate b/tools/deployment-cli-tools/harness-generate index 5bd1b9ac..e698b716 100755 --- a/tools/deployment-cli-tools/harness-generate +++ b/tools/deployment-cli-tools/harness-generate @@ -17,7 +17,10 @@ def main(): args = get_command_line_arguments() get_dependencies() - root_path = args.path.absolute() + root_path = args.path.absolute() if args.path else pathlib.Path.cwd().absolute() + app_name = root_path.name if args.path else args.app_name + if args.path and args.app_name: + logging.warning('Ignoring app_name flag because path was provided') # Check if applications folder exists, if not, go up until it's found while not (root_path / 'applications').exists(): @@ -32,11 +35,10 @@ def main(): generate_models(root_path, should_generate) if args.generate_servers: - generate_servers(root_path, should_generate, args.app_name) + generate_servers(root_path, should_generate, app_name) if args.generate_clients: - assert args.client_name is not None - generate_clients(root_path, should_generate, args.app_name, args.client_name, args.client_types) + generate_clients(root_path, should_generate, app_name, args.client_name, args.client_types) class GenerationMode(enum.Flag): @@ -46,7 +48,7 @@ class GenerationMode(enum.Flag): @classmethod def all(cls): - return functools.reduce(operator.or_, cls) + return functools.reduce(operator.or_, [cls.CLIENTS, cls.SERVERS]) @dataclass(frozen=True) @@ -72,18 +74,18 @@ class CommandLineArguments: def get_command_line_arguments() -> CommandLineArguments: - parser = argparse.ArgumentParser(description='Walks filesystem inside ./applications to create and update application scaffolding based on API specifications.') + parser = argparse.ArgumentParser(description='Walks the filesystem inside the ./applications folder to create and update applications scaffolding.') common_arguments = argparse.ArgumentParser(add_help=False) - common_arguments.add_argument('path', metavar='path', nargs='?', default=pathlib.Path.cwd(), type=pathlib.Path, - help='Base path of the application.') + common_arguments.add_argument('path', metavar='path', nargs='?', default=None, type=pathlib.Path, + help='Base path of the application. If used, the -a/--app_name flag is ignored.') common_arguments.add_argument('-i', '--interactive', dest='is_interactive', action="store_true", help='Asks before generate') common_arguments.add_argument('-a', '--app-name', dest='app_name', action="store", default=None, help='Generate only for a specific application') clients_arguments = argparse.ArgumentParser(add_help=False) - clients_arguments.add_argument('-cn', '--client-name', dest='client_name', action='store', default=LIB_NAME, + clients_arguments.add_argument('-cn', '--client-name', dest='client_name', action='store', default=None, help='specify client prefix name') client_type_group = clients_arguments.add_mutually_exclusive_group(required=False) client_type_group.add_argument('-t', '--ts-only', dest='client_types', action='store_const', const=ClientType.TS_CLIENT, @@ -95,20 +97,20 @@ def get_command_line_arguments() -> CommandLineArguments: subparsers = parser.add_subparsers(title='generation modes', required=True) all_parser = subparsers.add_parser('all', parents=[common_arguments, clients_arguments], - help='Generate models, server stubs and client libraries') + help='Generate server stubs and client libraries') all_parser.set_defaults(generation_mode=GenerationMode.all()) - models_parser = subparsers.add_parser('models', parents=[common_arguments], - help='Generate only model library') - models_parser.set_defaults(generation_mode=GenerationMode.MODELS) + clients_parser = subparsers.add_parser('clients', parents=[common_arguments, clients_arguments], + help='Generate only client libraries') + clients_parser.set_defaults(generation_mode=GenerationMode.CLIENTS) servers_parser = subparsers.add_parser('servers', parents=[common_arguments], help='Generate only server stubs') servers_parser.set_defaults(generation_mode=GenerationMode.SERVERS) - clients_parser = subparsers.add_parser('clients', parents=[common_arguments, clients_arguments], - help='Generate only client libraries') - clients_parser.set_defaults(generation_mode=GenerationMode.CLIENTS) + models_parser = subparsers.add_parser('models', parents=[common_arguments], + help='Special flag, used to regenerate only model libraries') + models_parser.set_defaults(generation_mode=GenerationMode.MODELS) args = parser.parse_args() From 2d03fd84b0532274c05e1aee104edf32ecb91e94 Mon Sep 17 00:00:00 2001 From: ddelpiano Date: Thu, 13 Mar 2025 13:29:29 +0100 Subject: [PATCH 87/93] documentation review and changes --- README.md | 223 +++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 195 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 1806489f..ff6c703b 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,9 @@ Can scaffold and maintain your cloud solution on top of Cloudharness without wri Kubernetes templates, with in place common utilities and applications already configured for you. What building your cloud solution with CloudHarness gives to you: + - Common framework and utilities to develop and deploy micro-service application + - Helm chart automatic generation - deployments - services @@ -23,11 +25,13 @@ What building your cloud solution with CloudHarness gives to you: - databases (postgreql) - access gatekeepers configuration - secrets and configmaps + * Automatic build and push of images * REST-API scaffolding building based on OpenApi * Continuous deployment script generation * Debug backend applications running on Kubernetes * Python cluster access utilities + * Prebuilt support applications and shared library to: * Log in and user management - based on Keycloak * Submit batch and asynchronous workflows - based on Argo @@ -43,17 +47,18 @@ What building your cloud solution with CloudHarness gives to you: The microservice architecture is a great to get code separation and flexible development, but may not be of easy implementation, especially for small development teams/projects. In particular, these questions may rise: - - How do I create a deployment for my microservices? - - How do I orchestrate my microservices? - - How to create consistent api documentation? - - Do I need to be an experienced devops to create a micro-service based application? - - Wouldn't it be nice to develop a plain database/backend/frontend application without infrastructure boilerplate but still be able to configure everything I want when needed? - - How to run batch operations like ETL processes easily and efficiently in a cloud environment? - - How to manage databases without being locked to a specific vendor solution? - - How to perform database backups? - - How to manage secret data? - - What about having a precounfigured account management application? - - Sooner rather than later I'll need an orchestration queue. Why not have that just ready to use? + +- How do I create a deployment for my microservices? +- How do I orchestrate my microservices? +- How to create consistent api documentation? +- Do I need to be an experienced devops to create a micro-service based application? +- Wouldn't it be nice to develop a plain database/backend/frontend application without infrastructure boilerplate but still be able to configure everything I want when needed? +- How to run batch operations like ETL processes easily and efficiently in a cloud environment? +- How to manage databases without being locked to a specific vendor solution? +- How to perform database backups? +- How to manage secret data? +- What about having a precounfigured account management application? +- Sooner rather than later I'll need an orchestration queue. Why not have that just ready to use? # Command line tools @@ -63,7 +68,7 @@ CloudHarness provides the following command line tools to help application scaff * `harness-application` - create a new CloudHarness REST application. * `harness-generate` - generates server and client code for all CloudHarness REST applications. * `harness-test` - run end to end tests - + # Get started ## Prerequisites @@ -71,22 +76,26 @@ CloudHarness provides the following command line tools to help application scaff ### Operative system Cloudharness can be used on all major operative systems. + - Linux: supported and tested - MacOS: supported and tested - Windows/WSL2: supported and tested - Windows native: mostly working, unsupported ### Python -Python 3.9 must be installed. + +Python 3.10 must be installed. It is recommended to setup a virtual environment. With conda: + ```bash -conda create --name ch python=3.12 +conda create --name ch python=3.10 conda activate ch ``` ### Docker + [Docker](https://www.docker.com) is required to build locally. ### Kubernetes command line client @@ -110,6 +119,7 @@ conda activate ch A node environment with npm is required for developing web applications and to run end to end tests. Recommended: + - node >= v14.0.0 - npm >= 8.0.0 @@ -120,26 +130,183 @@ A JRE is needed to run the code generators based on openapi-generator. For more info, see [here](https://openapi-generator.tech/docs/installation). ## CloudHarness command line tools + To use the cli tools, install requirements first: ```bash bash install.sh ``` -### Generate deployment - -To generate a deployment, run `harness-deployment`. See [below](#Deployment) for more. ### Create new REST application -To create a new REST application, run `harness-application` from the root of your solution. + +`harness-application` is a command-line tool used to create new applications based on predefined code templates. It allows users to quickly scaffold applications with backend, frontend, and database configurations. + +#### harness-application Usage + +```sh +harness-application [name] [-t TEMPLATE] +``` + +#### harness-application Arguments + +- `name` *(required)* – The name of the application to be created. + +#### harness-application Options + +- `-h, --help` – Displays the help message and exits. +- `-t TEMPLATES, --template TEMPLATES` – Specifies one or more templates to use when creating the application. + +#### Available Templates + +The following templates can be used with the `-t` flag: + +- **flask-server** – Backend Flask server based on OpenAPI. +- **webapp** – Full-stack React web application with both frontend and backend. +- **db-postgres** – PostgreSQL database setup. +- **db-neo4j** – Neo4j database setup. +- **db-mongo** – MongoDB database setup. +- **django-fastapi** – FastAPI and Django backend based on OpenAPI. +- **django-ninja** – Django Ninja backend. + +#### harness-application Examples + +##### Create a New Flask-Based Microservice Application + +```sh +harness-application myapp +``` + +##### Create a Full-Stack Web Application + +```sh +harness-application myapp -t webapp +``` + +##### Create a Web Application with Mongo Database + +```sh +harness-application myapp -t webapp -t db-mongo +``` + +##### Display Help Information + +```sh +harness-application --help +``` + +#### harness-application Notes + +- Multiple templates can be specified concatenating the -t parameter. +- The tool generates the necessary scaffolding for the chosen templates. +- Ensure you have the required dependencies installed before running the generated application. +- For more information, run `harness-application --help` or check out the additional documentation: + - [Applications README](./docs/applications/README.md) + - [Developer Guide](./docs/dev.md) ### Generate server and client code from openapi -To (re)generate the code for your applications, run `harness-generate` from the root. -The script will look for all openapi applications, and regenerate the Flask server code and documentation. -Note: the script will eventually override any manually modified file. To avoid that, define a file openapi-generator-ignore. + +To (re)generate the code for your applications, run `harness-generate`. +`harness-generate` is a command-line tool used to generate client code, server stubs, and model libraries for applications. It walks through the filesystem inside the `./applications` folder to create and update application scaffolding. The tool supports different generation modes and allows for both interactive and non-interactive usage. + +#### Usage + +```sh +harness-generate [mode] [-h] [-i] [-a APP_NAME] [-cn CLIENT_NAME] [-t | -p] [path] +``` + +#### harness-generate Arguments + +- `path` *(optional)* – The base path of the application. If provided, the `-a/--app-name` flag is ignored. + +#### harness-generate Options + +- `-h, --help` – Displays the help message and exits. +- `-i, --interactive` – Asks for confirmation before generating code. +- `-a APP_NAME, --app-name APP_NAME` – Specifies the application name to generate clients for. +- `-cn CLIENT_NAME, --client-name CLIENT_NAME` – Specifies a prefix for the client name. +- `-t, --ts-only` – Generates only TypeScript clients. +- `-p, --python-only` – Generates only Python clients. + +#### Generation Modes + +`harness-generate` supports the following modes: + +- **all** – Generates both server stubs and client libraries. +- **clients** – Generates only client libraries. +- **servers** – Generates only server stubs. +- **models** – Regenerates only model libraries. + +#### harness-generate Examples + +##### Generate Client and Server stubs for all applications + +```sh +harness-generate all +``` + +##### Generate Client and Server stubs for a Specific Application + +```sh +harness-generate all -a myApp +``` + +##### Generate Only Client Libraries + +```sh +harness-generate clients +``` + +##### Generate Only Server Stubs + +```sh +harness-generate servers +``` + +##### Regenerate Only Model Libraries (deprecated) + +```sh +harness-generate models +``` + +##### Generate TypeScript Clients Only and Server stubs + +```sh +harness-generate all -t +``` + +##### Generate Python Clients Only and Server stubs + +```sh +harness-generate all -p +``` + +##### Interactive Mode + +```sh +harness-generate all -i +``` + +#### harness-generate Notes + +- The tool scans the `./applications` directory for available applications. +- If `path` is provided, `-a/--app-name` is ignored. +- The `models` mode is a special flag used when regenerating only model libraries (deprecated). +- The tool supports interactive mode to confirm before generating clients. +- Use either `-t` or `-p`, but not both simultaneously. + +For further details, run: + +```sh +harness-generate --help +``` + +### Generate deployment + +To generate a deployment, run `harness-deployment`. See [below](#build-and-deploy) for more information. # Extend CloudHarness to build your project -CloudHarness is born to be extended. +CloudHarness is born to be extended. The quickest way to start is to install Cloud Harness, copy the *blueprint* folder and build from that with the cli tools, such as `harness-application`, `harness-generate`, `harness-deployment`. @@ -150,10 +317,11 @@ See the [developers documentation](docs/dev.md#start-your-project) for more info The script `harness-deployment` scans your applications and configurations to create the build and deploy artifacts. Created artifacts include: - - Helm chart (or docker compose configuration file) - - Skaffold build and run configuration - - Visual Studio Code debug and run configuration - - Codefresh pipeline yaml specification (optional) + +- Helm chart (or docker compose configuration file) +- Skaffold build and run configuration +- Visual Studio Code debug and run configuration +- Codefresh pipeline yaml specification (optional) With your project folder structure looking like @@ -193,5 +361,4 @@ Then, you can selectively add files related to configuration that you want to pe For more information about how to configure a deployment, see [here](./build-deploy/helm-configuration.md) - -[![Codefresh build status]( https://g.codefresh.io/api/badges/pipeline/tarelli/Cloudharness%2Funittests?type=cf-1&key=eyJhbGciOiJIUzI1NiJ9.NWFkNzMyNDIzNjQ1YWMwMDAxMTJkN2Rl.-gUEkJxH6NCCIRgSIgEikVDte-Q0BsGZKEs4uahgpzs)]( https://g.codefresh.io/pipelines/edit/new/builds?id=6034cfce1036693697cd602b&pipeline=unittests&projects=Cloudharness&projectId=6034cfb83bb11c399e85c71b) +[![Codefresh build status](https://g.codefresh.io/api/badges/pipeline/tarelli/Cloudharness%2Funittests?type=cf-1&key=eyJhbGciOiJIUzI1NiJ9.NWFkNzMyNDIzNjQ1YWMwMDAxMTJkN2Rl.-gUEkJxH6NCCIRgSIgEikVDte-Q0BsGZKEs4uahgpzs)](https://g.codefresh.io/pipelines/edit/new/builds?id=6034cfce1036693697cd602b&pipeline=unittests&projects=Cloudharness&projectId=6034cfb83bb11c399e85c71b) From 6eeaf32e14c7f6fee838b7d598be0115664d8505 Mon Sep 17 00:00:00 2001 From: ddelpiano Date: Thu, 13 Mar 2025 18:31:05 +0100 Subject: [PATCH 88/93] documentation edits --- README.md | 164 +---------------------- docs/applications/harness-application.md | 112 +++++++++++----- docs/applications/harness-generate.md | 96 +++++++++++++ 3 files changed, 183 insertions(+), 189 deletions(-) create mode 100644 docs/applications/harness-generate.md diff --git a/README.md b/README.md index ff6c703b..0ddee245 100644 --- a/README.md +++ b/README.md @@ -84,13 +84,13 @@ Cloudharness can be used on all major operative systems. ### Python -Python 3.10 must be installed. +Python 3.10+ must be installed. It is recommended to setup a virtual environment. With conda: ```bash -conda create --name ch python=3.10 +conda create --name ch python=3.12 conda activate ch ``` @@ -137,168 +137,16 @@ To use the cli tools, install requirements first: bash install.sh ``` -### Create new REST application +### Create a new REST application `harness-application` is a command-line tool used to create new applications based on predefined code templates. It allows users to quickly scaffold applications with backend, frontend, and database configurations. - -#### harness-application Usage - -```sh -harness-application [name] [-t TEMPLATE] -``` - -#### harness-application Arguments - -- `name` *(required)* – The name of the application to be created. - -#### harness-application Options - -- `-h, --help` – Displays the help message and exits. -- `-t TEMPLATES, --template TEMPLATES` – Specifies one or more templates to use when creating the application. - -#### Available Templates - -The following templates can be used with the `-t` flag: - -- **flask-server** – Backend Flask server based on OpenAPI. -- **webapp** – Full-stack React web application with both frontend and backend. -- **db-postgres** – PostgreSQL database setup. -- **db-neo4j** – Neo4j database setup. -- **db-mongo** – MongoDB database setup. -- **django-fastapi** – FastAPI and Django backend based on OpenAPI. -- **django-ninja** – Django Ninja backend. - -#### harness-application Examples - -##### Create a New Flask-Based Microservice Application - -```sh -harness-application myapp -``` - -##### Create a Full-Stack Web Application - -```sh -harness-application myapp -t webapp -``` - -##### Create a Web Application with Mongo Database - -```sh -harness-application myapp -t webapp -t db-mongo -``` - -##### Display Help Information - -```sh -harness-application --help -``` - -#### harness-application Notes - -- Multiple templates can be specified concatenating the -t parameter. -- The tool generates the necessary scaffolding for the chosen templates. -- Ensure you have the required dependencies installed before running the generated application. -- For more information, run `harness-application --help` or check out the additional documentation: - - [Applications README](./docs/applications/README.md) - - [Developer Guide](./docs/dev.md) +More information can be found [here](./docs/applications/harness-application.md). ### Generate server and client code from openapi To (re)generate the code for your applications, run `harness-generate`. -`harness-generate` is a command-line tool used to generate client code, server stubs, and model libraries for applications. It walks through the filesystem inside the `./applications` folder to create and update application scaffolding. The tool supports different generation modes and allows for both interactive and non-interactive usage. - -#### Usage - -```sh -harness-generate [mode] [-h] [-i] [-a APP_NAME] [-cn CLIENT_NAME] [-t | -p] [path] -``` - -#### harness-generate Arguments - -- `path` *(optional)* – The base path of the application. If provided, the `-a/--app-name` flag is ignored. - -#### harness-generate Options - -- `-h, --help` – Displays the help message and exits. -- `-i, --interactive` – Asks for confirmation before generating code. -- `-a APP_NAME, --app-name APP_NAME` – Specifies the application name to generate clients for. -- `-cn CLIENT_NAME, --client-name CLIENT_NAME` – Specifies a prefix for the client name. -- `-t, --ts-only` – Generates only TypeScript clients. -- `-p, --python-only` – Generates only Python clients. - -#### Generation Modes - -`harness-generate` supports the following modes: - -- **all** – Generates both server stubs and client libraries. -- **clients** – Generates only client libraries. -- **servers** – Generates only server stubs. -- **models** – Regenerates only model libraries. - -#### harness-generate Examples - -##### Generate Client and Server stubs for all applications - -```sh -harness-generate all -``` - -##### Generate Client and Server stubs for a Specific Application - -```sh -harness-generate all -a myApp -``` - -##### Generate Only Client Libraries - -```sh -harness-generate clients -``` - -##### Generate Only Server Stubs - -```sh -harness-generate servers -``` - -##### Regenerate Only Model Libraries (deprecated) - -```sh -harness-generate models -``` - -##### Generate TypeScript Clients Only and Server stubs - -```sh -harness-generate all -t -``` - -##### Generate Python Clients Only and Server stubs - -```sh -harness-generate all -p -``` - -##### Interactive Mode - -```sh -harness-generate all -i -``` - -#### harness-generate Notes - -- The tool scans the `./applications` directory for available applications. -- If `path` is provided, `-a/--app-name` is ignored. -- The `models` mode is a special flag used when regenerating only model libraries (deprecated). -- The tool supports interactive mode to confirm before generating clients. -- Use either `-t` or `-p`, but not both simultaneously. - -For further details, run: - -```sh -harness-generate --help -``` +`harness-generate` is a command-line tool used to generate client code, server stubs, and model libraries for applications. +More information can be found [here](./docs/applications/harness-generate.md) ### Generate deployment diff --git a/docs/applications/harness-application.md b/docs/applications/harness-application.md index e2110398..fdbb5f6c 100644 --- a/docs/applications/harness-application.md +++ b/docs/applications/harness-application.md @@ -1,53 +1,103 @@ -# Use harness-application to create a new application from templates +# Use harness-application to create a new application -## Choosing Templates +## Overview -If you create a new application, you can choose templates that are used to generate the application scaffold. +`harness-application` is a command-line tool used to create new applications from predefined code templates. It allows users to quickly scaffold applications with backend, frontend, and database configurations. -Running `harness-application --help` will list the currently available templates: +## Usage +```sh +harness-application [name] [-t TEMPLATE] ``` -usage: harness-application [-h] [-t TEMPLATES] name -Creates a new Application. +## Arguments + +- `name` *(required)* – The name of the application to be created. + +## Options + +- `-h, --help` – Displays the help message and exits. +- `-t TEMPLATES, --template TEMPLATES` – Specifies one or more templates to use when creating the application. + +## Choosing Templates -positional arguments: - name Application name +When creating a new application, you can choose templates that define its structure and components. Running `harness-application --help` will list the currently available templates: -optional arguments: - -h, --help show this help message and exit - -t TEMPLATES, --template TEMPLATES - Add a template name. Available templates: - base (always included) - flask-server (backend flask app based on openapi) - webapp (webapp including backend and frontend) - db-postgres - db-neo4j - db-mongo - django-app (fastapi django backend based on openapi) +```sh +usage: harness-application [-h] [-t TEMPLATES] name ``` ## Available Templates ### Base -* The `base` template is always included and used as foundation for any other template. +- The `base` template is always included and serves as the foundation for any other template. + +### Backend Templates + +#### Flask Server + +- The `flask-server` template consists of a backend built using [Flask](https://flask.palletsprojects.com/en/1.1.x/). + - Uses [Connexion](https://github.com/zalando/connexion) to map OpenAPI definitions to Flask routes. + - Served by [Gunicorn](https://gunicorn.org/) with 2 synchronous workers by default. + - Supports customization of the worker count and type. + +#### Django + +- The `django-fastapi` consists of a backend based on [FastAPI](https://fastapi.tiangolo.com/) and [Django](https://www.djangoproject.com/). + - Uses the [FastAPI code generator](https://github.com/koxudaxi/fastapi-code-generator) to map OpenAPI definitions. + - Served by [Uvicorn](https://www.uvicorn.org/) with 2 workers by default. +- The `django-ninja` consists of a backend based on [Django Ninja](https://django-ninja.dev/) + - Provides automatic OpenAPI schema generation. + - Supports Django's built-in ORM for seamless database integration. + - High performance due to Pydantic-based data validation. + - Simplifies request parsing and authentication. -### Flask Server -* It consists of a single backend, a Python [Flask](https://flask.palletsprojects.com/en/1.1.x/) application. -* The [Connexion](https://github.com/zalando/connexion) library maps the OpenAPI definition to Flask routing. -* Per default, [Gunicorn](https://gunicorn.org/) serves the Flask app with 2 synchronous workers. Depending on the application requirements, you can update the number of workers or choose a different [worker type](https://docs.gunicorn.org/en/stable/design.html). +### Full-Stack Templates +#### Webapp -### Webapp +- The `webapp` template extends the `base` template by adding a [React](https://reactjs.org/) frontend. + - The frontend bundle is served by the Python backend. + - React is used by default, but other frontend technologies can be integrated. -* The `webapp` template consists builds upon the `base` template extends it by a [React](https://reactjs.org/) frontend application. -* The generated frontend bundle is served by the Python backend. -* Per default, React is used as a frontend application, but you are free to choose a different frontend technology. +### Database Templates +- `db-postgres` – [PostgreSQL](https://www.postgresql.org/), a relational database. +- `db-neo4j` – [Neo4J](https://neo4j.com/), a graph database. +- `db-mongo` – [MongoDB](https://www.mongodb.com/), a NoSQL document-based database. -### Databases +## Examples + +### Create a New Flask-Based Microservice Application + +```sh +harness-application myapp +``` + +### Create a Full-Stack Web Application + +```sh +harness-application myapp -t webapp +``` + +### Create a Web Application with a Mongo Database + +```sh +harness-application myapp -t webapp -t db-mongo +``` + +### Display Help Information + +```sh +harness-application --help +``` -Additionally, you can choose one of the following database templates: -* `db-postgres` - [PostgreSQL](https://www.postgresql.org/), a relational database -* `db-neo4j`- [Neo4J](https://neo4j.com/), a graph database -* `db-mongo` - [MongoDB](https://www.mongodb.com/), a NoSQL document-based database +## Notes -### Django -* It consists of a single backend, a Python [FastAPI](https://fastapi.tiangolo.com/) application. -* The [FastAPI code generator](https://github.com/koxudaxi/fastapi-code-generator) maps the OpenAPI definition to FastAPI routing. -* The [Django framework](https://www.djangoproject.com/) encourages rapid development and clean, pragmatic design. -* Per default, [Uvicorn](https://www.uvicorn.org/) serves the FastAPI app with 2 workers. Depending on the application requirements, you can update the number of workers. +- Multiple templates can be specified by concatenating the `-t` parameter. +- The tool generates the necessary scaffolding for the chosen templates. +- Ensure you have the required dependencies installed before running the generated application. +- For more information, run `harness-application --help` or check out the additional documentation: + - [Applications README](./docs/applications/README.md) + - [Developer Guide](./docs/dev.md) diff --git a/docs/applications/harness-generate.md b/docs/applications/harness-generate.md new file mode 100644 index 00000000..c065cbe9 --- /dev/null +++ b/docs/applications/harness-generate.md @@ -0,0 +1,96 @@ +# Use harness-generate to generate server and client stubs + +To (re)generate the code for your applications, run `harness-generate`. +`harness-generate` is a command-line tool used to generate client code, server stubs, and model libraries for applications. It walks through the filesystem inside the `./applications` folder to create and update application scaffolding. The tool supports different generation modes and allows for both interactive and non-interactive usage. + +## Usage + +```sh +harness-generate [mode] [-h] [-i] [-a APP_NAME] [-cn CLIENT_NAME] [-t | -p] [path] +``` + +## harness-generate Arguments + +- `path` *(optional)* – The base path of the application. If provided, the `-a/--app-name` flag is ignored. + +## harness-generate Options + +- `-h, --help` – Displays the help message and exits. +- `-i, --interactive` – Asks for confirmation before generating code. +- `-a APP_NAME, --app-name APP_NAME` – Specifies the application name to generate clients for. +- `-cn CLIENT_NAME, --client-name CLIENT_NAME` – Specifies a prefix for the client name. +- `-t, --ts-only` – Generates only TypeScript clients. +- `-p, --python-only` – Generates only Python clients. + +## Generation Modes + +`harness-generate` supports the following modes: + +- **all** – Generates both server stubs and client libraries. +- **clients** – Generates only client libraries. +- **servers** – Generates only server stubs. +- **models** – Regenerates only model libraries. + +## harness-generate Examples + +### Generate Client and Server stubs for all applications + +```sh +harness-generate all +``` + +### Generate Client and Server stubs for a Specific Application + +```sh +harness-generate all -a myApp +``` + +### Generate Only Client Libraries + +```sh +harness-generate clients +``` + +### Generate Only Server Stubs + +```sh +harness-generate servers +``` + +### Regenerate Only Model Libraries (deprecated) + +```sh +harness-generate models +``` + +### Generate TypeScript Clients Only and Server stubs + +```sh +harness-generate all -t +``` + +### Generate Python Clients Only and Server stubs + +```sh +harness-generate all -p +``` + +### Interactive Mode + +```sh +harness-generate all -i +``` + +## harness-generate Notes + +- The tool scans the `./applications` directory for available applications. +- If `path` is provided, `-a/--app-name` is ignored. +- The `models` mode is a special flag used when regenerating only model libraries (deprecated). +- The tool supports interactive mode to confirm before generating clients. +- Use either `-t` or `-p`, but not both simultaneously. + +For further details, run: + +```sh +harness-generate --help +``` From 423145a52d4a830e1efde2d68ae84d34da4b11c0 Mon Sep 17 00:00:00 2001 From: Filippo Ledda Date: Fri, 14 Mar 2025 19:06:33 +0100 Subject: [PATCH 89/93] CH-169 try fix keycloak starting issue --- .gitignore | 2 +- applications/accounts/.dockerignore | 4 + applications/accounts/Dockerfile | 2 +- .../accounts/deploy/resources/realm.json | 8 +- applications/accounts/deploy/values-test.yaml | 20 + applications/accounts/deploy/values.yaml | 2 +- .../accounts/dev/disable-theme-cache.cli | 5 + applications/accounts/dev/docker-compose.yaml | 45 ++ applications/accounts/dev/realm.json | 669 ++++++++++++++++++ .../accounts/scripts/create_api_user.sh | 0 deployment/codefresh-test.yaml | 12 - deployment/docker-compose.yaml | 313 ++++++++ 12 files changed, 1063 insertions(+), 19 deletions(-) create mode 100644 applications/accounts/.dockerignore create mode 100644 applications/accounts/deploy/values-test.yaml create mode 100644 applications/accounts/dev/disable-theme-cache.cli create mode 100644 applications/accounts/dev/docker-compose.yaml create mode 100644 applications/accounts/dev/realm.json mode change 100644 => 100755 applications/accounts/scripts/create_api_user.sh create mode 100644 deployment/docker-compose.yaml diff --git a/.gitignore b/.gitignore index f4d38460..1238b4a6 100644 --- a/.gitignore +++ b/.gitignore @@ -8,7 +8,7 @@ node_modules *.DS_Store deployment/helm deployment/compose -docker-compose.yaml +/docker-compose.yaml *.egg-info *.idea /build diff --git a/applications/accounts/.dockerignore b/applications/accounts/.dockerignore new file mode 100644 index 00000000..d0c6ee4d --- /dev/null +++ b/applications/accounts/.dockerignore @@ -0,0 +1,4 @@ +/dev +.ch-manifest +README.md +/deploy \ No newline at end of file diff --git a/applications/accounts/Dockerfile b/applications/accounts/Dockerfile index 4e61bef4..c14b452f 100644 --- a/applications/accounts/Dockerfile +++ b/applications/accounts/Dockerfile @@ -1,4 +1,4 @@ -FROM quay.io/keycloak/keycloak:16.1.1 +FROM quay.io/keycloak/keycloak:16.1.0 # add kubectl USER root diff --git a/applications/accounts/deploy/resources/realm.json b/applications/accounts/deploy/resources/realm.json index c04b0b7f..005d4042 100644 --- a/applications/accounts/deploy/resources/realm.json +++ b/applications/accounts/deploy/resources/realm.json @@ -44,9 +44,9 @@ "resetPasswordAllowed": true, "editUsernameAllowed": true, "users": [ - {{- range $app := .Values.apps }} + {{- range $j, $app := .Values.apps }} {{- if (hasKey $app.harness "accounts") }} - {{- range $i, $user := $app.harness.accounts.users }}{{if $i}},{{end}} + {{- range $i, $user := $app.harness.accounts.users }}{{if (add $i $j)}},{{end}} {{ include "deploy_accounts_utils.user" (dict "root" $ "app" $app "user" $user) }} {{- end }} {{- end }} @@ -82,11 +82,11 @@ } ], "client": { - {{- range $app := .Values.apps }} + {{- range $j,$app := .Values.apps }} {{- if (hasKey $app.harness "accounts") }} {{ $app.harness.name | quote }}: [ {{- range $i, $role := $app.harness.accounts.roles }} - {{if $i}},{{end}} + {{if (add $i $j)}},{{end}} {{ include "deploy_accounts_utils.role" (dict "root" $ "app" $app "role" $role) }} {{- end }} ] diff --git a/applications/accounts/deploy/values-test.yaml b/applications/accounts/deploy/values-test.yaml new file mode 100644 index 00000000..a6fb91d0 --- /dev/null +++ b/applications/accounts/deploy/values-test.yaml @@ -0,0 +1,20 @@ + accounts: + roles: + - role1 + - role2 + - role3 + users: + - username: sample@testuser.com + clientRoles: + - role1 + realmRoles: + - administrator + - offline_access + - username: samples-test-user2 + email: sample2@testuser.com + password: test1 + clientRoles: + - role1 + realmRoles: + - offline_access + diff --git a/applications/accounts/deploy/values.yaml b/applications/accounts/deploy/values.yaml index 42e290cb..3e32ce4c 100644 --- a/applications/accounts/deploy/values.yaml +++ b/applications/accounts/deploy/values.yaml @@ -11,7 +11,7 @@ harness: cpu: "10m" limits: memory: "1024Mi" - cpu: "500m" + cpu: "2000m" service: auto: true port: 8080 diff --git a/applications/accounts/dev/disable-theme-cache.cli b/applications/accounts/dev/disable-theme-cache.cli new file mode 100644 index 00000000..4eca62b3 --- /dev/null +++ b/applications/accounts/dev/disable-theme-cache.cli @@ -0,0 +1,5 @@ +embed-server --std-out=echo --server-config=standalone-ha.xml +/subsystem=keycloak-server/theme=defaults/:write-attribute(name=cacheThemes,value=false) +/subsystem=keycloak-server/theme=defaults/:write-attribute(name=cacheTemplates,value=false) +/subsystem=keycloak-server/theme=defaults/:write-attribute(name=staticMaxAge,value=-1) +stop-embedded-server \ No newline at end of file diff --git a/applications/accounts/dev/docker-compose.yaml b/applications/accounts/dev/docker-compose.yaml new file mode 100644 index 00000000..fc043360 --- /dev/null +++ b/applications/accounts/dev/docker-compose.yaml @@ -0,0 +1,45 @@ +version: '3.2' + +services: + postgres: + image: postgres + environment: + POSTGRES_DB: keycloak + POSTGRES_USER: keycloak + POSTGRES_PASSWORD: password + PGDATA: /var/lib/postgresql/data/pgdata + volumes: + - pg_data:/var/lib/postgresql/data/pgdata + + keycloak: + image: quay.io/keycloak/keycloak:16.1.1 + environment: + DB_VENDOR: POSTGRES + DB_ADDR: postgres + DB_DATABASE: keycloak + DB_USER: keycloak + DB_SCHEMA: public + DB_PASSWORD: password + KEYCLOAK_USER: admin + KEYCLOAK_PASSWORD: Pa55w0rd + + ports: + - 8080:8080 + depends_on: + - postgres + volumes: + - type: bind + source: ../themes/custom + target: /opt/jboss/keycloak/themes/custom + # disable cache + - type: bind + source: ./disable-theme-cache.cli + target: /opt/jboss/startup-scripts/disable-theme-cache.cli + - type: bind + source: ../scripts/create_api_user.sh + target: /opt/jboss/startup-scripts/create_api_user.sh + - type: bind + source: ../plugins/metacell-admin-event-listener-bundle-1.0.0.ear + target: /opt/jboss/keycloak/standalone/deployments/metacell-admin-event-listener-bundle-1.0.0.ear +volumes: + pg_data: \ No newline at end of file diff --git a/applications/accounts/dev/realm.json b/applications/accounts/dev/realm.json new file mode 100644 index 00000000..045e6894 --- /dev/null +++ b/applications/accounts/dev/realm.json @@ -0,0 +1,669 @@ +{ + "id": "ch", + "realm": "ch", + "enabled": true, + "sslRequired": "none", + "loginTheme": "keycloak", + "accountTheme": "keycloak", + "adminTheme": "keycloak", + "emailTheme": "keycloak", + "registrationAllowed": true, + "registrationEmailAsUsername": false, + "rememberMe": true, + "verifyEmail": false, + "loginWithEmailAllowed": true, + "duplicateEmailsAllowed": false, + "resetPasswordAllowed": true, + "editUsernameAllowed": true, + "users": [ + ], + "roles": { + "realm": [ + { + "id": "70835ad6-1454-4bc5-86a4-f1597e776b75", + "name": "administrator", + "composite": false, + "clientRole": false, + "containerId": "ch", + "attributes": {} + }, + { + "id": "498353dd-88eb-4a5e-99b8-d912e0f20f23", + "name": "uma_authorization", + "description": "${role_uma_authorization}", + "composite": false, + "clientRole": false, + "containerId": "ch", + "attributes": {} + }, + { + "id": "f99970f1-958b-4bb8-8b39-0d7498b0ecc4", + "name": "offline_access", + "description": "${role_offline-access}", + "composite": false, + "clientRole": false, + "containerId": "ch", + "attributes": {} + } + ], + "client": { + } + }, + "clients": [ + { + "id": "9a6a2560-c6be-4493-8bd5-3fdc4522d82b", + "clientId": "rest-client", + "baseUrl": "http://accounts.ch.local", + "surrogateAuthRequired": false, + "enabled": true, + "clientAuthenticatorType": "client-secret", + "secret": "5678eb6e-9e2c-4ee5-bd54-34e7411339e8", + "redirectUris": [ + "*" + ], + "webOrigins": [ + "*" + ], + "standardFlowEnabled": true, + "directAccessGrantsEnabled": true, + "protocol": "openid-connect", + "attributes": { + "access.token.lifespan": "3600" + }, + "fullScopeAllowed": true, + "defaultClientScopes": [ + "web-origins", + "role_list", + "profile", + "roles", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ] + }, + { + "id": "accounts", + "clientId": "accounts", + "name": "accounts", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "secret": "GEdDm8xOhpVFXy8jrjzT", + "redirectUris": [ + "*" + ], + "webOrigins": [ + "*", + "+" + ], + "notBefore": 0, + "bearerOnly": false, + "consentRequired": false, + "standardFlowEnabled": true, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": true, + "serviceAccountsEnabled": false, + "publicClient": true, + "frontchannelLogout": false, + "protocol": "openid-connect", + "attributes": { + "backchannel.logout.session.required": "true", + "backchannel.logout.revoke.offline.tokens": "false" + }, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": true, + "nodeReRegistrationTimeout": -1, + "defaultClientScopes": [], + "optionalClientScopes": [] + }, + { + "id": "111caf43-3d26-484d-8dc9-7fa911ac221c", + "clientId": "web-client", + "baseUrl": "http://accounts.ch.local", + "surrogateAuthRequired": false, + "enabled": true, + "clientAuthenticatorType": "client-secret", + "secret": "452952ae-922c-4766-b912-7b106271e34b", + "redirectUris": [ + "*" + ], + "webOrigins": [ + "*", + "+" + ], + "standardFlowEnabled": true, + "directAccessGrantsEnabled": true, + "publicClient": true, + "protocol": "openid-connect", + "fullScopeAllowed": true, + "defaultClientScopes": [ + "web-origins", + "role_list", + "administrator-scope", + "profile", + "roles", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ] + } + ], + "clientScopes": [ + { + "id": "a8cddc84-c506-4196-8f2d-1bd5e8769f3c", + "name": "administrator-scope", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "true", + "display.on.consent.screen": "true" + }, + "protocolMappers": [ + { + "id": "45fc2547-1761-420b-b6a8-7dc882a51507", + "name": "administrator-audience", + "protocol": "openid-connect", + "protocolMapper": "oidc-audience-mapper", + "consentRequired": false, + "config": { + "included.client.audience": "web-client", + "id.token.claim": "true", + "access.token.claim": "true" + } + } + ] + }, + { + "id": "35c37cdc-6841-41e7-b90f-2964fc563998", + "name": "microprofile-jwt", + "description": "Microprofile - JWT built-in scope", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "true", + "display.on.consent.screen": "false" + }, + "protocolMappers": [ + { + "id": "611fb1bc-56cd-49d2-a11b-ddf05bd220db", + "name": "upn", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-property-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "username", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "upn", + "jsonType.label": "String" + } + }, + { + "id": "63850e7d-1031-447a-a8af-3df588a39350", + "name": "groups", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-realm-role-mapper", + "consentRequired": false, + "config": { + "multivalued": "true", + "user.attribute": "foo", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "groups", + "jsonType.label": "String" + } + } + ] + }, + { + "id": "dc927013-0448-4a29-ac72-7d6b019180d9", + "name": "web-origins", + "description": "OpenID Connect scope for add allowed web origins to the access token", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "false", + "display.on.consent.screen": "false", + "consent.screen.text": "" + }, + "protocolMappers": [ + { + "id": "3cc4569c-83b0-4bc9-af31-186c8081f8ac", + "name": "allowed web origins", + "protocol": "openid-connect", + "protocolMapper": "oidc-allowed-origins-mapper", + "consentRequired": false, + "config": {} + } + ] + }, + { + "id": "4bd583e6-9f6d-4846-9a94-2f02b1b4b1db", + "name": "roles", + "description": "OpenID Connect scope for add user roles to the access token", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "false", + "display.on.consent.screen": "true", + "consent.screen.text": "${rolesScopeConsentText}" + }, + "protocolMappers": [ + { + "id": "0d359e4f-3d4d-4ef3-88fd-2dd9f41da8cd", + "name": "client roles", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-client-role-mapper", + "consentRequired": false, + "config": { + "multivalued": "true", + "user.attribute": "foo", + "access.token.claim": "true", + "claim.name": "resource_access.${client_id}.roles", + "jsonType.label": "String" + } + }, + { + "id": "98ea5505-f703-49d2-b927-7715a7fc7a19", + "name": "realm roles", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-realm-role-mapper", + "consentRequired": false, + "config": { + "multivalued": "true", + "user.attribute": "foo", + "access.token.claim": "true", + "claim.name": "realm_access.roles", + "jsonType.label": "String" + } + }, + { + "id": "28b26ce3-7edc-47c2-982f-881f1c001ef3", + "name": "audience resolve", + "protocol": "openid-connect", + "protocolMapper": "oidc-audience-resolve-mapper", + "consentRequired": false, + "config": {} + } + ] + }, + { + "id": "e2606962-dd91-4926-af4e-cce6a036a04a", + "name": "phone", + "description": "OpenID Connect built-in scope: phone", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "true", + "display.on.consent.screen": "true", + "consent.screen.text": "${phoneScopeConsentText}" + }, + "protocolMappers": [ + { + "id": "c7e30f92-6026-4291-b526-3716662c26f1", + "name": "phone number verified", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "phoneNumberVerified", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "phone_number_verified", + "jsonType.label": "boolean" + } + }, + { + "id": "b1927570-c38d-49b8-9bbb-3cf9571f00be", + "name": "phone number", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "phoneNumber", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "phone_number", + "jsonType.label": "String" + } + } + ] + }, + { + "id": "6f532104-efc0-41d9-8fbc-9c78372d3f1b", + "name": "address", + "description": "OpenID Connect built-in scope: address", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "true", + "display.on.consent.screen": "true", + "consent.screen.text": "${addressScopeConsentText}" + }, + "protocolMappers": [ + { + "id": "548dd8e4-1ee8-4f7d-8934-439bdd1cc0bb", + "name": "address", + "protocol": "openid-connect", + "protocolMapper": "oidc-address-mapper", + "consentRequired": false, + "config": { + "user.attribute.formatted": "formatted", + "user.attribute.country": "country", + "user.attribute.postal_code": "postal_code", + "userinfo.token.claim": "true", + "user.attribute.street": "street", + "id.token.claim": "true", + "user.attribute.region": "region", + "access.token.claim": "true", + "user.attribute.locality": "locality" + } + } + ] + }, + { + "id": "b16d9232-a4e2-47d4-a368-5279a0d84913", + "name": "email", + "description": "OpenID Connect built-in scope: email", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "true", + "display.on.consent.screen": "true", + "consent.screen.text": "${emailScopeConsentText}" + }, + "protocolMappers": [ + { + "id": "4bd6701a-cc02-481e-83c5-e048ea5d83a9", + "name": "email", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-property-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "email", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "email", + "jsonType.label": "String" + } + }, + { + "id": "4cf00282-d385-456a-8943-4bdde6357c16", + "name": "email verified", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-property-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "emailVerified", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "email_verified", + "jsonType.label": "boolean" + } + } + ] + }, + { + "id": "1e9fa514-8ae1-4980-9ccc-2d2d2c43c7e6", + "name": "profile", + "description": "OpenID Connect built-in scope: profile", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "true", + "display.on.consent.screen": "true", + "consent.screen.text": "${profileScopeConsentText}" + }, + "protocolMappers": [ + { + "id": "2e186cd7-b7d5-4b63-b765-c77036183db6", + "name": "updated at", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "updatedAt", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "updated_at", + "jsonType.label": "String" + } + }, + { + "id": "86e94688-d91b-493b-809a-07005c7e6cab", + "name": "picture", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "picture", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "picture", + "jsonType.label": "String" + } + }, + { + "id": "8e65f9c7-a3c0-4bf6-9c4e-47be99464408", + "name": "zoneinfo", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "zoneinfo", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "zoneinfo", + "jsonType.label": "String" + } + }, + { + "id": "9eeaaeb3-93fc-439f-a8db-d6f3693a8ba1", + "name": "given name", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-property-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "firstName", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "given_name", + "jsonType.label": "String" + } + }, + { + "id": "34e60d98-fcde-49a2-b093-748464886a0d", + "name": "middle name", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "middleName", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "middle_name", + "jsonType.label": "String" + } + }, + { + "id": "08fa0341-5dd3-42e2-babb-1151c35b72c3", + "name": "nickname", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "nickname", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "nickname", + "jsonType.label": "String" + } + }, + { + "id": "9d9f1655-9b23-4e15-b244-aeffcb20c5ba", + "name": "gender", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "gender", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "gender", + "jsonType.label": "String" + } + }, + { + "id": "23b19dbb-5af2-494e-b462-e8f63d9266f4", + "name": "birthdate", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "birthdate", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "birthdate", + "jsonType.label": "String" + } + }, + { + "id": "b4644d65-ffbb-4e0b-8aac-238665af40dc", + "name": "locale", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "locale", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "locale", + "jsonType.label": "String" + } + }, + { + "id": "6366756e-bf69-4844-b127-60fa514ad768", + "name": "website", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "website", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "website", + "jsonType.label": "String" + } + }, + { + "id": "3d763f84-d417-4b4e-99e4-2b0e05bf861a", + "name": "family name", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-property-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "lastName", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "family_name", + "jsonType.label": "String" + } + }, + { + "id": "d05efa25-5348-4a14-9550-69791df4ec5e", + "name": "full name", + "protocol": "openid-connect", + "protocolMapper": "oidc-full-name-mapper", + "consentRequired": false, + "config": { + "id.token.claim": "true", + "access.token.claim": "true", + "userinfo.token.claim": "true" + } + }, + { + "id": "0d66e664-6b0c-45de-ba88-b2b86b23cacc", + "name": "profile", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "profile", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "profile", + "jsonType.label": "String" + } + }, + { + "id": "17d3b93d-993b-4768-892c-0b20f8462be3", + "name": "username", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-property-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "username", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "preferred_username", + "jsonType.label": "String" + } + } + ] + }, + { + "id": "a46716b3-8da1-4657-b703-13a5cd472c92", + "name": "role_list", + "description": "SAML role list", + "protocol": "saml", + "attributes": { + "consent.screen.text": "${samlRoleListScopeConsentText}", + "display.on.consent.screen": "true" + }, + "protocolMappers": [ + { + "id": "0ab50259-1e8b-40bd-9686-fb9a54dfc37d", + "name": "role list", + "protocol": "saml", + "protocolMapper": "saml-role-list-mapper", + "consentRequired": false, + "config": { + "single": "false", + "attribute.nameformat": "Basic", + "attribute.name": "Role" + } + } + ] + }, + { + "id": "0f9bd78c-129e-4f87-9cf7-8b68b628ea1b", + "name": "offline_access", + "description": "OpenID Connect built-in scope: offline_access", + "protocol": "openid-connect", + "attributes": { + "consent.screen.text": "${offlineAccessScopeConsentText}", + "display.on.consent.screen": "true" + } + } + ], + "keycloakVersion": "9.0.2" +} diff --git a/applications/accounts/scripts/create_api_user.sh b/applications/accounts/scripts/create_api_user.sh old mode 100644 new mode 100755 diff --git a/deployment/codefresh-test.yaml b/deployment/codefresh-test.yaml index f341aaa5..21bef923 100644 --- a/deployment/codefresh-test.yaml +++ b/deployment/codefresh-test.yaml @@ -485,18 +485,6 @@ steps: commands: - echo $APP_NAME scale: - volumemanager_api_test: - title: volumemanager api test - volumes: - - '${{CF_REPO_NAME}}/applications/volumemanager:/home/test' - - '${{CF_REPO_NAME}}/deployment/helm/values.yaml:/opt/cloudharness/resources/allvalues.yaml' - environment: - - APP_URL=https://volumemanager.${{DOMAIN}}/api - - USERNAME=volumes@testuser.com - - PASSWORD=test - commands: - - st --pre-run cloudharness_test.apitest_init run api/openapi.yaml --base-url - https://volumemanager.${{DOMAIN}}/api -c all samples_api_test: title: samples api test volumes: diff --git a/deployment/docker-compose.yaml b/deployment/docker-compose.yaml new file mode 100644 index 00000000..05df8c24 --- /dev/null +++ b/deployment/docker-compose.yaml @@ -0,0 +1,313 @@ +# Source: cloudharness/templates/auto-compose.yaml +version: '3.7' + +services: + traefik: + image: traefik:v2.10 + container_name: traefik + networks: + - ch + command: + - --log.level=INFO + - --api.insecure=true + - --providers.docker=true + - --providers.docker.exposedbydefault=false + - --entrypoints.web.address=:80 + - --entrypoints.websecure.address=:443 + - --providers.file.directory=/etc/traefik/dynamic_conf + ports: + - 80:80 + - 443:443 + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + - ./traefik/certs/:/certs/:ro + - ./traefik/traefik.yaml:/etc/traefik/dynamic_conf/conf.yml:ro + accounts: + networks: + - ch + image: cloudharness/accounts + + expose: + - 8080 + deploy: + mode: replicated + replicas: 1 + resources: + limits: + cpus: 0.500 + memory: 1024M + reservations: + cpus: 0.010 + memory: 512M + healthcheck: + test: [CMD, curl, -f, http://127.0.0.1:8080/auth/realms/ch/account] + interval: 1s + timeout: 3s + retries: 30 + environment: + - CH_CURRENT_APP_NAME=accounts + - CH_VERSION=2.4.0 + - CH_ACCOUNTS_SUBDOMAIN=accounts + - CH_ACCOUNTS_NAME=accounts + - CH_JUPYTERHUB_SUBDOMAIN=hub + - CH_JUPYTERHUB_NAME=jupyterhub + - CH_ARGO_SUBDOMAIN=argo + - CH_ARGO_NAME=argo + - CH_SAMPLES_SUBDOMAIN=samples + - CH_SAMPLES_PORT=80 + - CH_SAMPLES_NAME=samples + - CH_COMMON_SUBDOMAIN=common + - CH_COMMON_NAME=common + - CH_EVENTS_SUBDOMAIN=events + - CH_EVENTS_NAME=events + - CH_WORKFLOWS_SUBDOMAIN=workflows + - CH_WORKFLOWS_NAME=workflows + - CH_DOMAIN=ch + - CH_IMAGE_REGISTRY= + - CH_IMAGE_TAG= + - CH_ACCOUNTS_CLIENT_SECRET=5678eb6e-9e2c-4ee5-bd54-34e7411339e8 + - CH_ACCOUNTS_REALM=ch + - CH_ACCOUNTS_AUTH_DOMAIN=accounts.ch + - CH_ACCOUNTS_CLIENT_ID=rest-client + - DOMAIN=ch + - KEYCLOAK_IMPORT=/tmp/realm.json + - KEYCLOAK_USER=admin + - KEYCLOAK_PASSWORD=metacell + - PROXY_ADDRESS_FORWARDING=true + - DB_VENDOR=POSTGRES + - DB_ADDR=keycloak-postgres + - DB_DATABASE=auth_db + - DB_USER=user + - DB_PASSWORD=password + - JAVA_OPTS=-server -Xms64m -Xmx896m -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m + -Djava.net.preferIPv4Stack=true -Djboss.modules.system.pkgs=org.jboss.byteman + -Djava.awt.headless=true --add-exports=java.base/sun.nio.ch=ALL-UNNAMED --add-exports=jdk.unsupported/sun.misc=ALL-UNNAMED + --add-exports=jdk.unsupported/sun.reflect=ALL-UNNAMED + volumes: + - ./compose/allvalues.yaml:/opt/cloudharness/resources/allvalues.yaml:ro + - ./compose/resources/generated/auth/api_user_password:/opt/cloudharness/resources/auth/api_user_password + - type: bind + source: ./compose/resources/generated/accounts/realm.json + target: /tmp/realm.json + labels: + - traefik.enable=true + - traefik.http.services.accounts.loadbalancer.server.port=8080 + # - "traefik.http.middlewares.redirect-middleware.redirectscheme.scheme=https" + # - "traefik.http.routers..middlewares=redirect-middleware" + - traefik.http.routers.accounts.rule=Host(`accounts.ch`) + - traefik.http.routers.accounts.entrypoints=web + # Database type postgres named keycloak-postgres + keycloak-postgres: + networks: + ch: + image: postgres:10.4 + expose: + - '5432' + deploy: + resources: + limits: + cpus: 1.000 + memory: 2G + reservations: + cpus: 0.100 + memory: 512M + volumes: + - type: volume + source: keycloak-postgres + target: /data/db + - type: volume + source: dshm-keycloak-postgres + target: /dev/shm + environment: + - POSTGRES_DB=auth_db + - POSTGRES_USER=user + - POSTGRES_PASSWORD=password + - PGDATA=/data/db/pgdata + common: + networks: + - ch + image: cloudharness/common + + expose: + - 8080 + deploy: + mode: replicated + replicas: 1 + resources: + limits: + cpus: 0.200 + memory: 256M + reservations: + cpus: 0.050 + memory: 128M + # entrypoint: python /usr/src/app/common/__main__.py + environment: + - CH_CURRENT_APP_NAME=common + - CH_VERSION=2.4.0 + - CH_ACCOUNTS_SUBDOMAIN=accounts + - CH_ACCOUNTS_NAME=accounts + - CH_JUPYTERHUB_SUBDOMAIN=hub + - CH_JUPYTERHUB_NAME=jupyterhub + - CH_ARGO_SUBDOMAIN=argo + - CH_ARGO_NAME=argo + - CH_SAMPLES_SUBDOMAIN=samples + - CH_SAMPLES_PORT=80 + - CH_SAMPLES_NAME=samples + - CH_COMMON_SUBDOMAIN=common + - CH_COMMON_NAME=common + - CH_EVENTS_SUBDOMAIN=events + - CH_EVENTS_NAME=events + - CH_WORKFLOWS_SUBDOMAIN=workflows + - CH_WORKFLOWS_NAME=workflows + - CH_DOMAIN=ch + - CH_IMAGE_REGISTRY= + - CH_IMAGE_TAG= + - CH_ACCOUNTS_CLIENT_SECRET=5678eb6e-9e2c-4ee5-bd54-34e7411339e8 + - CH_ACCOUNTS_REALM=ch + - CH_ACCOUNTS_AUTH_DOMAIN=accounts.ch + - CH_ACCOUNTS_CLIENT_ID=rest-client + - DOMAIN=ch + volumes: + - ./compose/allvalues.yaml:/opt/cloudharness/resources/allvalues.yaml:ro + labels: + - traefik.enable=true + - traefik.http.services.common.loadbalancer.server.port=8080 + # - "traefik.http.middlewares.redirect-middleware.redirectscheme.scheme=https" + # - "traefik.http.routers..middlewares=redirect-middleware" + - traefik.http.routers.common.rule=Host(`common.ch`) + - traefik.http.routers.common.entrypoints=web + samples: + networks: + - ch + image: cloudharness/samples + + expose: + - 8080 + deploy: + mode: replicated + replicas: 1 + resources: + limits: + cpus: 0.500 + memory: 500M + reservations: + cpus: 0.010 + memory: 32M + # entrypoint: python /usr/src/app/samples/__main__.py + environment: + - CH_CURRENT_APP_NAME=samples + - CH_VERSION=2.4.0 + - CH_ACCOUNTS_SUBDOMAIN=accounts + - CH_ACCOUNTS_NAME=accounts + - CH_JUPYTERHUB_SUBDOMAIN=hub + - CH_JUPYTERHUB_NAME=jupyterhub + - CH_ARGO_SUBDOMAIN=argo + - CH_ARGO_NAME=argo + - CH_SAMPLES_SUBDOMAIN=samples + - CH_SAMPLES_PORT=80 + - CH_SAMPLES_NAME=samples + - CH_COMMON_SUBDOMAIN=common + - CH_COMMON_NAME=common + - CH_EVENTS_SUBDOMAIN=events + - CH_EVENTS_NAME=events + - CH_WORKFLOWS_SUBDOMAIN=workflows + - CH_WORKFLOWS_NAME=workflows + - CH_DOMAIN=ch + - CH_IMAGE_REGISTRY= + - CH_IMAGE_TAG= + - CH_ACCOUNTS_CLIENT_SECRET=5678eb6e-9e2c-4ee5-bd54-34e7411339e8 + - CH_ACCOUNTS_REALM=ch + - CH_ACCOUNTS_AUTH_DOMAIN=accounts.ch + - CH_ACCOUNTS_CLIENT_ID=rest-client + - DOMAIN=ch + - WORKERS=3 + + + links: + - workflows:workflows.ch +# - events:events.ch + - common:common.ch +# - jupyterhub:jupyterhub.ch + volumes: + - ./compose/allvalues.yaml:/opt/cloudharness/resources/allvalues.yaml:ro + - ./compose/resources/generated/auth/asecret:/opt/cloudharness/resources/auth/asecret + - type: volume + source: my-shared-volume + target: /tmp/myvolume + - type: bind + source: ./compose/resources/generated/samples/myConfig.json + target: /tmp/resources/myConfig.json + - type: bind + source: ./compose/resources/generated/samples/example.yaml + target: /usr/src/app/important_config.yaml + labels: + - traefik.enable=true + - traefik.http.services.samples.loadbalancer.server.port=8080 + # - "traefik.http.middlewares.redirect-middleware.redirectscheme.scheme=https" + # - "traefik.http.routers..middlewares=redirect-middleware" + - traefik.http.routers.samples.rule=Host(`samples.ch`) + - traefik.http.routers.samples.entrypoints=web + workflows: + networks: + - ch + image: cloudharness/workflows + + expose: + - 8080 + deploy: + mode: replicated + replicas: 1 + resources: + limits: + cpus: 0.500 + memory: 500M + reservations: + cpus: 0.010 + memory: 32M + # entrypoint: python /usr/src/app/workflows_api/__main__.py + environment: + - CH_CURRENT_APP_NAME=workflows + - CH_VERSION=2.4.0 + - CH_ACCOUNTS_SUBDOMAIN=accounts + - CH_ACCOUNTS_NAME=accounts + - CH_JUPYTERHUB_SUBDOMAIN=hub + - CH_JUPYTERHUB_NAME=jupyterhub + - CH_ARGO_SUBDOMAIN=argo + - CH_ARGO_NAME=argo + - CH_SAMPLES_SUBDOMAIN=samples + - CH_SAMPLES_PORT=80 + - CH_SAMPLES_NAME=samples + - CH_COMMON_SUBDOMAIN=common + - CH_COMMON_NAME=common + - CH_EVENTS_SUBDOMAIN=events + - CH_EVENTS_NAME=events + - CH_WORKFLOWS_SUBDOMAIN=workflows + - CH_WORKFLOWS_NAME=workflows + - CH_DOMAIN=ch + - CH_IMAGE_REGISTRY= + - CH_IMAGE_TAG= + - CH_ACCOUNTS_CLIENT_SECRET=5678eb6e-9e2c-4ee5-bd54-34e7411339e8 + - CH_ACCOUNTS_REALM=ch + - CH_ACCOUNTS_AUTH_DOMAIN=accounts.ch + - CH_ACCOUNTS_CLIENT_ID=rest-client + - DOMAIN=ch + + + volumes: + - ./compose/allvalues.yaml:/opt/cloudharness/resources/allvalues.yaml:ro + labels: + - traefik.enable=true + - traefik.http.services.workflows.loadbalancer.server.port=8080 + # - "traefik.http.middlewares.redirect-middleware.redirectscheme.scheme=https" + # - "traefik.http.routers..middlewares=redirect-middleware" + - traefik.http.routers.workflows.rule=Host(`workflows.ch`) + - traefik.http.routers.workflows.entrypoints=web + +# Network definition +networks: + ch: + name: ch_network +volumes: + keycloak-postgres: + dshm-keycloak-postgres: + my-shared-volume: From 169de78f06b76f3bfb472c64706a677f089000e1 Mon Sep 17 00:00:00 2001 From: Filippo Ledda Date: Fri, 14 Mar 2025 19:47:13 +0100 Subject: [PATCH 90/93] CH-169 try fix keycloak starting issue --- applications/accounts/deploy/resources/realm.json | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/applications/accounts/deploy/resources/realm.json b/applications/accounts/deploy/resources/realm.json index 005d4042..d9738fae 100644 --- a/applications/accounts/deploy/resources/realm.json +++ b/applications/accounts/deploy/resources/realm.json @@ -46,7 +46,8 @@ "users": [ {{- range $j, $app := .Values.apps }} {{- if (hasKey $app.harness "accounts") }} - {{- range $i, $user := $app.harness.accounts.users }}{{if (add $i $j)}},{{end}} + {{if $j}},{{end}} + {{- range $i, $user := $app.harness.accounts.users }}{{if $i}},{{end}} {{ include "deploy_accounts_utils.user" (dict "root" $ "app" $app "user" $user) }} {{- end }} {{- end }} @@ -82,11 +83,12 @@ } ], "client": { - {{- range $j,$app := .Values.apps }} + {{- range $j, $app := .Values.apps }} {{- if (hasKey $app.harness "accounts") }} + {{if $j}},{{end}} {{ $app.harness.name | quote }}: [ {{- range $i, $role := $app.harness.accounts.roles }} - {{if (add $i $j)}},{{end}} + {{if $i}},{{end}} {{ include "deploy_accounts_utils.role" (dict "root" $ "app" $app "role" $role) }} {{- end }} ] From 29d9cef65696c27f29213e313c416ec108a9f0fc Mon Sep 17 00:00:00 2001 From: Filippo Ledda Date: Fri, 14 Mar 2025 19:53:22 +0100 Subject: [PATCH 91/93] CH-169 try fix keycloak starting issue --- applications/accounts/deploy/resources/realm.json | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/applications/accounts/deploy/resources/realm.json b/applications/accounts/deploy/resources/realm.json index d9738fae..c5e592f5 100644 --- a/applications/accounts/deploy/resources/realm.json +++ b/applications/accounts/deploy/resources/realm.json @@ -44,10 +44,11 @@ "resetPasswordAllowed": true, "editUsernameAllowed": true, "users": [ - {{- range $j, $app := .Values.apps }} + {{- $j := 0}} + {{- range $app := .Values.apps }} {{- if (hasKey $app.harness "accounts") }} - {{if $j}},{{end}} - {{- range $i, $user := $app.harness.accounts.users }}{{if $i}},{{end}} + {{- $j := add1 $j }} + {{- range $i, $user := $app.harness.accounts.users }}{{if or $i $j}},{{end}} {{ include "deploy_accounts_utils.user" (dict "root" $ "app" $app "user" $user) }} {{- end }} {{- end }} @@ -83,12 +84,13 @@ } ], "client": { - {{- range $j, $app := .Values.apps }} + {{- $k := 0}} + {{- range $app := .Values.apps }} {{- if (hasKey $app.harness "accounts") }} - {{if $j}},{{end}} + {{- $k = add1 $k }} {{ $app.harness.name | quote }}: [ {{- range $i, $role := $app.harness.accounts.roles }} - {{if $i}},{{end}} + {{if or $i $k}},{{end}} {{ include "deploy_accounts_utils.role" (dict "root" $ "app" $app "role" $role) }} {{- end }} ] From ec5b8c140d93da1ba45cc657f8c3b12d51a972ca Mon Sep 17 00:00:00 2001 From: Filippo Ledda Date: Fri, 14 Mar 2025 20:00:34 +0100 Subject: [PATCH 92/93] CH-169 try fix keycloak starting issue --- applications/accounts/deploy/resources/realm.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/applications/accounts/deploy/resources/realm.json b/applications/accounts/deploy/resources/realm.json index c5e592f5..dbb04caf 100644 --- a/applications/accounts/deploy/resources/realm.json +++ b/applications/accounts/deploy/resources/realm.json @@ -47,11 +47,11 @@ {{- $j := 0}} {{- range $app := .Values.apps }} {{- if (hasKey $app.harness "accounts") }} - {{- $j := add1 $j }} {{- range $i, $user := $app.harness.accounts.users }}{{if or $i $j}},{{end}} {{ include "deploy_accounts_utils.user" (dict "root" $ "app" $app "user" $user) }} {{- end }} {{- end }} + {{- $j := add1 $j }} {{- end }} ], "roles": { @@ -87,13 +87,14 @@ {{- $k := 0}} {{- range $app := .Values.apps }} {{- if (hasKey $app.harness "accounts") }} - {{- $k = add1 $k }} + {{ $app.harness.name | quote }}: [ {{- range $i, $role := $app.harness.accounts.roles }} {{if or $i $k}},{{end}} {{ include "deploy_accounts_utils.role" (dict "root" $ "app" $app "role" $role) }} {{- end }} ] + {{- $k = add1 $k }} {{- end }} {{- end }} } From 14af0b69a778b46365cffa0abaa12d2d6b869535 Mon Sep 17 00:00:00 2001 From: Filippo Ledda Date: Sat, 15 Mar 2025 14:59:44 +0100 Subject: [PATCH 93/93] CH-169 try fix keycloak starting issue --- applications/accounts/deploy/resources/realm.json | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/applications/accounts/deploy/resources/realm.json b/applications/accounts/deploy/resources/realm.json index dbb04caf..f25643b6 100644 --- a/applications/accounts/deploy/resources/realm.json +++ b/applications/accounts/deploy/resources/realm.json @@ -47,11 +47,15 @@ {{- $j := 0}} {{- range $app := .Values.apps }} {{- if (hasKey $app.harness "accounts") }} - {{- range $i, $user := $app.harness.accounts.users }}{{if or $i $j}},{{end}} + {{- if $j}},{{end}} + {{- if len $app.harness.accounts.users}} + {{- $j = add1 $j }} + {{- end }} + {{- range $i, $user := $app.harness.accounts.users }}{{if $i}},{{end}} {{ include "deploy_accounts_utils.user" (dict "root" $ "app" $app "user" $user) }} {{- end }} {{- end }} - {{- $j := add1 $j }} + {{- end }} ], "roles": { @@ -86,12 +90,13 @@ "client": { {{- $k := 0}} {{- range $app := .Values.apps }} + {{- if (hasKey $app.harness "accounts") }} - + {{- if $k}},{{end}} {{ $app.harness.name | quote }}: [ {{- range $i, $role := $app.harness.accounts.roles }} - {{if or $i $k}},{{end}} - {{ include "deploy_accounts_utils.role" (dict "root" $ "app" $app "role" $role) }} + {{if $i}},{{end}} + {{- include "deploy_accounts_utils.role" (dict "root" $ "app" $app "role" $role) }} {{- end }} ] {{- $k = add1 $k }}