From 21a2194b0de099eafff354c2e6bfdc2d2c4159d5 Mon Sep 17 00:00:00 2001 From: Mohcine Tor Date: Sun, 19 Oct 2025 13:53:44 +0200 Subject: [PATCH 1/4] feat: refactor state and context logic to ensure compatibility with v5 --- Babylon/commands/namespace/get_contexts.py | 4 +- Babylon/config.py | 2 +- Babylon/utils/environment.py | 48 ++++++++++------------ 3 files changed, 25 insertions(+), 29 deletions(-) diff --git a/Babylon/commands/namespace/get_contexts.py b/Babylon/commands/namespace/get_contexts.py index 72db3366f..c33750f84 100644 --- a/Babylon/commands/namespace/get_contexts.py +++ b/Babylon/commands/namespace/get_contexts.py @@ -17,6 +17,6 @@ def get_contexts() -> CommandResponse: col_widths = [max(len(h), len(v)) + 2 for h, v in zip(headers, values)] header_line = "".join(h.ljust(w) for h, w in zip(headers, col_widths)) value_line = "".join(v.ljust(w) for v, w in zip(values, col_widths)) - logger.info(header_line) - logger.info(value_line) + print(header_line) + print(value_line) return CommandResponse.success() diff --git a/Babylon/config.py b/Babylon/config.py index ba36654d2..b33ef0880 100644 --- a/Babylon/config.py +++ b/Babylon/config.py @@ -5,7 +5,7 @@ from dynaconf import Dynaconf config_files = [ - 'acr', 'adt', 'adx', 'api', 'app', 'keycloak', 'azure', 'babylon', 'github', 'platform', 'powerbi', 'webapp' + 'acr', 'api', 'keycloak', 'babylon', 'powerbi' ] pwd = Path(os.getcwd()) diff --git a/Babylon/utils/environment.py b/Babylon/utils/environment.py index 53200c432..6b517553f 100644 --- a/Babylon/utils/environment.py +++ b/Babylon/utils/environment.py @@ -1,4 +1,3 @@ -from collections import defaultdict import json import os import re @@ -8,13 +7,13 @@ import logging import requests +from collections import defaultdict from pathlib import Path from hvac import Client from mako.template import Template from cryptography.fernet import Fernet from flatten_json import flatten from Babylon.config import config_files -from azure.storage.blob import BlobServiceClient from Babylon.utils import ORIGINAL_TEMPLATE_FOLDER_PATH from Babylon.utils.working_dir import WorkingDir from Babylon.utils.yaml_utils import yaml_to_json @@ -52,7 +51,7 @@ def __init__(self): self.remote = False self.pwd = Path.cwd() self.hvac_client = None - self.blob_client = None + # self.blob_client = None self.state_id: str = "" self.context_id: str = "" self.environ_id: str = "" @@ -75,7 +74,7 @@ def get_variables_for_logs(self): variables_file = self.pwd / "variables.yaml" vars = dict() if variables_file.exists(): - logger.debug(f"Loading variables from {variables_file}") + logger.debug(f"[babylon] Loading variables from {variables_file}") vars = yaml.safe_load(variables_file.open()) or dict() vars["secret_powerbi"] = "" vars["github_secret"] = "" @@ -98,28 +97,21 @@ def get_ns_from_text(self, content: str): vars = self.get_variables() payload = t.render(**vars) payload_dict = yaml.safe_load(payload) - context_id = payload_dict.get("context", "") - state_id = payload_dict.get("state_id", "") remote: bool = payload_dict.get("remote", self.remote) self.remote = remote - if not state_id: - logger.error("[babylon] state_id is mandatory") - sys.exit(1) plt_obj = payload_dict.get("platform", {}) platform_id = plt_obj.get("id", "") if not platform_id: - logger.error("[babylon] platform_id is mandatory") + logger.error("[babylon] Missing required 'platform_id' please check your 'variable file' ") sys.exit(1) platform_url = plt_obj.get("url", "") if not platform_url: - logger.error("[babylon] url is mandatory") + logger.error("[babylon] Missing required 'platform_url' please check your 'variable file' ") sys.exit(1) - self.set_state_id(state_id=state_id) - self.set_context(context_id=context_id) self.set_environ(environ_id=platform_id) self.set_server_id() self.set_org_name() - self.set_blob_client() + # self.set_blob_client() return platform_url def fill_template_jsondump(self, data: str, state: dict = None, ext_args: dict = None): @@ -198,16 +190,17 @@ def set_server_id(self): except Exception as e: logger.error(e) - def set_blob_client(self): - try: - state = self.get_state_from_vault_by_platform(self.environ_id) - storage_name = state["azure"]["storage_account_name"] - account_secret = self.get_platform_secret(self.environ_id, resource="storage", name="account") - prefix = f"DefaultEndpointsProtocol=https;AccountName={storage_name}" - connection_str = (f"{prefix};AccountKey={account_secret};EndpointSuffix=core.windows.net") - self.blob_client = BlobServiceClient.from_connection_string(connection_str) - except Exception as e: - logger.error(e) + # This is deactivated for now because we need to decide where the Babylon state should be stored + # def set_blob_client(self): + # try: + # state = self.get_state_from_vault_by_platform(self.environ_id) + # storage_name = state["azure"]["storage_account_name"] + # account_secret = self.get_platform_secret(self.environ_id, resource="storage", name="account") + # prefix = f"DefaultEndpointsProtocol=https;AccountName={storage_name}" + # connection_str = (f"{prefix};AccountKey={account_secret};EndpointSuffix=core.windows.net") + # self.blob_client = BlobServiceClient.from_connection_string(connection_str) + # except Exception as e: + # logger.error(e) def get_organization_secret(self, organization_name: str, name: str): data = self.hvac_client.read(path=f"organization/{organization_name}") @@ -390,6 +383,9 @@ def store_namespace_in_local(self): s = ns_dir / "namespace.yaml" ns = dict(state_id=self.state_id, context=self.context_id, platform=self.environ_id) s.write_bytes(data=yaml.dump(ns).encode("utf-8")) + self.set_state_id(state_id=self.state_id) + self.set_context(context_id=self.context_id) + self.set_environ(environ_id=self.environ_id) def get_namespace_from_local(self, context: str = "", platform: str = "", state_id: str = ""): ns_dir = Path().home() / ".config/cosmotech/babylon" @@ -408,7 +404,7 @@ def get_namespace_from_local(self, context: str = "", platform: str = "", state_ self.set_state_id(state_id=self.state_id) self.set_server_id() self.set_org_name() - self.set_blob_client() + # self.set_blob_client() return ns_data def retrieve_state_func(self, state_id: str = ""): @@ -462,7 +458,7 @@ def set_ns_from_yaml(self, content: str, state: dict = None, ext_args: dict = No self.set_environ(environ_id=platform_id) self.set_server_id() self.set_org_name() - self.set_blob_client() + # self.set_blob_client() return platform_url def get_metadata(self, vars: dict, content: str, state: dict): From 8d13aaeddc2f9fdb4a31ec007f0cec8772d35cb7 Mon Sep 17 00:00:00 2001 From: Mohcine Tor Date: Sun, 19 Oct 2025 13:55:07 +0200 Subject: [PATCH 2/4] refactor: improve request log message --- Babylon/commands/macro/apply.py | 7 ++++--- Babylon/commands/macro/deploy_organization.py | 21 ++++++++++++------- Babylon/commands/macro/deploy_workspace.py | 7 +++++-- Babylon/utils/request.py | 14 +++++++++++-- 4 files changed, 34 insertions(+), 15 deletions(-) diff --git a/Babylon/commands/macro/apply.py b/Babylon/commands/macro/apply.py index 712e13624..083d30a97 100644 --- a/Babylon/commands/macro/apply.py +++ b/Babylon/commands/macro/apply.py @@ -146,9 +146,10 @@ def apply( _ret.append(f" * Workspace : {services.get('api').get('workspace_id', '')}") for id in final_datasets: _ret.append(f" * Dataset : {id}") - if services.get("webapp").get("static_domain", ""): - _ret.append(" * WebApp ") - _ret.append(f" * Hostname : https://{services.get('webapp').get('static_domain', '')}") + # TODO: When the Helm chart for the web app is implemented in Babylon ! + # if services.get("webapp").get("static_domain", ""): + # _ret.append(" * WebApp ") + # _ret.append(f" * Hostname : https://{services.get('webapp').get('static_domain', '')}") vars = env.get_variables() current_working_directory = os.getcwd() logfile_path = os.path.join(current_working_directory, "babylon.log") diff --git a/Babylon/commands/macro/deploy_organization.py b/Babylon/commands/macro/deploy_organization.py index d79afbe8b..f6540280a 100644 --- a/Babylon/commands/macro/deploy_organization.py +++ b/Babylon/commands/macro/deploy_organization.py @@ -6,6 +6,7 @@ from click import echo, style from Babylon.utils.environment import Environment from Babylon.utils.credentials import get_keycloak_token +from Babylon.utils.response import CommandResponse from Babylon.commands.api.organizations.services.organization_api_svc import OrganizationService logger = getLogger("Babylon") @@ -20,7 +21,6 @@ def deploy_organization(namespace: str, file_content: str): platform_url = env.get_ns_from_text(content=namespace) state = env.retrieve_state_func(state_id=env.state_id) state["services"]["api"]["url"] = platform_url - state['services']['azure']['tenant_id'] = env.tenant_id content = env.fill_template(data=file_content, state=state) keycloak_token = get_keycloak_token() payload: dict = content.get("spec").get("payload", {}) @@ -32,12 +32,16 @@ def deploy_organization(namespace: str, file_content: str): if not state["services"]["api"]["organization_id"]: logger.info("[api] Creating organization") response = organization_service.create() + if response is None: + return CommandResponse.fail() organization = response.json() logger.info(json.dumps(organization, indent=2)) logger.info(f"[api] Organization {organization['id']} successfully created") else: logger.info(f"[api] Updating organization {state['services']['api']['organization_id']}") response = organization_service.update() + if response is None: + return CommandResponse.fail() response_json = response.json() old_security = response_json.get("security") security_spec = organization_service.update_security(old_security=old_security) @@ -49,10 +53,11 @@ def deploy_organization(namespace: str, file_content: str): env.store_state_in_local(state) if env.remote: env.store_state_in_cloud(state) - run_scripts = sidecars.get("run_scripts") - if run_scripts: - data = run_scripts.get("post_deploy.sh", "") - if data: - os.system(data) - if not organization.get("id"): - sys.exit(1) + if sidecars: + run_scripts = sidecars.get("run_scripts") + if run_scripts: + data = run_scripts.get("post_deploy.sh", "") + if data: + os.system(data) + if not organization.get("id"): + sys.exit(1) diff --git a/Babylon/commands/macro/deploy_workspace.py b/Babylon/commands/macro/deploy_workspace.py index 2db2d3db4..05c275d89 100644 --- a/Babylon/commands/macro/deploy_workspace.py +++ b/Babylon/commands/macro/deploy_workspace.py @@ -5,6 +5,7 @@ from click import echo, style from logging import getLogger +from Babylon.utils.response import CommandResponse from Babylon.utils.environment import Environment from Babylon.commands.api.workspaces.services.workspaces_api_svc import WorkspaceService from Babylon.commands.powerbi.report.service.powerbi_report_api_svc import ( @@ -59,8 +60,8 @@ def deploy_workspace(namespace: str, file_content: str, deploy_dir: pathlib.Path if not state["services"]["api"]["workspace_id"]: logger.info("[api] Creating workspace") response = workspace_svc.create() - if not response: - sys.exit(1) + if response is None: + return CommandResponse.fail() workspace = response.json() logger.info(json.dumps(workspace, indent=2)) logger.info(f"[api] Workspace {[workspace.get('id')]} successfully created") @@ -68,6 +69,8 @@ def deploy_workspace(namespace: str, file_content: str, deploy_dir: pathlib.Path else: logger.info(f"[api] Updating workspace {[state['services']['api']['workspace_id']]}") response = workspace_svc.update() + if response is None: + return CommandResponse.fail() response_json = response.json() old_security = response_json.get("security") security_spec = workspace_svc.update_security(old_security=old_security) diff --git a/Babylon/utils/request.py b/Babylon/utils/request.py index 7bade636c..ee615ad26 100644 --- a/Babylon/utils/request.py +++ b/Babylon/utils/request.py @@ -51,8 +51,18 @@ def oauth_request(url: str, except Exception as e: logger.warning(f"Request failed: {e}") return None + if response.status_code == 401: + logger.error("[api] Unauthorized (401): token missing or expired. Refresh token or check client credentials.") + return None + if response.status_code == 403: + logger.error( + "[api] Forbidden (403): token valid but lacks required roles/scopes. Check Keycloak client permissions.") + return None + if not response.ok: + logger.warning(f"[api] Request failed ({response.status_code}): {response.text}") + return None if response.status_code >= 300: - logger.warning(f"Failed: ({response.status_code}): {response.text}") + logger.warning(f"[api] Failed: ({response.status_code}): {response.text}") return None - logger.debug(f"Request success ({response.status_code}): {response.text}") + logger.debug(f"[api] Request success ({response.status_code}): {response.text}") return response From 26f1428e9cebb4f2504910a8198b05092b252cc2 Mon Sep 17 00:00:00 2001 From: Mohcine Tor Date: Mon, 20 Oct 2025 10:27:42 +0200 Subject: [PATCH 3/4] delete: tenant id from the state --- Babylon/commands/macro/deploy_solution.py | 1 - Babylon/commands/macro/deploy_workspace.py | 1 - 2 files changed, 2 deletions(-) diff --git a/Babylon/commands/macro/deploy_solution.py b/Babylon/commands/macro/deploy_solution.py index 107e04329..a57a072e8 100644 --- a/Babylon/commands/macro/deploy_solution.py +++ b/Babylon/commands/macro/deploy_solution.py @@ -21,7 +21,6 @@ def deploy_solution(namespace: str, file_content: str) -> bool: platform_url = env.get_ns_from_text(content=namespace) state = env.retrieve_state_func(state_id=env.state_id) state["services"]["api"]["url"] = platform_url - state["services"]["azure"]["tenant_id"] = env.tenant_id vars = env.get_variables() metadata = env.get_metadata(vars=vars, content=file_content, state=state) workspace_key = metadata.get( diff --git a/Babylon/commands/macro/deploy_workspace.py b/Babylon/commands/macro/deploy_workspace.py index 05c275d89..5576faebf 100644 --- a/Babylon/commands/macro/deploy_workspace.py +++ b/Babylon/commands/macro/deploy_workspace.py @@ -35,7 +35,6 @@ def deploy_workspace(namespace: str, file_content: str, deploy_dir: pathlib.Path platform_url = env.get_ns_from_text(content=namespace) state = env.retrieve_state_func(state_id=env.state_id) state["services"]["api"]["url"] = platform_url - state["services"]["azure"]["tenant_id"] = env.tenant_id vars = env.get_variables() metadata = env.get_metadata(vars=vars, content=file_content, state=state) workspace_key = metadata.get( From ce1df6a29f564575e2fff7559698ae6d852cf51f Mon Sep 17 00:00:00 2001 From: Mohcine Tor Date: Mon, 20 Oct 2025 15:50:06 +0200 Subject: [PATCH 4/4] feat: PROD-14893 --- Babylon/commands/__init__.py | 3 +- Babylon/commands/macro/apply.py | 16 ++++- Babylon/commands/macro/deploy_runner.py | 63 +++++++++++++++++++ Babylon/commands/macro/deploy_solution.py | 17 ++--- Babylon/commands/macro/deploy_workspace.py | 17 +++-- Babylon/commands/macro/init.py | 39 ++++++++++++ Babylon/config.py | 4 +- .../.templates/api/connector.adt.yaml | 28 --------- .../.templates/api/connector.storage.yaml | 27 -------- .../.templates/api/connector.twin.yaml | 19 ------ .../.templates/api/dataset.adt.yaml | 13 ---- .../.templates/api/dataset.twin.yaml | 11 ---- .../working_dir/.templates/yaml/Dataset.yaml | 8 +++ .../.templates/yaml/Organization.yaml | 9 +++ .../working_dir/.templates/yaml/Runner.yaml | 14 +++++ .../working_dir/.templates/yaml/Solution.yaml | 26 ++++++++ .../.templates/yaml/Workspace.yaml | 24 +++++++ .../.templates/yaml/variables.yaml | 31 +++++++++ Babylon/utils/environment.py | 2 +- 19 files changed, 249 insertions(+), 122 deletions(-) create mode 100644 Babylon/commands/macro/deploy_runner.py create mode 100644 Babylon/commands/macro/init.py delete mode 100644 Babylon/templates/working_dir/.templates/api/connector.adt.yaml delete mode 100644 Babylon/templates/working_dir/.templates/api/connector.storage.yaml delete mode 100644 Babylon/templates/working_dir/.templates/api/connector.twin.yaml delete mode 100644 Babylon/templates/working_dir/.templates/api/dataset.adt.yaml delete mode 100644 Babylon/templates/working_dir/.templates/api/dataset.twin.yaml create mode 100644 Babylon/templates/working_dir/.templates/yaml/Dataset.yaml create mode 100644 Babylon/templates/working_dir/.templates/yaml/Organization.yaml create mode 100644 Babylon/templates/working_dir/.templates/yaml/Runner.yaml create mode 100644 Babylon/templates/working_dir/.templates/yaml/Solution.yaml create mode 100644 Babylon/templates/working_dir/.templates/yaml/Workspace.yaml create mode 100644 Babylon/templates/working_dir/.templates/yaml/variables.yaml diff --git a/Babylon/commands/__init__.py b/Babylon/commands/__init__.py index f52e5a373..679cd290f 100644 --- a/Babylon/commands/__init__.py +++ b/Babylon/commands/__init__.py @@ -7,5 +7,6 @@ from .namespace import namespace from .macro.apply import apply from .macro.destroy import destroy +from .macro.init import init -list_groups = [api, azure, powerbi, webapp, vault, github, namespace, apply, destroy] +list_groups = [api, azure, powerbi, webapp, vault, github, namespace, apply, destroy, init] diff --git a/Babylon/commands/macro/apply.py b/Babylon/commands/macro/apply.py index 083d30a97..747cc6e64 100644 --- a/Babylon/commands/macro/apply.py +++ b/Babylon/commands/macro/apply.py @@ -12,6 +12,7 @@ from Babylon.commands.macro.deploy_solution import deploy_solution from Babylon.commands.macro.deploy_workspace import deploy_workspace from Babylon.commands.macro.deploy_organization import deploy_organization +from Babylon.commands.macro.deploy_runner import deploy_runner logger = getLogger("Babylon") env = Environment() @@ -33,6 +34,7 @@ @option("--workspace", is_flag=True, help="Deploy or update a workspace.") @option("--webapp", is_flag=True, help="Deploy or update a webapp.") @option("--dataset", is_flag=True, help="Deploy or update a dataset.") +@option("--runner", is_flag=True, help="Deploy or update a runner.") @option("--payload-only", is_flag=True, help="Specify if you want to specify payload only") def apply( deploy_dir: pathlib.Path, @@ -41,6 +43,7 @@ def apply( workspace: bool, webapp: bool, dataset: bool, + runner: bool, payload_only: bool, variables_files: Iterable[pathlib.Path], ): # type: ignore @@ -65,9 +68,10 @@ def apply( workspaces = list(filter(lambda x: x.get("kind") == "Workspace", resources)) webapps = list(filter(lambda x: x.get("kind") == "WebApp", resources)) datasets = list(filter(lambda x: x.get("kind") == "Dataset", resources)) + runners = list(filter(lambda x: x.get("kind") == "Runner", resources)) _ret = [""] final_datasets = [] - if not (organization or solution or workspace or webapp or dataset): + if not (organization or solution or workspace or webapp or dataset or runner): for o in organizations: content = o.get("content") namespace = o.get("namespace") @@ -121,6 +125,12 @@ def apply( file_content=content, deploy_dir=deploy_dir, payload_only=payload_only) + elif runner: + for r in runners: + content = r.get("content") + namespace = r.get("namespace") + deploy_runner(namespace=namespace, file_content=content) + elif webapp: for swa in webapps: content = swa.get("content") @@ -144,9 +154,11 @@ def apply( _ret.append(f" * Organization : {services.get('api').get('organization_id', '')}") _ret.append(f" * Solution : {services.get('api').get('solution_id', '')}") _ret.append(f" * Workspace : {services.get('api').get('workspace_id', '')}") + if runner: + _ret.append(f" * Runner : {services.get('api').get('runner_id', '')}") for id in final_datasets: _ret.append(f" * Dataset : {id}") - # TODO: When the Helm chart for the web app is implemented in Babylon ! + # TODO: When the Helm chart for the web app is implemented in Babylon ! # if services.get("webapp").get("static_domain", ""): # _ret.append(" * WebApp ") # _ret.append(f" * Hostname : https://{services.get('webapp').get('static_domain', '')}") diff --git a/Babylon/commands/macro/deploy_runner.py b/Babylon/commands/macro/deploy_runner.py new file mode 100644 index 000000000..a494b3084 --- /dev/null +++ b/Babylon/commands/macro/deploy_runner.py @@ -0,0 +1,63 @@ +import os +import sys +import json + +from logging import getLogger +from click import echo, style +from Babylon.utils.environment import Environment +from Babylon.utils.credentials import get_keycloak_token +from Babylon.utils.response import CommandResponse +from Babylon.commands.api.runners.services.runner_api_svc import RunnerService + +logger = getLogger("Babylon") +env = Environment() + + +def deploy_runner(namespace: str, file_content: str): + _ret = [""] + _ret.append("Runner deployment") + _ret.append("") + echo(style("\n".join(_ret), bold=True, fg="green")) + platform_url = env.get_ns_from_text(content=namespace) + state = env.retrieve_state_func(state_id=env.state_id) + state["services"]["api"]["url"] = platform_url + content = env.fill_template(data=file_content, state=state) + keycloak_token = get_keycloak_token() + payload: dict = content.get("spec").get("payload", {}) + state["services"]["api"]["runner_id"] = (payload.get("id") or state["services"]["api"]["runner_id"]) + spec = dict() + spec["payload"] = json.dumps(payload, indent=2, ensure_ascii=True) + runner_service = RunnerService(keycloak_token=keycloak_token, spec=spec, state=state["services"]) + sidecars = content.get("spec").get("sidecars", {}) + if not state["services"]["api"]["runner_id"]: + logger.info("[api] Creating runner") + response = runner_service.create() + if response is None: + return CommandResponse.fail() + organization = response.json() + logger.info(json.dumps(organization, indent=2)) + logger.info(f"[api] Runner {organization['id']} successfully created") + else: + logger.info(f"[api] Updating runner {state['services']['api']['runner_id']}") + response = runner_service.update() + if response is None: + return CommandResponse.fail() + response_json = response.json() + old_security = response_json.get("security") + security_spec = runner_service.update_security(old_security=old_security) + response_json["security"] = security_spec + organization = response_json + logger.info(json.dumps(organization, indent=2)) + logger.info(f"[api] Runner {organization['id']} successfully updated") + state["services"]["api"]["runner_id"] = organization.get("id") + env.store_state_in_local(state) + if env.remote: + env.store_state_in_cloud(state) + if sidecars: + run_scripts = sidecars.get("run_scripts") + if run_scripts: + data = run_scripts.get("post_deploy.sh", "") + if data: + os.system(data) + if not organization.get("id"): + sys.exit(1) diff --git a/Babylon/commands/macro/deploy_solution.py b/Babylon/commands/macro/deploy_solution.py index a57a072e8..e5081cc65 100644 --- a/Babylon/commands/macro/deploy_solution.py +++ b/Babylon/commands/macro/deploy_solution.py @@ -42,6 +42,7 @@ def deploy_solution(namespace: str, file_content: str) -> bool: spec = dict() spec["payload"] = json.dumps(payload, indent=2, ensure_ascii=True) solution_svc = SolutionService(keycloak_token=keycloak_token, spec=spec, state=state["services"]) + sidecars = content.get("spec").get("sidecars", {}) if not state["services"]["api"]["solution_id"]: logger.info("[api] Creating solution") response = solution_svc.create() @@ -67,11 +68,11 @@ def deploy_solution(namespace: str, file_content: str) -> bool: if env.remote: env.store_state_in_cloud(state) # update run_templates - sidecars = content.get("spec").get("sidecars", {}) - run_scripts = sidecars.get("run_scripts") - if run_scripts: - data = run_scripts.get("post_deploy.sh", "") - if data: - os.system(data) - if not solution.get("id"): - sys.exit(1) + if sidecars: + run_scripts = sidecars.get("run_scripts") + if run_scripts: + data = run_scripts.get("post_deploy.sh", "") + if data: + os.system(data) + if not solution.get("id"): + sys.exit(1) diff --git a/Babylon/commands/macro/deploy_workspace.py b/Babylon/commands/macro/deploy_workspace.py index 5576faebf..d32611f12 100644 --- a/Babylon/commands/macro/deploy_workspace.py +++ b/Babylon/commands/macro/deploy_workspace.py @@ -249,12 +249,11 @@ def deploy_workspace(namespace: str, file_content: str, deploy_dir: pathlib.Path specUpdated = dict() specUpdated["payload"] = json.dumps(payloadUpdated, indent=2, ensure_ascii=True) workspace_svc.update_with_payload(specUpdated) - run_scripts = sidecars.get("run_scripts") - if run_scripts: - data = run_scripts.get("post_deploy.sh", "") - if data: - print(data) - os.system(data) - if not workspace.get("id"): - logger.error("workspace id not found") - sys.exit(1) + if sidecars: + run_scripts = sidecars.get("run_scripts") + if run_scripts: + data = run_scripts.get("post_deploy.sh", "") + if data: + os.system(data) + if not workspace.get("id"): + sys.exit(1) diff --git a/Babylon/commands/macro/init.py b/Babylon/commands/macro/init.py new file mode 100644 index 000000000..87fca4106 --- /dev/null +++ b/Babylon/commands/macro/init.py @@ -0,0 +1,39 @@ +import os +import pathlib +import shutil + +from logging import getLogger +from click import command, option +from Babylon.utils.environment import Environment + +logger = getLogger("Babylon") +env = Environment() + + +@command() +@option("--project-folder", default="project", help="Name of the project folder to create (default: 'project').") +@option("--variables-file", default="variables.yaml", help="Name of the variables file (default: 'variables.yaml').") +def init(project_folder: str, variables_file: str): + """ + Create a Babylon project structure using YAML templates. + """ + project_path = pathlib.Path(os.getcwd()) / project_folder + variables_path = pathlib.Path(os.getcwd()) / variables_file + if project_path.exists(): + logger.info(f"[babylon] The directory '{project_path}' already exists") + return None + if variables_path.exists(): + logger.info(f"[babylon] 'variables.yaml' already exists at '{variables_path}'") + return None + project_yaml_files = ["Organization.yaml", "Solution.yaml", "Workspace.yaml", "Dataset.yaml", "Runner.yaml"] + try: + project_path.mkdir(parents=True, exist_ok=True) + for file in project_yaml_files: + deploy_file = pathlib.Path(env.convert_template_path(f"%templates%/yaml/{file}")) + destination = project_path / file + shutil.copy(deploy_file, destination) + variables_template = pathlib.Path(env.convert_template_path("%templates%/yaml/variables.yaml")) + shutil.copy(variables_template, variables_path) + logger.info(f"\n[babylon] Project successfully initialized at: {project_path}") + except Exception as e: + logger.error(f"[babylon] Error while initializing the project: {e}") diff --git a/Babylon/config.py b/Babylon/config.py index b33ef0880..83fdd021a 100644 --- a/Babylon/config.py +++ b/Babylon/config.py @@ -4,9 +4,7 @@ from pathlib import Path from dynaconf import Dynaconf -config_files = [ - 'acr', 'api', 'keycloak', 'babylon', 'powerbi' -] +config_files = ['acr', 'api', 'keycloak', 'babylon', 'powerbi'] pwd = Path(os.getcwd()) diff --git a/Babylon/templates/working_dir/.templates/api/connector.adt.yaml b/Babylon/templates/working_dir/.templates/api/connector.adt.yaml deleted file mode 100644 index 6011dfec4..000000000 --- a/Babylon/templates/working_dir/.templates/api/connector.adt.yaml +++ /dev/null @@ -1,28 +0,0 @@ -key: "${key}" -name: "${name}" -description: Connector for Azure Digital Twin. Converts data to CSV for a ScenarioRun -version: "${version}" -repository: cosmo-tech/azure-digital-twins-simulator-connector -tags: -- ADT -- Babylon -- "${tag}" -url: https://github.com/Cosmo-Tech/azure-digital-twins-simulator-connector -ioTypes: -- read -azureAuthenticationWithCustomerAppRegistration: -azureManagedIdentity: -parameterGroups: -- id: parameters - label: Parameters - parameters: - - id: AZURE_DIGITAL_TWINS_URL - label: Azure Digital Twins URL - valueType: string - envVar: AZURE_DIGITAL_TWINS_URL - - id: CSM_NUMBER_OF_THREAD - label: Number of thread used - valueType: int - options: - default: '1' - envVar: CSM_NUMBER_OF_THREAD diff --git a/Babylon/templates/working_dir/.templates/api/connector.storage.yaml b/Babylon/templates/working_dir/.templates/api/connector.storage.yaml deleted file mode 100644 index 89b425219..000000000 --- a/Babylon/templates/working_dir/.templates/api/connector.storage.yaml +++ /dev/null @@ -1,27 +0,0 @@ -key: "${key}" -name: "${name}" -description: Connector for Azure Storage -version: "${version}" -repository: cosmo-tech/azure-storage-simulator-connector -tags: -- Azure Storage -- Babylon -- "${tag}" -url: https://github.com/Cosmo-Tech/azure-storage-simulator-connector -ioTypes: -- read -azureAuthenticationWithCustomerAppRegistration: -azureManagedIdentity: -parameterGroups: -- id: parameters - label: Parameters - parameters: - - id: AZURE_STORAGE_CONNECTION_STRING - label: Azure Storage Connection String - valueType: string - default: "%STORAGE_CONNECTION_STRING%" - envVar: AZURE_STORAGE_CONNECTION_STRING - - id: AZURE_STORAGE_CONTAINER_BLOB_PREFIX - label: Azure Storage Path in the form container/path - valueType: string - envVar: AZURE_STORAGE_CONTAINER_BLOB_PREFIX diff --git a/Babylon/templates/working_dir/.templates/api/connector.twin.yaml b/Babylon/templates/working_dir/.templates/api/connector.twin.yaml deleted file mode 100644 index af5bc04ab..000000000 --- a/Babylon/templates/working_dir/.templates/api/connector.twin.yaml +++ /dev/null @@ -1,19 +0,0 @@ -key: ${key} -name: ${name} -description: Connector for TwinCache -version: ${version} -repository: cosmo-tech/twincache-connector -tags: -- TwinCache -- Babylon -- ${tag} -ioTypes: -- read -parameterGroups: -- id: parameters - label: Parameters - parameters: - - id: TWIN_CACHE_NAME - label: TwinCache entity name - valueType: string - envVar: TWIN_CACHE_NAME diff --git a/Babylon/templates/working_dir/.templates/api/dataset.adt.yaml b/Babylon/templates/working_dir/.templates/api/dataset.adt.yaml deleted file mode 100644 index f0e48280e..000000000 --- a/Babylon/templates/working_dir/.templates/api/dataset.adt.yaml +++ /dev/null @@ -1,13 +0,0 @@ -name: "${name}" -description: ${tag} reference model in ADT -tags: -- ADT -- Babylon -- Reference -- dataset -- ${tag} -connector: - id: "${connector_id}" - name: "Azure ADT Connector" - parametersValues: - AZURE_DIGITAL_TWINS_URL: "${cosmotech['adt']['digital_twin_url']}" \ No newline at end of file diff --git a/Babylon/templates/working_dir/.templates/api/dataset.twin.yaml b/Babylon/templates/working_dir/.templates/api/dataset.twin.yaml deleted file mode 100644 index 0983db018..000000000 --- a/Babylon/templates/working_dir/.templates/api/dataset.twin.yaml +++ /dev/null @@ -1,11 +0,0 @@ -name: "${name}" -description: "${tag} reference model in ADT" -tags: -- Babylon -- dataset -- "${tag}" -connector: - id: "${connector_id}" - parametersValues: - TWIN_CACHE_NAME: ${cosmotech['api']['connector']['twin_cache_name']} - LOG_LEVEL: 'DEBUG' \ No newline at end of file diff --git a/Babylon/templates/working_dir/.templates/yaml/Dataset.yaml b/Babylon/templates/working_dir/.templates/yaml/Dataset.yaml new file mode 100644 index 000000000..11fb14106 --- /dev/null +++ b/Babylon/templates/working_dir/.templates/yaml/Dataset.yaml @@ -0,0 +1,8 @@ +kind: Dataset +namespace: + platform: {{platform}} + remote: false +spec: + sidecars: + payload: + # TODO: waiting for the implementation of the dataset \ No newline at end of file diff --git a/Babylon/templates/working_dir/.templates/yaml/Organization.yaml b/Babylon/templates/working_dir/.templates/yaml/Organization.yaml new file mode 100644 index 000000000..2574a79c2 --- /dev/null +++ b/Babylon/templates/working_dir/.templates/yaml/Organization.yaml @@ -0,0 +1,9 @@ +kind: Organization +namespace: + platform: {{platform}} + remote: false +spec: + sidecars: + payload: + name: "{{organization_name}}" + security: {{security}} \ No newline at end of file diff --git a/Babylon/templates/working_dir/.templates/yaml/Runner.yaml b/Babylon/templates/working_dir/.templates/yaml/Runner.yaml new file mode 100644 index 000000000..3ea3947e1 --- /dev/null +++ b/Babylon/templates/working_dir/.templates/yaml/Runner.yaml @@ -0,0 +1,14 @@ +kind: Runner +namespace: + platform: {{platform}} + remote: false +spec: + sidecars: + payload: + name: "{{runner_name}}" + runTemplateId: "{{run_template_id}}" + solutionId: "{{services['api.solution_id']}}" + solutionName: "{{solution_name}}" + runTemplateName: "{{runTemplate_name}}" + ownerName: "{{owner_name}}" + security: {{security}} \ No newline at end of file diff --git a/Babylon/templates/working_dir/.templates/yaml/Solution.yaml b/Babylon/templates/working_dir/.templates/yaml/Solution.yaml new file mode 100644 index 000000000..c34ece39b --- /dev/null +++ b/Babylon/templates/working_dir/.templates/yaml/Solution.yaml @@ -0,0 +1,26 @@ +kind: Solution +namespace: + platform: {{platform}} + remote: false +metadata: + workspace_key: "{{workspace_key}}" + selector: + organization_id: "{{services['api.organization_id']}}" +spec: + sidecars: + payload: + key: "{{solution_key}}" + name: "{{solution_name}}" + description: "{{solution_description}}" + repository: cosmotech/brewerysamplesolution_simulator + version: latest + tags: + - brewery + runTemplates: + - id: "run_id" + name: "{{runTemplate_name}}" + csmSimulation: AzureWebApp/AzureWebApp_Simulation + run: true + preRun: true + parameterGroups: + security: {{security}} \ No newline at end of file diff --git a/Babylon/templates/working_dir/.templates/yaml/Workspace.yaml b/Babylon/templates/working_dir/.templates/yaml/Workspace.yaml new file mode 100644 index 000000000..347a7be74 --- /dev/null +++ b/Babylon/templates/working_dir/.templates/yaml/Workspace.yaml @@ -0,0 +1,24 @@ +kind: Workspace +namespace: + platform: {{platform}} + remote: false +spec: + sidecars: + payload: + key: "{{workspace_key}}" + name: "{{workspace_name}}" + description: "{{workspace_description}}" + solution: + solutionId: "{{services['api.solution_id']}}" + useDedicatedEventHubNamespace: true + sendScenarioMetadataToEventHub: true + sendInputToDataWarehouse: true + sendScenarioRunToEventHub: true + webApp: + url: "" + options: + charts: + dataSource: + datacontent: + menu: + security: {{security}} \ No newline at end of file diff --git a/Babylon/templates/working_dir/.templates/yaml/variables.yaml b/Babylon/templates/working_dir/.templates/yaml/variables.yaml new file mode 100644 index 000000000..0267925d7 --- /dev/null +++ b/Babylon/templates/working_dir/.templates/yaml/variables.yaml @@ -0,0 +1,31 @@ +# ========================================================= +# IMPORTANT: You can add variables here as needed! +# Make sure they are used in the manifest YAML. +# ========================================================= +vault_platform_id: sphinx +platform: + id: sphinx + url: https://warp.api.cosmotech.com/sphinx/v5 +location: westeurope +security: + default: none + accessControlList: + - id: mohcine.tor@cosmotech.com + role: admin +# Organization +organization_name: Babylon v5 Organization +# Solution +solution_name: Babylon v5 Solution +simulator_repository: cosmotech/brewerysamplesolution_simulator +solution_description: Brewery Testing babylon v5 Solution PLT +solution_key: breweryTesting +simulator_version: latest +# Workspace +workspace_name: Babylon v5 workspace +workspace_key: brewerytestingwork +workspace_description: Testing workspace for the brewery web application +# Runner +run_template_id: run_id +runner_name: Babylon v5 Runner +runTemplate_name: Standard simulation +owner_name: toto \ No newline at end of file diff --git a/Babylon/utils/environment.py b/Babylon/utils/environment.py index 6b517553f..40991b2b3 100644 --- a/Babylon/utils/environment.py +++ b/Babylon/utils/environment.py @@ -190,7 +190,7 @@ def set_server_id(self): except Exception as e: logger.error(e) - # This is deactivated for now because we need to decide where the Babylon state should be stored + # This is deactivated for now because we need to decide where the Babylon state should be stored # def set_blob_client(self): # try: # state = self.get_state_from_vault_by_platform(self.environ_id)