diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1b8f33a5..cfcd1c1a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,12 +1,6 @@ name: Continuous Integration -permissions: - contents: read env: - API_URL: ${{ secrets.API_URL }} - CLIENT_ID: ${{ secrets.CLIENT_ID }} - CLIENT_SECRET: ${{ secrets.CLIENT_SECRET }} - TOKEN_URL: ${{ secrets.TOKEN_URL }} STORAGE_NAME: ${{ secrets.STORAGE_NAME }} ACCOUNT_SECRET: ${{ secrets.ACCOUNT_SECRET }} @@ -19,6 +13,9 @@ on: branches: - "main" +permissions: + id-token: write + contents: read jobs: code-quality: runs-on: ubuntu-latest @@ -57,6 +54,18 @@ jobs: - uses: astral-sh/setup-uv@v7 with: enable-cache: true + + - uses: azure/login@v2 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + + - uses: azure/aks-set-context@v4 + with: + cluster-name: "${{ vars.CLUSTER_NAME }}" + resource-group: "${{ vars.CLUSTER_RESOURCE_GROUP }}" + - name: Install run: | uv venv diff --git a/Babylon/commands/macro/deploy.py b/Babylon/commands/macro/deploy.py index 3c1ee06a..26c801ad 100644 --- a/Babylon/commands/macro/deploy.py +++ b/Babylon/commands/macro/deploy.py @@ -1,3 +1,4 @@ +import subprocess from logging import getLogger from click import Abort, echo, style @@ -9,8 +10,14 @@ from cosmotech_api.models.workspace_security import WorkspaceSecurity from kubernetes import client, config +from Babylon.utils.environment import Environment + logger = getLogger(__name__) +env = Environment() + +# Helper functions for workspace deployment + def validate_inclusion_exclusion( include: tuple[str], @@ -137,20 +144,49 @@ def update_object_security( logger.error(f" [bold red]✘[/bold red] Failed to delete access control for id [magenta]{entry_id}[/magenta]: {e}") +# Helper functions for workspace deployment + + +def get_postgres_service_host(namespace: str) -> str: + """Discovers the PostgreSQL service name in a namespace to build its FQDN + + Note: This function assumes PostgreSQL is running within the same Kubernetes cluster. + External database clusters are not currently supported. + """ + try: + config.load_kube_config() + v1 = client.CoreV1Api() + services = v1.list_namespaced_service(namespace) + + for svc in services.items: + if "postgresql" in svc.metadata.name or svc.metadata.labels.get("app.kubernetes.io/name") == "postgresql": + logger.info(f" [dim]→ Found PostgreSQL service {svc.metadata.name}[/dim]") + return f"{svc.metadata.name}.{namespace}.svc.cluster.local" + + return f"postgresql.{namespace}.svc.cluster.local" + except Exception as e: + logger.warning(" [bold yellow]⚠[/bold yellow] Service discovery failed ! default will be used.") + logger.debug(f" Exception details: {e}", exc_info=True) + return f"postgresql.{namespace}.svc.cluster.local" + + +# Helper functions for web application deployment + + def dict_to_tfvars(payload: dict) -> str: """Convert a dictionary to Terraform HCL tfvars format (key = "value"). - + Currently handles simple data structures: - Booleans: converted to lowercase (true/false) - Numbers: integers and floats as-is - Strings: wrapped in double quotes - + Note: Complex nested structures (lists, dicts) are not yet supported. This is sufficient for current WebApp tfvars which only use simple scalar values. - + Args: payload (dict): Dictionary with simple key-value pairs - + Returns: str: Terraform HCL formatted variable assignments """ @@ -165,24 +201,51 @@ def dict_to_tfvars(payload: dict) -> str: return "\n".join(lines) -def get_postgres_service_host(namespace: str) -> str: - """Discovers the PostgreSQL service name in a namespace to build its FQDN - - Note: This function assumes PostgreSQL is running within the same Kubernetes cluster. - External database clusters are not currently supported. - """ +def _run_terraform_process(executable, cwd, payload, state): + """Helper function to reduce the size of the main function (Clean Code)""" try: - config.load_kube_config() - v1 = client.CoreV1Api() - services = v1.list_namespaced_service(namespace) - - for svc in services.items: - if "postgresql" in svc.metadata.name or svc.metadata.labels.get("app.kubernetes.io/name") == "postgresql": - logger.info(f" [dim]→ Found PostgreSQL service {svc.metadata.name}[/dim]") - return f"{svc.metadata.name}.{namespace}.svc.cluster.local" + process = subprocess.Popen(executable, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, bufsize=1) + + # Color mapping to avoid if/else statements in the loop + status_colors = { + "Initializing": "white", + "Upgrading": "white", + "Finding": "white", + "Refreshing": "white", + "Success": "green", + "complete": "green", + "Resources:": "green", + "Error": "red", + "error": "red", + } + + for line in process.stdout: + clean_line = line.strip() + if not clean_line: + continue + + color = next((status_colors[k] for k in status_colors if k in clean_line), "white") + is_bold = color == "red" + echo(style(f" {clean_line}", fg=color, bold=is_bold)) + + if process.wait() == 0: + _finalize_deployment(payload, state) + else: + logger.error(" [bold red]✘[/bold red] Deployment failed") - return f"postgresql.{namespace}.svc.cluster.local" except Exception as e: - logger.warning(" [bold yellow]⚠[/bold yellow] Service discovery failed ! default will be used.") - logger.debug(f" Exception details: {e}", exc_info=True) - return f"postgresql.{namespace}.svc.cluster.local" + logger.error(f" [bold red]✘[/bold red] Execution error: {e}") + + +def _finalize_deployment(payload, state): + """Handles the update of the final state""" + webapp_name = payload.get("webapp_name") + url = f"https://{payload.get('cluster_domain')}/tenant-{payload.get('tenant')}/webapp-{webapp_name}" + + services = state.setdefault("services", {}) + services["webapp"] = {"webapp_name": f"webapp-{webapp_name}", "webapp_url": url} + + logger.info(f" [bold green]✔[/bold green] WebApp [bold white]{webapp_name}[/bold white] deployed") + env.store_state_in_local(state) + if env.remote: + env.store_state_in_cloud(state) diff --git a/Babylon/commands/macro/deploy_webapp.py b/Babylon/commands/macro/deploy_webapp.py index fd5674b2..3b525d51 100644 --- a/Babylon/commands/macro/deploy_webapp.py +++ b/Babylon/commands/macro/deploy_webapp.py @@ -1,11 +1,11 @@ import os -import subprocess import sys from logging import getLogger +from re import MULTILINE, sub from click import echo, style -from Babylon.commands.macro.deploy import dict_to_tfvars +from Babylon.commands.macro.deploy import _run_terraform_process, dict_to_tfvars from Babylon.utils.environment import Environment logger = getLogger(__name__) @@ -19,64 +19,44 @@ def deploy_webapp(namespace: str, file_content: str): state = env.retrieve_state_func() content = env.fill_template(data=file_content, state=state) payload: dict = content.get("spec").get("payload", {}) - current_os = sys.platform tf_dir = env.working_dir.template_path.parent / "terraform-webapp" tfvars_path = tf_dir / "terraform.tfvars" - if current_os == "win32": - script_name = "_run-terraform.ps1" - executable = ["powershell.exe", "-ExecutionPolicy", "Bypass", "-File", f"./{script_name}"] - elif current_os == "linux": - script_name = "_run-terraform.sh" - executable = ["/bin/bash", f"./{script_name}"] - if (tf_dir / script_name).exists(): - os.chmod(tf_dir / script_name, 0o755) - else: - raise RuntimeError(f" Unsupported operating system: {current_os}") - + OS_CONFIGS = { + "win32": { + "script": "_run-terraform.ps1", + "exec": lambda s: ["powershell.exe", "-ExecutionPolicy", "Bypass", "-File", f"./{s}"], + }, + "linux": {"script": "_run-terraform.sh", "exec": lambda s: ["/bin/bash", f"./{s}"]}, + } + config = OS_CONFIGS.get(sys.platform) + if not config: + raise RuntimeError(f"Unsupported operating system: {sys.platform}") + + script_name = config["script"] script_path = tf_dir / script_name + executable = config["exec"](script_name) if not script_path.exists(): - logger.error(f" [bold red]✘[/bold red]Script not found at {script_path}") + logger.error(f" [bold red]✘[/bold red] Script not found at {script_path}") + return + try: + content = script_path.read_text() + updated_content = sub(r"^#\s*(terraform apply.*)", r"\1", content, flags=MULTILINE) + + if content != updated_content: + script_path.write_text(updated_content) + if sys.platform == "linux": + os.chmod(script_path, 0o755) + except IOError as e: + logger.error(f" [bold red]✘[/bold red] Script modification failed: {e}") return + try: - hcl_content = dict_to_tfvars(payload) - with open(tfvars_path, "w") as f: - f.write(hcl_content) + tfvars_path.write_text(dict_to_tfvars(payload)) except Exception as e: logger.error(f" [bold red]✘[/bold red] Failed to write tfvars: {e}") return + logger.info(" [dim]→ Running Terraform deployment...[/dim]") - try: - process = subprocess.Popen(executable, cwd=tf_dir, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, bufsize=1) - for line in process.stdout: - clean_line = line.strip() - if not clean_line: - continue - if any(key in clean_line for key in ["Initializing", "Upgrading", "Finding", "Refreshing"]): - echo(style(f" {clean_line}", fg="white")) - elif any(key in clean_line for key in ["Success", "complete", "Resources:"]): - echo(style(f" {clean_line}", fg="green")) - elif "Error" in clean_line or "error" in clean_line: - echo(style(f" {clean_line}", fg="red", bold=True)) - else: - echo(style(f" {clean_line}", fg="white")) - return_code = process.wait() - webapp_name = payload.get("webapp_name") - cluster_domain = payload.get("cluster_domain") - tenant_name = payload.get("tenant") - webapp_url = f"https://{cluster_domain}/{tenant_name}/webapp-{webapp_name}" - services = state.get("services", {}) - if "webapp" not in services: - services["webapp"] = {} - services["webapp"]["webapp_name"] = f"webapp-{webapp_name}" - services["webapp"]["webapp_url"] = webapp_url - if return_code == 0: - logger.info(f" [bold green]✔[/bold green] WebApp [bold white]{webapp_name}[/bold white] deployed successfully") - env.store_state_in_local(state) - if env.remote: - env.store_state_in_cloud(state) - else: - logger.error(f" [bold red]✘[/bold red] Deployment failed with exit code {return_code}") - except Exception as e: - logger.error(f" [bold red]✘[/bold red] Execution error: {e}") + _run_terraform_process(executable, tf_dir, payload, state) diff --git a/Babylon/commands/macro/destroy.py b/Babylon/commands/macro/destroy.py index fe065a7f..faacd6cc 100644 --- a/Babylon/commands/macro/destroy.py +++ b/Babylon/commands/macro/destroy.py @@ -22,7 +22,7 @@ env = Environment() -def _destroy_schema(schema_name: str, state: dict) -> bool: +def _destroy_schema(schema_name: str, state: dict) -> None: """ Destroy PostgreSQL schema for a workspace. """ @@ -134,18 +134,18 @@ def _destroy_webapp(state: dict): text=True, bufsize=1, ) - + line_handlers = { "Destroy complete!": "green", "Resources:": "green", "Error": "red", } - + for line in process.stdout: clean_line = line.strip() if not clean_line: continue - + color = next((color for key, color in line_handlers.items() if key in clean_line), "white") bold = color == "red" echo(style(f" {clean_line}", fg=color, bold=bold)) diff --git a/Babylon/templates/working_dir/.templates/yaml/Solution.yaml b/Babylon/templates/working_dir/.templates/yaml/Solution.yaml index a94a2a7c..476e07ab 100644 --- a/Babylon/templates/working_dir/.templates/yaml/Solution.yaml +++ b/Babylon/templates/working_dir/.templates/yaml/Solution.yaml @@ -10,9 +10,9 @@ spec: key: "{{solution_key}}" name: "{{solution_name}}" description: "{{solution_description}}" - repository: cosmotech/brewerysamplesolution_simulator - version: latest - sdkVersion: '10.4.0' + repository: "{{simulator_repository}}" + version: "{{simulator_version}}" + sdkVersion: '12.0.0' alwaysPull: true url: 'https://webapp-url.com' tags: diff --git a/Babylon/templates/working_dir/.templates/yaml/Workspace.yaml b/Babylon/templates/working_dir/.templates/yaml/Workspace.yaml index dca0dad4..b791fd67 100644 --- a/Babylon/templates/working_dir/.templates/yaml/Workspace.yaml +++ b/Babylon/templates/working_dir/.templates/yaml/Workspace.yaml @@ -5,7 +5,7 @@ spec: sidecars: postgres: schema: - create: false + create: true jobs: - name: k8s_job.yaml path: postgres/jobs @@ -20,7 +20,7 @@ spec: sendInputToDataWarehouse: true sendScenarioRunToEventHub: true additionalData: - webApp: + webapp: solution: runTemplateFilter: defaultRunTemplateDataset: null @@ -36,5 +36,6 @@ spec: supportUrl: 'https://support.cosmotech.com' organizationUrl: 'https://cosmotech.com' documentationUrl: 'https://portal.cosmotech.com/resources/platform-resources/web-app-user-guide' - datasetManager: + datasetManager: 'removeToDisableDatasetManager' + datasourceFilter: [] security: {{security}} \ No newline at end of file diff --git a/Babylon/templates/working_dir/.templates/yaml/variables.yaml b/Babylon/templates/working_dir/.templates/yaml/variables.yaml index 24b100fb..c77bb6d5 100644 --- a/Babylon/templates/working_dir/.templates/yaml/variables.yaml +++ b/Babylon/templates/working_dir/.templates/yaml/variables.yaml @@ -6,18 +6,18 @@ organization_name: Babylon v5 Organization # Solution solution_name: Babylon v5 Solution -simulator_repository: cosmotech/brewerysamplesolution_simulator +simulator_repository: tenant-sphinx/brewerysamplesolution_simulator solution_description: Brewery Testing babylon v5 Solution PLT solution_key: breweryTesting -simulator_version: latest +simulator_version: 3.0.0 # Workspace workspace_name: Babylon v5 workspace workspace_key: brewerytestingwork workspace_description: Testing workspace for the brewery web application # Webapp cloud_provider: azure -cluster_name: warpvy52ww -cluster_domain: warp.api.cosmotech.com +cluster_name: aks-dev-luxor +cluster_domain: aks-dev-luxor.azure.platform.cosmotech.com tenant: sphinx webapp_name: business organization_id: o-xxxxxxxxxxx diff --git a/Babylon/utils/environment.py b/Babylon/utils/environment.py index 5b005564..3ca0ab66 100644 --- a/Babylon/utils/environment.py +++ b/Babylon/utils/environment.py @@ -48,7 +48,7 @@ def __call__(cls, *args, **kwargs): class Environment(metaclass=SingletonMeta): # Azure Blob Storage configuration STATE_CONTAINER = "babylon-states" - + def __init__(self): self.remote = False self.pwd = Path.cwd() diff --git a/pyproject.toml b/pyproject.toml index 30898ffb..23822d16 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,7 @@ dependencies = [ "dynaconf", "azure-mgmt-authorization>=4.0.0", "kubernetes>=35.0.0", - "cosmotech-api==5.0.0rc5" + "cosmotech-api==5.0.0" ] dynamic = ["version"] readme = {file = "README.md", content-type = "text/markdown"} diff --git a/tests/e2e/test_e2e.sh b/tests/e2e/test_e2e.sh index 1f8a3c3a..19e5fd46 100755 --- a/tests/e2e/test_e2e.sh +++ b/tests/e2e/test_e2e.sh @@ -10,7 +10,7 @@ fi # Set testing namespace export CONTEXT="e2e" -export TENANT="sphinx" +export TENANT="tenant-sphinx" export STATE="teststate" babylon namespace use -c ${CONTEXT} -t ${TENANT} -s $STATE diff --git a/tests/integration/data/runner.yaml b/tests/integration/data/runner.yaml index d0216c0f..5f736f94 100644 --- a/tests/integration/data/runner.yaml +++ b/tests/integration/data/runner.yaml @@ -1,5 +1,5 @@ name: "Babylon testing runner" -runTemplateId: "minimal" +runTemplateId: "standalone" solutionId: "sol-7xgdvwq6959j" solutionName: "Updated description" runTemplateName: "Run template with Brewery parameters" diff --git a/tests/integration/data/solution.yaml b/tests/integration/data/solution.yaml index de18aa1c..94af01a4 100644 --- a/tests/integration/data/solution.yaml +++ b/tests/integration/data/solution.yaml @@ -1,13 +1,13 @@ # Solution Template key: testing name: Dev Brewery Solution -repository: cosmotech/brewerysamplesolution_simulator -version: latest +repository: tenant-sphinx/brewerysamplesolution_simulator +version: 3.0.0 alwaysPull: true parameters: [] parameterGroups: [] runTemplates: - - id: 'minimal' + - id: 'standalone' security: default: viewer accessControlList: diff --git a/tests/integration/test_api_endpoints.sh b/tests/integration/test_api_endpoints.sh index a229fa13..f5ec73d7 100755 --- a/tests/integration/test_api_endpoints.sh +++ b/tests/integration/test_api_endpoints.sh @@ -12,7 +12,7 @@ mkdir output # Set testing namespace export CONTEXT="integration" -export TENANT="sphinx" +export TENANT="tenant-sphinx" export STATE="teststate" babylon namespace use -c ${CONTEXT} -t ${TENANT} -s $STATE diff --git a/uv.lock b/uv.lock index c12634a1..76bf1c23 100644 --- a/uv.lock +++ b/uv.lock @@ -169,7 +169,7 @@ requires-dist = [ { name = "azure-storage-blob", specifier = ">=12.28.0" }, { name = "click" }, { name = "click-log" }, - { name = "cosmotech-api", specifier = "==5.0.0rc5" }, + { name = "cosmotech-api", specifier = "==5.0.0" }, { name = "cryptography" }, { name = "dynaconf" }, { name = "flatten-json" }, @@ -383,7 +383,7 @@ wheels = [ [[package]] name = "cosmotech-api" -version = "5.0.0rc5" +version = "5.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pydantic" }, @@ -391,9 +391,9 @@ dependencies = [ { name = "typing-extensions" }, { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/04/36/4f3635f3b7e75d6eaa6aa66ca72cb199011e74004d8227e514e64f80d85f/cosmotech_api-5.0.0rc5.tar.gz", hash = "sha256:73c76d8371fc41d8866ab514c242f4ae41956ea9eb41cd7a1974a3db8935a862", size = 213222, upload-time = "2026-01-20T10:14:21.209Z" } +sdist = { url = "https://files.pythonhosted.org/packages/85/07/4cfc51935a2cf9278f6f51d1fb59b402c26895dafc5923fdebe99f77afcb/cosmotech_api-5.0.0.tar.gz", hash = "sha256:182b6801a4a5008feeb7e20b367c5420005421978ef79e69edeacf7495456d5a", size = 219103, upload-time = "2026-02-23T15:50:44.042Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/59/ea/de404b4eb09248bb052ad97557bea8c57e7a71fa048ef316b65ad6f2ebf5/cosmotech_api-5.0.0rc5-py3-none-any.whl", hash = "sha256:2bb7d10f5d5533fd484698590ca3c6b7be1a43d6e1534f0ba067f4c9c27db367", size = 623676, upload-time = "2026-01-20T10:14:19.393Z" }, + { url = "https://files.pythonhosted.org/packages/d9/7e/1294914773dd6407395d22ae9734a68455443d55e526ceaff3d9a08b0b3c/cosmotech_api-5.0.0-py3-none-any.whl", hash = "sha256:1e41ea2c9ab09ea678e1861e5751c6976430989f51f34496dce4b8e1772c0b0f", size = 625091, upload-time = "2026-02-23T15:50:42.546Z" }, ] [[package]]