Mtor/improve code

.github/workflows/ci.yml

@@ -1,12 +1,6 @@
 name: Continuous Integration
-permissions:
-  contents: read
 
 env:
-  API_URL: ${{ secrets.API_URL }}
-  CLIENT_ID: ${{ secrets.CLIENT_ID }}
-  CLIENT_SECRET: ${{ secrets.CLIENT_SECRET }}
-  TOKEN_URL: ${{ secrets.TOKEN_URL }}
   STORAGE_NAME: ${{ secrets.STORAGE_NAME }}
   ACCOUNT_SECRET: ${{ secrets.ACCOUNT_SECRET }}
 
@@ -19,6 +13,9 @@ on:
     branches:
       - "main"
 
+permissions:
+  id-token: write
+  contents: read
 jobs:
   code-quality:
     runs-on: ubuntu-latest
@@ -57,6 +54,18 @@ jobs:
       - uses: astral-sh/setup-uv@v7
         with:
           enable-cache: true
+
+      - uses: azure/login@v2
+        with:
+          client-id: ${{ secrets.AZURE_CLIENT_ID }}
+          tenant-id: ${{ secrets.AZURE_TENANT_ID }}
+          subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+
+      - uses: azure/aks-set-context@v4
+        with:
+          cluster-name: "${{ vars.CLUSTER_NAME }}"
+          resource-group: "${{ vars.CLUSTER_RESOURCE_GROUP }}"
+
       - name: Install
         run: |
           uv venv

Babylon/commands/macro/deploy.py

@@ -1,3 +1,4 @@
+import subprocess
 from logging import getLogger
 
 from click import Abort, echo, style
@@ -9,8 +10,14 @@
 from cosmotech_api.models.workspace_security import WorkspaceSecurity
 from kubernetes import client, config
 
+from Babylon.utils.environment import Environment
+
 logger = getLogger(__name__)
 
+env = Environment()
+
+# Helper functions for workspace deployment
+
 
 def validate_inclusion_exclusion(
     include: tuple[str],
@@ -137,20 +144,49 @@ def update_object_security(
             logger.error(f"  [bold red]✘[/bold red] Failed to delete access control for id [magenta]{entry_id}[/magenta]: {e}")
 
 
+# Helper functions for workspace deployment
+
+
+def get_postgres_service_host(namespace: str) -> str:
+    """Discovers the PostgreSQL service name in a namespace to build its FQDN
+
+    Note: This function assumes PostgreSQL is running within the same Kubernetes cluster.
+    External database clusters are not currently supported.
+    """
+    try:
+        config.load_kube_config()
+        v1 = client.CoreV1Api()
+        services = v1.list_namespaced_service(namespace)
+
+        for svc in services.items:
+            if "postgresql" in svc.metadata.name or svc.metadata.labels.get("app.kubernetes.io/name") == "postgresql":
+                logger.info(f"  [dim]→ Found PostgreSQL service {svc.metadata.name}[/dim]")
+                return f"{svc.metadata.name}.{namespace}.svc.cluster.local"
+
+        return f"postgresql.{namespace}.svc.cluster.local"
+    except Exception as e:
+        logger.warning("  [bold yellow]⚠[/bold yellow] Service discovery failed ! default will be used.")
+        logger.debug(f"  Exception details: {e}", exc_info=True)
+        return f"postgresql.{namespace}.svc.cluster.local"
+
+
+# Helper functions for web application deployment
+
+
 def dict_to_tfvars(payload: dict) -> str:
     """Convert a dictionary to Terraform HCL tfvars format (key = "value").
-    
+
     Currently handles simple data structures:
     - Booleans: converted to lowercase (true/false)
     - Numbers: integers and floats as-is
     - Strings: wrapped in double quotes
-    
+
     Note: Complex nested structures (lists, dicts) are not yet supported.
     This is sufficient for current WebApp tfvars which only use simple scalar values.
-    
+
     Args:
         payload (dict): Dictionary with simple key-value pairs
-        
+
     Returns:
         str: Terraform HCL formatted variable assignments
     """
@@ -165,24 +201,51 @@ def dict_to_tfvars(payload: dict) -> str:
     return "\n".join(lines)
 
 
-def get_postgres_service_host(namespace: str) -> str:
-    """Discovers the PostgreSQL service name in a namespace to build its FQDN
-
-    Note: This function assumes PostgreSQL is running within the same Kubernetes cluster.
-    External database clusters are not currently supported.
-    """
+def _run_terraform_process(executable, cwd, payload, state):
+    """Helper function to reduce the size of the main function (Clean Code)"""
     try:
-        config.load_kube_config()
-        v1 = client.CoreV1Api()
-        services = v1.list_namespaced_service(namespace)
-
-        for svc in services.items:
-            if "postgresql" in svc.metadata.name or svc.metadata.labels.get("app.kubernetes.io/name") == "postgresql":
-                logger.info(f"  [dim]→ Found PostgreSQL service {svc.metadata.name}[/dim]")
-                return f"{svc.metadata.name}.{namespace}.svc.cluster.local"
+        process = subprocess.Popen(executable, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, bufsize=1)
+
+        # Color mapping to avoid if/else statements in the loop
+        status_colors = {
+            "Initializing": "white",
+            "Upgrading": "white",
+            "Finding": "white",
+            "Refreshing": "white",
+            "Success": "green",
+            "complete": "green",
+            "Resources:": "green",
+            "Error": "red",
+            "error": "red",
+        }
+
+        for line in process.stdout:
+            clean_line = line.strip()
+            if not clean_line:
+                continue
+
+            color = next((status_colors[k] for k in status_colors if k in clean_line), "white")
+            is_bold = color == "red"
+            echo(style(f"   {clean_line}", fg=color, bold=is_bold))
+
+        if process.wait() == 0:
+            _finalize_deployment(payload, state)
+        else:
+            logger.error("  [bold red]✘[/bold red] Deployment failed")
 
-        return f"postgresql.{namespace}.svc.cluster.local"
     except Exception as e:
-        logger.warning("  [bold yellow]⚠[/bold yellow] Service discovery failed ! default will be used.")
-        logger.debug(f"  Exception details: {e}", exc_info=True)
-        return f"postgresql.{namespace}.svc.cluster.local"
+        logger.error(f"  [bold red]✘[/bold red] Execution error: {e}")
+
+
+def _finalize_deployment(payload, state):
+    """Handles the update of the final state"""
+    webapp_name = payload.get("webapp_name")
+    url = f"https://{payload.get('cluster_domain')}/tenant-{payload.get('tenant')}/webapp-{webapp_name}"
+
+    services = state.setdefault("services", {})
+    services["webapp"] = {"webapp_name": f"webapp-{webapp_name}", "webapp_url": url}
+
+    logger.info(f"  [bold green]✔[/bold green] WebApp [bold white]{webapp_name}[/bold white] deployed")
+    env.store_state_in_local(state)
+    if env.remote:
+        env.store_state_in_cloud(state)

Babylon/commands/macro/deploy_webapp.py

@@ -1,11 +1,11 @@
 import os
-import subprocess
 import sys
 from logging import getLogger
+from re import MULTILINE, sub
 
 from click import echo, style
 
-from Babylon.commands.macro.deploy import dict_to_tfvars
+from Babylon.commands.macro.deploy import _run_terraform_process, dict_to_tfvars
 from Babylon.utils.environment import Environment
 
 logger = getLogger(__name__)
@@ -19,64 +19,44 @@ def deploy_webapp(namespace: str, file_content: str):
     state = env.retrieve_state_func()
     content = env.fill_template(data=file_content, state=state)
     payload: dict = content.get("spec").get("payload", {})
-    current_os = sys.platform
     tf_dir = env.working_dir.template_path.parent / "terraform-webapp"
     tfvars_path = tf_dir / "terraform.tfvars"
 
-    if current_os == "win32":
-        script_name = "_run-terraform.ps1"
-        executable = ["powershell.exe", "-ExecutionPolicy", "Bypass", "-File", f"./{script_name}"]
-    elif current_os == "linux":
-        script_name = "_run-terraform.sh"
-        executable = ["/bin/bash", f"./{script_name}"]
-        if (tf_dir / script_name).exists():
-            os.chmod(tf_dir / script_name, 0o755)
-    else:
-        raise RuntimeError(f"  Unsupported operating system: {current_os}")
-
+    OS_CONFIGS = {
+        "win32": {
+            "script": "_run-terraform.ps1",
+            "exec": lambda s: ["powershell.exe", "-ExecutionPolicy", "Bypass", "-File", f"./{s}"],
+        },
+        "linux": {"script": "_run-terraform.sh", "exec": lambda s: ["/bin/bash", f"./{s}"]},
+    }
+    config = OS_CONFIGS.get(sys.platform)
+    if not config:
+        raise RuntimeError(f"Unsupported operating system: {sys.platform}")
+
+    script_name = config["script"]
     script_path = tf_dir / script_name
+    executable = config["exec"](script_name)
 
     if not script_path.exists():
-        logger.error(f"  [bold red]✘[/bold red]Script not found at {script_path}")
+        logger.error(f"  [bold red]✘[/bold red] Script not found at {script_path}")
+        return
+    try:
+        content = script_path.read_text()
+        updated_content = sub(r"^#\s*(terraform apply.*)", r"\1", content, flags=MULTILINE)
+
+        if content != updated_content:
+            script_path.write_text(updated_content)
+            if sys.platform == "linux":
+                os.chmod(script_path, 0o755)
+    except IOError as e:
+        logger.error(f"  [bold red]✘[/bold red] Script modification failed: {e}")
         return
+
     try:
-        hcl_content = dict_to_tfvars(payload)
-        with open(tfvars_path, "w") as f:
-            f.write(hcl_content)
+        tfvars_path.write_text(dict_to_tfvars(payload))
     except Exception as e:
         logger.error(f"  [bold red]✘[/bold red] Failed to write tfvars: {e}")
         return
+
     logger.info("  [dim]→ Running Terraform deployment...[/dim]")
-    try:
-        process = subprocess.Popen(executable, cwd=tf_dir, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, bufsize=1)
-        for line in process.stdout:
-            clean_line = line.strip()
-            if not clean_line:
-                continue
-            if any(key in clean_line for key in ["Initializing", "Upgrading", "Finding", "Refreshing"]):
-                echo(style(f"   {clean_line}", fg="white"))
-            elif any(key in clean_line for key in ["Success", "complete", "Resources:"]):
-                echo(style(f"   {clean_line}", fg="green"))
-            elif "Error" in clean_line or "error" in clean_line:
-                echo(style(f"   {clean_line}", fg="red", bold=True))
-            else:
-                echo(style(f"   {clean_line}", fg="white"))
-        return_code = process.wait()
-        webapp_name = payload.get("webapp_name")
-        cluster_domain = payload.get("cluster_domain")
-        tenant_name = payload.get("tenant")
-        webapp_url = f"https://{cluster_domain}/{tenant_name}/webapp-{webapp_name}"
-        services = state.get("services", {})
-        if "webapp" not in services:
-            services["webapp"] = {}
-        services["webapp"]["webapp_name"] = f"webapp-{webapp_name}"
-        services["webapp"]["webapp_url"] = webapp_url
-        if return_code == 0:
-            logger.info(f"  [bold green]✔[/bold green] WebApp [bold white]{webapp_name}[/bold white] deployed successfully")
-            env.store_state_in_local(state)
-            if env.remote:
-                env.store_state_in_cloud(state)
-        else:
-            logger.error(f"  [bold red]✘[/bold red] Deployment failed with exit code {return_code}")
-    except Exception as e:
-        logger.error(f"  [bold red]✘[/bold red] Execution error: {e}")
+    _run_terraform_process(executable, tf_dir, payload, state)

Babylon/commands/macro/destroy.py

@@ -22,7 +22,7 @@
 env = Environment()
 
 
-def _destroy_schema(schema_name: str, state: dict) -> bool:
+def _destroy_schema(schema_name: str, state: dict) -> None:
     """
     Destroy PostgreSQL schema for a workspace.
     """
@@ -134,18 +134,18 @@ def _destroy_webapp(state: dict):
             text=True,
             bufsize=1,
         )
-        
+
         line_handlers = {
             "Destroy complete!": "green",
             "Resources:": "green",
             "Error": "red",
         }
-        
+
         for line in process.stdout:
             clean_line = line.strip()
             if not clean_line:
                 continue
-            
+
             color = next((color for key, color in line_handlers.items() if key in clean_line), "white")
             bold = color == "red"
             echo(style(f"   {clean_line}", fg=color, bold=bold))

Babylon/templates/working_dir/.templates/yaml/Solution.yaml

@@ -10,9 +10,9 @@ spec:
     key: "{{solution_key}}"
     name: "{{solution_name}}"
     description: "{{solution_description}}"
-    repository: cosmotech/brewerysamplesolution_simulator
-    version: latest
-    sdkVersion: '10.4.0'
+    repository: "{{simulator_repository}}"
+    version: "{{simulator_version}}"
+    sdkVersion: '12.0.0'
     alwaysPull: true
     url: 'https://webapp-url.com'
     tags:

Babylon/templates/working_dir/.templates/yaml/Workspace.yaml

@@ -5,7 +5,7 @@ spec:
   sidecars:
     postgres:
         schema:
-          create: false
+          create: true
           jobs:
             - name: k8s_job.yaml
               path: postgres/jobs
@@ -20,7 +20,7 @@ spec:
     sendInputToDataWarehouse: true
     sendScenarioRunToEventHub: true
     additionalData:
-      webApp:
+      webapp:
         solution:
           runTemplateFilter:
           defaultRunTemplateDataset: null
@@ -36,5 +36,6 @@ spec:
           supportUrl: 'https://support.cosmotech.com'
           organizationUrl: 'https://cosmotech.com'
           documentationUrl: 'https://portal.cosmotech.com/resources/platform-resources/web-app-user-guide'
-        datasetManager:
+        datasetManager: 'removeToDisableDatasetManager'
+        datasourceFilter: []
     security: {{security}}

Babylon/templates/working_dir/.templates/yaml/variables.yaml

@@ -6,18 +6,18 @@
 organization_name: Babylon v5 Organization
 # Solution
 solution_name: Babylon v5 Solution
-simulator_repository: cosmotech/brewerysamplesolution_simulator
+simulator_repository: tenant-sphinx/brewerysamplesolution_simulator
 solution_description: Brewery Testing babylon v5 Solution PLT
 solution_key: breweryTesting
-simulator_version: latest
+simulator_version: 3.0.0
 # Workspace
 workspace_name: Babylon v5 workspace
 workspace_key: brewerytestingwork
 workspace_description: Testing workspace for the brewery web application
 # Webapp
 cloud_provider: azure
-cluster_name: warpvy52ww
-cluster_domain: warp.api.cosmotech.com
+cluster_name: aks-dev-luxor
+cluster_domain: aks-dev-luxor.azure.platform.cosmotech.com
 tenant: sphinx
 webapp_name: business
 organization_id: o-xxxxxxxxxxx

Babylon/utils/environment.py

@@ -48,7 +48,7 @@ def __call__(cls, *args, **kwargs):
 class Environment(metaclass=SingletonMeta):
     # Azure Blob Storage configuration
     STATE_CONTAINER = "babylon-states"
-    
+
     def __init__(self):
         self.remote = False
         self.pwd = Path.cwd()