Mtor/prod 15615

.github/workflows/ci.yml

@@ -55,7 +55,7 @@ jobs:
         with:
           enable-cache: true
 
-      - uses: azure/login@v2
+      - uses: azure/login@v3
         with:
           client-id: ${{ secrets.AZURE_CLIENT_ID }}
           tenant-id: ${{ secrets.AZURE_TENANT_ID }}

Babylon/commands/macro/deploy.py

@@ -1,5 +1,7 @@
 import subprocess
+from base64 import b64encode
 from logging import getLogger
+from textwrap import dedent
 
 from click import Abort, echo, style
 from cosmotech_api.models.organization_access_control import OrganizationAccessControl
@@ -170,6 +172,161 @@
         return f"postgresql.{namespace}.svc.cluster.local"
 
 
+def create_workspace_secret(
+    namespace: str,
+    organization_id: str,
+    workspace_id: str,
+    writer_password: str,
+) -> bool:
+    """Create a Kubernetes Secret for a workspace containing API and PostgreSQL credentials.
+
+    The secret is named ``<organization_id>-<workspace_id>`` and holds all
+    environment variables required by workspace.
+
+    Returns:
+        bool: True if the secret was created or already exists, False on error.
+    """
+    secret_name = f"{organization_id}-{workspace_id}"
+    data = {
+        "POSTGRES_USER_PASSWORD": writer_password,
+    }
+    encoded_data = {k: b64encode(v.encode("utf-8")).decode("utf-8") for k, v in data.items()}
+
+    secret = client.V1Secret(
+        api_version="v1",
+        kind="Secret",
+        metadata=client.V1ObjectMeta(name=secret_name, namespace=namespace),
+        type="Opaque",
+        data=encoded_data,
+    )
+
+    try:
+        config.load_kube_config()
+        v1 = client.CoreV1Api()
+        v1.create_namespaced_secret(namespace=namespace, body=secret)
+        logger.info(f"  [bold green]✔[/bold green] Secret [magenta]{secret_name}[/magenta] created")
+        return True
+    except client.ApiException as e:
+        if e.status == 409:
+            logger.warning(f"  [yellow]⚠[/yellow] [dim]Secret [magenta]{secret_name}[/magenta] already exists[/dim]")
+            return True
+        logger.error(f"  [bold red]✘[/bold red] Failed to create secret {secret_name}: {e.reason}")
+        return False
+    except Exception as e:
+        logger.error(f"  [bold red]✘[/bold red] Unexpected error creating secret {secret_name}")
+        logger.debug(f"  Detail: {e}", exc_info=True)
+        return False
+
+
+def create_coal_configmap(
+    namespace: str,
+    organization_id: str,
+    workspace_id: str,
+    db_host: str,
+    db_port: str,
+    db_name: str,
+    schema_name: str,
+    writer_username: str,
+) -> bool:
+    """Create a CoAL ConfigMap for a workspace.
+
+    The ConfigMap is named ``<organization_id>-<workspace_id>-coal-config`` and
+    contains a ``coal-config.toml`` key with Postgres output configuration.  The
+    ``user_password`` value is deliberately set to the literal string
+    ``env.POSTGRES_USER_PASSWORD`` so that the CoAL runtime resolves it from the
+    environment at execution time.
+
+    Returns:
+        bool: True if the ConfigMap was created or already exists, False on error.
+    """
+    configmap_name = f"{organization_id}-{workspace_id}-coal-config"
+    coal_toml = dedent(f"""\
+        [[outputs]]
+        type = "postgres"
+        [outputs.conf.postgres]
+        host = "{db_host}"
+        port = "{db_port}"
+        db_name = "{db_name}"
+        db_schema = "{schema_name}"
+        user_name = "{writer_username}"
+        user_password = "env.POSTGRES_USER_PASSWORD"
+    """)
+
+    configmap = client.V1ConfigMap(
+        api_version="v1",
+        kind="ConfigMap",
+        metadata=client.V1ObjectMeta(name=configmap_name, namespace=namespace),
+        data={"coal-config.toml": coal_toml},
+    )
+
+    try:
+        config.load_kube_config()
+        v1 = client.CoreV1Api()
+        v1.create_namespaced_config_map(namespace=namespace, body=configmap)
+        logger.info(f"  [bold green]✔[/bold green] ConfigMap [magenta]{configmap_name}[/magenta] created")
+        return True
+    except client.ApiException as e:
+        if e.status == 409:
+            logger.warning(f"  [yellow]⚠[/yellow] [dim]ConfigMap [magenta]{configmap_name}[/magenta] already exists[/dim]")
+            return True
+        logger.error(f"  [bold red]✘[/bold red] Failed to create ConfigMap {configmap_name}: {e.reason}")
+        return False
+    except Exception as e:
+        logger.error(f"  [bold red]✘[/bold red] Unexpected error creating ConfigMap {configmap_name}")
+        logger.debug(f"  Detail: {e}", exc_info=True)
+        return False
+
+
+def delete_kubernetes_resources(namespace: str, organization_id: str, workspace_id: str) -> None:
+    """Delete the Workspace Secret and CoAL ConfigMap created during deployment.
+
+    Targets:
+      - Secret:    ``<organization_id>-<workspace_id>``
+      - ConfigMap: ``<organization_id>-<workspace_id>-coal-config``
+
+    If a resource is already gone (404), a warning is logged and execution
+    continues without error.
+    """
+    secret_name = f"{organization_id}-{workspace_id}"
+    configmap_name = f"{organization_id}-{workspace_id}-coal-config"
+
+    try:
+        config.load_kube_config()
+        v1 = client.CoreV1Api()
+    except Exception as e:
+        logger.error("  [bold red]✘[/bold red] Failed to initialise Kubernetes client")
+        logger.debug(f"  Detail: {e}", exc_info=True)
+        return
+
+    # --- Delete Secret ---
+    try:
+        logger.info("  [dim]→ Deleting workspace Secret ...[/dim]")
+        v1.delete_namespaced_secret(name=secret_name, namespace=namespace)
+        logger.info(f"  [bold green]✔[/bold green] Secret [magenta]{secret_name}[/magenta] deleted")
+    except client.ApiException as e:
+        if e.status == 404:
+            logger.warning("  [yellow]⚠[/yellow] [dim]Secret not found (already deleted)[/dim]")
+        else:
+            logger.error(f"  [bold red]✘[/bold red] Failed to delete secret {secret_name}: {e.reason}")
+    except Exception as e:
+        logger.error(f"  [bold red]✘[/bold red] Unexpected error deleting secret {secret_name}")
+        logger.debug(f"  Detail: {e}", exc_info=True)
+
+    # --- Delete ConfigMap ---
+    try:
+        logger.info("  [dim]→ Deleting workspace ConfigMap ...[/dim]")
+        v1.delete_namespaced_config_map(name=configmap_name, namespace=namespace)
+        logger.info(f"  [bold green]✔[/bold green] ConfigMap [magenta]{configmap_name}[/magenta] deleted")
+    except client.ApiException as e:
+        if e.status == 404:
+            logger.warning("  [yellow]⚠[/yellow] [dim]ConfigMap not found (already deleted)[/dim]")
+        else:
+            logger.error(f"  [bold red]✘[/bold red] Failed to delete ConfigMap {configmap_name}: {e.reason}")
+    except Exception as e:
+        logger.error(f"  [bold red]✘[/bold red] Unexpected error deleting ConfigMap {configmap_name}")
+        logger.debug(f"  Detail: {e}", exc_info=True)
+
+
 # Helper functions for web application deployment
 
 
@@ -240,7 +397,7 @@
 def _finalize_deployment(payload, state):
     """Handles the update of the final state"""
     webapp_name = payload.get("webapp_name")
-    url = f"https://{payload.get('cluster_domain')}/tenant-{payload.get('tenant')}/webapp-{webapp_name}"
+    url = f"https://{payload.get('cluster_name')}.{payload.get('domain_zone')}/tenant-{payload.get('tenant')}/webapp-{webapp_name}"
 
     services = state.setdefault("services", {})
     services["webapp"] = {"webapp_name": f"webapp-{webapp_name}", "webapp_url": url}

Babylon/commands/macro/deploy_workspace.py

@@ -14,7 +14,12 @@
 from yaml import safe_load
 
 from Babylon.commands.api.workspace import get_workspace_api_instance
-from Babylon.commands.macro.deploy import get_postgres_service_host, update_object_security
+from Babylon.commands.macro.deploy import (
+    create_coal_configmap,
+    create_workspace_secret,
+    get_postgres_service_host,
+    update_object_security,
+)
 from Babylon.utils.credentials import get_keycloak_token
 from Babylon.utils.environment import Environment
 from Babylon.utils.response import CommandResponse
@@ -93,7 +98,7 @@ def deploy_workspace(namespace: str, file_content: str, deploy_dir: PathlibPath)
     should_create_schema = schema_config.get("create", False)
     if should_create_schema:
         db_host = get_postgres_service_host(env.environ_id)
-        logger.info(f"  [dim]→ Initializing PostgreSQL schema for workspace {workspace_id}...[/dim]")
+        logger.info(f"  [dim]→ Initializing PostgreSQL schema for workspace [bold cyan]{workspace_id}[/bold cyan]...[/dim]")
         pg_config = env.get_config_from_k8s_secret_by_tenant("postgresql-config", env.environ_id)
         api_config = env.get_config_from_k8s_secret_by_tenant("postgresql-cosmotechapi", env.environ_id)
         if pg_config and api_config:
@@ -102,11 +107,11 @@ def deploy_workspace(namespace: str, file_content: str, deploy_dir: PathlibPath)
                 "namespace": env.environ_id,
                 "db_host": db_host,
                 "db_port": "5432",
-                "cosmotech_api_database": api_config.get("database-name"),
-                "cosmotech_api_admin_username": api_config.get("admin-username"),
-                "cosmotech_api_admin_password": api_config.get("admin-password"),
-                "cosmotech_api_writer_username": api_config.get("writer-username"),
-                "cosmotech_api_reader_username": api_config.get("reader-username"),
+                "cosmotech_api_database": api_config.get("database-name", ""),
+                "cosmotech_api_admin_username": api_config.get("admin-username", ""),
+                "cosmotech_api_admin_password": api_config.get("admin-password", ""),
+                "cosmotech_api_writer_username": api_config.get("writer-username", ""),
+                "cosmotech_api_reader_username": api_config.get("reader-username", ""),
                 "workspace_schema": schema_name,
                 "job_name": workspace_id,
             }
@@ -187,6 +192,32 @@ def deploy_workspace(namespace: str, file_content: str, deploy_dir: PathlibPath)
                         logger.error("  [bold red]✘[/bold red] Unexpected error please check babylon logs file for details")
                         logger.debug(f"  [bold red]✘[/bold red] {e}")
 
+            # --- Workspace Secret & CoAL ConfigMap ---
+            organization_id = api_section["organization_id"]
+            writer_username = api_config.get("writer-username", "")
+            writer_password = api_config.get("writer-password", "")
+            db_name = api_config.get("database-name", "")
+
+            logger.info(f"  [dim]→ Creating workspace secret for [cyan]{workspace_id}[/cyan]...[/dim]")
+            create_workspace_secret(
+                namespace=env.environ_id,
+                organization_id=organization_id,
+                workspace_id=workspace_id,
+                writer_password=writer_password,
+            )
+
+            logger.info(f"  [dim]→ Creating CoAL ConfigMap for [cyan]{workspace_id}[/cyan]...[/dim]")
+            create_coal_configmap(
+                namespace=env.environ_id,
+                organization_id=organization_id,
+                workspace_id=workspace_id,
+                db_host=db_host,
+                db_port="5432",
+                db_name=db_name,
+                schema_name=schema_name,
+                writer_username=writer_username,
+            )
+
     # --- State Persistence ---
     # Ensure the local and remote states are synchronized after successful API calls
     env.store_state_in_local(state)

Babylon/commands/macro/destroy.py

@@ -12,7 +12,7 @@
 from Babylon.commands.api.organization import get_organization_api_instance
 from Babylon.commands.api.solution import get_solution_api_instance
 from Babylon.commands.api.workspace import get_workspace_api_instance
-from Babylon.commands.macro.deploy import get_postgres_service_host, resolve_inclusion_exclusion
+from Babylon.commands.macro.deploy import delete_kubernetes_resources, get_postgres_service_host, resolve_inclusion_exclusion
 from Babylon.utils.credentials import get_keycloak_token
 from Babylon.utils.decorators import injectcontext, retrieve_state
 from Babylon.utils.environment import Environment
@@ -31,7 +31,7 @@ def _destroy_schema(schema_name: str, state: dict) -> None:
         return
     workspace_id_tmp = f"{schema_name.replace('_', '-')}"
     db_host = get_postgres_service_host(env.environ_id)
-    logger.info(f"  [dim]→ Destroying postgreSQL schema for workspace {workspace_id_tmp}...[/dim]")
+    logger.info(f"  [dim]→ Destroying postgreSQL schema for workspace [bold cyan]{workspace_id_tmp}[/bold cyan]...[/dim]")
 
     pg_config = env.get_config_from_k8s_secret_by_tenant("postgresql-config", env.environ_id)
     api_config = env.get_config_from_k8s_secret_by_tenant("postgresql-cosmotechapi", env.environ_id)
@@ -212,6 +212,11 @@ def destroy(state: dict, include: tuple[str], exclude: tuple[str]):
 
     if workspace:
         _destroy_schema(schema_state["schema_name"], state)
+        delete_kubernetes_resources(
+            namespace=env.environ_id,
+            organization_id=org_id,
+            workspace_id=api_state["workspace_id"],
+        )
         api = get_workspace_api_instance(config=config, keycloak_token=keycloak_token)
         _delete_resource(api.delete_workspace, "Workspace", org_id, api_state["workspace_id"], state, "workspace_id")
 

Babylon/templates/working_dir/.templates/yaml/Webapp.yaml

@@ -5,7 +5,7 @@ spec:
   payload:
     cloud_provider: "azure"
     cluster_name: "{{cluster_name}}"
-    cluster_domain: "{{cluster_domain}}"
+    domain_zone: "{{domain_zone}}"
     tenant: "{{tenant}}"
     webapp_name: "{{webapp_name}}"
     organization_id: "{{services['api.organization_id']}}"

Babylon/templates/working_dir/.templates/yaml/variables.yaml

@@ -17,7 +17,7 @@ workspace_description: Testing workspace for the brewery web application
 # Webapp
 cloud_provider: azure
 cluster_name: aks-dev-luxor
-cluster_domain: aks-dev-luxor.azure.platform.cosmotech.com
+domain_zone: azure.platform.cosmotech.com
 tenant: sphinx
 webapp_name: business
 organization_id: o-xxxxxxxxxxx

Babylon/utils/environment.py

@@ -156,10 +156,43 @@ def get_config_from_k8s_secret_by_tenant(self, secret_name: str, tenant: str):
         except ApiException:
             logger.error("\n  [bold red]✘[/bold red] Resource Not Found")
             logger.error(f"  Secret [green]{secret_name}[/green] could not be found in namespace [green]{tenant}[/green]")
+
+            # Show current kubectl context to help users spot a misconfiguration
+            try:
+                from kubernetes.config.kube_config import list_kube_config_contexts
+
+                _, active_context = list_kube_config_contexts()
+                current_k8s_ctx = active_context["name"] if active_context else "unknown"
+            except Exception:
+                current_k8s_ctx = "unknown"
+
+            # Show current Babylon namespace from local config
+            ns_file = self.state_dir / "namespace.yaml"
+            if ns_file.exists():
+                try:
+                    ns_data = safe_load(ns_file.open("r").read()) or {}
+                    babylon_ctx = ns_data.get("context", "")
+                    babylon_tenant = ns_data.get("tenant", "")
+                    babylon_state = ns_data.get("state_id", "")
+                    babylon_ns_info = (
+                        f"context=[bold cyan]{babylon_ctx}[/bold cyan]  "
+                        f"tenant=[bold cyan]{babylon_tenant}[/bold cyan]  "
+                        f"state-id=[bold cyan]{babylon_state}[/bold cyan]"
+                    )
+                except Exception:
+                    babylon_ctx = babylon_tenant = babylon_state = ""
+                    babylon_ns_info = "[dim]unavailable[/dim]"
+            else:
+                babylon_ctx = babylon_tenant = babylon_state = ""
+                babylon_ns_info = "[dim]not set[/dim]"
+
             logger.info("\n [bold white]💡 Troubleshooting:[/bold white]")
-            logger.info("  • Please ensure your kubeconfig is valid")
-            logger.info("  • Check that your context is correctly set [cyan]kubectl config current-context[/cyan]")
-            logger.info("  • You can set context using [cyan]kubectl config use-context <context-name>[/cyan]")
+            logger.info(f"  • Active kubectl context : [cyan]{current_k8s_ctx}[/cyan]")
+            logger.info(f"  • Active Babylon namespace: {babylon_ns_info}")
+            logger.info("  • If the kubectl context is wrong, switch it:")
+            logger.info("    [cyan]kubectl config use-context <context-name>[/cyan]")
+            logger.info("  • If the Babylon namespace is wrong, switch it:")
+            logger.info("    [cyan]babylon namespace use -c <context> -t <tenant> -s <state-id>[/cyan]")
             sys.exit(1)
         except Exception:
             logger.error(

Babylon/version.py

@@ -1,4 +1,4 @@
-VERSION = "5.0.0"
+VERSION = "5.1.0"
 
 
 def get_version():

pyproject.toml

@@ -24,7 +24,7 @@ dependencies = [
   "dynaconf",
   "azure-mgmt-authorization>=4.0.0",
   "kubernetes>=35.0.0",
-  "cosmotech-api==5.0.0"
+  "cosmotech-api==5.0.1"
 ]
 dynamic = ["version"]
 readme = {file = "README.md", content-type = "text/markdown"}
@@ -33,7 +33,7 @@ license = "MIT"
 
 [dependency-groups]
 dev = [
-  "ruff>=0.14.13",
+  "ruff>=0.15.7",
   "pytest>=9.0.2",
 ]
 doc = [