fix: handle race condition when joining first storage node

sunbeam-python/sunbeam/core/steps.py

@@ -72,6 +72,7 @@ def __init__(
         roles: list[Role] | list[list[Role]] | None = None,
         banner: str = "",
         description: str = "",
+        wait: bool = True,
     ):
         super().__init__(banner, description)
         self.deployment = deployment
@@ -83,6 +84,7 @@ def __init__(
         self.application = application
         self.model = model
         self.roles = roles or []
+        self.wait = wait
 
     def extra_tfvars(self) -> dict:
         """Extra terraform vars to pass to terraform apply."""
@@ -147,16 +149,17 @@ def run(self, context: StepContext) -> Result:
 
         # Note(gboutry): application is in state unknown when it's deployed
         # without units
-        try:
-            self.jhelper.wait_application_ready(
-                self.application,
-                self.model,
-                accepted_status=self.get_accepted_application_status(),
-                timeout=self.get_application_timeout(),
-            )
-        except TimeoutError as e:
-            LOG.warning(str(e))
-            return Result(ResultType.FAILED, str(e))
+        if self.wait:
+            try:
+                self.jhelper.wait_application_ready(
+                    self.application,
+                    self.model,
+                    accepted_status=self.get_accepted_application_status(),
+                    timeout=self.get_application_timeout(),
+                )
+            except TimeoutError as e:
+                LOG.warning(str(e))
+                return Result(ResultType.FAILED, str(e))
 
         return Result(ResultType.COMPLETED)
 

sunbeam-python/sunbeam/provider/local/commands.py

@@ -1599,42 +1599,29 @@ def join(  # noqa: C901
             )
 
     if is_storage_node:
-        plan4.append(TerraformInitStep(microceph_tfhelper))
-        plan4.append(
-            DeployMicrocephApplicationStep(
-                deployment,
-                client,
-                microceph_tfhelper,
-                jhelper,
-                manifest,
-                deployment.openstack_machines_model,
-            )
+        # Check if this is the first storage node joining
+        is_first_storage_node = (
+            len(client.cluster.list_nodes_by_role(Role.STORAGE.name.lower())) <= 1
         )
-        plan4.append(
-            ConfigureMicrocephOSDStep(
-                client,
-                name,
-                jhelper,
-                deployment.openstack_machines_model,
-                accept_defaults=accept_defaults,
-                manifest=manifest,
-            )
-        )
-        plan4.append(
-            DeployCinderVolumeApplicationStep(
-                deployment,
-                client,
-                cinder_volume_tfhelper,
-                jhelper,
-                manifest,
-                deployment.openstack_machines_model,
+
+        if is_first_storage_node:
+            # Deploy MicroCeph WITHOUT waiting
+            # The charm will be stuck in "waiting" because it needs traefik-route-rgw
+            # integration which doesn't exist as enable-ceph=False.
+            plan4.append(TerraformInitStep(microceph_tfhelper))
+            plan4.append(
+                DeployMicrocephApplicationStep(
+                    deployment,
+                    client,
+                    microceph_tfhelper,
+                    jhelper,
+                    manifest,
+                    deployment.openstack_machines_model,
+                    wait=False,
+                )
             )
-        )
 
-        # Re-deploy control plane if this is the first storage node joining
-        # the cluster to enable mandatory storage services
-        storage_nodes = client.cluster.list_nodes_by_role(Role.STORAGE.name.lower())
-        if len(storage_nodes) == 1:
+            # Redeploy control plane with enable-ceph=true
             plan4.append(
                 DeployControlPlaneStep(
                     deployment,
@@ -1647,9 +1634,8 @@ def join(  # noqa: C901
                 )
             )
 
-            # Redeploy of Microceph is required to fill terraform vars
-            # related to traefik-rgw/keystone-endpoints offers from
-            # openstack model
+            # Redeploy MicroCeph with output from OS TF variables related to
+            # traefik-rgw/keystone-endpoints offers from openstack model
             microceph_tfhelper = deployment.get_tfhelper("microceph-plan")
             plan4.append(TerraformInitStep(microceph_tfhelper))
             plan4.append(
@@ -1662,18 +1648,41 @@ def join(  # noqa: C901
                     deployment.openstack_machines_model,
                 )
             )
-            # Fill AMQP / Keystone / MySQL offers from openstack model
+        else:
+            # Deploy MicroCeph directly with blocking as OS TF variables are already set
+            plan4.append(TerraformInitStep(microceph_tfhelper))
             plan4.append(
-                DeployCinderVolumeApplicationStep(
+                DeployMicrocephApplicationStep(
                     deployment,
                     client,
-                    cinder_volume_tfhelper,
+                    microceph_tfhelper,
                     jhelper,
                     manifest,
                     deployment.openstack_machines_model,
                 )
             )
 
+        plan4.append(
+            ConfigureMicrocephOSDStep(
+                client,
+                name,
+                jhelper,
+                deployment.openstack_machines_model,
+                accept_defaults=accept_defaults,
+                manifest=manifest,
+            )
+        )
+        plan4.append(
+            DeployCinderVolumeApplicationStep(
+                deployment,
+                client,
+                cinder_volume_tfhelper,
+                jhelper,
+                manifest,
+                deployment.openstack_machines_model,
+            )
+        )
+
         plan4.append(
             ReapplyHypervisorOptionalIntegrationsStep(
                 deployment,

sunbeam-python/sunbeam/steps/microceph.py

@@ -103,6 +103,7 @@ def __init__(
         jhelper: JujuHelper,
         manifest: Manifest,
         model: str,
+        wait: bool = True,
     ):
         super().__init__(
             deployment,
@@ -116,6 +117,7 @@ def __init__(
             [Role.STORAGE],
             "Deploy MicroCeph",
             "Deploying MicroCeph",
+            wait=wait,
         )
 
     def get_application_timeout(self) -> int:

sunbeam-python/tests/unit/sunbeam/storage/test_steps.py

@@ -7,7 +7,9 @@
 import pydantic
 import pytest
 
+from sunbeam.core.common import ResultType
 from sunbeam.core.questions import PasswordPromptQuestion, PromptQuestion
+from sunbeam.core.steps import DeployMachineApplicationStep
 from sunbeam.storage.models import SecretDictField
 from sunbeam.storage.steps import (
     DeploySpecificCinderVolumeStep,
@@ -356,3 +358,34 @@ def test_run_extra_tfvars_precedence(
             ]
             is True
         )
+
+        def test_deploy_machine_application_step_run_skips_wait(
+            basic_deployment,
+            basic_client,
+            basic_tfhelper,
+            basic_jhelper,
+            basic_manifest,
+            test_model,
+            step_context,
+        ):
+            step = DeployMachineApplicationStep(
+                basic_deployment,
+                basic_client,
+                basic_tfhelper,
+                basic_jhelper,
+                basic_manifest,
+                config="test-config",
+                application="test-app",
+                model=test_model,
+                roles=[],
+                wait=False,
+            )
+
+            basic_jhelper.get_model_uuid.return_value = "model-uuid"
+            basic_client.cluster.list_nodes_by_role.return_value = []
+
+            result = step.run(step_context)
+
+            assert result.result_type == ResultType.COMPLETED
+            basic_tfhelper.update_tfvars_and_apply_tf.assert_called_once()
+            basic_jhelper.wait_application_ready.assert_not_called()