feat: bind k8s cluster endpoint to internal space

sunbeam-python/sunbeam/steps/k8s.py

@@ -81,12 +81,14 @@
     import lightkube.config.kubeconfig as l_kubeconfig
     import lightkube.core.client as l_client
     import lightkube.core.exceptions as l_exceptions
+    import lightkube.types as l_patch_type
     from lightkube.models import meta_v1
     from lightkube.resources import apps_v1, autoscaling_v2, core_v1
 else:
     l_kubeconfig = LazyImport("lightkube.config.kubeconfig")
     l_client = LazyImport("lightkube.core.client")
     l_exceptions = LazyImport("lightkube.core.exceptions")
+    l_patch_type = LazyImport("lightkube.types")
     meta_v1 = LazyImport("lightkube.models.meta_v1")
     apps_v1 = LazyImport("lightkube.resources.apps_v1")
     autoscaling_v2 = LazyImport("lightkube.resources.autoscaling_v2")
@@ -300,6 +302,10 @@ def extra_tfvars(self) -> dict:
         tfvars = {
             "endpoint_bindings": [
                 {"space": self.deployment.get_space(Networks.MANAGEMENT)},
+                {
+                    "endpoint": "cluster",
+                    "space": self.deployment.get_space(Networks.INTERNAL),
+                },
             ],
             "k8s_config": self._get_k8s_config_tfvars(),
         }
@@ -332,9 +338,9 @@ class EnsureK8SUnitsTaggedStep(BaseStep):
     This step ensures that every k8s node is tagged with the
     HOSTNAME_LABEL, to ensure sunbeam can query the correct nodes
     afterwards.
-    Match is done on management ip address. Node IP in k8s is guaranteed by
-    the cluster space binding, which is always bound to the management
-    space.
+    Match is done on the IP addresses from the space configured as
+    Networks.INTERNAL. Node IP in k8s is guaranteed by the cluster
+    space binding.
     """
 
     def __init__(
@@ -355,18 +361,16 @@ def __init__(
         self.fqdn = fqdn
         self.to_update: dict[str, str] = {}
 
-    def _get_management_ips(
+    def _get_cluster_ips(
         self, juju_machine: "jubilant.statustypes.MachineStatus"
     ) -> list[str]:
-        management_space = self.deployment.get_space(Networks.MANAGEMENT)
-        management_networks = self.jhelper.get_space_networks(
-            self.model, management_space
-        )
+        cluster_space = self.deployment.get_space(Networks.INTERNAL)
+        cluster_networks = self.jhelper.get_space_networks(self.model, cluster_space)
 
         return _get_machines_space_ips(
             juju_machine.network_interfaces,
-            management_space,
-            management_networks,
+            cluster_space,
+            cluster_networks,
         )
 
     @tenacity.retry(
@@ -442,18 +446,18 @@ def _get_k8s_node_to_update(
                 raise SunbeamException(
                     f"{sunbeam_name!r} not found in Juju, expected id {machine_id!r}"
                 )
-            management_ips = self._get_management_ips(juju_machine)
-            if not management_ips:
-                LOG.debug("No management IPs found for machine %s", machine_id)
-                raise SunbeamException(f"{sunbeam_name!r} has no management IPs")
+            cluster_ips = self._get_cluster_ips(juju_machine)
+            if not cluster_ips:
+                LOG.debug("No cluster IPs found for machine %s", machine_id)
+                raise SunbeamException(f"{sunbeam_name!r} has no cluster IPs")
 
             try:
-                k8s_node = self._find_matching_k8s_node(sunbeam_name, management_ips)
+                k8s_node = self._find_matching_k8s_node(sunbeam_name, cluster_ips)
             except ValueError:
                 LOG.debug(
-                    "No matching k8s node found for %s, management IPs %s",
+                    "No matching k8s node found for %s, cluster IPs %s",
                     sunbeam_name,
-                    management_ips,
+                    cluster_ips,
                 )
                 raise SunbeamException(f"{sunbeam_name} has no matching k8s node")
             except K8SError as e:
@@ -1657,6 +1661,7 @@ def run(self, context: StepContext) -> Result:
                         }
                     },
                     namespace=self._CILIUM_NAMESPACE,
+                    patch_type=l_patch_type.PatchType.MERGE,
                 )
             except l_exceptions.ApiError:
                 LOG.debug(

sunbeam-python/tests/unit/sunbeam/provider/maas/test_maas.py

@@ -1555,7 +1555,10 @@ def test_extra_tfvars_with_ranges(self, deployment_k8s):
         step.ranges = "10.0.0.0/28"
         step.client.cluster.get_config.return_value = "{}"
         expected_tfvars = {
-            "endpoint_bindings": [{"space": "data"}],
+            "endpoint_bindings": [
+                {"space": "data"},
+                {"endpoint": "cluster", "space": "internal_space"},
+            ],
             "k8s_config": {
                 "load-balancer-cidrs": "10.0.0.0/28",
                 "load-balancer-enabled": True,

sunbeam-python/tests/unit/sunbeam/steps/test_k8s.py

@@ -11,9 +11,11 @@
 import pytest
 import tenacity
 from lightkube import ApiError
+from lightkube.types import PatchType
 
 from sunbeam.clusterd.service import ConfigItemNotFoundException
 from sunbeam.core.common import ResultType
+from sunbeam.core.deployment import Networks
 from sunbeam.core.juju import (
     ActionFailedException,
     ApplicationNotFoundException,
@@ -28,6 +30,7 @@
     K8S_CLOUD_SUFFIX,
     AddK8SCloudStep,
     AddK8SCredentialStep,
+    DeployK8SApplicationStep,
     EnsureCiliumDeviceByHostStep,
     EnsureDefaultL2AdvertisementMutedStep,
     EnsureK8SUnitsTaggedStep,
@@ -63,7 +66,13 @@ def deployment_with_space():
     """Deployment mock with space configuration."""
     deployment = Mock()
     deployment.name = "test-deployment"
-    deployment.get_space.return_value = "management"
+
+    def get_space(network):
+        if network == Networks.INTERNAL:
+            return "internal"
+        return "management"
+
+    deployment.get_space.side_effect = get_space
     return deployment
 
 
@@ -927,12 +936,12 @@ def test_is_skip_no_nodes_to_update(self, step, client, jhelper, step_context):
         jhelper.get_machines.return_value = {
             "1": Mock(
                 network_interfaces={
-                    "eth0": Mock(space="management", ip_addresses=["10.0.0.1"])
+                    "eth0": Mock(space="internal", ip_addresses=["10.0.0.1"])
                 }
             ),
             "2": Mock(
                 network_interfaces={
-                    "eth0": Mock(space="management", ip_addresses=["10.0.0.2"])
+                    "eth0": Mock(space="internal", ip_addresses=["10.0.0.2"])
                 }
             ),
         }
@@ -959,12 +968,12 @@ def test_is_skip_nodes_to_update(self, step, client, jhelper, step_context):
         jhelper.get_machines.return_value = {
             "1": Mock(
                 network_interfaces={
-                    "eth0": Mock(space="management", ip_addresses=["10.0.0.1"])
+                    "eth0": Mock(space="internal", ip_addresses=["10.0.0.1"])
                 }
             ),
             "2": Mock(
                 network_interfaces={
-                    "eth0": Mock(space="management", ip_addresses=["10.0.0.2"])
+                    "eth0": Mock(space="internal", ip_addresses=["10.0.0.2"])
                 }
             ),
         }
@@ -994,12 +1003,12 @@ def test_is_skip_nodes_to_update_with_fqdn(
         jhelper.get_machines.return_value = {
             "1": Mock(
                 network_interfaces={
-                    "eth0": Mock(space="management", ip_addresses=["10.0.0.1"])
+                    "eth0": Mock(space="internal", ip_addresses=["10.0.0.1"])
                 }
             ),
             "2": Mock(
                 network_interfaces={
-                    "eth0": Mock(space="management", ip_addresses=["10.0.0.2"])
+                    "eth0": Mock(space="internal", ip_addresses=["10.0.0.2"])
                 }
             ),
         }
@@ -1023,12 +1032,12 @@ def test_is_skip_k8s_api_error(self, step, client, jhelper, step_context):
         jhelper.get_machines.return_value = {
             "1": Mock(
                 network_interfaces={
-                    "eth0": Mock(space="management", ip_addresses=["10.0.0.1"])
+                    "eth0": Mock(space="internal", ip_addresses=["10.0.0.1"])
                 }
             ),
             "2": Mock(
                 network_interfaces={
-                    "eth0": Mock(space="management", ip_addresses=["10.0.0.2"])
+                    "eth0": Mock(space="internal", ip_addresses=["10.0.0.2"])
                 }
             ),
         }
@@ -1085,6 +1094,39 @@ def test_run_apply_failure(self, step):
         assert result.result_type == ResultType.FAILED
 
 
+class TestDeployK8SApplicationStep:
+    @pytest.fixture
+    def deployment(self, deployment_with_space):
+        deployment_with_space.openstack_machines_model = "test-model"
+        return deployment_with_space
+
+    @pytest.fixture
+    def manifest(self, basic_manifest):
+        basic_manifest.core.software.charms.get.return_value = None
+        return basic_manifest
+
+    @pytest.fixture
+    def step(self, deployment, basic_client, basic_tfhelper, basic_jhelper, manifest):
+        basic_client.cluster.get_config.return_value = "{}"
+        return DeployK8SApplicationStep(
+            deployment,
+            basic_client,
+            basic_tfhelper,
+            basic_jhelper,
+            manifest,
+            "test-model",
+        )
+
+    def test_extra_tfvars_binds_cluster_endpoint_to_internal_space(self, step):
+        assert step.extra_tfvars()["endpoint_bindings"] == [
+            {"space": "management"},
+            {"endpoint": "cluster", "space": "internal"},
+        ]
+
+    def test_get_k8s_config_tfvars_does_not_manage_cluster_annotations(self, step):
+        assert "cluster-annotations" not in step._get_k8s_config_tfvars()
+
+
 class TestGetKubeClient:
     @pytest.fixture
     def client(self, basic_client):
@@ -1804,7 +1846,19 @@ def list_side_effect(*args, **kwargs):
         result = step.run(None)
 
         step.kube.apply.assert_called_once()
-        step.kube.patch.assert_called_once()  # clears restart-pending
+        step.kube.patch.assert_called_once_with(
+            step.cilium_node_config_resource,
+            "cilium-devices-node1",
+            {
+                "metadata": {
+                    "annotations": {
+                        "sunbeam/restart-pending": "false",
+                    }
+                }
+            },
+            namespace="kube-system",
+            patch_type=PatchType.MERGE,
+        )
         assert result.result_type == ResultType.COMPLETED
 
     def test_run_updates_config(self, step):