Waiting for the agents to reach goal state

Author: lsierant

docker/mongodb-kubernetes-tests/kubetester/__init__.py

@@ -131,7 +131,6 @@ def create_or_update_configmap(
     data: Dict[str, str],
     api_client: Optional[kubernetes.client.ApiClient] = None,
 ) -> str:
-    print("Logging inside create_or_update configmap")
     try:
         create_configmap(namespace, name, data, api_client)
     except kubernetes.client.ApiException as e:

docker/mongodb-kubernetes-tests/kubetester/kubetester.py

@@ -978,16 +978,6 @@ def get_automation_status(group_id=None, group_name=None):
 
         return response.json()
 
-    @staticmethod
-    def get_automation_status(group_id=None, group_name=None):
-        if group_id is None:
-            group_id = KubernetesTester.get_om_group_id(group_name=group_name)
-
-        url = build_automation_status_endpoint(KubernetesTester.get_om_base_url(), group_id)
-        response = KubernetesTester.om_request("get", url)
-
-        return response.json()
-
     @staticmethod
     def get_monitoring_config(group_id=None):
         if group_id is None:

docker/mongodb-kubernetes-tests/kubetester/omtester.py

@@ -15,15 +15,23 @@
 import requests
 import semver
 from kubetester.automation_config_tester import AutomationConfigTester
-from kubetester.kubetester import build_agent_auth, build_auth, run_periodically
+from kubetester.kubetester import (
+    KubernetesTester,
+    build_agent_auth,
+    build_auth,
+    run_periodically,
+)
 from kubetester.mongotester import BackgroundHealthChecker
 from kubetester.om_queryable_backups import OMQueryableBackup
 from opentelemetry import trace
 from requests.adapters import HTTPAdapter, Retry
+from tests import test_logger
 from tests.common.ops_manager.cloud_manager import is_cloud_qa
 
 skip_if_cloud_manager = pytest.mark.skipif(is_cloud_qa(), reason="Do not run in Cloud Manager")
 
+logger = test_logger.get_test_logger(__name__)
+
 
 class BackupStatus(str, Enum):
     """Enum for backup statuses in Ops Manager. Note that 'str' is inherited to fix json serialization issues"""
@@ -414,7 +422,7 @@ def om_request():
                 span.set_attribute(key=f"mck.om.request.retries", value=retries - retry_count)
                 return resp
             except Exception as e:
-                print(f"Encountered exception: {e} on retry number {retries-retry_count}")
+                print(f"Encountered exception: {e} on retry number {retries - retry_count}")
                 span.set_attribute(key=f"mck.om.request.exception", value=str(e))
                 last_exception = e
                 time.sleep(1)
@@ -678,6 +686,42 @@ def api_update_version_manifest(self, major_version: str = "8.0"):
         body = requests.get(url=f"https://opsmanager.mongodb.com/static/version_manifest/{major_version}.json").json()
         self.om_request("put", "/versionManifest", json_object=body)
 
+    def api_get_automation_status(self) -> dict[str, str]:
+        return self.om_request("get", f"/groups/{self.context.project_id}/automationStatus").json()
+
+    def wait_agents_ready(self, timeout: Optional[int] = 600):
+        """Waits until all the agents reached the goal automation config version."""
+        log_prefix = f"[{self.context.group_name}/{self.context.project_id}] "
+
+        def agents_are_ready():
+            auto_status = self.api_get_automation_status()
+            goal_version = auto_status.get("goalVersion")
+
+            logger.info(f"{log_prefix}Checking if all agent processes have reached goal version: {goal_version}")
+            processes_not_ready = []
+            for process in auto_status.get("processes", []):
+                process_name = process.get("name", "unknown")
+                process_version = process.get("lastGoalVersionAchieved")
+                if process_version != goal_version:
+                    logger.info(
+                        f"{log_prefix}Process {process_name} at version {process_version}, expected {goal_version}"
+                    )
+                    processes_not_ready.append(process_name)
+
+            all_processes_ready = len(processes_not_ready) == 0
+            if all_processes_ready:
+                logger.info(f"{log_prefix}All agent processes have reached the goal version")
+            else:
+                logger.info(f"{log_prefix}{len(processes_not_ready)} processes have not yet reached the goal version")
+
+            return all_processes_ready
+
+        KubernetesTester.wait_until(
+            agents_are_ready,
+            timeout=timeout,
+            sleep_time=3,
+        )
+
 
 class OMBackgroundTester(BackgroundHealthChecker):
     """

docker/mongodb-kubernetes-tests/tests/search/search_enterprise_tls.py

@@ -1,7 +1,6 @@
 import pymongo
 import yaml
-from kubernetes import client
-from kubetester import create_or_update_secret, run_periodically, try_load, wait_until
+from kubetester import create_or_update_secret, run_periodically, try_load
 from kubetester.certs import create_mongodb_tls_certs, create_tls_certs
 from kubetester.kubetester import KubernetesTester
 from kubetester.kubetester import fixture as yaml_fixture
@@ -212,11 +211,12 @@ def check_mongod_parameters():
     run_periodically(check_mongod_parameters, timeout=200)
 
 
-# After picking up MongoDBSearch CR, MongoDB reconciler will add mongod parameters.
-# But it will not immediately mark the MongoDB CR as Pending
-# spinning
+# After picking up MongoDBSearch CR, MongoDB reconciler will add mongod parameters to each process.
+# Due to how MongoDB reconciler works (blocking on waiting for agents and not changing the status to pending)
+# the phase won't be updated to Pending and we need to wait by checking agents' status directly in OM.
 @mark.e2e_search_enterprise_tls
-def test_wait_for_database_resource_ready(mdb: MongoDB):
+def test_wait_for_agents_ready(mdb: MongoDB):
+    mdb.get_om_tester().wait_agents_ready()
     mdb.assert_reaches_phase(Phase.Running, timeout=300)