Fix couple multicluster flaky tests (#562)

# Summary

Couple small fixes that together with @lsierant we have found during
recent investigation. Details are provided in the PR comments.

## Proof of Work

Passing CI.

## Checklist

- [ ] Have you linked a jira ticket and/or is the ticket in the title?
- [ ] Have you checked whether your jira ticket required DOCSP changes?
- [x] Have you added changelog file?
    - use `skip-changelog` label if not needed
- refer to [Changelog files and Release
Notes](https://github.com/mongodb/mongodb-kubernetes/blob/master/CONTRIBUTING.md#changelog-files-and-release-notes)
section in CONTRIBUTING.md for more details

Author: MaciejKaras

.evergreen-tasks.yml

@@ -1116,7 +1116,6 @@ tasks:
 
   - name: e2e_multi_cluster_validation
     tags: [ "patch-run" ]
-    exec_timeout_secs: 1000
     commands:
       - func: e2e_test
 

docker/mongodb-kubernetes-tests/tests/multicluster_appdb/multicluster_appdb_disaster_recovery.py

@@ -152,11 +152,12 @@ def test_delete_om_and_appdb_statefulset_in_failed_cluster(
         # delete OM to simulate losing Ops Manager application
         # this is only for testing unavailability of the OM application, it's not testing losing OM cluster
         # we don't delete here any additional resources (secrets, configmaps) that are required for a proper OM recovery testing
+        # it will be immediately recreated by the operator, so we cannot check if it was deleted
         delete_statefulset(
             ops_manager.namespace,
             ops_manager.name,
             propagation_policy="Background",
-            api_client=central_cluster_client,
+            api_client=get_member_cluster_api_client(OM_MEMBER_CLUSTER_NAME),
         )
     except kubernetes.client.ApiException as e:
         if e.status != 404:
@@ -184,14 +185,6 @@ def statefulset_is_deleted(namespace: str, name: str, api_client=Optional[kubern
             else:
                 raise e
 
-    run_periodically(
-        lambda: statefulset_is_deleted(
-            ops_manager.namespace,
-            ops_manager.name,
-            api_client=get_member_cluster_api_client(OM_MEMBER_CLUSTER_NAME),
-        ),
-        timeout=120,
-    )
     run_periodically(
         lambda: statefulset_is_deleted(
             ops_manager.namespace,

scripts/funcs/multicluster

@@ -128,26 +128,6 @@ EOF
   sleep 1
 
   local service_account_name="operator-tests-multi-cluster-service-account"
-
-  local secret_name
-  secret_name="$(kubectl --context "${CENTRAL_CLUSTER}" get secret -n "${NAMESPACE}" | { grep "${service_account_name}" || test $? = 1; } | awk '{ print $1 }')"
-  if [[ "${secret_name}" == "" ]]; then
-    secret_name="${service_account_name}-token-secret"
-    create_service_account_token_secret "${CENTRAL_CLUSTER}" "${service_account_name}" "${secret_name}"
-  fi
-
-  local central_cluster_token
-  central_cluster_token="$(kubectl --context "${CENTRAL_CLUSTER}" get secret "${secret_name}" -o jsonpath='{ .data.token}' -n "${NAMESPACE}" | base64 -d)"
-  echo "Creating Multi Cluster configuration secret"
-
-  configuration_params=(
-    "--from-literal=central_cluster=${CENTRAL_CLUSTER}"
-  )
-
-  configuration_params+=(
-    "--from-literal=${CENTRAL_CLUSTER}=${central_cluster_token}"
-  )
-
   local secret_name
   secret_name="$(kubectl --context "${CENTRAL_CLUSTER}" get secret -n "${NAMESPACE}" | { grep "${service_account_name}" || test $? = 1; } | awk '{ print $1 }')"
   if [[ "${secret_name}" == "" ]]; then
@@ -175,7 +155,18 @@ EOF
       create_service_account_token_secret "${member_cluster}" "${service_account_name}" "${secret_name}"
     fi
 
-    member_cluster_token="$(kubectl --context "${member_cluster}" get secret "${secret_name}" -o jsonpath='{ .data.token}' -n "${NAMESPACE}" | base64 -d)"
+    # Retry up to 10 times if .data.token is not yet populated
+    for _ in {1..10}; do
+      member_cluster_token="$(kubectl --context "${member_cluster}" get secret "${secret_name}" -o jsonpath='{ .data.token }' -n "${NAMESPACE}" | base64 -d)"
+      if [[ -n "${member_cluster_token}" ]]; then
+        break
+      fi
+      sleep 1
+    done
+    if [[ -z "${member_cluster_token}" ]]; then
+      echo "Error: .data.token not populated for secret ${secret_name} in cluster ${member_cluster}"
+      exit 1
+    fi
     # for 2 cluster tests central cluster is the first member, so we cannot add this as it will result in duplicate key and error in create secret
     if [[ "${member_cluster}" != "${CENTRAL_CLUSTER}" ]]; then
       configuration_params+=(