diff --git a/pmm-tests/check_client_upgrade.py b/pmm-tests/check_client_upgrade.py index 8080d535..9cb6ef53 100644 --- a/pmm-tests/check_client_upgrade.py +++ b/pmm-tests/check_client_upgrade.py @@ -1,132 +1,125 @@ import subprocess import sys -arguments = sys.argv -print(arguments) - -containers = subprocess.run(["docker", "ps", "-a"], capture_output=True, text=True).stdout.splitlines() - -def verify_agent_status(list, service_name): - if any('Waiting' in name or 'Done' in name or 'Unknown' in name or 'Initialization Error' in name or 'Stopping' in name for name in list): - errors.append(f"Agent status contains wrong status in {service_name} container. Error in: {list}") - if all('Running' not in name for name in list): - errors.append(f"Agent status does not contain running status in {service_name} container. Error in: {list}") - -def get_pmm_admin_status(service_type): - container_name = containers[i][containers[i].index(service_type):] - print(f"Getting admin status for container name is: {container_name}") - print(subprocess.run(["docker", "exec", container_name, "pmm-admin", "status"], capture_output=True, text=True).stdout.splitlines()) - return subprocess.run(["docker", "exec", container_name, "pmm-admin", "status"], capture_output=True, text=True).stdout.splitlines() - -def get_pmm_admin_list(service_type): - container_name = containers[i][containers[i].index(service_type):] - print(f"Getting admin list for container name is: {container_name}") - print(subprocess.run(["docker", "exec", container_name, "pmm-admin", "list"], capture_output=True, text=True).stdout.splitlines()) - return subprocess.run(["docker", "exec", container_name, "pmm-admin", "list"], capture_output=True, text=True).stdout.splitlines() - -psContainerStatus = [] -pgContainerStatus = [] -firstMongoReplicaStatus = [] -secondMongoReplicaStatus = [] -thirdMongoReplicaStatus = [] -psSSLStatus = [] -pdpgsqlSSLStatus = [] -psmdbSSLStatus = [] -localClientStatus = subprocess.run(["pmm-admin", "status"], capture_output=True, text=True).stdout.splitlines() - -psContainerList = [] -pgContainerList = [] -firstMongoReplicaList = [] -secondMongoReplicaList = [] -thirdMongoReplicaList = [] -psSSLList = [] -pdpgsqlSSLList = [] -psmdbSSLList = [] -localClientList = subprocess.run(["pmm-admin", "list"], capture_output=True, text=True).stdout.splitlines() +BAD_STATES = ["Waiting", "Done", "Unknown", "Initialization Error", "Stopping"] -errors = [] +# Container name patterns and labels +CONTAINER_PATTERNS = [ + ("ps_pmm_", "ps"), + ("pgsql_pgss_pmm", "pgsql"), + ("rs101", "first mongo"), + ("rs102", "second mongo"), + ("rs103", "third mongo"), +] + +# Get expected version from command-line arguments +EXPECTED_VERSION = sys.argv[1].strip() if len(sys.argv) > 1 else None +assert EXPECTED_VERSION, "āŒ Expected version must be passed as the first argument." -for i in range(len(containers)): - if "ps_pmm_" in containers[i]: - psContainerStatus = get_pmm_admin_status("ps_pmm") - psContainerList = get_pmm_admin_list("ps_pmm") - elif "pgsql_pgss_pmm" in containers[i]: - pgContainerStatus = get_pmm_admin_status("pgsql_pgss_pmm") - pgContainerList = get_pmm_admin_list("pgsql_pgss_pmm") - elif "rs101" in containers[i]: - firstMongoReplicaStatus = get_pmm_admin_status("rs101") - firstMongoReplicaList = get_pmm_admin_list("rs101") - elif "rs102" in containers[i]: - secondMongoReplicaStatus = get_pmm_admin_status("rs102") - secondMongoReplicaList = get_pmm_admin_list("rs102") - elif "rs103" in containers[i]: - thirdMongoReplicaStatus = get_pmm_admin_status("rs103") - thirdMongoReplicaList = get_pmm_admin_list("rs103") - elif "mysql_ssl" in containers[i]: - psSSLStatus = get_pmm_admin_status("mysql_ssl") - psSSLList = get_pmm_admin_list("mysql_ssl") - elif "pdpgsql_pgsm_ssl" in containers[i]: - pdpgsqlSSLStatus = get_pmm_admin_status("pdpgsql_pgsm_ssl") - pdpgsqlSSLList = get_pmm_admin_list("pdpgsql_pgsm_ssl") - elif "psmdb-server" in containers[i]: - psmdbSSLStatus = get_pmm_admin_status("psmdb-server") - psmdbSSLList = get_pmm_admin_list("psmdb-server") - -if len(psContainerStatus) > 0: - verify_agent_status(psContainerStatus, "Percona Server") - verify_agent_status(psContainerList, "Percona Server") - -if len(pgContainerStatus) > 0: - verify_agent_status(pgContainerStatus, "Percona Distribution for PostgreSQL") - verify_agent_status(pgContainerList, "Percona Distribution for PostgreSQL") - -if len(psSSLStatus) > 0: - verify_agent_status(psSSLStatus, "Percona Server SSl") - verify_agent_status(psSSLList, "Percona Server SSl") - -if len(pdpgsqlSSLStatus) > 0: - verify_agent_status(pdpgsqlSSLStatus, "Percona Distribution for PostgreSQL SSL") - verify_agent_status(pdpgsqlSSLList, "Percona Distribution for PostgreSQL SSL") - -if len(psmdbSSLStatus) > 0: - verify_agent_status(psmdbSSLStatus, "Percona Server for MongoDB instance SSL status") - verify_agent_status(psmdbSSLList, "Percona Server for MongoDB instance SSL list") - -if len(firstMongoReplicaStatus) > 0: - verify_agent_status(firstMongoReplicaStatus, "Percona Server for MongoDB instance 1") - verify_agent_status(firstMongoReplicaList, "Percona Server for MongoDB instance 1") - -if len(secondMongoReplicaStatus) > 0: - verify_agent_status(secondMongoReplicaStatus, "Percona Server for MongoDB instance 2") - verify_agent_status(secondMongoReplicaList, "Percona Server for MongoDB instance 2") - -if len(thirdMongoReplicaStatus) > 0: - verify_agent_status(thirdMongoReplicaStatus, "Percona Server for MongoDB instance 3") - verify_agent_status(thirdMongoReplicaList, "Percona Server for MongoDB instance 3") - -if len(localClientStatus) > 0: - verify_agent_status(localClientStatus, "Local Client") - verify_agent_status(localClientList, "Local Client") - -if len(errors) > 0: - raise Exception("Some errors in pmm-admin status: ".join(errors)) - -expected_version=arguments[1].replace("\\r\\n", "") -admin_version = subprocess.run(["pmm-admin status | grep pmm-admin | awk -F' ' '{print $3}'"], capture_output=True, text=True, shell=True).stdout.replace("\\r\\n", "").strip() - -if admin_version != expected_version: - print(f"admin version is: {admin_version} and expected version is: {expected_version}") - errors.append(f"Version of pmm admin is not correct expected: {expected_version} actual: {admin_version}") - -agent_version = subprocess.run(["pmm-admin status | grep pmm-agent | awk -F' ' '{print $3}'"], capture_output=True, text=True, shell=True).stdout.replace("\\r\\n", "").strip() - -if agent_version != expected_version: - print(f"agent version is: {agent_version} and expected version is: {expected_version}") - errors.append(f"Version of pmm agent is not correct expected: {expected_version} actual: {agent_version}") - -if admin_version != agent_version: - errors.append(f"PMM admin version: {admin_version} does not equal PMM agent version {agent_version}") - -if len(errors) > 0: - raise Exception("Errors in pmm-admin and pmm-agent versions: ".join(errors)) +errors = [] +def run_command(cmd, shell=False): + """Run a command and return its output as lines.""" + result = subprocess.run(cmd, capture_output=True, text=True, shell=shell) + return result.stdout.splitlines() + + +def get_container_name(pattern): + """Find and return the container name that matches a given pattern.""" + containers = run_command(["docker", "ps", "-a"]) + for line in containers: + if pattern in line: + return line.split()[-1] # container name is usually the last column + return None + + +def contains_bad_state(lines): + """Check if any line contains a bad PMM agent state.""" + return any(state in line for line in lines for state in BAD_STATES) + + +def check_pmm_output(container_name, label, command): + """Run pmm-admin command and check for bad states.""" + output = run_command(["docker", "exec", container_name, "pmm-admin", command]) + if contains_bad_state(output): + errors.append(f"āŒ Bad state found in {label} container ({command})") + return output + + +def check_local_pmm_output(command, label): + """Run pmm-admin command locally and check for bad states.""" + output = run_command(["pmm-admin", command]) + if contains_bad_state(output): + errors.append(f"āŒ Bad state found in local PMM client ({label})") + return output + + +def get_version_from_output(lines, keyword, label): + """Extract version for a given keyword from a list of lines.""" + for line in lines: + if keyword in line: + parts = line.split() + if len(parts) >= 3: + return parts[2] + errors.append(f"āŒ Could not determine version for {keyword} in {label}") + return None + + +def main(): + print("šŸ” Validating PMM containers and local PMM client...\n") + + # Check containers + for pattern, label in CONTAINER_PATTERNS: + container_name = get_container_name(pattern) + assert container_name, f"āŒ Could not find container matching pattern '{pattern}'" + print(f"āœ… Found container '{container_name}' for {label}") + + # Check status and list + status_output = check_pmm_output(container_name, label, "status") + list_output = check_pmm_output(container_name, label, "list") + + # Version checks inside container + admin_version = get_version_from_output(status_output, "pmm-admin", label) + agent_version = get_version_from_output(status_output, "pmm-agent", label) + + if admin_version and admin_version != EXPECTED_VERSION: + errors.append(f"āŒ PMM admin version in {label} container mismatch: expected {EXPECTED_VERSION}, got {admin_version}") + if agent_version and agent_version != EXPECTED_VERSION: + errors.append(f"āŒ PMM agent version in {label} container mismatch: expected {EXPECTED_VERSION}, got {agent_version}") + if admin_version and agent_version and admin_version != agent_version: + errors.append(f"āŒ Version mismatch in {label} container: pmm-admin={admin_version}, pmm-agent={agent_version}") + else: + print(f"āœ… Version checks passed in {label} container") + + print("\nāœ… All container checks done.\n") + + # Check local PMM client + local_status_output = check_local_pmm_output("status", "status") + check_local_pmm_output("list", "list") + + local_admin_version = get_version_from_output(local_status_output, "pmm-admin", "local client") + local_agent_version = get_version_from_output(local_status_output, "pmm-agent", "local client") + + assert local_admin_version, "āŒ Could not determine local pmm-admin version" + assert local_agent_version, "āŒ Could not determine local pmm-agent version" + + if local_admin_version != EXPECTED_VERSION: + errors.append(f"āŒ Local PMM admin version mismatch: expected {EXPECTED_VERSION}, got {local_admin_version}") + if local_agent_version != EXPECTED_VERSION: + errors.append(f"āŒ Local PMM agent version mismatch: expected {EXPECTED_VERSION}, got {local_agent_version}") + if local_admin_version != local_agent_version: + errors.append(f"āŒ Local version mismatch: pmm-admin={local_admin_version}, pmm-agent={local_agent_version}") + else: + print("āœ… Local PMM admin and agent versions match") + + print("\nšŸ“¦ Final Results:\n") + if errors: + for err in errors: + print(err) + raise AssertionError("šŸš« Validation failed. See errors above.") + else: + print("šŸŽ‰ All checks passed! PMM containers and local client are healthy and versions are correct.") + + +if __name__ == "__main__": + main()