Skip to content

PMM-7 check_client_upgrade.py tweaks proposal #779

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: v3
Choose a base branch
from
Open

PMM-7 check_client_upgrade.py tweaks proposal #779

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
245 changes: 119 additions & 126 deletions pmm-tests/check_client_upgrade.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,132 +1,125 @@
import subprocess
import sys

arguments = sys.argv
print(arguments)

containers = subprocess.run(["docker", "ps", "-a"], capture_output=True, text=True).stdout.splitlines()

def verify_agent_status(list, service_name):
if any('Waiting' in name or 'Done' in name or 'Unknown' in name or 'Initialization Error' in name or 'Stopping' in name for name in list):
errors.append(f"Agent status contains wrong status in {service_name} container. Error in: {list}")
if all('Running' not in name for name in list):
errors.append(f"Agent status does not contain running status in {service_name} container. Error in: {list}")

def get_pmm_admin_status(service_type):
container_name = containers[i][containers[i].index(service_type):]
print(f"Getting admin status for container name is: {container_name}")
print(subprocess.run(["docker", "exec", container_name, "pmm-admin", "status"], capture_output=True, text=True).stdout.splitlines())
return subprocess.run(["docker", "exec", container_name, "pmm-admin", "status"], capture_output=True, text=True).stdout.splitlines()

def get_pmm_admin_list(service_type):
container_name = containers[i][containers[i].index(service_type):]
print(f"Getting admin list for container name is: {container_name}")
print(subprocess.run(["docker", "exec", container_name, "pmm-admin", "list"], capture_output=True, text=True).stdout.splitlines())
return subprocess.run(["docker", "exec", container_name, "pmm-admin", "list"], capture_output=True, text=True).stdout.splitlines()

psContainerStatus = []
pgContainerStatus = []
firstMongoReplicaStatus = []
secondMongoReplicaStatus = []
thirdMongoReplicaStatus = []
psSSLStatus = []
pdpgsqlSSLStatus = []
psmdbSSLStatus = []
localClientStatus = subprocess.run(["pmm-admin", "status"], capture_output=True, text=True).stdout.splitlines()

psContainerList = []
pgContainerList = []
firstMongoReplicaList = []
secondMongoReplicaList = []
thirdMongoReplicaList = []
psSSLList = []
pdpgsqlSSLList = []
psmdbSSLList = []
localClientList = subprocess.run(["pmm-admin", "list"], capture_output=True, text=True).stdout.splitlines()
BAD_STATES = ["Waiting", "Done", "Unknown", "Initialization Error", "Stopping"]

errors = []
# Container name patterns and labels
CONTAINER_PATTERNS = [
("ps_pmm_", "ps"),
("pgsql_pgss_pmm", "pgsql"),
("rs101", "first mongo"),
("rs102", "second mongo"),
("rs103", "third mongo"),
]

# Get expected version from command-line arguments
EXPECTED_VERSION = sys.argv[1].strip() if len(sys.argv) > 1 else None
assert EXPECTED_VERSION, "❌ Expected version must be passed as the first argument."

for i in range(len(containers)):
if "ps_pmm_" in containers[i]:
psContainerStatus = get_pmm_admin_status("ps_pmm")
psContainerList = get_pmm_admin_list("ps_pmm")
elif "pgsql_pgss_pmm" in containers[i]:
pgContainerStatus = get_pmm_admin_status("pgsql_pgss_pmm")
pgContainerList = get_pmm_admin_list("pgsql_pgss_pmm")
elif "rs101" in containers[i]:
firstMongoReplicaStatus = get_pmm_admin_status("rs101")
firstMongoReplicaList = get_pmm_admin_list("rs101")
elif "rs102" in containers[i]:
secondMongoReplicaStatus = get_pmm_admin_status("rs102")
secondMongoReplicaList = get_pmm_admin_list("rs102")
elif "rs103" in containers[i]:
thirdMongoReplicaStatus = get_pmm_admin_status("rs103")
thirdMongoReplicaList = get_pmm_admin_list("rs103")
elif "mysql_ssl" in containers[i]:
psSSLStatus = get_pmm_admin_status("mysql_ssl")
psSSLList = get_pmm_admin_list("mysql_ssl")
elif "pdpgsql_pgsm_ssl" in containers[i]:
pdpgsqlSSLStatus = get_pmm_admin_status("pdpgsql_pgsm_ssl")
pdpgsqlSSLList = get_pmm_admin_list("pdpgsql_pgsm_ssl")
elif "psmdb-server" in containers[i]:
psmdbSSLStatus = get_pmm_admin_status("psmdb-server")
psmdbSSLList = get_pmm_admin_list("psmdb-server")

if len(psContainerStatus) > 0:
verify_agent_status(psContainerStatus, "Percona Server")
verify_agent_status(psContainerList, "Percona Server")

if len(pgContainerStatus) > 0:
verify_agent_status(pgContainerStatus, "Percona Distribution for PostgreSQL")
verify_agent_status(pgContainerList, "Percona Distribution for PostgreSQL")

if len(psSSLStatus) > 0:
verify_agent_status(psSSLStatus, "Percona Server SSl")
verify_agent_status(psSSLList, "Percona Server SSl")

if len(pdpgsqlSSLStatus) > 0:
verify_agent_status(pdpgsqlSSLStatus, "Percona Distribution for PostgreSQL SSL")
verify_agent_status(pdpgsqlSSLList, "Percona Distribution for PostgreSQL SSL")

if len(psmdbSSLStatus) > 0:
verify_agent_status(psmdbSSLStatus, "Percona Server for MongoDB instance SSL status")
verify_agent_status(psmdbSSLList, "Percona Server for MongoDB instance SSL list")

if len(firstMongoReplicaStatus) > 0:
verify_agent_status(firstMongoReplicaStatus, "Percona Server for MongoDB instance 1")
verify_agent_status(firstMongoReplicaList, "Percona Server for MongoDB instance 1")

if len(secondMongoReplicaStatus) > 0:
verify_agent_status(secondMongoReplicaStatus, "Percona Server for MongoDB instance 2")
verify_agent_status(secondMongoReplicaList, "Percona Server for MongoDB instance 2")

if len(thirdMongoReplicaStatus) > 0:
verify_agent_status(thirdMongoReplicaStatus, "Percona Server for MongoDB instance 3")
verify_agent_status(thirdMongoReplicaList, "Percona Server for MongoDB instance 3")

if len(localClientStatus) > 0:
verify_agent_status(localClientStatus, "Local Client")
verify_agent_status(localClientList, "Local Client")

if len(errors) > 0:
raise Exception("Some errors in pmm-admin status: ".join(errors))

expected_version=arguments[1].replace("\\r\\n", "")
admin_version = subprocess.run(["pmm-admin status | grep pmm-admin | awk -F' ' '{print $3}'"], capture_output=True, text=True, shell=True).stdout.replace("\\r\\n", "").strip()

if admin_version != expected_version:
print(f"admin version is: {admin_version} and expected version is: {expected_version}")
errors.append(f"Version of pmm admin is not correct expected: {expected_version} actual: {admin_version}")

agent_version = subprocess.run(["pmm-admin status | grep pmm-agent | awk -F' ' '{print $3}'"], capture_output=True, text=True, shell=True).stdout.replace("\\r\\n", "").strip()

if agent_version != expected_version:
print(f"agent version is: {agent_version} and expected version is: {expected_version}")
errors.append(f"Version of pmm agent is not correct expected: {expected_version} actual: {agent_version}")

if admin_version != agent_version:
errors.append(f"PMM admin version: {admin_version} does not equal PMM agent version {agent_version}")

if len(errors) > 0:
raise Exception("Errors in pmm-admin and pmm-agent versions: ".join(errors))
errors = []

def run_command(cmd, shell=False):
"""Run a command and return its output as lines."""
result = subprocess.run(cmd, capture_output=True, text=True, shell=shell)
return result.stdout.splitlines()


def get_container_name(pattern):
"""Find and return the container name that matches a given pattern."""
containers = run_command(["docker", "ps", "-a"])
for line in containers:
if pattern in line:
return line.split()[-1] # container name is usually the last column
return None


def contains_bad_state(lines):
"""Check if any line contains a bad PMM agent state."""
return any(state in line for line in lines for state in BAD_STATES)


def check_pmm_output(container_name, label, command):
"""Run pmm-admin command and check for bad states."""
output = run_command(["docker", "exec", container_name, "pmm-admin", command])
if contains_bad_state(output):
errors.append(f"❌ Bad state found in {label} container ({command})")
return output


def check_local_pmm_output(command, label):
"""Run pmm-admin command locally and check for bad states."""
output = run_command(["pmm-admin", command])
if contains_bad_state(output):
errors.append(f"❌ Bad state found in local PMM client ({label})")
return output


def get_version_from_output(lines, keyword, label):
"""Extract version for a given keyword from a list of lines."""
for line in lines:
if keyword in line:
parts = line.split()
if len(parts) >= 3:
return parts[2]
errors.append(f"❌ Could not determine version for {keyword} in {label}")
return None


def main():
print("🔍 Validating PMM containers and local PMM client...\n")

# Check containers
for pattern, label in CONTAINER_PATTERNS:
container_name = get_container_name(pattern)
assert container_name, f"❌ Could not find container matching pattern '{pattern}'"
print(f"✅ Found container '{container_name}' for {label}")

# Check status and list
status_output = check_pmm_output(container_name, label, "status")
list_output = check_pmm_output(container_name, label, "list")

# Version checks inside container
admin_version = get_version_from_output(status_output, "pmm-admin", label)
agent_version = get_version_from_output(status_output, "pmm-agent", label)

if admin_version and admin_version != EXPECTED_VERSION:
errors.append(f"❌ PMM admin version in {label} container mismatch: expected {EXPECTED_VERSION}, got {admin_version}")
if agent_version and agent_version != EXPECTED_VERSION:
errors.append(f"❌ PMM agent version in {label} container mismatch: expected {EXPECTED_VERSION}, got {agent_version}")
if admin_version and agent_version and admin_version != agent_version:
errors.append(f"❌ Version mismatch in {label} container: pmm-admin={admin_version}, pmm-agent={agent_version}")
else:
print(f"✅ Version checks passed in {label} container")

print("\n✅ All container checks done.\n")

# Check local PMM client
local_status_output = check_local_pmm_output("status", "status")
check_local_pmm_output("list", "list")

local_admin_version = get_version_from_output(local_status_output, "pmm-admin", "local client")
local_agent_version = get_version_from_output(local_status_output, "pmm-agent", "local client")

assert local_admin_version, "❌ Could not determine local pmm-admin version"
assert local_agent_version, "❌ Could not determine local pmm-agent version"

if local_admin_version != EXPECTED_VERSION:
errors.append(f"❌ Local PMM admin version mismatch: expected {EXPECTED_VERSION}, got {local_admin_version}")
if local_agent_version != EXPECTED_VERSION:
errors.append(f"❌ Local PMM agent version mismatch: expected {EXPECTED_VERSION}, got {local_agent_version}")
if local_admin_version != local_agent_version:
errors.append(f"❌ Local version mismatch: pmm-admin={local_admin_version}, pmm-agent={local_agent_version}")
else:
print("✅ Local PMM admin and agent versions match")

print("\n📦 Final Results:\n")
if errors:
for err in errors:
print(err)
raise AssertionError("🚫 Validation failed. See errors above.")
else:
print("🎉 All checks passed! PMM containers and local client are healthy and versions are correct.")


if __name__ == "__main__":
main()