From b3aa34ea8adceb3d108c26fc3e95e20fcdfcd2ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Wed, 11 Feb 2026 17:20:40 +0100 Subject: [PATCH 01/11] Fix metadata --- kubernetes/metadata.yaml | 2 +- machines/metadata.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/kubernetes/metadata.yaml b/kubernetes/metadata.yaml index 1281587c0..8f3bd8b57 100644 --- a/kubernetes/metadata.yaml +++ b/kubernetes/metadata.yaml @@ -14,7 +14,7 @@ docs: https://canonical-charmed-mysql-k8s.readthedocs-hosted.com/ source: https://github.com/canonical/mysql-operators issues: https://github.com/canonical/mysql-operators/issues website: - - https://ubuntu.com/data/mysql + - https://canonical.com/data/mysql - https://charmhub.io/mysql-k8s - https://github.com/canonical/mysql-operators maintainers: diff --git a/machines/metadata.yaml b/machines/metadata.yaml index 7f0ed8599..96bd2f59c 100644 --- a/machines/metadata.yaml +++ b/machines/metadata.yaml @@ -14,7 +14,7 @@ docs: https://canonical-charmed-mysql.readthedocs-hosted.com/ source: https://github.com/canonical/mysql-operators issues: https://github.com/canonical/mysql-operators/issues website: - - https://ubuntu.com/data/mysql + - https://canonical.com/data/mysql - https://charmhub.io/mysql - https://github.com/canonical/mysql-operators maintainers: From a0f06fba8c2012da84ec5ff7cad979a6904297d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Mon, 20 Apr 2026 16:39:46 +0200 Subject: [PATCH 02/11] Improve support for ty --- kubernetes/lib/charms/mysql/v0/backups.py | 2 +- kubernetes/pyproject.toml | 3 +++ machines/lib/charms/mysql/v0/backups.py | 2 +- machines/pyproject.toml | 3 +++ machines/src/charm.py | 2 +- 5 files changed, 9 insertions(+), 3 deletions(-) diff --git a/kubernetes/lib/charms/mysql/v0/backups.py b/kubernetes/lib/charms/mysql/v0/backups.py index 5f216c94e..2fe1570f5 100644 --- a/kubernetes/lib/charms/mysql/v0/backups.py +++ b/kubernetes/lib/charms/mysql/v0/backups.py @@ -118,7 +118,7 @@ def is_unit_blocked(self) -> bool: ) if typing.TYPE_CHECKING: - from mysql import MySQLCharmBase + from .mysql import MySQLCharmBase class MySQLBackups(Object): diff --git a/kubernetes/pyproject.toml b/kubernetes/pyproject.toml index 676d07254..1ac2b69c6 100644 --- a/kubernetes/pyproject.toml +++ b/kubernetes/pyproject.toml @@ -144,3 +144,6 @@ max-complexity = 10 [tool.ruff.lint.pydocstyle] convention = "google" + +[tool.ty.environment] +extra-paths = ["./lib"] diff --git a/machines/lib/charms/mysql/v0/backups.py b/machines/lib/charms/mysql/v0/backups.py index 54f84b202..acf536025 100644 --- a/machines/lib/charms/mysql/v0/backups.py +++ b/machines/lib/charms/mysql/v0/backups.py @@ -117,7 +117,7 @@ def is_unit_blocked(self) -> bool: ) if typing.TYPE_CHECKING: - from mysql import MySQLCharmBase + from .mysql import MySQLCharmBase class MySQLBackups(Object): diff --git a/machines/pyproject.toml b/machines/pyproject.toml index 696bde588..26f7ee03b 100644 --- a/machines/pyproject.toml +++ b/machines/pyproject.toml @@ -142,3 +142,6 @@ max-complexity = 10 [tool.ruff.lint.pydocstyle] convention = "google" + +[tool.ty.environment] +extra-paths = ["./lib"] diff --git a/machines/src/charm.py b/machines/src/charm.py index 5090ee494..c35e2162f 100755 --- a/machines/src/charm.py +++ b/machines/src/charm.py @@ -634,7 +634,7 @@ def _on_cos_agent_relation_broken(self, _: RelationBrokenEvent) -> None: # ======================= @property - def _mysql(self): + def _mysql(self) -> MySQL: """Returns an instance of the MySQL object.""" return MySQL( self.unit_fqdn, From ac2228a785b7f56bae3c4bc2cd50e537ccf16bf6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Fri, 17 Apr 2026 16:15:46 +0200 Subject: [PATCH 03/11] Make Ceph backup test amenable for local development --- .../integration/test_backup_ceph.py | 80 ++++++++++++------- kubernetes/tox.ini | 2 + .../integration/test_backup_ceph.py | 41 +++++++--- machines/tox.ini | 2 + 4 files changed, 82 insertions(+), 43 deletions(-) diff --git a/kubernetes/tests/integration/integration/test_backup_ceph.py b/kubernetes/tests/integration/integration/test_backup_ceph.py index 719e8b288..7b72c0f44 100644 --- a/kubernetes/tests/integration/integration/test_backup_ceph.py +++ b/kubernetes/tests/integration/integration/test_backup_ceph.py @@ -83,8 +83,8 @@ class MicrocephConnectionInformation: access_key_id: str secret_access_key: str bucket: str - ca_cert_base64: str - region: str + ca_cert_base64: str | None = None + region: str = "default" @pytest.fixture(scope="session") @@ -111,6 +111,8 @@ def microceph(certs_path, host_ip) -> MicrocephConnectionInformation: os.environ["CEPH_ACCESS_KEY"], os.environ["CEPH_SECRET_KEY"], MICROCEPH_BUCKET, + os.environ.get("CEPH_CA_CERT"), # Optional for HTTP-only local dev + os.environ.get("CEPH_REGION", "default"), ) logger.info("Setting up TLS certificates") subprocess.run(f"openssl genrsa -out {certs_path}/ca.key 2048".split(), check=True) @@ -217,47 +219,57 @@ def microceph(certs_path, host_ip) -> MicrocephConnectionInformation: @pytest.fixture(scope="session") -def cloud_credentials(microceph) -> dict[str, str]: - """Read cloud credentials.""" - return { - "access-key": microceph.access_key_id, - "secret-key": microceph.secret_access_key, - } - - -@pytest.fixture(scope="session") -def cloud_configs(microceph) -> dict[str, str]: - return { +def cloud_configs_ceph(microceph) -> tuple[dict[str, str], dict[str, str]]: + configs = { "endpoint": microceph.endpoint_url, "bucket": microceph.bucket, "path": "mysql-k8s", - "region": "default", - "tls-ca-chain": microceph.ca_cert_base64, + "region": microceph.region, + } + # Only add TLS CA chain if provided (for HTTPS endpoints) + if microceph.ca_cert_base64: + configs["tls-ca-chain"] = microceph.ca_cert_base64 + + credentials = { + "access-key": microceph.access_key_id, + "secret-key": microceph.secret_access_key, } + return configs, credentials @pytest.fixture(scope="session", autouse=True) -def clean_backups_from_buckets(cloud_credentials, cloud_configs): +def clean_backups_from_buckets(cloud_configs_ceph, request): """Teardown to clean up created backups from clouds.""" yield - logger.info("Cleaning backups from buckets") + # Skip cleanup if any tests failed + if request.session.testsfailed > 0: + logger.warning( + f"Keeping backups for debugging ({request.session.testsfailed} test(s) failed)" + ) + return + + # All tests passed and completed - proceed with cleanup + cloud_configs, cloud_credentials = cloud_configs_ceph + + logger.info("Cleaning backups from cloud buckets") session = boto3.session.Session( # pyright: ignore aws_access_key_id=cloud_credentials["access-key"], aws_secret_access_key=cloud_credentials["secret-key"], region_name=cloud_configs["region"], ) - with tempfile.NamedTemporaryFile() as ca_file: - ca_chain = base64.b64decode(cloud_configs["tls-ca-chain"]) - ca_file.write(ca_chain) - ca_file.flush() - - s3 = session.resource( - "s3", - endpoint_url=cloud_configs["endpoint"], - verify=ca_file.name, - ) - bucket = s3.Bucket(cloud_configs["bucket"]) + + with tempfile.TemporaryDirectory() as tmpdir: + s3_extra_kwargs = {} + if "tls-ca-chain" in cloud_configs: + ca_path = Path(tmpdir) / "ca.crt" + ca_chain = base64.b64decode(cloud_configs["tls-ca-chain"]) + ca_path.write_bytes(ca_chain) + s3_extra_kwargs["verify"] = str(ca_path) + + bucket = session.resource( + "s3", endpoint_url=cloud_configs["endpoint"], **s3_extra_kwargs + ).Bucket(cloud_configs["bucket"]) # GCS doesn't support batch delete operation, so delete the objects one by one backup_path = str(Path(cloud_configs["path"]) / CLOUD) @@ -307,10 +319,12 @@ def test_build_and_deploy(juju: Juju, charm) -> None: ) -def test_backup(juju: Juju, cloud_credentials, cloud_configs) -> None: +def test_backup(juju: Juju, cloud_configs_ceph) -> None: """Test to create a backup and list backups.""" global backup_id, value_before_backup, value_after_backup + cloud_configs, cloud_credentials = cloud_configs_ceph + app_units = get_app_units(juju, DATABASE_APP_NAME) zeroth_unit_name = app_units[0] @@ -360,8 +374,10 @@ def test_backup(juju: Juju, cloud_credentials, cloud_configs) -> None: verify_mysql_test_data(juju, DATABASE_APP_NAME, TABLE_NAME, value_after_backup) -def test_restore_on_same_cluster(juju: Juju, cloud_credentials, cloud_configs) -> None: +def test_restore_on_same_cluster(juju: Juju, cloud_configs_ceph) -> None: """Test to restore a backup to the same mysql cluster.""" + cloud_configs, cloud_credentials = cloud_configs_ceph + logger.info("Scaling mysql application to 1 unit") scale_app_units(juju, DATABASE_APP_NAME, 1) @@ -449,8 +465,10 @@ def test_restore_on_same_cluster(juju: Juju, cloud_credentials, cloud_configs) - ), "cluster should migrate to blocked status after restore" -def test_restore_on_new_cluster(juju: Juju, charm, cloud_credentials, cloud_configs) -> None: +def test_restore_on_new_cluster(juju: Juju, charm, cloud_configs_ceph) -> None: """Test to restore a backup on a new mysql cluster.""" + cloud_configs, cloud_credentials = cloud_configs_ceph + logger.info("Deploying a new mysql cluster") new_mysql_application_name = "another-mysql-k8s" diff --git a/kubernetes/tox.ini b/kubernetes/tox.ini index 7609a3df5..df0a6239d 100644 --- a/kubernetes/tox.ini +++ b/kubernetes/tox.ini @@ -78,6 +78,8 @@ pass_env = CEPH_ENDPOINT_URL CEPH_ACCESS_KEY CEPH_SECRET_KEY + CEPH_CA_CERT + CEPH_REGION commands_pre = poetry install --only integration commands = diff --git a/machines/tests/integration/integration/test_backup_ceph.py b/machines/tests/integration/integration/test_backup_ceph.py index cf861e93d..b7469465f 100644 --- a/machines/tests/integration/integration/test_backup_ceph.py +++ b/machines/tests/integration/integration/test_backup_ceph.py @@ -82,8 +82,8 @@ class MicrocephConnectionInformation: access_key_id: str secret_access_key: str bucket: str - ca_cert_base64: str - region: str + ca_cert_base64: str | None = None + region: str = "default" @pytest.fixture(scope="session") @@ -110,6 +110,8 @@ def microceph(certs_path, host_ip) -> MicrocephConnectionInformation: os.environ["CEPH_ACCESS_KEY"], os.environ["CEPH_SECRET_KEY"], MICROCEPH_BUCKET, + os.environ.get("CEPH_CA_CERT"), # Optional for HTTP-only local dev + os.environ.get("CEPH_REGION", "default"), ) logger.info("Setting up TLS certificates") subprocess.run(f"openssl genrsa -out {certs_path}/ca.key 2048".split(), check=True) @@ -221,9 +223,12 @@ def cloud_configs_ceph(microceph) -> tuple[dict[str, str], dict[str, str]]: "endpoint": microceph.endpoint_url, "bucket": microceph.bucket, "path": "mysql", - "region": "default", - "tls-ca-chain": microceph.ca_cert_base64, + "region": microceph.region, } + # Only add TLS CA chain if provided (for HTTPS endpoints) + if microceph.ca_cert_base64: + configs["tls-ca-chain"] = microceph.ca_cert_base64 + credentials = { "access-key": microceph.access_key_id, "secret-key": microceph.secret_access_key, @@ -232,10 +237,18 @@ def cloud_configs_ceph(microceph) -> tuple[dict[str, str], dict[str, str]]: @pytest.fixture(scope="session", autouse=True) -def clean_backups_from_buckets(cloud_configs_ceph): +def clean_backups_from_buckets(cloud_configs_ceph, request): """Teardown to clean up created backups from clouds.""" yield + # Skip cleanup if any tests failed + if request.session.testsfailed > 0: + logger.warning( + f"Keeping backups for debugging ({request.session.testsfailed} test(s) failed)" + ) + return + + # All tests passed and completed - proceed with cleanup cloud_configs, cloud_credentials = cloud_configs_ceph logger.info("Cleaning backups from cloud buckets") @@ -245,13 +258,17 @@ def clean_backups_from_buckets(cloud_configs_ceph): region_name=cloud_configs["region"], ) - with tempfile.NamedTemporaryFile() as ca_file: - ca_chain = base64.b64decode(cloud_configs["tls-ca-chain"]) - ca_file.write(ca_chain) - ca_file.flush() - - s3 = session.resource("s3", endpoint_url=cloud_configs["endpoint"], verify=ca_file.name) - bucket = s3.Bucket(cloud_configs["bucket"]) + with tempfile.TemporaryDirectory() as tmpdir: + s3_extra_kwargs = {} + if "tls-ca-chain" in cloud_configs: + ca_path = Path(tmpdir) / "ca.crt" + ca_chain = base64.b64decode(cloud_configs["tls-ca-chain"]) + ca_path.write_bytes(ca_chain) + s3_extra_kwargs["verify"] = str(ca_path) + + bucket = session.resource( + "s3", endpoint_url=cloud_configs["endpoint"], **s3_extra_kwargs + ).Bucket(cloud_configs["bucket"]) # GCS doesn't support batch delete operation, so delete the objects one by one backup_path = str(Path(cloud_configs["path"]) / backup_id) diff --git a/machines/tox.ini b/machines/tox.ini index 7c6578064..578b2992b 100644 --- a/machines/tox.ini +++ b/machines/tox.ini @@ -84,6 +84,8 @@ pass_env = CEPH_ENDPOINT_URL CEPH_ACCESS_KEY CEPH_SECRET_KEY + CEPH_CA_CERT + CEPH_REGION commands_pre = poetry install --only integration commands = From d5d87d6f8ea5eddfbd43d6949fb91c80b71c6520 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Wed, 15 Apr 2026 14:06:49 +0200 Subject: [PATCH 04/11] [VM] Add better logging to stop_mysql_process_gracefully --- machines/tests/integration/helpers_ha.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/machines/tests/integration/helpers_ha.py b/machines/tests/integration/helpers_ha.py index 7ab945c12..663953ac5 100644 --- a/machines/tests/integration/helpers_ha.py +++ b/machines/tests/integration/helpers_ha.py @@ -3,6 +3,7 @@ # See LICENSE file for licensing details. import json +import logging import subprocess import uuid from collections.abc import Callable, Generator @@ -15,6 +16,7 @@ from jubilant.statustypes import Status from tenacity import ( Retrying, + before_sleep_log, retry, stop_after_attempt, stop_after_delay, @@ -33,6 +35,8 @@ JujuModelStatusFn = Callable[[Status], bool] JujuAppsStatusFn = Callable[[Status, str], bool] +logger = logging.getLogger(__name__) + def check_mysql_instances_online( juju: Juju, @@ -443,7 +447,11 @@ def stop_mysql_process_gracefully(juju: Juju, unit_name: str) -> None: ) # Hold execution until process is stopped - for attempt in Retrying(stop=stop_after_attempt(10), wait=wait_fixed(5)): + for attempt in Retrying( + stop=stop_after_attempt(10), + wait=wait_fixed(5), + before_sleep=before_sleep_log(logger, logging.WARNING), + ): with attempt: if get_unit_process_id(juju, unit_name, "mysqld") is not None: raise Exception("Failed to stop the mysqld process") From 91d958570a4aacd43b89335a70dfd70880a28d39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Thu, 16 Apr 2026 16:07:11 +0200 Subject: [PATCH 05/11] [VM] Improve observability of backup command --- machines/src/mysql_vm_helpers.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/machines/src/mysql_vm_helpers.py b/machines/src/mysql_vm_helpers.py index 3eb4fa4d1..eafd7a90e 100644 --- a/machines/src/mysql_vm_helpers.py +++ b/machines/src/mysql_vm_helpers.py @@ -620,19 +620,23 @@ def _execute_commands( stdout += line return_code = process.wait() + + # Read stdout and stderr before checking return code + if not stdout and process.stdout: + stdout = process.stdout.read() + if not stderr and process.stderr: + stderr = process.stderr.read() + if return_code != 0: message = ( "Failed command: " f"{self.strip_off_passwords(' '.join(commands))};" - f" {user=}; {group=}" + f" {user=}; {group=}; " + f"stdout: {stdout.strip()}; " + f"stderr: {stderr.strip()}" ) logger.error(message) - raise MySQLExecError from None - - if not stdout and process.stdout: - stdout = process.stdout.read() - if not stderr and process.stderr: - stderr = process.stderr.read() + raise MySQLExecError(message) from None return (stdout.strip(), stderr.strip()) From a6436537a2ec22d10280c4b37afd47feb2c63ae1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Tue, 21 Apr 2026 12:28:50 +0200 Subject: [PATCH 06/11] [VM] Fix LSP in backup methods of MySQL class --- kubernetes/lib/charms/mysql/v0/backups.py | 2 +- kubernetes/lib/charms/mysql/v0/mysql.py | 1 + kubernetes/src/mysql_k8s_helpers.py | 83 +++++++++++----- machines/lib/charms/mysql/v0/backups.py | 2 +- machines/lib/charms/mysql/v0/mysql.py | 1 + machines/src/mysql_vm_helpers.py | 110 ++++++++++++++-------- 6 files changed, 134 insertions(+), 65 deletions(-) diff --git a/kubernetes/lib/charms/mysql/v0/backups.py b/kubernetes/lib/charms/mysql/v0/backups.py index 2fe1570f5..c6460cdea 100644 --- a/kubernetes/lib/charms/mysql/v0/backups.py +++ b/kubernetes/lib/charms/mysql/v0/backups.py @@ -727,7 +727,7 @@ def _clean_data_dir_and_start_mysqld(self) -> tuple[bool, str]: self.charm._mysql.delete_temp_backup_directory() # Old backups may contain the temp backup directory (as previously, the temp # backup directory was created in the mysql data directory to reduce IOPS latency) - self.charm._mysql.delete_temp_backup_directory(from_directory=MYSQL_DATA_DIR) + self.charm._mysql.delete_temp_backup_directory(tmp_base_directory=MYSQL_DATA_DIR) except MySQLDeleteTempRestoreDirectoryError: return False, "Failed to delete the temp restore directory" except MySQLDeleteTempBackupDirectoryError: diff --git a/kubernetes/lib/charms/mysql/v0/mysql.py b/kubernetes/lib/charms/mysql/v0/mysql.py index 85a6cbe86..26261ce01 100644 --- a/kubernetes/lib/charms/mysql/v0/mysql.py +++ b/kubernetes/lib/charms/mysql/v0/mysql.py @@ -2868,6 +2868,7 @@ def _execute_commands( user: str | None = None, group: str | None = None, env_extra: dict | None = None, + timeout: float | None = None, stream_output: str | None = None, ) -> tuple[str, str]: """Execute commands on the server where MySQL is running.""" diff --git a/kubernetes/src/mysql_k8s_helpers.py b/kubernetes/src/mysql_k8s_helpers.py index 5722de153..3774a2f1f 100644 --- a/kubernetes/src/mysql_k8s_helpers.py +++ b/kubernetes/src/mysql_k8s_helpers.py @@ -319,12 +319,17 @@ def execute_backup_commands( group, ) - def delete_temp_backup_directory(self, from_directory: str = MYSQL_DATA_DIR) -> None: + def delete_temp_backup_directory( + self, + tmp_base_directory: str = MYSQL_DATA_DIR, + user=MYSQL_SYSTEM_USER, + group=MYSQL_SYSTEM_GROUP, + ) -> None: """Delete the temp backup directory in the data directory.""" super().delete_temp_backup_directory( - from_directory, - user=MYSQL_SYSTEM_USER, - group=MYSQL_SYSTEM_GROUP, + tmp_base_directory, + user, + group, ) def retrieve_backup_with_xbcloud( @@ -334,8 +339,8 @@ def retrieve_backup_with_xbcloud( temp_restore_directory: str = MYSQL_DATA_DIR, xbcloud_location: str = CHARMED_MYSQL_XBCLOUD_LOCATION, xbstream_location: str = CHARMED_MYSQL_XBSTREAM_LOCATION, - user: str = MYSQL_SYSTEM_USER, - group: str = MYSQL_SYSTEM_GROUP, + user: str | None = MYSQL_SYSTEM_USER, + group: str | None = MYSQL_SYSTEM_GROUP, ) -> tuple[str, str, str]: """Retrieve the specified backup from S3. @@ -352,42 +357,68 @@ def retrieve_backup_with_xbcloud( group, ) - def prepare_backup_for_restore(self, backup_location: str) -> tuple[str, str]: + def prepare_backup_for_restore( + self, + backup_location: str, + xtrabackup_location: str = CHARMED_MYSQL_XTRABACKUP_LOCATION, + xtrabackup_plugin_dir: str = XTRABACKUP_PLUGIN_DIR, + user=MYSQL_SYSTEM_USER, + group=MYSQL_SYSTEM_GROUP, + ) -> tuple[str, str]: """Prepare the backup in the provided dir for restore.""" return super().prepare_backup_for_restore( backup_location, - CHARMED_MYSQL_XTRABACKUP_LOCATION, - XTRABACKUP_PLUGIN_DIR, - user=MYSQL_SYSTEM_USER, - group=MYSQL_SYSTEM_GROUP, + xtrabackup_location, + xtrabackup_plugin_dir, + user, + group, ) - def empty_data_files(self) -> None: + def empty_data_files( + self, + mysql_data_directory=MYSQL_DATA_DIR, + user=MYSQL_SYSTEM_USER, + group=MYSQL_SYSTEM_GROUP, + ) -> None: """Empty the mysql data directory in preparation of backup restore.""" super().empty_data_files( - MYSQL_DATA_DIR, - user=MYSQL_SYSTEM_USER, - group=MYSQL_SYSTEM_GROUP, + mysql_data_directory, + user, + group, ) - def restore_backup(self, backup_location: str) -> tuple[str, str]: + def restore_backup( + self, + backup_location: str, + xtrabackup_location: str = CHARMED_MYSQL_XTRABACKUP_LOCATION, + defaults_config_file: str = MYSQLD_DEFAULTS_CONFIG_FILE, + mysql_data_directory: str = MYSQL_DATA_DIR, + xtrabackup_plugin_directory: str = XTRABACKUP_PLUGIN_DIR, + user=MYSQL_SYSTEM_USER, + group=MYSQL_SYSTEM_GROUP, + ) -> tuple[str, str]: """Restore the provided prepared backup.""" return super().restore_backup( backup_location, - CHARMED_MYSQL_XTRABACKUP_LOCATION, - MYSQLD_DEFAULTS_CONFIG_FILE, - MYSQL_DATA_DIR, - XTRABACKUP_PLUGIN_DIR, - user=MYSQL_SYSTEM_USER, - group=MYSQL_SYSTEM_GROUP, + xtrabackup_location, + defaults_config_file, + mysql_data_directory, + xtrabackup_plugin_directory, + user, + group, ) - def delete_temp_restore_directory(self) -> None: + def delete_temp_restore_directory( + self, + temp_restore_directory: str = MYSQL_DATA_DIR, + user=MYSQL_SYSTEM_USER, + group=MYSQL_SYSTEM_GROUP, + ) -> None: """Delete the temp restore directory from the mysql data directory.""" super().delete_temp_restore_directory( - MYSQL_DATA_DIR, - user=MYSQL_SYSTEM_USER, - group=MYSQL_SYSTEM_GROUP, + temp_restore_directory, + user, + group, ) @retry( diff --git a/machines/lib/charms/mysql/v0/backups.py b/machines/lib/charms/mysql/v0/backups.py index acf536025..f41986170 100644 --- a/machines/lib/charms/mysql/v0/backups.py +++ b/machines/lib/charms/mysql/v0/backups.py @@ -726,7 +726,7 @@ def _clean_data_dir_and_start_mysqld(self) -> tuple[bool, str]: self.charm._mysql.delete_temp_backup_directory() # Old backups may contain the temp backup directory (as previously, the temp # backup directory was created in the mysql data directory to reduce IOPS latency) - self.charm._mysql.delete_temp_backup_directory(from_directory=MYSQL_DATA_DIR) + self.charm._mysql.delete_temp_backup_directory(tmp_base_directory=MYSQL_DATA_DIR) except MySQLDeleteTempRestoreDirectoryError: return False, "Failed to delete the temp restore directory" except MySQLDeleteTempBackupDirectoryError: diff --git a/machines/lib/charms/mysql/v0/mysql.py b/machines/lib/charms/mysql/v0/mysql.py index 5172138cd..0159ae96f 100644 --- a/machines/lib/charms/mysql/v0/mysql.py +++ b/machines/lib/charms/mysql/v0/mysql.py @@ -2872,6 +2872,7 @@ def _execute_commands( user: str | None = None, group: str | None = None, env_extra: dict | None = None, + timeout: float | None = None, stream_output: str | None = None, ) -> tuple[str, str]: """Execute commands on the server where MySQL is running.""" diff --git a/machines/src/mysql_vm_helpers.py b/machines/src/mysql_vm_helpers.py index eafd7a90e..06c4fe1ff 100644 --- a/machines/src/mysql_vm_helpers.py +++ b/machines/src/mysql_vm_helpers.py @@ -426,36 +426,47 @@ def wait_until_mysql_connection(self, check_port: bool = True) -> None: logger.debug("MySQL connection possible") - def execute_backup_commands( # type: ignore + def execute_backup_commands( self, - s3_directory: str, + s3_path: str, s3_parameters: dict[str, str], + xtrabackup_location: str = CHARMED_MYSQL_XTRABACKUP_LOCATION, + xbcloud_location: str = CHARMED_MYSQL_XBCLOUD_LOCATION, + xtrabackup_plugin_dir: str = XTRABACKUP_PLUGIN_DIR, + mysqld_socket_file: str = MYSQLD_SOCK_FILE, + tmp_base_directory: str = CHARMED_MYSQL_COMMON_DIRECTORY, + defaults_config_file: str = MYSQLD_DEFAULTS_CONFIG_FILE, + user: str | None = ROOT_SYSTEM_USER, + group: str | None = ROOT_SYSTEM_USER, ) -> tuple[str, str]: """Executes commands to create a backup.""" return super().execute_backup_commands( - s3_directory, + s3_path, s3_parameters, - CHARMED_MYSQL_XTRABACKUP_LOCATION, - CHARMED_MYSQL_XBCLOUD_LOCATION, - XTRABACKUP_PLUGIN_DIR, - MYSQLD_SOCK_FILE, - CHARMED_MYSQL_COMMON_DIRECTORY, - MYSQLD_DEFAULTS_CONFIG_FILE, - user=ROOT_SYSTEM_USER, - group=ROOT_SYSTEM_USER, + xtrabackup_location, + xbcloud_location, + xtrabackup_plugin_dir, + mysqld_socket_file, + tmp_base_directory, + defaults_config_file, + user, + group, ) - def delete_temp_backup_directory( # type: ignore - self, from_directory: str = CHARMED_MYSQL_COMMON_DIRECTORY + def delete_temp_backup_directory( + self, + tmp_base_directory: str = CHARMED_MYSQL_COMMON_DIRECTORY, + user: str | None = ROOT_SYSTEM_USER, + group: str | None = ROOT_SYSTEM_USER, ) -> None: """Delete the temp backup directory.""" super().delete_temp_backup_directory( - from_directory, - user=ROOT_SYSTEM_USER, - group=ROOT_SYSTEM_USER, + tmp_base_directory, + user=user, + group=group, ) - def retrieve_backup_with_xbcloud( # type: ignore + def retrieve_backup_with_xbcloud( self, backup_id: str, s3_parameters: dict[str, str], @@ -476,27 +487,45 @@ def retrieve_backup_with_xbcloud( # type: ignore group, ) - def prepare_backup_for_restore(self, backup_location: str) -> tuple[str, str]: + def prepare_backup_for_restore( + self, + backup_location: str, + xtrabackup_location: str = CHARMED_MYSQL_XTRABACKUP_LOCATION, + xtrabackup_plugin_dir: str = XTRABACKUP_PLUGIN_DIR, + user=ROOT_SYSTEM_USER, + group=ROOT_SYSTEM_USER, + ) -> tuple[str, str]: """Prepare the download backup for restore with xtrabackup --prepare.""" return super().prepare_backup_for_restore( backup_location, - CHARMED_MYSQL_XTRABACKUP_LOCATION, - XTRABACKUP_PLUGIN_DIR, - user=ROOT_SYSTEM_USER, - group=ROOT_SYSTEM_USER, + xtrabackup_location, + xtrabackup_plugin_dir, + user=user, + group=group, ) - def empty_data_files(self) -> None: + def empty_data_files( + self, + mysql_data_directory: str = MYSQL_DATA_DIR, + user: str | None = ROOT_SYSTEM_USER, + group: str | None = ROOT_SYSTEM_USER, + ) -> None: """Empty the mysql data directory in preparation of the restore.""" super().empty_data_files( - MYSQL_DATA_DIR, - user=ROOT_SYSTEM_USER, - group=ROOT_SYSTEM_USER, + mysql_data_directory, + user, + group, ) def restore_backup( self, backup_location: str, + xtrabackup_location: str = CHARMED_MYSQL_XTRABACKUP_LOCATION, + defaults_config_file: str = MYSQLD_DEFAULTS_CONFIG_FILE, + mysql_data_directory: str = MYSQL_DATA_DIR, + xtrabackup_plugin_directory: str = XTRABACKUP_PLUGIN_DIR, + user: str | None = ROOT_SYSTEM_USER, + group: str | None = ROOT_SYSTEM_USER, ) -> tuple[str, str]: """Restore the provided prepared backup.""" # TODO: remove workaround for changing permissions and ownership of data @@ -518,12 +547,12 @@ def restore_backup( stdout, stderr = super().restore_backup( backup_location, - CHARMED_MYSQL_XTRABACKUP_LOCATION, - MYSQLD_DEFAULTS_CONFIG_FILE, - MYSQL_DATA_DIR, - XTRABACKUP_PLUGIN_DIR, - user=ROOT_SYSTEM_USER, - group=ROOT_SYSTEM_USER, + xtrabackup_location, + defaults_config_file, + mysql_data_directory, + xtrabackup_plugin_directory, + user, + group, ) try: @@ -560,12 +589,17 @@ def restore_backup( return (stdout, stderr) - def delete_temp_restore_directory(self) -> None: + def delete_temp_restore_directory( + self, + temp_restore_directory: str = CHARMED_MYSQL_COMMON_DIRECTORY, + user: str | None = ROOT_SYSTEM_USER, + group: str | None = ROOT_SYSTEM_USER, + ) -> None: """Delete the temp restore directory from the mysql data directory.""" super().delete_temp_restore_directory( - CHARMED_MYSQL_COMMON_DIRECTORY, - user=ROOT_SYSTEM_USER, - group=ROOT_SYSTEM_USER, + temp_restore_directory, + user, + group, ) def _execute_commands( @@ -575,6 +609,7 @@ def _execute_commands( user: str | None = None, group: str | None = None, env_extra: dict | None = None, + timeout: float | None = None, stream_output: str | None = None, ) -> tuple[str, str]: """Execute commands on the server where mysql is running. @@ -585,6 +620,7 @@ def _execute_commands( user: the user with which to execute the commands group: the group with which to execute the commands env_extra: the environment variables to add to the current process' environment + timeout: command timeout (unused) stream_output: whether to stream the output to stdout, stderr or None Returns: tuple of (stdout, stderr) @@ -638,7 +674,7 @@ def _execute_commands( logger.error(message) raise MySQLExecError(message) from None - return (stdout.strip(), stderr.strip()) + return stdout.strip(), stderr.strip() def is_mysqld_running(self) -> bool: """Returns whether mysqld is running.""" From fc8955382bd7e59ed4c9eb10e3086c7830cd03e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Tue, 21 Apr 2026 12:47:59 +0200 Subject: [PATCH 07/11] Fix type hints of TLS codepaths --- kubernetes/lib/charms/mysql/v0/mysql.py | 8 ++++---- kubernetes/lib/charms/mysql/v0/s3_helpers.py | 10 +++++----- machines/lib/charms/mysql/v0/mysql.py | 8 ++++---- machines/lib/charms/mysql/v0/s3_helpers.py | 6 +++--- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/kubernetes/lib/charms/mysql/v0/mysql.py b/kubernetes/lib/charms/mysql/v0/mysql.py index 26261ce01..2f9c6057f 100644 --- a/kubernetes/lib/charms/mysql/v0/mysql.py +++ b/kubernetes/lib/charms/mysql/v0/mysql.py @@ -2475,7 +2475,7 @@ def create_sa_pem_file( def execute_backup_commands( self, s3_path: str, - s3_parameters: dict[str, str], + s3_parameters: dict, xtrabackup_location: str, xbcloud_location: str, xtrabackup_plugin_dir: str, @@ -2488,7 +2488,7 @@ def execute_backup_commands( """Executes commands to create a backup with the given args.""" nproc_command = ["nproc"] make_temp_dir_command = f"mktemp --directory {tmp_base_directory}/xtra_backup_XXXX".split() - ca_chain = s3_parameters.get("tls-ca-chain") + ca_chain: list[str] | None = s3_parameters.get("tls-ca-chain") ca_file_location = None try: nproc, _ = self._execute_commands(nproc_command) @@ -2593,7 +2593,7 @@ def delete_temp_backup_directory( def retrieve_backup_with_xbcloud( self, backup_id: str, - s3_parameters: dict[str, str], + s3_parameters: dict, temp_restore_directory: str, xbcloud_location: str, xbstream_location: str, @@ -2605,7 +2605,7 @@ def retrieve_backup_with_xbcloud( make_temp_dir_command = ( f"mktemp --directory {temp_restore_directory}/#mysql_sst_XXXX".split() ) - ca_chain = s3_parameters.get("tls-ca-chain") + ca_chain: list[str] | None = s3_parameters.get("tls-ca-chain") ca_file_location = None try: nproc, _ = self._execute_commands(nproc_command) diff --git a/kubernetes/lib/charms/mysql/v0/s3_helpers.py b/kubernetes/lib/charms/mysql/v0/s3_helpers.py index 3f0fd00fe..435d922d5 100644 --- a/kubernetes/lib/charms/mysql/v0/s3_helpers.py +++ b/kubernetes/lib/charms/mysql/v0/s3_helpers.py @@ -110,7 +110,7 @@ def upload_content_to_s3(content: str, content_path: str, s3_parameters: dict) - try: logger.info(f"Uploading content to bucket={s3_parameters['bucket']}, path={content_path}") - ca_chain = s3_parameters.get("tls-ca-chain") + ca_chain: list[str] | None = s3_parameters.get("tls-ca-chain") with tempfile.NamedTemporaryFile() as content_file, tempfile.NamedTemporaryFile() if ca_chain else nullcontext() as ca_file: content_file.write(content.encode("utf-8")) content_file.flush() @@ -146,7 +146,7 @@ def _read_content_from_s3(content_path: str, s3_parameters: dict) -> str | None: """ try: logger.info(f"Reading content from bucket={s3_parameters['bucket']}, path={content_path}") - ca_chain = s3_parameters.get("tls-ca-chain") + ca_chain: list[str] | None = s3_parameters.get("tls-ca-chain") with tempfile.NamedTemporaryFile() if ca_chain else nullcontext() as ca_file, BytesIO() as buf: if ca_file: ca = "\n".join(ca_chain) @@ -277,7 +277,7 @@ def list_backups_in_s3_path(s3_parameters: dict) -> list[tuple[str, str]]: logger.info( f"Listing subdirectories from S3 bucket={s3_parameters['bucket']}, path={s3_parameters['path']}" ) - ca_chain = s3_parameters.get("tls-ca-chain") + ca_chain: list[str] | None = s3_parameters.get("tls-ca-chain") with _temporary_ca_file(ca_chain) as ca_file_name: s3_client = boto3.client( "s3", @@ -306,7 +306,7 @@ def list_backups_in_s3_path(s3_parameters: dict) -> list[tuple[str, str]]: raise -def fetch_and_check_existence_of_s3_path(path: str, s3_parameters: dict[str, str]) -> bool: +def fetch_and_check_existence_of_s3_path(path: str, s3_parameters: dict) -> bool: """Checks the existence of a provided S3 path by fetching the object. Args: @@ -319,7 +319,7 @@ def fetch_and_check_existence_of_s3_path(path: str, s3_parameters: dict[str, str Raises: any exceptions raised by boto3 """ - ca_chain = s3_parameters.get("tls-ca-chain") + ca_chain: list[str] | None = s3_parameters.get("tls-ca-chain") with _temporary_ca_file(ca_chain) as ca_file_name: s3_client = boto3.client( "s3", diff --git a/machines/lib/charms/mysql/v0/mysql.py b/machines/lib/charms/mysql/v0/mysql.py index 0159ae96f..212ae1fad 100644 --- a/machines/lib/charms/mysql/v0/mysql.py +++ b/machines/lib/charms/mysql/v0/mysql.py @@ -2475,7 +2475,7 @@ def create_ca_pem_file( def execute_backup_commands( self, s3_path: str, - s3_parameters: dict[str, str], + s3_parameters: dict, xtrabackup_location: str, xbcloud_location: str, xtrabackup_plugin_dir: str, @@ -2488,7 +2488,7 @@ def execute_backup_commands( """Executes commands to create a backup with the given args.""" nproc_command = ["nproc"] make_temp_dir_command = f"mktemp --directory {tmp_base_directory}/xtra_backup_XXXX".split() - ca_chain = s3_parameters.get("tls-ca-chain") + ca_chain: list[str] | None = s3_parameters.get("tls-ca-chain") ca_file_location = None try: nproc, _ = self._execute_commands(nproc_command) @@ -2593,7 +2593,7 @@ def delete_temp_backup_directory( def retrieve_backup_with_xbcloud( self, backup_id: str, - s3_parameters: dict[str, str], + s3_parameters: dict, temp_restore_directory: str, xbcloud_location: str, xbstream_location: str, @@ -2605,7 +2605,7 @@ def retrieve_backup_with_xbcloud( make_temp_dir_command = ( f"mktemp --directory {temp_restore_directory}/#mysql_sst_XXXX".split() ) - ca_chain = s3_parameters.get("tls-ca-chain") + ca_chain: list[str] | None = s3_parameters.get("tls-ca-chain") ca_file_location = None try: nproc, _ = self._execute_commands(nproc_command) diff --git a/machines/lib/charms/mysql/v0/s3_helpers.py b/machines/lib/charms/mysql/v0/s3_helpers.py index d55c2d8d8..4ed657c14 100644 --- a/machines/lib/charms/mysql/v0/s3_helpers.py +++ b/machines/lib/charms/mysql/v0/s3_helpers.py @@ -275,7 +275,7 @@ def list_backups_in_s3_path(s3_parameters: dict) -> list[tuple[str, str]]: logger.info( f"Listing subdirectories from S3 bucket={s3_parameters['bucket']}, path={s3_parameters['path']}" ) - ca_chain = s3_parameters.get("tls-ca-chain") + ca_chain: list[str] | None = s3_parameters.get("tls-ca-chain") with _temporary_ca_file(ca_chain) as ca_file_name: s3_client = boto3.client( "s3", @@ -304,7 +304,7 @@ def list_backups_in_s3_path(s3_parameters: dict) -> list[tuple[str, str]]: raise -def fetch_and_check_existence_of_s3_path(path: str, s3_parameters: dict[str, str]) -> bool: +def fetch_and_check_existence_of_s3_path(path: str, s3_parameters: dict) -> bool: """Checks the existence of a provided S3 path by fetching the object. Args: @@ -317,7 +317,7 @@ def fetch_and_check_existence_of_s3_path(path: str, s3_parameters: dict[str, str Raises: any exceptions raised by boto3 """ - ca_chain = s3_parameters.get("tls-ca-chain") + ca_chain: list[str] | None = s3_parameters.get("tls-ca-chain") with _temporary_ca_file(ca_chain) as ca_file_name: s3_client = boto3.client( "s3", From 8812072a9fc1e619042bb3db4f0926f8bb23aebf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Tue, 21 Apr 2026 12:56:34 +0200 Subject: [PATCH 08/11] [K8s] Typo --- kubernetes/lib/charms/mysql/v0/mysql.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kubernetes/lib/charms/mysql/v0/mysql.py b/kubernetes/lib/charms/mysql/v0/mysql.py index 2f9c6057f..ab8aeef36 100644 --- a/kubernetes/lib/charms/mysql/v0/mysql.py +++ b/kubernetes/lib/charms/mysql/v0/mysql.py @@ -2448,7 +2448,7 @@ def get_available_memory(self) -> int: """Platform dependent method to get the available memory for mysql-server.""" raise NotImplementedError - def create_sa_pem_file( + def create_ca_pem_file( self, ca_chain: list[str], tmp_base_directory: str, @@ -2496,7 +2496,7 @@ def execute_backup_commands( make_temp_dir_command, user=user, group=group ) if ca_chain: - ca_file_location = self.create_sa_pem_file( + ca_file_location = self.create_ca_pem_file( ca_chain, tmp_base_directory, user=user, group=group ) except MySQLExecError as e: @@ -2615,7 +2615,7 @@ def retrieve_backup_with_xbcloud( group=group, ) if ca_chain: - ca_file_location = self.create_sa_pem_file( + ca_file_location = self.create_ca_pem_file( ca_chain, temp_restore_directory, user=user, group=group ) except MySQLExecError as e: From fb41c7e3a445d2e5bb2987a828a4105872eb9734 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Tue, 21 Apr 2026 13:02:11 +0200 Subject: [PATCH 09/11] Separate TLS handling more cleanly in backups --- kubernetes/lib/charms/mysql/v0/mysql.py | 55 +++++++++++++++++-------- machines/lib/charms/mysql/v0/mysql.py | 39 ++++++++++++------ 2 files changed, 64 insertions(+), 30 deletions(-) diff --git a/kubernetes/lib/charms/mysql/v0/mysql.py b/kubernetes/lib/charms/mysql/v0/mysql.py index ab8aeef36..e4e207db2 100644 --- a/kubernetes/lib/charms/mysql/v0/mysql.py +++ b/kubernetes/lib/charms/mysql/v0/mysql.py @@ -2488,19 +2488,11 @@ def execute_backup_commands( """Executes commands to create a backup with the given args.""" nproc_command = ["nproc"] make_temp_dir_command = f"mktemp --directory {tmp_base_directory}/xtra_backup_XXXX".split() - ca_chain: list[str] | None = s3_parameters.get("tls-ca-chain") - ca_file_location = None try: nproc, _ = self._execute_commands(nproc_command) - tmp_dir, _ = self._execute_commands( - make_temp_dir_command, user=user, group=group - ) - if ca_chain: - ca_file_location = self.create_ca_pem_file( - ca_chain, tmp_base_directory, user=user, group=group - ) + tmp_dir, _ = self._execute_commands(make_temp_dir_command, user=user, group=group) except MySQLExecError as e: - logger.error("Failed to execute commands prior to running backup") + logger.error(f"Failed to execute commands prior to running backup, reason: {e}") raise MySQLExecuteBackupCommandsError from e except Exception as e: # Catch all other exceptions to prevent the database being stuck in @@ -2508,6 +2500,22 @@ def execute_backup_commands( logger.error("Failed unexpectedly to execute commands prior to running backup") raise MySQLExecuteBackupCommandsError from e + ca_chain: list[str] | None = s3_parameters.get("tls-ca-chain") + ca_file_location = None + if ca_chain: + try: + ca_file_location = self.create_ca_pem_file( + ca_chain, tmp_base_directory, user=user, group=group + ) + except MySQLExecError as e: + logger.error(f"Failed to execute commands prior to running backup, reason: {e}") + raise MySQLExecuteBackupCommandsError from e + except Exception as e: + # Catch all other exceptions to prevent the database being stuck in + # a bad state due to pre-backup operations + logger.error("Failed unexpectedly to execute commands prior to running backup") + raise MySQLExecuteBackupCommandsError from e + # TODO: remove flags --no-server-version-check # when MySQL and XtraBackup versions are in sync xtrabackup_commands = [ @@ -2536,7 +2544,9 @@ def execute_backup_commands( f"--s3-api-version={s3_parameters['s3-api-version']}", f"--s3-bucket-lookup={s3_parameters['s3-uri-style']}", ] - xtrabackup_commands.append(f"--cacert={ca_file_location}") if ca_chain and ca_file_location else None + xtrabackup_commands.append( + f"--cacert={ca_file_location}" + ) if ca_chain and ca_file_location else None xtrabackup_commands.append(f"{s3_path}") try: logger.debug( @@ -2605,8 +2615,6 @@ def retrieve_backup_with_xbcloud( make_temp_dir_command = ( f"mktemp --directory {temp_restore_directory}/#mysql_sst_XXXX".split() ) - ca_chain: list[str] | None = s3_parameters.get("tls-ca-chain") - ca_file_location = None try: nproc, _ = self._execute_commands(nproc_command) tmp_dir, _ = self._execute_commands( @@ -2614,13 +2622,21 @@ def retrieve_backup_with_xbcloud( user=user, group=group, ) - if ca_chain: - ca_file_location = self.create_ca_pem_file( - ca_chain, temp_restore_directory, user=user, group=group - ) except MySQLExecError as e: logger.error("Failed to execute commands prior to running xbcloud get") raise MySQLRetrieveBackupWithXBCloudError(e.message) from e + + ca_chain: list[str] | None = s3_parameters.get("tls-ca-chain") + ca_file_location = None + if ca_chain: + try: + ca_file_location = self.create_ca_pem_file( + ca_chain, temp_restore_directory, user=user, group=group + ) + except MySQLExecError as e: + logger.error("Failed to execute commands prior to running xbcloud get") + raise MySQLRetrieveBackupWithXBCloudError(e.message) from e + retrieve_backup_command = [ f"{xbcloud_location} get", "--curl-retriable-errors=7", @@ -2633,7 +2649,9 @@ def retrieve_backup_with_xbcloud( f"--s3-api-version={s3_parameters['s3-api-version']}", f"{s3_parameters['path']}/{backup_id}", ] - retrieve_backup_command.append(f"--cacert={ca_file_location}") if ca_chain and ca_file_location else None + retrieve_backup_command.append( + f"--cacert={ca_file_location}" + ) if ca_chain and ca_file_location else None retrieve_backup_command += [ f"| {xbstream_location}", "--decompress", @@ -2641,6 +2659,7 @@ def retrieve_backup_with_xbcloud( f"-C {tmp_dir}", f"--parallel={nproc}", ] + try: logger.debug(f"Command to retrieve backup: {' '.join(retrieve_backup_command)}") diff --git a/machines/lib/charms/mysql/v0/mysql.py b/machines/lib/charms/mysql/v0/mysql.py index 212ae1fad..e4e207db2 100644 --- a/machines/lib/charms/mysql/v0/mysql.py +++ b/machines/lib/charms/mysql/v0/mysql.py @@ -2488,15 +2488,9 @@ def execute_backup_commands( """Executes commands to create a backup with the given args.""" nproc_command = ["nproc"] make_temp_dir_command = f"mktemp --directory {tmp_base_directory}/xtra_backup_XXXX".split() - ca_chain: list[str] | None = s3_parameters.get("tls-ca-chain") - ca_file_location = None try: nproc, _ = self._execute_commands(nproc_command) tmp_dir, _ = self._execute_commands(make_temp_dir_command, user=user, group=group) - if ca_chain: - ca_file_location = self.create_ca_pem_file( - ca_chain, tmp_base_directory, user=user, group=group - ) except MySQLExecError as e: logger.error(f"Failed to execute commands prior to running backup, reason: {e}") raise MySQLExecuteBackupCommandsError from e @@ -2506,6 +2500,22 @@ def execute_backup_commands( logger.error("Failed unexpectedly to execute commands prior to running backup") raise MySQLExecuteBackupCommandsError from e + ca_chain: list[str] | None = s3_parameters.get("tls-ca-chain") + ca_file_location = None + if ca_chain: + try: + ca_file_location = self.create_ca_pem_file( + ca_chain, tmp_base_directory, user=user, group=group + ) + except MySQLExecError as e: + logger.error(f"Failed to execute commands prior to running backup, reason: {e}") + raise MySQLExecuteBackupCommandsError from e + except Exception as e: + # Catch all other exceptions to prevent the database being stuck in + # a bad state due to pre-backup operations + logger.error("Failed unexpectedly to execute commands prior to running backup") + raise MySQLExecuteBackupCommandsError from e + # TODO: remove flags --no-server-version-check # when MySQL and XtraBackup versions are in sync xtrabackup_commands = [ @@ -2605,8 +2615,6 @@ def retrieve_backup_with_xbcloud( make_temp_dir_command = ( f"mktemp --directory {temp_restore_directory}/#mysql_sst_XXXX".split() ) - ca_chain: list[str] | None = s3_parameters.get("tls-ca-chain") - ca_file_location = None try: nproc, _ = self._execute_commands(nproc_command) tmp_dir, _ = self._execute_commands( @@ -2614,14 +2622,21 @@ def retrieve_backup_with_xbcloud( user=user, group=group, ) - if ca_chain: - ca_file_location = self.create_ca_pem_file( - ca_chain, temp_restore_directory, user=user, group=group - ) except MySQLExecError as e: logger.error("Failed to execute commands prior to running xbcloud get") raise MySQLRetrieveBackupWithXBCloudError(e.message) from e + ca_chain: list[str] | None = s3_parameters.get("tls-ca-chain") + ca_file_location = None + if ca_chain: + try: + ca_file_location = self.create_ca_pem_file( + ca_chain, temp_restore_directory, user=user, group=group + ) + except MySQLExecError as e: + logger.error("Failed to execute commands prior to running xbcloud get") + raise MySQLRetrieveBackupWithXBCloudError(e.message) from e + retrieve_backup_command = [ f"{xbcloud_location} get", "--curl-retriable-errors=7", From 11ac34022df44369beb6c2097f6a1cfd836242bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Tue, 21 Apr 2026 15:13:11 +0200 Subject: [PATCH 10/11] Improve observability --- kubernetes/lib/charms/mysql/v0/backups.py | 5 ++++- machines/lib/charms/mysql/v0/backups.py | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/kubernetes/lib/charms/mysql/v0/backups.py b/kubernetes/lib/charms/mysql/v0/backups.py index c6460cdea..72b09f265 100644 --- a/kubernetes/lib/charms/mysql/v0/backups.py +++ b/kubernetes/lib/charms/mysql/v0/backups.py @@ -265,7 +265,10 @@ def _pre_create_backup_checks(self, event: ActionEvent) -> bool: return False if "s3-block-message" in self.charm.app_peer_data: - logger.error("Backup failed: S3 relation is blocked for write") + logger.error( + "Backup failed: S3 relation is blocked for write: %s", + self.charm.app_peer_data["s3-block-message"], + ) event.fail("S3 relation is blocked for write") return False diff --git a/machines/lib/charms/mysql/v0/backups.py b/machines/lib/charms/mysql/v0/backups.py index f41986170..4eb3c706d 100644 --- a/machines/lib/charms/mysql/v0/backups.py +++ b/machines/lib/charms/mysql/v0/backups.py @@ -264,7 +264,10 @@ def _pre_create_backup_checks(self, event: ActionEvent) -> bool: return False if "s3-block-message" in self.charm.app_peer_data: - logger.error("Backup failed: S3 relation is blocked for write") + logger.error( + "Backup failed: S3 relation is blocked for write: %s", + self.charm.app_peer_data["s3-block-message"], + ) event.fail("S3 relation is blocked for write") return False From 80206624e84a670abe5974e82f9f30a7bed54ede Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Wed, 22 Apr 2026 09:43:24 +0200 Subject: [PATCH 11/11] Remove ty config --- kubernetes/pyproject.toml | 3 --- machines/pyproject.toml | 3 --- 2 files changed, 6 deletions(-) diff --git a/kubernetes/pyproject.toml b/kubernetes/pyproject.toml index 1ac2b69c6..676d07254 100644 --- a/kubernetes/pyproject.toml +++ b/kubernetes/pyproject.toml @@ -144,6 +144,3 @@ max-complexity = 10 [tool.ruff.lint.pydocstyle] convention = "google" - -[tool.ty.environment] -extra-paths = ["./lib"] diff --git a/machines/pyproject.toml b/machines/pyproject.toml index 26f7ee03b..696bde588 100644 --- a/machines/pyproject.toml +++ b/machines/pyproject.toml @@ -142,6 +142,3 @@ max-complexity = 10 [tool.ruff.lint.pydocstyle] convention = "google" - -[tool.ty.environment] -extra-paths = ["./lib"]