From 1177a7fdafe27b1055423ae7458f05e2b297b265 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Wed, 11 Feb 2026 17:20:40 +0100 Subject: [PATCH 01/44] Fix metadata --- kubernetes/metadata.yaml | 2 +- machines/metadata.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/kubernetes/metadata.yaml b/kubernetes/metadata.yaml index 1281587c0..8f3bd8b57 100644 --- a/kubernetes/metadata.yaml +++ b/kubernetes/metadata.yaml @@ -14,7 +14,7 @@ docs: https://canonical-charmed-mysql-k8s.readthedocs-hosted.com/ source: https://github.com/canonical/mysql-operators issues: https://github.com/canonical/mysql-operators/issues website: - - https://ubuntu.com/data/mysql + - https://canonical.com/data/mysql - https://charmhub.io/mysql-k8s - https://github.com/canonical/mysql-operators maintainers: diff --git a/machines/metadata.yaml b/machines/metadata.yaml index 7f0ed8599..96bd2f59c 100644 --- a/machines/metadata.yaml +++ b/machines/metadata.yaml @@ -14,7 +14,7 @@ docs: https://canonical-charmed-mysql.readthedocs-hosted.com/ source: https://github.com/canonical/mysql-operators issues: https://github.com/canonical/mysql-operators/issues website: - - https://ubuntu.com/data/mysql + - https://canonical.com/data/mysql - https://charmhub.io/mysql - https://github.com/canonical/mysql-operators maintainers: From f2659b7d51da1e4d2c2147fb48560042b27263ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Thu, 12 Feb 2026 16:34:02 +0100 Subject: [PATCH 02/44] Add simple test --- .../integration/integration/test_storage.py | 74 +++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 kubernetes/tests/integration/integration/test_storage.py diff --git a/kubernetes/tests/integration/integration/test_storage.py b/kubernetes/tests/integration/integration/test_storage.py new file mode 100644 index 000000000..03c6d20f6 --- /dev/null +++ b/kubernetes/tests/integration/integration/test_storage.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +# Copyright 2026 Canonical Ltd. +# See LICENSE file for licensing details. + +import logging + +import jubilant +from jubilant import Juju + +from constants import CONTAINER_NAME, MYSQL_DATA_DIR + +from ..helpers_ha import ( + CHARM_METADATA, + MINUTE_SECS, + wait_for_apps_status, +) + +logger = logging.getLogger(__name__) + +DATABASE_APP_NAME = "mysql-k8s" +CLUSTER_NAME = "test_cluster" +TIMEOUT = 15 * MINUTE_SECS + + +def test_build_and_deploy(juju: Juju, charm) -> None: + logger.info(f"Deploying {DATABASE_APP_NAME} with 1 unit") + juju.deploy( + charm, + DATABASE_APP_NAME, + base="ubuntu@22.04", + config={"cluster-name": CLUSTER_NAME, "profile": "testing"}, + resources={"mysql-image": CHARM_METADATA["resources"]["mysql-image"]["upstream-source"]}, + num_units=1, + trust=True, + ) + + juju.wait( + ready=wait_for_apps_status(jubilant.all_active, DATABASE_APP_NAME), + timeout=TIMEOUT, + ) + + +def test_charm_lists_expected_storage(juju: Juju) -> None: + expected_storages = ["database"] + + assert len(juju.status().storage.storage) == len(expected_storages) + + +def test_data_directory_has_expected_contents_after_initialization(juju: Juju) -> None: + expected_content = { + "'#innodb_redo'", + "'#innodb_temp'", + "auto.cnf", + "ca-key.pem", + "ca.pem", + "client-cert.pem", + "client-key.pem", + "ib_buffer_pool", + "mysql", + "mysql.ibd", + "performance_schema", + "private_key.pem", + "public_key.pem", + "server-cert.pem", + "server-key.pem", + "sys", + "undo_001", + "undo_002", + } + + result = juju.ssh(f"{DATABASE_APP_NAME}/0", "ls", MYSQL_DATA_DIR, container=CONTAINER_NAME) + actual_content = set(result.strip().split()) + + assert expected_content <= actual_content From f62e0b2f256d24eed3f3467712d336652c58033f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Fri, 13 Feb 2026 11:56:41 +0100 Subject: [PATCH 03/44] Properly parametrize MySQL datadir --- kubernetes/lib/charms/mysql/v0/mysql.py | 2 ++ kubernetes/src/charm.py | 3 ++- kubernetes/src/constants.py | 2 +- kubernetes/tests/integration/integration/test_tls.py | 6 +++--- kubernetes/tests/unit/test_charm.py | 3 ++- 5 files changed, 10 insertions(+), 6 deletions(-) diff --git a/kubernetes/lib/charms/mysql/v0/mysql.py b/kubernetes/lib/charms/mysql/v0/mysql.py index 85a6cbe86..ecab2de2c 100644 --- a/kubernetes/lib/charms/mysql/v0/mysql.py +++ b/kubernetes/lib/charms/mysql/v0/mysql.py @@ -78,6 +78,7 @@ def __init__( import ops from charms.data_platform_libs.v0.data_interfaces import DataPeerData, DataPeerUnitData from constants import ( + MYSQL_DATA_DIR, BACKUPS_PASSWORD_KEY, BACKUPS_USERNAME, CHARMED_MYSQL_PITR_HELPER, @@ -1161,6 +1162,7 @@ def render_mysqld_configuration( # noqa: C901 # do not enable slow query logs, but specify a log file path in case # the admin enables them manually config["mysqld"] = { + "datadir": MYSQL_DATA_DIR, # All interfaces bind expected "bind_address": "0.0.0.0", # noqa: S104 "mysqlx_bind_address": "0.0.0.0", # noqa: S104 diff --git a/kubernetes/src/charm.py b/kubernetes/src/charm.py index f9aff5f16..d0c7ad22b 100755 --- a/kubernetes/src/charm.py +++ b/kubernetes/src/charm.py @@ -75,6 +75,7 @@ MONITORING_PASSWORD_KEY, MONITORING_USERNAME, MYSQL_BINLOGS_COLLECTOR_SERVICE, + MYSQL_DATA_DIR, MYSQL_LOG_ERROR, MYSQL_LOG_FILES, MYSQL_LOG_SERVICE, @@ -227,7 +228,7 @@ def _pebble_layer(self) -> Layer: mysqld_cmd = [ MYSQLD_LOCATION, "--basedir=/usr", - "--datadir=/var/lib/mysql", + f"--datadir={MYSQL_DATA_DIR}", "--plugin-dir=/usr/lib/mysql/plugin", f"--log-error={MYSQL_LOG_ERROR}", f"--pid-file={self.unit_label}.pid", diff --git a/kubernetes/src/constants.py b/kubernetes/src/constants.py index 0310e2e1a..6a2752ca7 100644 --- a/kubernetes/src/constants.py +++ b/kubernetes/src/constants.py @@ -27,7 +27,7 @@ TLS_SSL_CERT_FILE = "custom-server-cert.pem" MYSQL_CLI_LOCATION = "/usr/bin/mysql" MYSQLSH_LOCATION = "/usr/bin/mysqlsh" -MYSQL_DATA_DIR = "/var/lib/mysql" +MYSQL_DATA_DIR = "/var/lib/mysql" # Corresponds to the database storage mount MYSQLD_SOCK_FILE = "/var/run/mysqld/mysqld.sock" MYSQLD_CONFIG_FILE = "/etc/mysql/mysql.conf.d/z-custom.cnf" MYSQLD_INIT_CONFIG_FILE = "/etc/mysql/mysql.conf.d/z-custom-init-file.cnf" diff --git a/kubernetes/tests/integration/integration/test_tls.py b/kubernetes/tests/integration/integration/test_tls.py index b244bda1b..a0f0c933e 100644 --- a/kubernetes/tests/integration/integration/test_tls.py +++ b/kubernetes/tests/integration/integration/test_tls.py @@ -8,7 +8,7 @@ import jubilant from jubilant import Juju -from constants import CONTAINER_NAME, REPLICATION_USERNAME, TLS_SSL_CERT_FILE +from constants import CONTAINER_NAME, MYSQL_DATA_DIR, REPLICATION_USERNAME, TLS_SSL_CERT_FILE from ..helpers import is_connection_possible from ..helpers_ha import ( @@ -138,7 +138,7 @@ def test_rotate_tls_key(juju: Juju) -> None: for unit_name in app_units: original_tls[unit_name] = {} original_tls[unit_name]["cert"] = unit_file_md5( - juju, unit_name, f"/var/lib/mysql/{TLS_SSL_CERT_FILE}" + juju, unit_name, f"{MYSQL_DATA_DIR}/{TLS_SSL_CERT_FILE}" ) # set key using auto-generated key for each unit @@ -155,7 +155,7 @@ def test_rotate_tls_key(juju: Juju) -> None: # After updating both the external key and the internal key a new certificate request will be # made; then the certificates should be available and updated. for unit_name in app_units: - new_cert_md5 = unit_file_md5(juju, unit_name, f"/var/lib/mysql/{TLS_SSL_CERT_FILE}") + new_cert_md5 = unit_file_md5(juju, unit_name, f"{MYSQL_DATA_DIR}/{TLS_SSL_CERT_FILE}") assert new_cert_md5 != original_tls[unit_name]["cert"], ( f"cert for {unit_name} was not updated." diff --git a/kubernetes/tests/unit/test_charm.py b/kubernetes/tests/unit/test_charm.py index d5ad58676..678d4a723 100644 --- a/kubernetes/tests/unit/test_charm.py +++ b/kubernetes/tests/unit/test_charm.py @@ -15,6 +15,7 @@ BACKUPS_PASSWORD_KEY, DEFAULT_PASSWORD_LENGTH, MONITORING_PASSWORD_KEY, + MYSQL_DATA_DIR, MYSQLD_LOCATION, OPERATOR_PASSWORD_KEY, REPLICATION_PASSWORD_KEY, @@ -56,7 +57,7 @@ def layer_dict(self, with_mysqld_exporter: bool = False): mysqld_cmd = [ MYSQLD_LOCATION, "--basedir=/usr", - "--datadir=/var/lib/mysql", + f"--datadir={MYSQL_DATA_DIR}", "--plugin-dir=/usr/lib/mysql/plugin", "--log-error=/var/log/mysql/error.log", f"--pid-file={self.charm.unit_label}.pid", From 6aae614606ce245e778ba63baa675ed16c892035 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Fri, 13 Feb 2026 12:45:58 +0100 Subject: [PATCH 04/44] Separate temporary tablespace storage --- kubernetes/lib/charms/mysql/v0/mysql.py | 2 ++ kubernetes/metadata.yaml | 11 ++++++++--- kubernetes/src/charm.py | 6 ++---- kubernetes/src/constants.py | 3 ++- kubernetes/src/mysql_k8s_helpers.py | 2 +- .../integration/integration/test_storage.py | 16 +++++++++++++--- kubernetes/tests/unit/test_charm.py | 4 ++-- 7 files changed, 30 insertions(+), 14 deletions(-) diff --git a/kubernetes/lib/charms/mysql/v0/mysql.py b/kubernetes/lib/charms/mysql/v0/mysql.py index ecab2de2c..2b1802208 100644 --- a/kubernetes/lib/charms/mysql/v0/mysql.py +++ b/kubernetes/lib/charms/mysql/v0/mysql.py @@ -79,6 +79,7 @@ def __init__( from charms.data_platform_libs.v0.data_interfaces import DataPeerData, DataPeerUnitData from constants import ( MYSQL_DATA_DIR, + MYSQL_TEMP_DIR, BACKUPS_PASSWORD_KEY, BACKUPS_USERNAME, CHARMED_MYSQL_PITR_HELPER, @@ -1163,6 +1164,7 @@ def render_mysqld_configuration( # noqa: C901 # the admin enables them manually config["mysqld"] = { "datadir": MYSQL_DATA_DIR, + "innodb_temp_tablespaces_dir": MYSQL_TEMP_DIR, # All interfaces bind expected "bind_address": "0.0.0.0", # noqa: S104 "mysqlx_bind_address": "0.0.0.0", # noqa: S104 diff --git a/kubernetes/metadata.yaml b/kubernetes/metadata.yaml index 8f3bd8b57..6df4d3434 100644 --- a/kubernetes/metadata.yaml +++ b/kubernetes/metadata.yaml @@ -27,8 +27,10 @@ containers: gid: 584788 resource: mysql-image mounts: - - storage: database - location: /var/lib/mysql + - storage: data + location: /var/lib/mysql/data + - storage: temp + location: /var/lib/mysql/temp resources: mysql-image: @@ -81,9 +83,12 @@ requires: optional: true storage: - database: + data: type: filesystem description: Persistent storage for MySQL data + temp: + type: filesystem + description: Persistent storage for InnoDB temporary tablespaces assumes: - k8s-api diff --git a/kubernetes/src/charm.py b/kubernetes/src/charm.py index d0c7ad22b..19de55b7a 100755 --- a/kubernetes/src/charm.py +++ b/kubernetes/src/charm.py @@ -129,9 +129,7 @@ def __init__(self, *args): self.framework.observe(self.on.leader_elected, self._on_leader_elected) self.framework.observe(self.on.config_changed, self._on_config_changed) self.framework.observe(self.on.update_status, self._on_update_status) - self.framework.observe( - self.on.database_storage_detaching, self._on_database_storage_detaching - ) + self.framework.observe(self.on.data_storage_detaching, self._on_data_storage_detaching) self.framework.observe(self.on[PEER].relation_joined, self._on_peer_relation_joined) self.framework.observe(self.on[PEER].relation_changed, self._on_peer_relation_changed) @@ -1065,7 +1063,7 @@ def _on_peer_relation_departed(self, event: RelationDepartedEvent) -> None: if not self._mysql.reconcile_binlogs_collection(force_restart=True): logger.error("Failed to reconcile binlogs collection during peer departed event") - def _on_database_storage_detaching(self, _) -> None: + def _on_data_storage_detaching(self, _) -> None: """Handle the database storage detaching event.""" # Only executes if the unit was initialised if not self.unit_initialized(): diff --git a/kubernetes/src/constants.py b/kubernetes/src/constants.py index 6a2752ca7..ee6d30e77 100644 --- a/kubernetes/src/constants.py +++ b/kubernetes/src/constants.py @@ -27,7 +27,8 @@ TLS_SSL_CERT_FILE = "custom-server-cert.pem" MYSQL_CLI_LOCATION = "/usr/bin/mysql" MYSQLSH_LOCATION = "/usr/bin/mysqlsh" -MYSQL_DATA_DIR = "/var/lib/mysql" # Corresponds to the database storage mount +MYSQL_DATA_DIR = "/var/lib/mysql/data" # Corresponds to the data storage mount +MYSQL_TEMP_DIR = "/var/lib/mysql/temp" # Corresponds to the temp storage mount MYSQLD_SOCK_FILE = "/var/run/mysqld/mysqld.sock" MYSQLD_CONFIG_FILE = "/etc/mysql/mysql.conf.d/z-custom.cnf" MYSQLD_INIT_CONFIG_FILE = "/etc/mysql/mysql.conf.d/z-custom-init-file.cnf" diff --git a/kubernetes/src/mysql_k8s_helpers.py b/kubernetes/src/mysql_k8s_helpers.py index 5722de153..24febf110 100644 --- a/kubernetes/src/mysql_k8s_helpers.py +++ b/kubernetes/src/mysql_k8s_helpers.py @@ -578,7 +578,7 @@ def is_data_dir_initialised(self) -> bool: # minimal expected content for an integral mysqld data-dir expected_content = { "#innodb_redo", - "#innodb_temp", + # "#innodb_temp", # stored separately "auto.cnf", "ca-key.pem", "ca.pem", diff --git a/kubernetes/tests/integration/integration/test_storage.py b/kubernetes/tests/integration/integration/test_storage.py index 03c6d20f6..fbd84d27b 100644 --- a/kubernetes/tests/integration/integration/test_storage.py +++ b/kubernetes/tests/integration/integration/test_storage.py @@ -7,7 +7,7 @@ import jubilant from jubilant import Juju -from constants import CONTAINER_NAME, MYSQL_DATA_DIR +from constants import CONTAINER_NAME, MYSQL_DATA_DIR, MYSQL_TEMP_DIR from ..helpers_ha import ( CHARM_METADATA, @@ -41,7 +41,7 @@ def test_build_and_deploy(juju: Juju, charm) -> None: def test_charm_lists_expected_storage(juju: Juju) -> None: - expected_storages = ["database"] + expected_storages = ["data", "temp"] assert len(juju.status().storage.storage) == len(expected_storages) @@ -49,7 +49,6 @@ def test_charm_lists_expected_storage(juju: Juju) -> None: def test_data_directory_has_expected_contents_after_initialization(juju: Juju) -> None: expected_content = { "'#innodb_redo'", - "'#innodb_temp'", "auto.cnf", "ca-key.pem", "ca.pem", @@ -67,8 +66,19 @@ def test_data_directory_has_expected_contents_after_initialization(juju: Juju) - "undo_001", "undo_002", } + excluded_content = { + "'#innodb_temp'", + } result = juju.ssh(f"{DATABASE_APP_NAME}/0", "ls", MYSQL_DATA_DIR, container=CONTAINER_NAME) actual_content = set(result.strip().split()) assert expected_content <= actual_content + assert excluded_content.isdisjoint(actual_content) + + +def test_temp_directory_has_only_expected_file_extensions_after_initialization(juju: Juju) -> None: + result = juju.ssh(f"{DATABASE_APP_NAME}/0", "ls", MYSQL_TEMP_DIR, container=CONTAINER_NAME) + actual_content = set(result.strip().split()) + + assert all(fname.endswith(".ibt") for fname in actual_content) diff --git a/kubernetes/tests/unit/test_charm.py b/kubernetes/tests/unit/test_charm.py index 678d4a723..b1c2f38e0 100644 --- a/kubernetes/tests/unit/test_charm.py +++ b/kubernetes/tests/unit/test_charm.py @@ -337,7 +337,7 @@ def test_mysql_property(self, _, mock_get_unit_address): @patch("mysql_k8s_helpers.MySQL.remove_instance") @patch("mysql_k8s_helpers.MySQL.get_primary_label") @patch("mysql_k8s_helpers.MySQL.is_instance_in_cluster", return_value=True) - def test_database_storage_detaching( + def test_data_storage_detaching( self, mock_is_instance_in_cluster, mock_get_primary_label, @@ -353,7 +353,7 @@ def test_database_storage_detaching( ) mock_get_primary_label.return_value = self.charm.unit_label - self.charm._on_database_storage_detaching(None) + self.charm._on_data_storage_detaching(None) mock_remove_instance.assert_called_once_with(self.charm.unit_label, from_instance=None) self.assertEqual( From b1da3180dd01bc3e9fd39ec4ee56b53e5d76593a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Fri, 13 Feb 2026 12:52:57 +0100 Subject: [PATCH 05/44] Separate binlogs storage --- kubernetes/lib/charms/mysql/v0/mysql.py | 2 ++ kubernetes/metadata.yaml | 5 +++++ kubernetes/src/constants.py | 1 + .../tests/integration/integration/test_storage.py | 12 ++++++++++-- 4 files changed, 18 insertions(+), 2 deletions(-) diff --git a/kubernetes/lib/charms/mysql/v0/mysql.py b/kubernetes/lib/charms/mysql/v0/mysql.py index 2b1802208..d7fa86604 100644 --- a/kubernetes/lib/charms/mysql/v0/mysql.py +++ b/kubernetes/lib/charms/mysql/v0/mysql.py @@ -80,6 +80,7 @@ def __init__( from constants import ( MYSQL_DATA_DIR, MYSQL_TEMP_DIR, + MYSQL_BINLOGS_DIR, BACKUPS_PASSWORD_KEY, BACKUPS_USERNAME, CHARMED_MYSQL_PITR_HELPER, @@ -1165,6 +1166,7 @@ def render_mysqld_configuration( # noqa: C901 config["mysqld"] = { "datadir": MYSQL_DATA_DIR, "innodb_temp_tablespaces_dir": MYSQL_TEMP_DIR, + "log_bin": f"{MYSQL_BINLOGS_DIR}/binlog", # All interfaces bind expected "bind_address": "0.0.0.0", # noqa: S104 "mysqlx_bind_address": "0.0.0.0", # noqa: S104 diff --git a/kubernetes/metadata.yaml b/kubernetes/metadata.yaml index 6df4d3434..5232695d2 100644 --- a/kubernetes/metadata.yaml +++ b/kubernetes/metadata.yaml @@ -31,6 +31,8 @@ containers: location: /var/lib/mysql/data - storage: temp location: /var/lib/mysql/temp + - storage: binlogs + location: /var/lib/mysql/binlogs resources: mysql-image: @@ -89,6 +91,9 @@ storage: temp: type: filesystem description: Persistent storage for InnoDB temporary tablespaces + binlogs: + type: filesystem + description: Persistent storage for MySQL binary log files, used for replication and backups assumes: - k8s-api diff --git a/kubernetes/src/constants.py b/kubernetes/src/constants.py index ee6d30e77..21029c24b 100644 --- a/kubernetes/src/constants.py +++ b/kubernetes/src/constants.py @@ -29,6 +29,7 @@ MYSQLSH_LOCATION = "/usr/bin/mysqlsh" MYSQL_DATA_DIR = "/var/lib/mysql/data" # Corresponds to the data storage mount MYSQL_TEMP_DIR = "/var/lib/mysql/temp" # Corresponds to the temp storage mount +MYSQL_BINLOGS_DIR = "/var/lib/mysql/binlogs" # Corresponds to the binlogs storage mount MYSQLD_SOCK_FILE = "/var/run/mysqld/mysqld.sock" MYSQLD_CONFIG_FILE = "/etc/mysql/mysql.conf.d/z-custom.cnf" MYSQLD_INIT_CONFIG_FILE = "/etc/mysql/mysql.conf.d/z-custom-init-file.cnf" diff --git a/kubernetes/tests/integration/integration/test_storage.py b/kubernetes/tests/integration/integration/test_storage.py index fbd84d27b..d1910f73f 100644 --- a/kubernetes/tests/integration/integration/test_storage.py +++ b/kubernetes/tests/integration/integration/test_storage.py @@ -7,7 +7,7 @@ import jubilant from jubilant import Juju -from constants import CONTAINER_NAME, MYSQL_DATA_DIR, MYSQL_TEMP_DIR +from constants import CONTAINER_NAME, MYSQL_BINLOGS_DIR, MYSQL_DATA_DIR, MYSQL_TEMP_DIR from ..helpers_ha import ( CHARM_METADATA, @@ -41,7 +41,7 @@ def test_build_and_deploy(juju: Juju, charm) -> None: def test_charm_lists_expected_storage(juju: Juju) -> None: - expected_storages = ["data", "temp"] + expected_storages = ["data", "temp", "binlogs"] assert len(juju.status().storage.storage) == len(expected_storages) @@ -82,3 +82,11 @@ def test_temp_directory_has_only_expected_file_extensions_after_initialization(j actual_content = set(result.strip().split()) assert all(fname.endswith(".ibt") for fname in actual_content) + + +def test_binlogs_directory_has_only_expected_file_names_after_initialization(juju: Juju) -> None: + result = juju.ssh(f"{DATABASE_APP_NAME}/0", "ls", MYSQL_BINLOGS_DIR, container=CONTAINER_NAME) + actual_content = set(result.strip().split()) + + assert all(fname.startswith("binlog") for fname in actual_content) + assert "binlog.index" in actual_content From 970a9f8eb22f98fc5256d9b425a64ff2fa8529ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Fri, 13 Feb 2026 16:00:09 +0100 Subject: [PATCH 06/44] Separate redologs storage --- kubernetes/lib/charms/mysql/v0/mysql.py | 3 ++ kubernetes/metadata.yaml | 5 +++ kubernetes/src/constants.py | 1 + kubernetes/src/mysql_k8s_helpers.py | 2 +- .../integration/integration/test_storage.py | 40 ++++++++++++++++--- 5 files changed, 45 insertions(+), 6 deletions(-) diff --git a/kubernetes/lib/charms/mysql/v0/mysql.py b/kubernetes/lib/charms/mysql/v0/mysql.py index d7fa86604..a112d85b3 100644 --- a/kubernetes/lib/charms/mysql/v0/mysql.py +++ b/kubernetes/lib/charms/mysql/v0/mysql.py @@ -81,6 +81,7 @@ def __init__( MYSQL_DATA_DIR, MYSQL_TEMP_DIR, MYSQL_BINLOGS_DIR, + MYSQL_REDOLOGS_DIR, BACKUPS_PASSWORD_KEY, BACKUPS_USERNAME, CHARMED_MYSQL_PITR_HELPER, @@ -1167,6 +1168,8 @@ def render_mysqld_configuration( # noqa: C901 "datadir": MYSQL_DATA_DIR, "innodb_temp_tablespaces_dir": MYSQL_TEMP_DIR, "log_bin": f"{MYSQL_BINLOGS_DIR}/binlog", + "innodb_log_group_home_dir": MYSQL_REDOLOGS_DIR, + "innodb_undo_directory": MYSQL_REDOLOGS_DIR, # All interfaces bind expected "bind_address": "0.0.0.0", # noqa: S104 "mysqlx_bind_address": "0.0.0.0", # noqa: S104 diff --git a/kubernetes/metadata.yaml b/kubernetes/metadata.yaml index 5232695d2..b66cda96e 100644 --- a/kubernetes/metadata.yaml +++ b/kubernetes/metadata.yaml @@ -33,6 +33,8 @@ containers: location: /var/lib/mysql/temp - storage: binlogs location: /var/lib/mysql/binlogs + - storage: redologs + location: /var/lib/mysql/redologs resources: mysql-image: @@ -94,6 +96,9 @@ storage: binlogs: type: filesystem description: Persistent storage for MySQL binary log files, used for replication and backups + redologs: + type: filesystem + description: Persistent storage for MySQL redo log and undo log files assumes: - k8s-api diff --git a/kubernetes/src/constants.py b/kubernetes/src/constants.py index 21029c24b..c31603427 100644 --- a/kubernetes/src/constants.py +++ b/kubernetes/src/constants.py @@ -30,6 +30,7 @@ MYSQL_DATA_DIR = "/var/lib/mysql/data" # Corresponds to the data storage mount MYSQL_TEMP_DIR = "/var/lib/mysql/temp" # Corresponds to the temp storage mount MYSQL_BINLOGS_DIR = "/var/lib/mysql/binlogs" # Corresponds to the binlogs storage mount +MYSQL_REDOLOGS_DIR = "/var/lib/mysql/redologs" # Corresponds to the redologs storage mount MYSQLD_SOCK_FILE = "/var/run/mysqld/mysqld.sock" MYSQLD_CONFIG_FILE = "/etc/mysql/mysql.conf.d/z-custom.cnf" MYSQLD_INIT_CONFIG_FILE = "/etc/mysql/mysql.conf.d/z-custom-init-file.cnf" diff --git a/kubernetes/src/mysql_k8s_helpers.py b/kubernetes/src/mysql_k8s_helpers.py index 24febf110..b93195f66 100644 --- a/kubernetes/src/mysql_k8s_helpers.py +++ b/kubernetes/src/mysql_k8s_helpers.py @@ -577,7 +577,7 @@ def is_data_dir_initialised(self) -> bool: # minimal expected content for an integral mysqld data-dir expected_content = { - "#innodb_redo", + # "#innodb_redo", # stored separately # "#innodb_temp", # stored separately "auto.cnf", "ca-key.pem", diff --git a/kubernetes/tests/integration/integration/test_storage.py b/kubernetes/tests/integration/integration/test_storage.py index d1910f73f..3020e1fd8 100644 --- a/kubernetes/tests/integration/integration/test_storage.py +++ b/kubernetes/tests/integration/integration/test_storage.py @@ -3,11 +3,18 @@ # See LICENSE file for licensing details. import logging +import re import jubilant from jubilant import Juju -from constants import CONTAINER_NAME, MYSQL_BINLOGS_DIR, MYSQL_DATA_DIR, MYSQL_TEMP_DIR +from constants import ( + CONTAINER_NAME, + MYSQL_BINLOGS_DIR, + MYSQL_DATA_DIR, + MYSQL_REDOLOGS_DIR, + MYSQL_TEMP_DIR, +) from ..helpers_ha import ( CHARM_METADATA, @@ -41,14 +48,13 @@ def test_build_and_deploy(juju: Juju, charm) -> None: def test_charm_lists_expected_storage(juju: Juju) -> None: - expected_storages = ["data", "temp", "binlogs"] + expected_storages = ["data", "temp", "binlogs", "redologs"] assert len(juju.status().storage.storage) == len(expected_storages) def test_data_directory_has_expected_contents_after_initialization(juju: Juju) -> None: expected_content = { - "'#innodb_redo'", "auto.cnf", "ca-key.pem", "ca.pem", @@ -63,11 +69,12 @@ def test_data_directory_has_expected_contents_after_initialization(juju: Juju) - "server-cert.pem", "server-key.pem", "sys", - "undo_001", - "undo_002", } excluded_content = { "'#innodb_temp'", + "'#innodb_redo'", + "undo_001", + "undo_002", } result = juju.ssh(f"{DATABASE_APP_NAME}/0", "ls", MYSQL_DATA_DIR, container=CONTAINER_NAME) @@ -90,3 +97,26 @@ def test_binlogs_directory_has_only_expected_file_names_after_initialization(juj assert all(fname.startswith("binlog") for fname in actual_content) assert "binlog.index" in actual_content + + +def test_redologs_directory_has_only_expected_files_after_initialization( + juju: Juju, +) -> None: + redolog_pattern = re.compile(r"^\'\#ib_redo\d+") + undolog_pattern = re.compile(r"^undo_\d+$") + result = juju.ssh(f"{DATABASE_APP_NAME}/0", "ls", MYSQL_REDOLOGS_DIR, container=CONTAINER_NAME) + actual_content = set(result.strip().split()) + + assert all( + (undolog_pattern.match(fname) or (fname == "'#innodb_redo'")) for fname in actual_content + ) + + result = juju.ssh( + f"{DATABASE_APP_NAME}/0", + "ls", + f"{MYSQL_REDOLOGS_DIR}/#innodb_redo", + container=CONTAINER_NAME, + ) + actual_content = set(result.strip().split()) + + assert all(redolog_pattern.match(fname) for fname in actual_content) From 66a0e8f9bab343fe5cbb2622919658667af06500 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Fri, 13 Feb 2026 16:18:59 +0100 Subject: [PATCH 07/44] Separate logs storage --- kubernetes/metadata.yaml | 5 +++++ kubernetes/src/constants.py | 2 +- .../integration/integration/test_storage.py | 18 +++++++++++++++++- kubernetes/tests/unit/test_charm.py | 3 ++- .../tests/unit/test_mysql_k8s_helpers.py | 1 + 5 files changed, 26 insertions(+), 3 deletions(-) diff --git a/kubernetes/metadata.yaml b/kubernetes/metadata.yaml index b66cda96e..626b4a1ee 100644 --- a/kubernetes/metadata.yaml +++ b/kubernetes/metadata.yaml @@ -35,6 +35,8 @@ containers: location: /var/lib/mysql/binlogs - storage: redologs location: /var/lib/mysql/redologs + - storage: logs + location: /var/log/mysql resources: mysql-image: @@ -99,6 +101,9 @@ storage: redologs: type: filesystem description: Persistent storage for MySQL redo log and undo log files + logs: + type: filesystem + description: Persistent storage for MySQL error logs, general query logs, and slow query logs assumes: - k8s-api diff --git a/kubernetes/src/constants.py b/kubernetes/src/constants.py index c31603427..5b780fc66 100644 --- a/kubernetes/src/constants.py +++ b/kubernetes/src/constants.py @@ -34,7 +34,7 @@ MYSQLD_SOCK_FILE = "/var/run/mysqld/mysqld.sock" MYSQLD_CONFIG_FILE = "/etc/mysql/mysql.conf.d/z-custom.cnf" MYSQLD_INIT_CONFIG_FILE = "/etc/mysql/mysql.conf.d/z-custom-init-file.cnf" -MYSQL_LOG_DIR = "/var/log/mysql" +MYSQL_LOG_DIR = "/var/log/mysql" # Corresponds to the logs storage mount MYSQL_LOG_ERROR = f"{MYSQL_LOG_DIR}/error.log" MYSQL_LOG_FILES = [ MYSQL_LOG_ERROR, diff --git a/kubernetes/tests/integration/integration/test_storage.py b/kubernetes/tests/integration/integration/test_storage.py index 3020e1fd8..ef5e0ba47 100644 --- a/kubernetes/tests/integration/integration/test_storage.py +++ b/kubernetes/tests/integration/integration/test_storage.py @@ -12,6 +12,7 @@ CONTAINER_NAME, MYSQL_BINLOGS_DIR, MYSQL_DATA_DIR, + MYSQL_LOG_DIR, MYSQL_REDOLOGS_DIR, MYSQL_TEMP_DIR, ) @@ -48,7 +49,7 @@ def test_build_and_deploy(juju: Juju, charm) -> None: def test_charm_lists_expected_storage(juju: Juju) -> None: - expected_storages = ["data", "temp", "binlogs", "redologs"] + expected_storages = ["data", "temp", "binlogs", "redologs", "logs"] assert len(juju.status().storage.storage) == len(expected_storages) @@ -120,3 +121,18 @@ def test_redologs_directory_has_only_expected_files_after_initialization( actual_content = set(result.strip().split()) assert all(redolog_pattern.match(fname) for fname in actual_content) + + +def test_logs_directory_has_only_expected_contents_after_initialization( + juju: Juju, +) -> None: + expected_content = { + "archive_audit", + "archive_error", + "audit.log", + "error.log", + } + result = juju.ssh(f"{DATABASE_APP_NAME}/0", "ls", MYSQL_LOG_DIR, container=CONTAINER_NAME) + actual_content = set(result.strip().split()) + + assert expected_content <= actual_content diff --git a/kubernetes/tests/unit/test_charm.py b/kubernetes/tests/unit/test_charm.py index b1c2f38e0..bccb8bf44 100644 --- a/kubernetes/tests/unit/test_charm.py +++ b/kubernetes/tests/unit/test_charm.py @@ -16,6 +16,7 @@ DEFAULT_PASSWORD_LENGTH, MONITORING_PASSWORD_KEY, MYSQL_DATA_DIR, + MYSQL_LOG_DIR, MYSQLD_LOCATION, OPERATOR_PASSWORD_KEY, REPLICATION_PASSWORD_KEY, @@ -59,7 +60,7 @@ def layer_dict(self, with_mysqld_exporter: bool = False): "--basedir=/usr", f"--datadir={MYSQL_DATA_DIR}", "--plugin-dir=/usr/lib/mysql/plugin", - "--log-error=/var/log/mysql/error.log", + f"--log-error={MYSQL_LOG_DIR}/error.log", f"--pid-file={self.charm.unit_label}.pid", ] return { diff --git a/kubernetes/tests/unit/test_mysql_k8s_helpers.py b/kubernetes/tests/unit/test_mysql_k8s_helpers.py index 21357e086..254379432 100644 --- a/kubernetes/tests/unit/test_mysql_k8s_helpers.py +++ b/kubernetes/tests/unit/test_mysql_k8s_helpers.py @@ -127,6 +127,7 @@ def test_wait_until_unit_removed_from_cluster_exception(self, _get_cluster_statu @patch("ops.model.Container") def test_log_rotate_config(self, _container): """Test log_rotate_config.""" + # TODO: Parametrize with MYSQL_LOG_DIR? rendered_logrotate_config = ( "# Create dedicated subdirectory for rotated files\n" "createolddir 770 mysql mysql\n\n" From 2ac5bcba9fe0cc47cb9e6aa59bd890058f3a166c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Fri, 13 Feb 2026 16:27:54 +0100 Subject: [PATCH 08/44] Refactor --- .../integration/integration/test_storage.py | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/kubernetes/tests/integration/integration/test_storage.py b/kubernetes/tests/integration/integration/test_storage.py index ef5e0ba47..6fb2f0b96 100644 --- a/kubernetes/tests/integration/integration/test_storage.py +++ b/kubernetes/tests/integration/integration/test_storage.py @@ -78,23 +78,20 @@ def test_data_directory_has_expected_contents_after_initialization(juju: Juju) - "undo_002", } - result = juju.ssh(f"{DATABASE_APP_NAME}/0", "ls", MYSQL_DATA_DIR, container=CONTAINER_NAME) - actual_content = set(result.strip().split()) + actual_content = set(list_container_files(juju, f"{DATABASE_APP_NAME}/0", MYSQL_DATA_DIR)) assert expected_content <= actual_content assert excluded_content.isdisjoint(actual_content) def test_temp_directory_has_only_expected_file_extensions_after_initialization(juju: Juju) -> None: - result = juju.ssh(f"{DATABASE_APP_NAME}/0", "ls", MYSQL_TEMP_DIR, container=CONTAINER_NAME) - actual_content = set(result.strip().split()) + actual_content = set(list_container_files(juju, f"{DATABASE_APP_NAME}/0", MYSQL_TEMP_DIR)) assert all(fname.endswith(".ibt") for fname in actual_content) def test_binlogs_directory_has_only_expected_file_names_after_initialization(juju: Juju) -> None: - result = juju.ssh(f"{DATABASE_APP_NAME}/0", "ls", MYSQL_BINLOGS_DIR, container=CONTAINER_NAME) - actual_content = set(result.strip().split()) + actual_content = set(list_container_files(juju, f"{DATABASE_APP_NAME}/0", MYSQL_BINLOGS_DIR)) assert all(fname.startswith("binlog") for fname in actual_content) assert "binlog.index" in actual_content @@ -105,8 +102,8 @@ def test_redologs_directory_has_only_expected_files_after_initialization( ) -> None: redolog_pattern = re.compile(r"^\'\#ib_redo\d+") undolog_pattern = re.compile(r"^undo_\d+$") - result = juju.ssh(f"{DATABASE_APP_NAME}/0", "ls", MYSQL_REDOLOGS_DIR, container=CONTAINER_NAME) - actual_content = set(result.strip().split()) + + actual_content = set(list_container_files(juju, f"{DATABASE_APP_NAME}/0", MYSQL_REDOLOGS_DIR)) assert all( (undolog_pattern.match(fname) or (fname == "'#innodb_redo'")) for fname in actual_content @@ -132,7 +129,13 @@ def test_logs_directory_has_only_expected_contents_after_initialization( "audit.log", "error.log", } - result = juju.ssh(f"{DATABASE_APP_NAME}/0", "ls", MYSQL_LOG_DIR, container=CONTAINER_NAME) - actual_content = set(result.strip().split()) + actual_content = set(list_container_files(juju, f"{DATABASE_APP_NAME}/0", MYSQL_LOG_DIR)) assert expected_content <= actual_content + + +def list_container_files( + juju, unit_name: str, path: str, container: str = CONTAINER_NAME +) -> list[str]: + result = juju.ssh(unit_name, "ls", path, container=container) + return result.strip().split() From dc0b444aa664163cc9c079b83dcc217d23c09177 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Fri, 13 Feb 2026 16:40:19 +0100 Subject: [PATCH 09/44] Add spread task --- .../tests/spread/integration/test_storage.py/task.yaml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 kubernetes/tests/spread/integration/test_storage.py/task.yaml diff --git a/kubernetes/tests/spread/integration/test_storage.py/task.yaml b/kubernetes/tests/spread/integration/test_storage.py/task.yaml new file mode 100644 index 000000000..f46310078 --- /dev/null +++ b/kubernetes/tests/spread/integration/test_storage.py/task.yaml @@ -0,0 +1,7 @@ +summary: test_storage.py +environment: + TEST_MODULE: test_storage.py +execute: | + tox run -e integration -- "tests/integration/integration/$TEST_MODULE" --alluredir="$SPREAD_TASK/allure-results" +artifacts: + - allure-results From 1bacabf3fbeaf4bbc6f1b81c72041f812f531ead Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Tue, 17 Feb 2026 09:54:33 +0100 Subject: [PATCH 10/44] Fix mysql data dir contents check ahead of initialization --- kubernetes/src/mysql_k8s_helpers.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/kubernetes/src/mysql_k8s_helpers.py b/kubernetes/src/mysql_k8s_helpers.py index b93195f66..593645651 100644 --- a/kubernetes/src/mysql_k8s_helpers.py +++ b/kubernetes/src/mysql_k8s_helpers.py @@ -593,9 +593,10 @@ def is_data_dir_initialised(self) -> bool: "server-cert.pem", "server-key.pem", "sys", - "undo_001", - "undo_002", + # "undo_001", # stored separately + # "undo_002", # stored separately } + logger.debug("mysql data dir contents: %s", content_set) return expected_content <= content_set except (ExecError, APIError): From cdab4e03133be2a93e134271face6be9d73f4543 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Wed, 18 Feb 2026 16:59:37 +0100 Subject: [PATCH 11/44] Proper cleanup of other extra data directories before restoring backups --- kubernetes/lib/charms/mysql/v0/mysql.py | 24 +++++++++++++++++++++--- kubernetes/src/mysql_k8s_helpers.py | 4 ++++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/kubernetes/lib/charms/mysql/v0/mysql.py b/kubernetes/lib/charms/mysql/v0/mysql.py index a112d85b3..ee0988244 100644 --- a/kubernetes/lib/charms/mysql/v0/mysql.py +++ b/kubernetes/lib/charms/mysql/v0/mysql.py @@ -2721,8 +2721,9 @@ def empty_data_files( mysql_data_directory: str, user: str | None = None, group: str | None = None, + extra_dirs: list[str] | None = None, ) -> None: - """Empty the mysql data directory in preparation of backup restore.""" + """Empty the mysql data directories in preparation of backup restore.""" empty_data_files_command = [ "find", mysql_data_directory, @@ -2742,11 +2743,28 @@ def empty_data_files( user=user, group=group, ) + + if extra_dirs: + for extra_dir in extra_dirs: + logger.debug(f"Emptying extra directory {extra_dir}") + self._execute_commands( + [ + "find", + extra_dir, + "-not", + "-path", + extra_dir, + "-delete", + ], + user=user, + group=group, + ) + except MySQLExecError as e: - logger.error("Failed to empty data directory in prep for backup restore") + logger.error("Failed to empty data directories in prep for backup restore") raise MySQLEmptyDataDirectoryError(e.message) from e except Exception as e: - logger.error("Failed to empty data directory in prep for backup restore") + logger.error("Failed to empty data directories in prep for backup restore") raise MySQLEmptyDataDirectoryError from e def restore_backup( diff --git a/kubernetes/src/mysql_k8s_helpers.py b/kubernetes/src/mysql_k8s_helpers.py index 593645651..60df9b8ec 100644 --- a/kubernetes/src/mysql_k8s_helpers.py +++ b/kubernetes/src/mysql_k8s_helpers.py @@ -36,11 +36,14 @@ CONTAINER_NAME, LOG_ROTATE_CONFIG_FILE, MYSQL_BINLOGS_COLLECTOR_SERVICE, + MYSQL_BINLOGS_DIR, MYSQL_DATA_DIR, MYSQL_LOG_DIR, MYSQL_LOG_ERROR, + MYSQL_REDOLOGS_DIR, MYSQL_SYSTEM_GROUP, MYSQL_SYSTEM_USER, + MYSQL_TEMP_DIR, MYSQLD_DEFAULTS_CONFIG_FILE, MYSQLD_INIT_CONFIG_FILE, MYSQLD_LOCATION, @@ -368,6 +371,7 @@ def empty_data_files(self) -> None: MYSQL_DATA_DIR, user=MYSQL_SYSTEM_USER, group=MYSQL_SYSTEM_GROUP, + extra_dirs=[MYSQL_TEMP_DIR, MYSQL_BINLOGS_DIR, MYSQL_REDOLOGS_DIR], ) def restore_backup(self, backup_location: str) -> tuple[str, str]: From db1a0bf099e90d767308bd3cd06989fe53020b38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Fri, 10 Apr 2026 17:58:47 +0200 Subject: [PATCH 12/44] [K8S] Consolidate all logs under one volume --- kubernetes/lib/charms/mysql/v0/mysql.py | 22 ++++--- kubernetes/metadata.yaml | 17 ++---- kubernetes/src/constants.py | 11 ++-- kubernetes/src/mysql_k8s_helpers.py | 8 +-- .../test_replication_logs_rotation.py | 10 ++-- .../integration/integration/test_storage.py | 57 +++++++------------ kubernetes/tests/unit/test_charm.py | 6 +- .../tests/unit/test_mysql_k8s_helpers.py | 15 +++-- 8 files changed, 60 insertions(+), 86 deletions(-) diff --git a/kubernetes/lib/charms/mysql/v0/mysql.py b/kubernetes/lib/charms/mysql/v0/mysql.py index ee0988244..0dd990222 100644 --- a/kubernetes/lib/charms/mysql/v0/mysql.py +++ b/kubernetes/lib/charms/mysql/v0/mysql.py @@ -79,9 +79,8 @@ def __init__( from charms.data_platform_libs.v0.data_interfaces import DataPeerData, DataPeerUnitData from constants import ( MYSQL_DATA_DIR, + MYSQL_LOGS_DIR, MYSQL_TEMP_DIR, - MYSQL_BINLOGS_DIR, - MYSQL_REDOLOGS_DIR, BACKUPS_PASSWORD_KEY, BACKUPS_USERNAME, CHARMED_MYSQL_PITR_HELPER, @@ -1109,7 +1108,6 @@ def render_mysqld_configuration( # noqa: C901 memory_limit: int | None = None, experimental_max_connections: int | None = None, binlog_retention_days: int, - snap_common: str = "", ) -> tuple[str, dict]: """Render mysqld ini configuration file.""" max_connections = None @@ -1158,18 +1156,17 @@ def render_mysqld_configuration( # noqa: C901 # disable memory instruments if we have less than 2GiB of RAM performance_schema_instrument = "'memory/%=OFF'" - logging_path = f"{snap_common}/var/log/mysql" binlog_retention_seconds = binlog_retention_days * 24 * 60 * 60 config = configparser.ConfigParser(interpolation=None) # do not enable slow query logs, but specify a log file path in case # the admin enables them manually - config["mysqld"] = { + base_config = { "datadir": MYSQL_DATA_DIR, "innodb_temp_tablespaces_dir": MYSQL_TEMP_DIR, - "log_bin": f"{MYSQL_BINLOGS_DIR}/binlog", - "innodb_log_group_home_dir": MYSQL_REDOLOGS_DIR, - "innodb_undo_directory": MYSQL_REDOLOGS_DIR, + "log_bin": f"{MYSQL_LOGS_DIR}/binlog", + "innodb_log_group_home_dir": MYSQL_LOGS_DIR, + "innodb_undo_directory": MYSQL_LOGS_DIR, # All interfaces bind expected "bind_address": "0.0.0.0", # noqa: S104 "mysqlx_bind_address": "0.0.0.0", # noqa: S104 @@ -1178,11 +1175,11 @@ def render_mysqld_configuration( # noqa: C901 "max_connections": max_connections, "innodb_buffer_pool_size": innodb_buffer_pool_size, "log_error_services": "log_filter_internal;log_sink_internal", - "log_error": f"{logging_path}/error.log", + "log_error": f"{MYSQL_LOGS_DIR}/error.log", "general_log": "OFF", - "general_log_file": f"{logging_path}/general.log", + "general_log_file": f"{MYSQL_LOGS_DIR}/general.log", "loose-group_replication_paxos_single_leader": "ON", - "slow_query_log_file": f"{logging_path}/slow.log", + "slow_query_log_file": f"{MYSQL_LOGS_DIR}/slow.log", "binlog_expire_logs_seconds": f"{binlog_retention_seconds}", "gtid_mode": "ON", "enforce_gtid_consistency": "ON", @@ -1196,9 +1193,10 @@ def render_mysqld_configuration( # noqa: C901 "loose-validate_password.policy": "MEDIUM", "loose-validate_password.special_char_count": 0, } + config["mysqld"] = base_config # ty:ignore[invalid-assignment] if audit_log_enabled: - config["mysqld"]["loose-audit_log_filter.file"] = f"{logging_path}/audit.log" + config["mysqld"]["loose-audit_log_filter.file"] = f"{MYSQL_LOGS_DIR}/audit.log" config["mysqld"]["loose-audit_log_filter.format"] = "JSON" config["mysqld"]["loose-audit_log_filter.policy"] = audit_log_policy.upper() if audit_log_strategy == "async": diff --git a/kubernetes/metadata.yaml b/kubernetes/metadata.yaml index 626b4a1ee..a5c97af85 100644 --- a/kubernetes/metadata.yaml +++ b/kubernetes/metadata.yaml @@ -29,14 +29,11 @@ containers: mounts: - storage: data location: /var/lib/mysql/data + - storage: logs + # location: /var/log/mysql + location: /var/lib/mysql/logs - storage: temp location: /var/lib/mysql/temp - - storage: binlogs - location: /var/lib/mysql/binlogs - - storage: redologs - location: /var/lib/mysql/redologs - - storage: logs - location: /var/log/mysql resources: mysql-image: @@ -95,15 +92,9 @@ storage: temp: type: filesystem description: Persistent storage for InnoDB temporary tablespaces - binlogs: - type: filesystem - description: Persistent storage for MySQL binary log files, used for replication and backups - redologs: - type: filesystem - description: Persistent storage for MySQL redo log and undo log files logs: type: filesystem - description: Persistent storage for MySQL error logs, general query logs, and slow query logs + description: Persistent storage for MySQL error logs, general query logs, slow query logs, binary logs, redo logs and undo logs assumes: - k8s-api diff --git a/kubernetes/src/constants.py b/kubernetes/src/constants.py index 5b780fc66..4a0973649 100644 --- a/kubernetes/src/constants.py +++ b/kubernetes/src/constants.py @@ -28,18 +28,17 @@ MYSQL_CLI_LOCATION = "/usr/bin/mysql" MYSQLSH_LOCATION = "/usr/bin/mysqlsh" MYSQL_DATA_DIR = "/var/lib/mysql/data" # Corresponds to the data storage mount +# MYSQL_LOGS_DIR = "/var/log/mysql" +MYSQL_LOGS_DIR = "/var/lib/mysql/logs" # Corresponds to the logs storage mount MYSQL_TEMP_DIR = "/var/lib/mysql/temp" # Corresponds to the temp storage mount -MYSQL_BINLOGS_DIR = "/var/lib/mysql/binlogs" # Corresponds to the binlogs storage mount -MYSQL_REDOLOGS_DIR = "/var/lib/mysql/redologs" # Corresponds to the redologs storage mount MYSQLD_SOCK_FILE = "/var/run/mysqld/mysqld.sock" MYSQLD_CONFIG_FILE = "/etc/mysql/mysql.conf.d/z-custom.cnf" MYSQLD_INIT_CONFIG_FILE = "/etc/mysql/mysql.conf.d/z-custom-init-file.cnf" -MYSQL_LOG_DIR = "/var/log/mysql" # Corresponds to the logs storage mount -MYSQL_LOG_ERROR = f"{MYSQL_LOG_DIR}/error.log" +MYSQL_LOG_ERROR = f"{MYSQL_LOGS_DIR}/error.log" MYSQL_LOG_FILES = [ MYSQL_LOG_ERROR, - f"{MYSQL_LOG_DIR}/audit.log", - f"{MYSQL_LOG_DIR}/general.log", + f"{MYSQL_LOGS_DIR}/audit.log", + f"{MYSQL_LOGS_DIR}/general.log", ] MYSQL_SYSTEM_USER = "mysql" MYSQL_SYSTEM_GROUP = "mysql" diff --git a/kubernetes/src/mysql_k8s_helpers.py b/kubernetes/src/mysql_k8s_helpers.py index 60df9b8ec..682080afc 100644 --- a/kubernetes/src/mysql_k8s_helpers.py +++ b/kubernetes/src/mysql_k8s_helpers.py @@ -36,11 +36,9 @@ CONTAINER_NAME, LOG_ROTATE_CONFIG_FILE, MYSQL_BINLOGS_COLLECTOR_SERVICE, - MYSQL_BINLOGS_DIR, MYSQL_DATA_DIR, - MYSQL_LOG_DIR, MYSQL_LOG_ERROR, - MYSQL_REDOLOGS_DIR, + MYSQL_LOGS_DIR, MYSQL_SYSTEM_GROUP, MYSQL_SYSTEM_USER, MYSQL_TEMP_DIR, @@ -280,7 +278,7 @@ def setup_logrotate_config( rendered = template.render( system_user=MYSQL_SYSTEM_USER, system_group=MYSQL_SYSTEM_GROUP, - log_dir=MYSQL_LOG_DIR, + log_dir=MYSQL_LOGS_DIR, logs_retention_period=logs_retention_period, logs_rotations=logs_rotations, logs_compression_enabled=logs_compression, @@ -371,7 +369,7 @@ def empty_data_files(self) -> None: MYSQL_DATA_DIR, user=MYSQL_SYSTEM_USER, group=MYSQL_SYSTEM_GROUP, - extra_dirs=[MYSQL_TEMP_DIR, MYSQL_BINLOGS_DIR, MYSQL_REDOLOGS_DIR], + extra_dirs=[MYSQL_TEMP_DIR, MYSQL_LOGS_DIR], # TODO: Verify ) def restore_backup(self, backup_location: str) -> tuple[str, str]: diff --git a/kubernetes/tests/integration/integration/high_availability/test_replication_logs_rotation.py b/kubernetes/tests/integration/integration/high_availability/test_replication_logs_rotation.py index 0efb68b66..adcf27a6f 100644 --- a/kubernetes/tests/integration/integration/high_availability/test_replication_logs_rotation.py +++ b/kubernetes/tests/integration/integration/high_availability/test_replication_logs_rotation.py @@ -16,7 +16,7 @@ wait_fixed, ) -from constants import CONTAINER_NAME, MYSQL_LOG_DIR +from constants import CONTAINER_NAME, MYSQL_LOGS_DIR from ... import architecture from ...helpers_ha import ( @@ -99,7 +99,7 @@ def test_log_rotation(juju: Juju) -> None: juju=juju, unit_name=mysql_app_leader, container=CONTAINER_NAME, - file_path=f"{MYSQL_LOG_DIR}/archive_{log_type}", + file_path=f"{MYSQL_LOGS_DIR}/archive_{log_type}", ) logging.info("Writing some data to the text log files") @@ -107,7 +107,7 @@ def test_log_rotation(juju: Juju) -> None: juju=juju, unit_name=mysql_app_leader, container=CONTAINER_NAME, - file_path=f"{MYSQL_LOG_DIR}/{log_type}.log", + file_path=f"{MYSQL_LOGS_DIR}/{log_type}.log", file_data=f"{log_type} content", ) @@ -120,11 +120,11 @@ def test_log_rotation(juju: Juju) -> None: juju=juju, unit_name=mysql_app_leader, container=CONTAINER_NAME, - file_path=f"{MYSQL_LOG_DIR}/{log_type}.log", + file_path=f"{MYSQL_LOGS_DIR}/{log_type}.log", ) assert f"{log_type} content" not in active_log_file_data - archive_log_dir = f"{MYSQL_LOG_DIR}/archive_{log_type}" + archive_log_dir = f"{MYSQL_LOGS_DIR}/archive_{log_type}" archive_log_files_listed = list_unit_files( juju=juju, unit_name=mysql_app_leader, diff --git a/kubernetes/tests/integration/integration/test_storage.py b/kubernetes/tests/integration/integration/test_storage.py index 6fb2f0b96..306b185b1 100644 --- a/kubernetes/tests/integration/integration/test_storage.py +++ b/kubernetes/tests/integration/integration/test_storage.py @@ -10,10 +10,8 @@ from constants import ( CONTAINER_NAME, - MYSQL_BINLOGS_DIR, MYSQL_DATA_DIR, - MYSQL_LOG_DIR, - MYSQL_REDOLOGS_DIR, + MYSQL_LOGS_DIR, MYSQL_TEMP_DIR, ) @@ -35,7 +33,7 @@ def test_build_and_deploy(juju: Juju, charm) -> None: juju.deploy( charm, DATABASE_APP_NAME, - base="ubuntu@22.04", + base="ubuntu@24.04", config={"cluster-name": CLUSTER_NAME, "profile": "testing"}, resources={"mysql-image": CHARM_METADATA["resources"]["mysql-image"]["upstream-source"]}, num_units=1, @@ -49,7 +47,7 @@ def test_build_and_deploy(juju: Juju, charm) -> None: def test_charm_lists_expected_storage(juju: Juju) -> None: - expected_storages = ["data", "temp", "binlogs", "redologs", "logs"] + expected_storages = ["data", "temp", "logs"] assert len(juju.status().storage.storage) == len(expected_storages) @@ -90,36 +88,6 @@ def test_temp_directory_has_only_expected_file_extensions_after_initialization(j assert all(fname.endswith(".ibt") for fname in actual_content) -def test_binlogs_directory_has_only_expected_file_names_after_initialization(juju: Juju) -> None: - actual_content = set(list_container_files(juju, f"{DATABASE_APP_NAME}/0", MYSQL_BINLOGS_DIR)) - - assert all(fname.startswith("binlog") for fname in actual_content) - assert "binlog.index" in actual_content - - -def test_redologs_directory_has_only_expected_files_after_initialization( - juju: Juju, -) -> None: - redolog_pattern = re.compile(r"^\'\#ib_redo\d+") - undolog_pattern = re.compile(r"^undo_\d+$") - - actual_content = set(list_container_files(juju, f"{DATABASE_APP_NAME}/0", MYSQL_REDOLOGS_DIR)) - - assert all( - (undolog_pattern.match(fname) or (fname == "'#innodb_redo'")) for fname in actual_content - ) - - result = juju.ssh( - f"{DATABASE_APP_NAME}/0", - "ls", - f"{MYSQL_REDOLOGS_DIR}/#innodb_redo", - container=CONTAINER_NAME, - ) - actual_content = set(result.strip().split()) - - assert all(redolog_pattern.match(fname) for fname in actual_content) - - def test_logs_directory_has_only_expected_contents_after_initialization( juju: Juju, ) -> None: @@ -128,10 +96,27 @@ def test_logs_directory_has_only_expected_contents_after_initialization( "archive_error", "audit.log", "error.log", + "binlog.index", + "'#innodb_redo'", } - actual_content = set(list_container_files(juju, f"{DATABASE_APP_NAME}/0", MYSQL_LOG_DIR)) + + actual_content = set(list_container_files(juju, f"{DATABASE_APP_NAME}/0", MYSQL_LOGS_DIR)) assert expected_content <= actual_content + remaining_content = actual_content - expected_content + + undolog_pattern = re.compile(r"^undo_\d+$") + assert all( + (undolog_pattern.match(fname) or fname.startswith("binlog") or fname.startswith("audit")) + for fname in remaining_content + ) + + redolog_pattern = re.compile(r"^\'\#ib_redo\d+") + actual_content = set( + list_container_files(juju, f"{DATABASE_APP_NAME}/0", f"{MYSQL_LOGS_DIR}/#innodb_redo") + ) + + assert all(redolog_pattern.match(fname) for fname in actual_content) def list_container_files( diff --git a/kubernetes/tests/unit/test_charm.py b/kubernetes/tests/unit/test_charm.py index bccb8bf44..198d364b6 100644 --- a/kubernetes/tests/unit/test_charm.py +++ b/kubernetes/tests/unit/test_charm.py @@ -16,7 +16,7 @@ DEFAULT_PASSWORD_LENGTH, MONITORING_PASSWORD_KEY, MYSQL_DATA_DIR, - MYSQL_LOG_DIR, + MYSQL_LOGS_DIR, MYSQLD_LOCATION, OPERATOR_PASSWORD_KEY, REPLICATION_PASSWORD_KEY, @@ -60,7 +60,7 @@ def layer_dict(self, with_mysqld_exporter: bool = False): "--basedir=/usr", f"--datadir={MYSQL_DATA_DIR}", "--plugin-dir=/usr/lib/mysql/plugin", - f"--log-error={MYSQL_LOG_DIR}/error.log", + f"--log-error={MYSQL_LOGS_DIR}/error.log", f"--pid-file={self.charm.unit_label}.pid", ] return { @@ -82,7 +82,7 @@ def layer_dict(self, with_mysqld_exporter: bool = False): "mysql": { "override": "replace", "summary": "tail log", - "command": "tail -F /var/log/mysql/error.log", + "command": f"tail -F {MYSQL_LOGS_DIR}/error.log", "startup": "enabled", }, "mysqld_exporter": { diff --git a/kubernetes/tests/unit/test_mysql_k8s_helpers.py b/kubernetes/tests/unit/test_mysql_k8s_helpers.py index 254379432..e3635de77 100644 --- a/kubernetes/tests/unit/test_mysql_k8s_helpers.py +++ b/kubernetes/tests/unit/test_mysql_k8s_helpers.py @@ -7,7 +7,7 @@ import tenacity from ops.pebble import ExecError, PathError -from constants import PEER +from constants import MYSQL_LOGS_DIR, PEER from mysql_k8s_helpers import ( MySQL, MySQLInitialiseMySQLDError, @@ -127,7 +127,6 @@ def test_wait_until_unit_removed_from_cluster_exception(self, _get_cluster_statu @patch("ops.model.Container") def test_log_rotate_config(self, _container): """Test log_rotate_config.""" - # TODO: Parametrize with MYSQL_LOG_DIR? rendered_logrotate_config = ( "# Create dedicated subdirectory for rotated files\n" "createolddir 770 mysql mysql\n\n" @@ -146,16 +145,20 @@ def test_log_rotate_config(self, _container): "nomail\n" "nosharedscripts\n" "nocopytruncate\n\n\n" - "/var/log/mysql/error.log {\n" + f"{MYSQL_LOGS_DIR}/error.log" + " {\n" " olddir archive_error\n" "}\n\n" - "/var/log/mysql/general.log {\n" + f"{MYSQL_LOGS_DIR}/general.log" + " {\n" " olddir archive_general\n" "}\n\n" - "/var/log/mysql/slowquery.log {\n" + f"{MYSQL_LOGS_DIR}/slowquery.log" + " {\n" " olddir archive_slowquery\n" "}\n\n" - "/var/log/mysql/audit.log {\n" + f"{MYSQL_LOGS_DIR}/audit.log" + " {\n" " olddir archive_audit\n" "}\n" ) From c0467b6277620d761ce48ba541d6a9b92554c5b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Fri, 10 Apr 2026 18:41:15 +0200 Subject: [PATCH 13/44] [VM] Implement separation of storage --- machines/lib/charms/mysql/v0/mysql.py | 21 ++- machines/metadata.yaml | 16 ++- machines/src/charm.py | 9 +- machines/src/constants.py | 13 +- machines/src/mysql_vm_helpers.py | 10 +- .../scripts/clean-data-dir.sh | 2 +- .../integration/integration/test_storage.py | 128 ++++++++++++++++++ .../tests/integration/integration/test_tls.py | 8 +- .../tests/unit/test_log_rotation_setup.py | 6 +- machines/tests/unit/test_mysql.py | 19 ++- machines/tests/unit/test_mysqlsh_helpers.py | 15 +- 11 files changed, 207 insertions(+), 40 deletions(-) create mode 100644 machines/tests/integration/integration/test_storage.py diff --git a/machines/lib/charms/mysql/v0/mysql.py b/machines/lib/charms/mysql/v0/mysql.py index 5172138cd..b1f4ccbaf 100644 --- a/machines/lib/charms/mysql/v0/mysql.py +++ b/machines/lib/charms/mysql/v0/mysql.py @@ -87,6 +87,9 @@ def __init__( MAX_PASSWORD_LENGTH, MONITORING_PASSWORD_KEY, MONITORING_USERNAME, + MYSQL_DATA_DIR, + MYSQL_LOGS_DIR, + MYSQL_TEMP_DIR, OPERATOR_PASSWORD_KEY, OPERATOR_USERNAME, PEER, @@ -1105,7 +1108,6 @@ def render_mysqld_configuration( # noqa: C901 memory_limit: int | None = None, experimental_max_connections: int | None = None, binlog_retention_days: int, - snap_common: str = "", ) -> tuple[str, dict]: """Render mysqld ini configuration file.""" max_connections = None @@ -1154,13 +1156,17 @@ def render_mysqld_configuration( # noqa: C901 # disable memory instruments if we have less than 2GiB of RAM performance_schema_instrument = "'memory/%=OFF'" - logging_path = f"{snap_common}/var/log/mysql" binlog_retention_seconds = binlog_retention_days * 24 * 60 * 60 config = configparser.ConfigParser(interpolation=None) # do not enable slow query logs, but specify a log file path in case # the admin enables them manually - config["mysqld"] = { + base_config = { + "datadir": MYSQL_DATA_DIR, + "innodb_temp_tablespaces_dir": MYSQL_TEMP_DIR, + "log_bin": f"{MYSQL_LOGS_DIR}/binlog", + "innodb_log_group_home_dir": MYSQL_LOGS_DIR, + "innodb_undo_directory": MYSQL_LOGS_DIR, # All interfaces bind expected "bind_address": "0.0.0.0", # noqa: S104 "mysqlx_bind_address": "0.0.0.0", # noqa: S104 @@ -1169,11 +1175,11 @@ def render_mysqld_configuration( # noqa: C901 "max_connections": max_connections, "innodb_buffer_pool_size": innodb_buffer_pool_size, "log_error_services": "log_filter_internal;log_sink_internal", - "log_error": f"{logging_path}/error.log", + "log_error": f"{MYSQL_LOGS_DIR}/error.log", "general_log": "OFF", - "general_log_file": f"{logging_path}/general.log", + "general_log_file": f"{MYSQL_LOGS_DIR}/general.log", "loose-group_replication_paxos_single_leader": "ON", - "slow_query_log_file": f"{logging_path}/slow.log", + "slow_query_log_file": f"{MYSQL_LOGS_DIR}/slow.log", "binlog_expire_logs_seconds": f"{binlog_retention_seconds}", "gtid_mode": "ON", "enforce_gtid_consistency": "ON", @@ -1187,9 +1193,10 @@ def render_mysqld_configuration( # noqa: C901 "loose-validate_password.policy": "MEDIUM", "loose-validate_password.special_char_count": 0, } + config["mysqld"] = base_config # ty:ignore[invalid-assignment] if audit_log_enabled: - config["mysqld"]["loose-audit_log_filter.file"] = f"{logging_path}/audit.log" + config["mysqld"]["loose-audit_log_filter.file"] = f"{MYSQL_LOGS_DIR}/audit.log" config["mysqld"]["loose-audit_log_filter.format"] = "JSON" config["mysqld"]["loose-audit_log_filter.policy"] = audit_log_policy.upper() if audit_log_strategy == "async": diff --git a/machines/metadata.yaml b/machines/metadata.yaml index 96bd2f59c..df74c08d1 100644 --- a/machines/metadata.yaml +++ b/machines/metadata.yaml @@ -59,10 +59,20 @@ requires: optional: true storage: - database: + data: + type: filesystem + description: Persistent storage for MySQL data + location: /var/snap/charmed-mysql/common/var/lib/mysql/data + logs: type: filesystem - description: Persistent storage for data - location: /var/snap/charmed-mysql/common + description: Persistent storage for MySQL data + # location: /var/snap/charmed-mysql/common/var/log/mysql + location: /var/snap/charmed-mysql/common/var/lib/mysql/logs + temp: + type: filesystem + description: Persistent storage for InnoDB temporary tablespaces + location: /var/snap/charmed-mysql/common/var/lib/mysql/temp + assumes: - juju diff --git a/machines/src/charm.py b/machines/src/charm.py index 5090ee494..35bf5e6b9 100755 --- a/machines/src/charm.py +++ b/machines/src/charm.py @@ -155,9 +155,10 @@ def __init__(self, *args): self.framework.observe(self.on.config_changed, self._on_config_changed) self.framework.observe(self.on.start, self._on_start) self.framework.observe(self.on.update_status, self._on_update_status) - self.framework.observe( - self.on.database_storage_detaching, self._on_database_storage_detaching - ) + + self.framework.observe(self.on.data_storage_detaching, self._on_storage_detaching) + self.framework.observe(self.on.logs_storage_detaching, self._on_storage_detaching) + self.framework.observe(self.on.temp_storage_detaching, self._on_storage_detaching) self.framework.observe(self.on[PEER].relation_changed, self._on_peer_relation_changed) self.framework.observe(self.on[PEER].relation_departed, self._on_peer_relation_departed) @@ -367,7 +368,7 @@ def _on_peer_relation_departed(self, event: RelationDepartedEvent) -> None: if not self._mysql.reconcile_binlogs_collection(force_restart=True): logger.error("Failed to reconcile binlogs collection during peer departed event") - def _on_database_storage_detaching(self, _) -> None: + def _on_storage_detaching(self, _) -> None: """Handle the database storage detaching event.""" # Only executes if the unit was initialised if not self.unit_initialized(): diff --git a/machines/src/constants.py b/machines/src/constants.py index c72b3ffad..c747f3b66 100644 --- a/machines/src/constants.py +++ b/machines/src/constants.py @@ -38,7 +38,18 @@ MYSQLD_DEFAULTS_CONFIG_FILE = f"{CHARMED_MYSQL_DATA_DIRECTORY}/etc/mysql/mysql.cnf" MYSQLD_CUSTOM_CONFIG_FILE = f"{MYSQLD_CONFIG_DIRECTORY}/z-custom-mysqld.cnf" MYSQL_SYSTEM_USER = "snap_daemon" -MYSQL_DATA_DIR = f"{CHARMED_MYSQL_COMMON_DIRECTORY}/var/lib/mysql" + +MYSQL_DATA_DIR = ( + f"{CHARMED_MYSQL_COMMON_DIRECTORY}/var/lib/mysql/data" # Corresponds to the data storage mount +) +# MYSQL_LOGS_DIR = f"{CHARMED_MYSQL_COMMON_DIRECTORY}/var/log/mysql" +MYSQL_LOGS_DIR = ( + f"{CHARMED_MYSQL_COMMON_DIRECTORY}/var/lib/mysql/logs" # Corresponds to the logs storage mount +) +MYSQL_TEMP_DIR = ( + f"{CHARMED_MYSQL_COMMON_DIRECTORY}/var/lib/mysql/temp" # Corresponds to the temp storage mount +) + CHARMED_MYSQL_XTRABACKUP_LOCATION = "/snap/bin/charmed-mysql.xtrabackup" CHARMED_MYSQL_XBCLOUD_LOCATION = "/snap/bin/charmed-mysql.xbcloud" CHARMED_MYSQL_XBSTREAM_LOCATION = "/snap/bin/charmed-mysql.xbstream" diff --git a/machines/src/mysql_vm_helpers.py b/machines/src/mysql_vm_helpers.py index 3eb4fa4d1..b720db5f7 100644 --- a/machines/src/mysql_vm_helpers.py +++ b/machines/src/mysql_vm_helpers.py @@ -45,6 +45,7 @@ CHARMED_MYSQLD_SERVICE, CHARMED_MYSQLSH, MYSQL_DATA_DIR, + MYSQL_LOGS_DIR, MYSQL_SYSTEM_USER, MYSQLD_CONFIG_DIRECTORY, MYSQLD_CUSTOM_CONFIG_FILE, @@ -249,7 +250,6 @@ def write_mysqld_config(self) -> dict: audit_log_enabled=self.charm.config.plugin_audit_enabled, audit_log_strategy=self.charm.config.plugin_audit_strategy, audit_log_policy=self.charm.config.logs_audit_policy, - snap_common=CHARMED_MYSQL_COMMON_DIRECTORY, memory_limit=memory_limit, binlog_retention_days=self.charm.config.binlog_retention_days, experimental_max_connections=self.charm.config.experimental_max_connections, @@ -285,7 +285,7 @@ def setup_logrotate_and_cron( config_path = "/etc/logrotate.d/flush_mysql_logs" script_path = f"{self.charm.charm_dir}/logrotation.sh" cron_path = "/etc/cron.d/flush_mysql_logs" - logs_dir = f"{CHARMED_MYSQL_COMMON_DIRECTORY}/var/log/mysql" + logs_dir = f"{MYSQL_LOGS_DIR}" # days * minutes/day = amount of rotated files to keep logs_rotations = logs_retention_period * 1440 @@ -312,7 +312,7 @@ def setup_logrotate_and_cron( template = jinja2.Template(file.read()) logrotation_script_content = template.render( - log_path=f"{CHARMED_MYSQL_COMMON_DIRECTORY}/var/log/mysql", + log_path=f"{MYSQL_LOGS_DIR}", enabled_log_files=enabled_log_files, logrotate_conf=config_path, owner=MYSQL_SYSTEM_USER, @@ -872,9 +872,9 @@ def read_file_content(self, path: str) -> str | None: @staticmethod def fetch_error_log() -> str | None: """Fetch the mysqld error log.""" - if os.path.exists(f"{CHARMED_MYSQL_COMMON_DIRECTORY}/var/log/mysql/error.log"): + if os.path.exists(f"{MYSQL_LOGS_DIR}/error.log"): # can be empty if just rotated - with open(f"{CHARMED_MYSQL_COMMON_DIRECTORY}/var/log/mysql/error.log") as fd: + with open(f"{MYSQL_LOGS_DIR}/error.log") as fd: return fd.read() @staticmethod diff --git a/machines/tests/integration/integration/high_availability/scripts/clean-data-dir.sh b/machines/tests/integration/integration/high_availability/scripts/clean-data-dir.sh index 92b183ff3..f14e1c1b7 100755 --- a/machines/tests/integration/integration/high_availability/scripts/clean-data-dir.sh +++ b/machines/tests/integration/integration/high_availability/scripts/clean-data-dir.sh @@ -1,4 +1,4 @@ #!/usr/bin/env bash set -Eeuo pipefail -rm -rf /var/lib/mysql/* +rm -rf /var/lib/mysql/* # TODO: Verify diff --git a/machines/tests/integration/integration/test_storage.py b/machines/tests/integration/integration/test_storage.py new file mode 100644 index 000000000..5bc854079 --- /dev/null +++ b/machines/tests/integration/integration/test_storage.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +# Copyright 2026 Canonical Ltd. +# See LICENSE file for licensing details. + +import logging +import re + +import jubilant +from jubilant import Juju + +from constants import ( + MYSQL_DATA_DIR, + MYSQL_LOGS_DIR, + MYSQL_TEMP_DIR, +) + +from ..helpers_ha import ( + MINUTE_SECS, + wait_for_apps_status, +) + +logger = logging.getLogger(__name__) + +DATABASE_APP_NAME = "mysql" +CLUSTER_NAME = "test_cluster" +TIMEOUT = 15 * MINUTE_SECS + + +def test_build_and_deploy(juju: Juju, charm) -> None: + logger.info(f"Deploying {DATABASE_APP_NAME} with 1 unit") + juju.deploy( + charm, + DATABASE_APP_NAME, + base="ubuntu@24.04", + config={"cluster-name": CLUSTER_NAME, "profile": "testing"}, + num_units=1, + trust=True, + ) + + juju.wait( + ready=wait_for_apps_status(jubilant.all_active, DATABASE_APP_NAME), + timeout=TIMEOUT, + ) + + +def test_charm_lists_expected_storage(juju: Juju) -> None: + expected_storages = ["data", "temp", "logs"] + + assert len(juju.status().storage.storage) == len(expected_storages) + + +def test_data_directory_has_expected_contents_after_initialization(juju: Juju) -> None: + expected_content = { + "auto.cnf", + "ca-key.pem", + "ca.pem", + "client-cert.pem", + "client-key.pem", + "ib_buffer_pool", + "mysql", + "mysql.ibd", + "performance_schema", + "private_key.pem", + "public_key.pem", + "server-cert.pem", + "server-key.pem", + "sys", + } + excluded_content = { + "'#innodb_temp'", + "'#innodb_redo'", + "undo_001", + "undo_002", + } + + actual_content = set(list_vm_files(juju, f"{DATABASE_APP_NAME}/0", MYSQL_DATA_DIR)) + + assert expected_content <= actual_content + assert excluded_content.isdisjoint(actual_content) + + +def test_temp_directory_has_only_expected_file_extensions_after_initialization(juju: Juju) -> None: + actual_content = set(list_vm_files(juju, f"{DATABASE_APP_NAME}/0", MYSQL_TEMP_DIR)) + + assert all(fname.endswith(".ibt") for fname in actual_content) + + +def test_logs_directory_has_only_expected_contents_after_initialization( + juju: Juju, +) -> None: + expected_content = { + "archive_audit", + "archive_error", + "audit.log", + "error.log", + "binlog.index", + "'#innodb_redo'", + } + + actual_content = set(list_vm_files(juju, f"{DATABASE_APP_NAME}/0", MYSQL_LOGS_DIR)) + + assert expected_content <= actual_content + remaining_content = actual_content - expected_content + + undolog_pattern = re.compile(r"^undo_\d+$") + assert all( + (undolog_pattern.match(fname) or fname.startswith("binlog") or fname.startswith("audit")) + for fname in remaining_content + ) + + redolog_pattern = re.compile(r"^\'\#ib_redo\d+") + result = juju.ssh( + f"{DATABASE_APP_NAME}/0", + "ls", + f"{MYSQL_LOGS_DIR}/#innodb_redo", + ) + actual_content = set(result.strip().split()) + + assert all(redolog_pattern.match(fname) for fname in actual_content) + + +def list_vm_files( + juju, + unit_name: str, + path: str, +) -> list[str]: + result = juju.ssh(unit_name, "ls", path) + return result.strip().split() diff --git a/machines/tests/integration/integration/test_tls.py b/machines/tests/integration/integration/test_tls.py index a6ab45a6e..4eea5def1 100644 --- a/machines/tests/integration/integration/test_tls.py +++ b/machines/tests/integration/integration/test_tls.py @@ -8,7 +8,7 @@ import jubilant from jubilant import Juju -from constants import REPLICATION_USERNAME, TLS_SSL_CERT_FILE +from constants import MYSQL_DATA_DIR, REPLICATION_USERNAME, TLS_SSL_CERT_FILE from ..helpers import ( is_connection_possible, @@ -141,7 +141,7 @@ def test_rotate_tls_key(juju: Juju) -> None: for unit_name in app_units: original_tls[unit_name] = {} original_tls[unit_name]["cert"] = unit_file_md5( - juju, unit_name, f"/var/snap/charmed-mysql/common/var/lib/mysql/{TLS_SSL_CERT_FILE}" + juju, unit_name, f"{MYSQL_DATA_DIR}/{TLS_SSL_CERT_FILE}" ) # set key using auto-generated key for each unit @@ -161,9 +161,7 @@ def test_rotate_tls_key(juju: Juju) -> None: # After updating both the external key and the internal key a new certificate request will be # made; then the certificates should be available and updated. for unit_name in app_units: - new_cert_md5 = unit_file_md5( - juju, unit_name, f"/var/snap/charmed-mysql/common/var/lib/mysql/{TLS_SSL_CERT_FILE}" - ) + new_cert_md5 = unit_file_md5(juju, unit_name, f"{MYSQL_DATA_DIR}/{TLS_SSL_CERT_FILE}") assert new_cert_md5 != original_tls[unit_name]["cert"], ( f"cert for {unit_name} was not updated." diff --git a/machines/tests/unit/test_log_rotation_setup.py b/machines/tests/unit/test_log_rotation_setup.py index 99becb56f..b14eae45d 100644 --- a/machines/tests/unit/test_log_rotation_setup.py +++ b/machines/tests/unit/test_log_rotation_setup.py @@ -7,7 +7,7 @@ from ops.testing import Harness from charm import MySQLOperatorCharm -from constants import COS_AGENT_RELATION_NAME, PEER +from constants import COS_AGENT_RELATION_NAME, MYSQL_LOGS_DIR, PEER class TestLogRotationSetup(unittest.TestCase): @@ -44,9 +44,7 @@ def test_log_syncing( ): self.harness.update_config({"logs-retention-period": "auto"}) self.harness.add_relation(COS_AGENT_RELATION_NAME, "grafana-agent") - positions = ( - "positions:\n '/var/snap/charmed-mysql/common/var/log/mysql/error.log': '466'\n" - ) + positions = f"positions:\n '{MYSQL_LOGS_DIR}/error.log': '466'\n" event = MagicMock() mock_setup.assert_called_once() mock_setup.reset_mock() diff --git a/machines/tests/unit/test_mysql.py b/machines/tests/unit/test_mysql.py index ddb86dc9f..a97204d11 100644 --- a/machines/tests/unit/test_mysql.py +++ b/machines/tests/unit/test_mysql.py @@ -68,7 +68,11 @@ InstanceState, ) -from constants import CHARMED_MYSQLSH, MYSQLD_SOCK_FILE +from constants import ( + CHARMED_MYSQL_COMMON_DIRECTORY, + CHARMED_MYSQLSH, + MYSQLD_SOCK_FILE, +) SHORT_CLUSTER_STATUS = { "defaultReplicaSet": { @@ -1714,6 +1718,11 @@ def test_render_mysqld_configuration(self, _get_available_memory): _get_available_memory.return_value = 32341442560 expected_config = { + "datadir": f"{CHARMED_MYSQL_COMMON_DIRECTORY}/var/lib/mysql/data", + "innodb_temp_tablespaces_dir": f"{CHARMED_MYSQL_COMMON_DIRECTORY}/var/lib/mysql/temp", + "log_bin": f"{CHARMED_MYSQL_COMMON_DIRECTORY}/var/lib/mysql/logs/binlog", + "innodb_log_group_home_dir": f"{CHARMED_MYSQL_COMMON_DIRECTORY}/var/lib/mysql/logs", + "innodb_undo_directory": f"{CHARMED_MYSQL_COMMON_DIRECTORY}/var/lib/mysql/logs", "bind_address": "0.0.0.0", "mysqlx_bind_address": "0.0.0.0", "admin_address": "127.0.0.1", @@ -1721,15 +1730,15 @@ def test_render_mysqld_configuration(self, _get_available_memory): "max_connections": "724", "innodb_buffer_pool_size": "23219666944", "log_error_services": "log_filter_internal;log_sink_internal", - "log_error": "/var/log/mysql/error.log", + "log_error": f"{CHARMED_MYSQL_COMMON_DIRECTORY}/var/lib/mysql/logs/error.log", "general_log": "OFF", - "general_log_file": "/var/log/mysql/general.log", - "slow_query_log_file": "/var/log/mysql/slow.log", + "general_log_file": f"{CHARMED_MYSQL_COMMON_DIRECTORY}/var/lib/mysql/logs/general.log", + "slow_query_log_file": f"{CHARMED_MYSQL_COMMON_DIRECTORY}/var/lib/mysql/logs/slow.log", "binlog_expire_logs_seconds": "604800", "loose-audit_log_filter.format": "JSON", "loose-audit_log_filter.policy": "LOGINS", "loose-audit_log_filter.strategy": "ASYNCHRONOUS", - "loose-audit_log_filter.file": "/var/log/mysql/audit.log", + "loose-audit_log_filter.file": f"{CHARMED_MYSQL_COMMON_DIRECTORY}/var/lib/mysql/logs/audit.log", "loose-group_replication_paxos_single_leader": "ON", "innodb_buffer_pool_chunk_size": "2902458368", "gtid_mode": "ON", diff --git a/machines/tests/unit/test_mysqlsh_helpers.py b/machines/tests/unit/test_mysqlsh_helpers.py index a0e2a699b..35fff1ebe 100644 --- a/machines/tests/unit/test_mysqlsh_helpers.py +++ b/machines/tests/unit/test_mysqlsh_helpers.py @@ -229,6 +229,11 @@ def test_write_mysqld_config( config = "\n".join(( "[mysqld]", + "datadir = /var/snap/charmed-mysql/common/var/lib/mysql/data", + "innodb_temp_tablespaces_dir = /var/snap/charmed-mysql/common/var/lib/mysql/temp", + "log_bin = /var/snap/charmed-mysql/common/var/lib/mysql/logs/binlog", + "innodb_log_group_home_dir = /var/snap/charmed-mysql/common/var/lib/mysql/logs", + "innodb_undo_directory = /var/snap/charmed-mysql/common/var/lib/mysql/logs", "bind_address = 0.0.0.0", "mysqlx_bind_address = 0.0.0.0", "admin_address = 127.0.0.1", @@ -236,11 +241,11 @@ def test_write_mysqld_config( "max_connections = 111", "innodb_buffer_pool_size = 1234", "log_error_services = log_filter_internal;log_sink_internal", - "log_error = /var/snap/charmed-mysql/common/var/log/mysql/error.log", + "log_error = /var/snap/charmed-mysql/common/var/lib/mysql/logs/error.log", "general_log = OFF", - "general_log_file = /var/snap/charmed-mysql/common/var/log/mysql/general.log", + "general_log_file = /var/snap/charmed-mysql/common/var/lib/mysql/logs/general.log", "loose-group_replication_paxos_single_leader = ON", - "slow_query_log_file = /var/snap/charmed-mysql/common/var/log/mysql/slow.log", + "slow_query_log_file = /var/snap/charmed-mysql/common/var/lib/mysql/logs/slow.log", "binlog_expire_logs_seconds = 604800", "gtid_mode = ON", "enforce_gtid_consistency = ON", @@ -252,7 +257,7 @@ def test_write_mysqld_config( "loose-validate_password.number_count = 1", "loose-validate_password.policy = MEDIUM", "loose-validate_password.special_char_count = 0", - "loose-audit_log_filter.file = /var/snap/charmed-mysql/common/var/log/mysql/audit.log", + "loose-audit_log_filter.file = /var/snap/charmed-mysql/common/var/lib/mysql/logs/audit.log", "loose-audit_log_filter.format = JSON", "loose-audit_log_filter.policy = LOGINS", "loose-audit_log_filter.strategy = ASYNCHRONOUS", @@ -486,7 +491,7 @@ def test_initialise_mysqld(self, _subprocess_run, _reset_data_dir): "/usr/bin/sudo", "/snap/bin/charmed-mysql.mysqld-initialize", "--datadir", - "/var/snap/charmed-mysql/common/var/lib/mysql", + "/var/snap/charmed-mysql/common/var/lib/mysql/data", ], check=True, ) From 8656b7fe73fa42d3b8a694afbe6447908e582647 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Tue, 14 Apr 2026 17:20:59 +0200 Subject: [PATCH 14/44] [VM] Clear contents without deleting --- machines/src/mysql_vm_helpers.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/machines/src/mysql_vm_helpers.py b/machines/src/mysql_vm_helpers.py index b720db5f7..b6735cc85 100644 --- a/machines/src/mysql_vm_helpers.py +++ b/machines/src/mysql_vm_helpers.py @@ -47,6 +47,7 @@ MYSQL_DATA_DIR, MYSQL_LOGS_DIR, MYSQL_SYSTEM_USER, + MYSQL_TEMP_DIR, MYSQLD_CONFIG_DIRECTORY, MYSQLD_CUSTOM_CONFIG_FILE, MYSQLD_DEFAULTS_CONFIG_FILE, @@ -879,20 +880,29 @@ def fetch_error_log() -> str | None: @staticmethod def reset_data_dir() -> None: - """Reset the data directory.""" + """Remove all files from the data directory.""" logger.warning(f"Resetting data directory: {MYSQL_DATA_DIR}") - # Remove the data directory - shutil.rmtree(MYSQL_DATA_DIR, ignore_errors=False) - - # Recreate the data directory - os.makedirs(MYSQL_DATA_DIR) + # Remove the contents of the data directory + for root, dirs, files in pathlib.Path(MYSQL_DATA_DIR).walk(top_down=False): + for name in files: + (root / name).unlink() + for name in dirs: + (root / name).rmdir() # Change ownership of the data directory shutil.chown(MYSQL_DATA_DIR, user=MYSQL_SYSTEM_USER, group="root") def is_volume_mounted() -> bool: + """Returns if data directory is attached.""" + return all( + _is_volume_mounted(directory) + for directory in (MYSQL_DATA_DIR, MYSQL_LOGS_DIR, MYSQL_TEMP_DIR) + ) + + +def _is_volume_mounted(path: str) -> bool: """Returns if data directory is attached.""" try: for attempt in Retrying(stop=stop_after_attempt(10), wait=wait_fixed(12)): @@ -901,7 +911,7 @@ def is_volume_mounted() -> bool: subprocess.check_call([ # noqa: S603 "/usr/bin/mountpoint", "-q", - CHARMED_MYSQL_COMMON_DIRECTORY, + path, ]) except RetryError: return False From 3ba7dd82849582df282e3a43d7fad388c2d605d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Tue, 14 Apr 2026 18:18:10 +0200 Subject: [PATCH 15/44] [VM] Synchronize initialization flags with runtime configuration --- machines/lib/charms/mysql/v0/mysql.py | 1 + machines/src/mysql_vm_helpers.py | 6 ++++++ machines/tests/unit/test_mysql.py | 1 + machines/tests/unit/test_mysqlsh_helpers.py | 15 ++++++++++----- 4 files changed, 18 insertions(+), 5 deletions(-) diff --git a/machines/lib/charms/mysql/v0/mysql.py b/machines/lib/charms/mysql/v0/mysql.py index b1f4ccbaf..af901e762 100644 --- a/machines/lib/charms/mysql/v0/mysql.py +++ b/machines/lib/charms/mysql/v0/mysql.py @@ -1165,6 +1165,7 @@ def render_mysqld_configuration( # noqa: C901 "datadir": MYSQL_DATA_DIR, "innodb_temp_tablespaces_dir": MYSQL_TEMP_DIR, "log_bin": f"{MYSQL_LOGS_DIR}/binlog", + "log_bin_index": f"{MYSQL_LOGS_DIR}/binlog.index", "innodb_log_group_home_dir": MYSQL_LOGS_DIR, "innodb_undo_directory": MYSQL_LOGS_DIR, # All interfaces bind expected diff --git a/machines/src/mysql_vm_helpers.py b/machines/src/mysql_vm_helpers.py index b6735cc85..faec4ba7f 100644 --- a/machines/src/mysql_vm_helpers.py +++ b/machines/src/mysql_vm_helpers.py @@ -336,6 +336,12 @@ def initialise_mysqld(self) -> None: "/snap/bin/charmed-mysql.mysqld-initialize", "--datadir", MYSQL_DATA_DIR, + "--innodb-log-group-home-dir", + MYSQL_LOGS_DIR, + "--innodb-undo-directory", + MYSQL_LOGS_DIR, + "--innodb-temp-tablespaces-dir", + MYSQL_TEMP_DIR, ] try: diff --git a/machines/tests/unit/test_mysql.py b/machines/tests/unit/test_mysql.py index a97204d11..73992af4c 100644 --- a/machines/tests/unit/test_mysql.py +++ b/machines/tests/unit/test_mysql.py @@ -1721,6 +1721,7 @@ def test_render_mysqld_configuration(self, _get_available_memory): "datadir": f"{CHARMED_MYSQL_COMMON_DIRECTORY}/var/lib/mysql/data", "innodb_temp_tablespaces_dir": f"{CHARMED_MYSQL_COMMON_DIRECTORY}/var/lib/mysql/temp", "log_bin": f"{CHARMED_MYSQL_COMMON_DIRECTORY}/var/lib/mysql/logs/binlog", + "log_bin_index": f"{CHARMED_MYSQL_COMMON_DIRECTORY}/var/lib/mysql/logs/binlog.index", "innodb_log_group_home_dir": f"{CHARMED_MYSQL_COMMON_DIRECTORY}/var/lib/mysql/logs", "innodb_undo_directory": f"{CHARMED_MYSQL_COMMON_DIRECTORY}/var/lib/mysql/logs", "bind_address": "0.0.0.0", diff --git a/machines/tests/unit/test_mysqlsh_helpers.py b/machines/tests/unit/test_mysqlsh_helpers.py index 35fff1ebe..75383e35f 100644 --- a/machines/tests/unit/test_mysqlsh_helpers.py +++ b/machines/tests/unit/test_mysqlsh_helpers.py @@ -232,6 +232,7 @@ def test_write_mysqld_config( "datadir = /var/snap/charmed-mysql/common/var/lib/mysql/data", "innodb_temp_tablespaces_dir = /var/snap/charmed-mysql/common/var/lib/mysql/temp", "log_bin = /var/snap/charmed-mysql/common/var/lib/mysql/logs/binlog", + "log_bin_index = /var/snap/charmed-mysql/common/var/lib/mysql/logs/binlog.index", "innodb_log_group_home_dir = /var/snap/charmed-mysql/common/var/lib/mysql/logs", "innodb_undo_directory = /var/snap/charmed-mysql/common/var/lib/mysql/logs", "bind_address = 0.0.0.0", @@ -470,14 +471,12 @@ def test_get_available_memory(self): ): self.mysql.get_available_memory() - @patch("shutil.rmtree") - @patch("os.makedirs") @patch("shutil.chown") - def test_reset_data_dir(self, _chown, _makedirs, _rmtree): + @patch("pathlib.Path.walk", return_value=iter([])) + def test_reset_data_dir(self, _walk, _chown): self.mysql.reset_data_dir() + _walk.assert_called_once() _chown.assert_called_once() - _makedirs.assert_called_once() - _rmtree.assert_called_once() @patch("mysql_vm_helpers.MySQL.reset_data_dir") @patch("subprocess.run") @@ -492,6 +491,12 @@ def test_initialise_mysqld(self, _subprocess_run, _reset_data_dir): "/snap/bin/charmed-mysql.mysqld-initialize", "--datadir", "/var/snap/charmed-mysql/common/var/lib/mysql/data", + "--innodb-log-group-home-dir", + "/var/snap/charmed-mysql/common/var/lib/mysql/logs", + "--innodb-undo-directory", + "/var/snap/charmed-mysql/common/var/lib/mysql/logs", + "--innodb-temp-tablespaces-dir", + "/var/snap/charmed-mysql/common/var/lib/mysql/temp", ], check=True, ) From d5045a73373088a581cbef8f66eb46816137d05f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Tue, 14 Apr 2026 19:48:55 +0200 Subject: [PATCH 16/44] [VM] Rework file retrieval in storage test --- .../integration/integration/test_storage.py | 23 +++++++++---------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/machines/tests/integration/integration/test_storage.py b/machines/tests/integration/integration/test_storage.py index 5bc854079..12ec5b2e3 100644 --- a/machines/tests/integration/integration/test_storage.py +++ b/machines/tests/integration/integration/test_storage.py @@ -67,8 +67,8 @@ def test_data_directory_has_expected_contents_after_initialization(juju: Juju) - "sys", } excluded_content = { - "'#innodb_temp'", - "'#innodb_redo'", + "#innodb_temp", + "#innodb_redo", "undo_001", "undo_002", } @@ -94,7 +94,7 @@ def test_logs_directory_has_only_expected_contents_after_initialization( "audit.log", "error.log", "binlog.index", - "'#innodb_redo'", + "#innodb_redo", } actual_content = set(list_vm_files(juju, f"{DATABASE_APP_NAME}/0", MYSQL_LOGS_DIR)) @@ -108,21 +108,20 @@ def test_logs_directory_has_only_expected_contents_after_initialization( for fname in remaining_content ) - redolog_pattern = re.compile(r"^\'\#ib_redo\d+") - result = juju.ssh( - f"{DATABASE_APP_NAME}/0", - "ls", - f"{MYSQL_LOGS_DIR}/#innodb_redo", + redolog_pattern = re.compile(r"^#ib_redo\d+") + actual_content = set( + list_vm_files(juju, f"{DATABASE_APP_NAME}/0", f"{MYSQL_LOGS_DIR}/#innodb_redo") ) - actual_content = set(result.strip().split()) assert all(redolog_pattern.match(fname) for fname in actual_content) def list_vm_files( - juju, + juju: jubilant.Juju, unit_name: str, path: str, ) -> list[str]: - result = juju.ssh(unit_name, "ls", path) - return result.strip().split() + task = juju.exec("ls", path, unit=unit_name) + task.raise_on_failure() + + return task.stdout.split() From e98a4c723c0609c37db0704f1a9acf66368c904a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Wed, 15 Apr 2026 08:04:24 +0200 Subject: [PATCH 17/44] [K8s] Synchronize initialization flags with runtime configuration --- kubernetes/lib/charms/mysql/v0/mysql.py | 1 + kubernetes/src/mysql_k8s_helpers.py | 8 ++++++++ kubernetes/tests/unit/test_mysql_k8s_helpers.py | 15 ++++++++++++++- 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/kubernetes/lib/charms/mysql/v0/mysql.py b/kubernetes/lib/charms/mysql/v0/mysql.py index 0dd990222..6f80b4e2f 100644 --- a/kubernetes/lib/charms/mysql/v0/mysql.py +++ b/kubernetes/lib/charms/mysql/v0/mysql.py @@ -1165,6 +1165,7 @@ def render_mysqld_configuration( # noqa: C901 "datadir": MYSQL_DATA_DIR, "innodb_temp_tablespaces_dir": MYSQL_TEMP_DIR, "log_bin": f"{MYSQL_LOGS_DIR}/binlog", + "log_bin_index": f"{MYSQL_LOGS_DIR}/binlog.index", "innodb_log_group_home_dir": MYSQL_LOGS_DIR, "innodb_undo_directory": MYSQL_LOGS_DIR, # All interfaces bind expected diff --git a/kubernetes/src/mysql_k8s_helpers.py b/kubernetes/src/mysql_k8s_helpers.py index 682080afc..720876cf0 100644 --- a/kubernetes/src/mysql_k8s_helpers.py +++ b/kubernetes/src/mysql_k8s_helpers.py @@ -185,6 +185,14 @@ def initialise_mysqld(self) -> None: "--initialize", "-u", MYSQL_SYSTEM_USER, + "--datadir", + MYSQL_DATA_DIR, + "--innodb-log-group-home-dir", + MYSQL_LOGS_DIR, + "--innodb-undo-directory", + MYSQL_LOGS_DIR, + "--innodb-temp-tablespaces-dir", + MYSQL_TEMP_DIR, ] try: diff --git a/kubernetes/tests/unit/test_mysql_k8s_helpers.py b/kubernetes/tests/unit/test_mysql_k8s_helpers.py index e3635de77..ba3496657 100644 --- a/kubernetes/tests/unit/test_mysql_k8s_helpers.py +++ b/kubernetes/tests/unit/test_mysql_k8s_helpers.py @@ -68,7 +68,20 @@ def test_initialise_mysqld(self, _container, _process): self.mysql.initialise_mysqld() _container.exec.assert_called_once_with( - command=["/usr/sbin/mysqld", "--initialize", "-u", "mysql"], + command=[ + "/usr/sbin/mysqld", + "--initialize", + "-u", + "mysql", + "--datadir", + "/var/lib/mysql/data", + "--innodb-log-group-home-dir", + "/var/lib/mysql/logs", + "--innodb-undo-directory", + "/var/lib/mysql/logs", + "--innodb-temp-tablespaces-dir", + "/var/lib/mysql/temp", + ], user="mysql", group="mysql", ) From e4ba4b794d035e8833393cb4cb28256f096da61b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Wed, 15 Apr 2026 10:18:39 +0200 Subject: [PATCH 18/44] [K8s] Adjust storage test --- kubernetes/tests/integration/integration/test_storage.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/kubernetes/tests/integration/integration/test_storage.py b/kubernetes/tests/integration/integration/test_storage.py index 306b185b1..ddccc0265 100644 --- a/kubernetes/tests/integration/integration/test_storage.py +++ b/kubernetes/tests/integration/integration/test_storage.py @@ -70,8 +70,8 @@ def test_data_directory_has_expected_contents_after_initialization(juju: Juju) - "sys", } excluded_content = { - "'#innodb_temp'", - "'#innodb_redo'", + "#innodb_temp", + "#innodb_redo", "undo_001", "undo_002", } @@ -92,12 +92,10 @@ def test_logs_directory_has_only_expected_contents_after_initialization( juju: Juju, ) -> None: expected_content = { - "archive_audit", - "archive_error", "audit.log", "error.log", "binlog.index", - "'#innodb_redo'", + "#innodb_redo", } actual_content = set(list_container_files(juju, f"{DATABASE_APP_NAME}/0", MYSQL_LOGS_DIR)) From 641820fa33eef4bf212251423acb652231fa0cf7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Wed, 15 Apr 2026 10:18:56 +0200 Subject: [PATCH 19/44] [VM] Adjust storage test --- machines/tests/integration/integration/test_storage.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/machines/tests/integration/integration/test_storage.py b/machines/tests/integration/integration/test_storage.py index 12ec5b2e3..24e0103c1 100644 --- a/machines/tests/integration/integration/test_storage.py +++ b/machines/tests/integration/integration/test_storage.py @@ -89,8 +89,6 @@ def test_logs_directory_has_only_expected_contents_after_initialization( juju: Juju, ) -> None: expected_content = { - "archive_audit", - "archive_error", "audit.log", "error.log", "binlog.index", From 41661bc534dcc628f1c477c4309a2ebab1285b79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Wed, 15 Apr 2026 10:47:34 +0200 Subject: [PATCH 20/44] [VM] Add archive storage for rotated logs --- machines/metadata.yaml | 4 ++++ machines/src/charm.py | 1 + machines/src/constants.py | 1 + machines/src/mysql_vm_helpers.py | 7 ++++--- machines/templates/logrotate.j2 | 7 +++++-- .../test_replication_logs_rotation.py | 11 +++++------ .../tests/integration/integration/test_storage.py | 2 +- 7 files changed, 21 insertions(+), 12 deletions(-) diff --git a/machines/metadata.yaml b/machines/metadata.yaml index df74c08d1..e56fd515f 100644 --- a/machines/metadata.yaml +++ b/machines/metadata.yaml @@ -59,6 +59,10 @@ requires: optional: true storage: + archive: + type: filesystem + description: Persistent storage for rotated logs and other archival purposes + location: /var/snap/charmed-mysql/common/var/lib/mysql/archive data: type: filesystem description: Persistent storage for MySQL data diff --git a/machines/src/charm.py b/machines/src/charm.py index 35bf5e6b9..b38587ed1 100755 --- a/machines/src/charm.py +++ b/machines/src/charm.py @@ -156,6 +156,7 @@ def __init__(self, *args): self.framework.observe(self.on.start, self._on_start) self.framework.observe(self.on.update_status, self._on_update_status) + self.framework.observe(self.on.archive_storage_detaching, self._on_storage_detaching) self.framework.observe(self.on.data_storage_detaching, self._on_storage_detaching) self.framework.observe(self.on.logs_storage_detaching, self._on_storage_detaching) self.framework.observe(self.on.temp_storage_detaching, self._on_storage_detaching) diff --git a/machines/src/constants.py b/machines/src/constants.py index c747f3b66..6b252d156 100644 --- a/machines/src/constants.py +++ b/machines/src/constants.py @@ -39,6 +39,7 @@ MYSQLD_CUSTOM_CONFIG_FILE = f"{MYSQLD_CONFIG_DIRECTORY}/z-custom-mysqld.cnf" MYSQL_SYSTEM_USER = "snap_daemon" +MYSQL_ARCHIVE_DIR = f"{CHARMED_MYSQL_COMMON_DIRECTORY}/var/lib/mysql/archive" # Corresponds to the archive storage mount MYSQL_DATA_DIR = ( f"{CHARMED_MYSQL_COMMON_DIRECTORY}/var/lib/mysql/data" # Corresponds to the data storage mount ) diff --git a/machines/src/mysql_vm_helpers.py b/machines/src/mysql_vm_helpers.py index faec4ba7f..fa49e336a 100644 --- a/machines/src/mysql_vm_helpers.py +++ b/machines/src/mysql_vm_helpers.py @@ -44,6 +44,7 @@ CHARMED_MYSQLD_EXPORTER_SERVICE, CHARMED_MYSQLD_SERVICE, CHARMED_MYSQLSH, + MYSQL_ARCHIVE_DIR, MYSQL_DATA_DIR, MYSQL_LOGS_DIR, MYSQL_SYSTEM_USER, @@ -286,7 +287,6 @@ def setup_logrotate_and_cron( config_path = "/etc/logrotate.d/flush_mysql_logs" script_path = f"{self.charm.charm_dir}/logrotation.sh" cron_path = "/etc/cron.d/flush_mysql_logs" - logs_dir = f"{MYSQL_LOGS_DIR}" # days * minutes/day = amount of rotated files to keep logs_rotations = logs_retention_period * 1440 @@ -296,7 +296,8 @@ def setup_logrotate_and_cron( logrotate_conf_content = template.render( system_user=MYSQL_SYSTEM_USER, - log_dir=logs_dir, + log_dir=MYSQL_LOGS_DIR, + archive_dir=MYSQL_ARCHIVE_DIR, charm_directory=self.charm.charm_dir, unit_name=self.charm.unit.name, enabled_log_files=enabled_log_files, @@ -313,7 +314,7 @@ def setup_logrotate_and_cron( template = jinja2.Template(file.read()) logrotation_script_content = template.render( - log_path=f"{MYSQL_LOGS_DIR}", + log_path=MYSQL_LOGS_DIR, enabled_log_files=enabled_log_files, logrotate_conf=config_path, owner=MYSQL_SYSTEM_USER, diff --git a/machines/templates/logrotate.j2 b/machines/templates/logrotate.j2 index 53934806a..ad616ea43 100644 --- a/machines/templates/logrotate.j2 +++ b/machines/templates/logrotate.j2 @@ -26,11 +26,14 @@ ifempty missingok nomail nosharedscripts -nocopytruncate +# Needed because rotated logs might be in a different filesystem, +# `nocopytruncate` fails with `failed to rename ...: Invalid cross-device link` +copy +copytruncate {% for log in enabled_log_files %} {{ log_dir }}/{{ log }}.log { - olddir archive_{{ log }} + olddir {{ archive_dir }}/archive_{{ log }} postrotate juju_command=/usr/bin/juju-run if command -v /usr/bin/juju-exec; then juju_command=/usr/bin/juju-exec; fi diff --git a/machines/tests/integration/integration/high_availability/test_replication_logs_rotation.py b/machines/tests/integration/integration/high_availability/test_replication_logs_rotation.py index 7961ed411..6676e1963 100644 --- a/machines/tests/integration/integration/high_availability/test_replication_logs_rotation.py +++ b/machines/tests/integration/integration/high_availability/test_replication_logs_rotation.py @@ -14,7 +14,7 @@ wait_fixed, ) -from constants import CHARMED_MYSQL_COMMON_DIRECTORY +from constants import MYSQL_ARCHIVE_DIR, MYSQL_LOGS_DIR from ...helpers_ha import ( get_app_leader, @@ -70,7 +70,6 @@ def test_log_rotation(juju: Juju) -> None: log_types = ["audit", "error"] mysql_app_leader = get_app_leader(juju, MYSQL_APP_NAME) - mysql_logs_path = f"{CHARMED_MYSQL_COMMON_DIRECTORY}/var/log/mysql" logging.info("Removing the cron file") delete_unit_file(juju, mysql_app_leader, "/etc/cron.d/flush_mysql_logs") @@ -79,7 +78,7 @@ def test_log_rotation(juju: Juju) -> None: stop_unit_flush_logs_job(juju, mysql_app_leader) for log_type in log_types: - archive_log_dir = f"{mysql_logs_path}/archive_{log_type}" + archive_log_dir = f"{MYSQL_ARCHIVE_DIR}/archive_{log_type}" logging.info("Removing existing archive directories") delete_unit_file(juju, mysql_app_leader, archive_log_dir) @@ -88,7 +87,7 @@ def test_log_rotation(juju: Juju) -> None: write_unit_file( juju=juju, unit_name=mysql_app_leader, - file_path=f"{mysql_logs_path}/{log_type}.log", + file_path=f"{MYSQL_LOGS_DIR}/{log_type}.log", file_data=f"{log_type} content", ) @@ -100,11 +99,11 @@ def test_log_rotation(juju: Juju) -> None: active_log_file_data = read_unit_file( juju=juju, unit_name=mysql_app_leader, - file_path=f"{mysql_logs_path}/{log_type}.log", + file_path=f"{MYSQL_LOGS_DIR}/{log_type}.log", ) assert f"{log_type} content" not in active_log_file_data - archive_log_dir = f"{mysql_logs_path}/archive_{log_type}" + archive_log_dir = f"{MYSQL_ARCHIVE_DIR}/archive_{log_type}" archive_log_files_listed = list_unit_files(juju, mysql_app_leader, archive_log_dir) assert len(archive_log_files_listed) == 1 diff --git a/machines/tests/integration/integration/test_storage.py b/machines/tests/integration/integration/test_storage.py index 24e0103c1..b168ef8ea 100644 --- a/machines/tests/integration/integration/test_storage.py +++ b/machines/tests/integration/integration/test_storage.py @@ -44,7 +44,7 @@ def test_build_and_deploy(juju: Juju, charm) -> None: def test_charm_lists_expected_storage(juju: Juju) -> None: - expected_storages = ["data", "temp", "logs"] + expected_storages = {"archive", "data", "temp", "logs"} assert len(juju.status().storage.storage) == len(expected_storages) From 5b8be755224db6c78512484c6b8d500e5aaeb098 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Wed, 15 Apr 2026 10:50:07 +0200 Subject: [PATCH 21/44] [K8s] Add archive storage for rotated logs --- kubernetes/metadata.yaml | 5 +++++ kubernetes/src/constants.py | 1 + kubernetes/src/mysql_k8s_helpers.py | 2 ++ kubernetes/templates/logrotate.j2 | 7 +++++-- .../test_replication_logs_rotation.py | 10 ++++++---- .../tests/integration/integration/test_storage.py | 2 +- kubernetes/tests/unit/test_mysql_k8s_helpers.py | 13 ++++++++----- 7 files changed, 28 insertions(+), 12 deletions(-) diff --git a/kubernetes/metadata.yaml b/kubernetes/metadata.yaml index a5c97af85..2d78a4dfe 100644 --- a/kubernetes/metadata.yaml +++ b/kubernetes/metadata.yaml @@ -27,6 +27,8 @@ containers: gid: 584788 resource: mysql-image mounts: + - storage: archive + location: /var/lib/mysql/archive - storage: data location: /var/lib/mysql/data - storage: logs @@ -86,6 +88,9 @@ requires: optional: true storage: + archive: + type: filesystem + description: Persistent storage for rotated logs and other archival purposes data: type: filesystem description: Persistent storage for MySQL data diff --git a/kubernetes/src/constants.py b/kubernetes/src/constants.py index 4a0973649..dd1a2978a 100644 --- a/kubernetes/src/constants.py +++ b/kubernetes/src/constants.py @@ -27,6 +27,7 @@ TLS_SSL_CERT_FILE = "custom-server-cert.pem" MYSQL_CLI_LOCATION = "/usr/bin/mysql" MYSQLSH_LOCATION = "/usr/bin/mysqlsh" +MYSQL_ARCHIVE_DIR = "/var/lib/mysql/archive" # Corresponds to the archive storage mount MYSQL_DATA_DIR = "/var/lib/mysql/data" # Corresponds to the data storage mount # MYSQL_LOGS_DIR = "/var/log/mysql" MYSQL_LOGS_DIR = "/var/lib/mysql/logs" # Corresponds to the logs storage mount diff --git a/kubernetes/src/mysql_k8s_helpers.py b/kubernetes/src/mysql_k8s_helpers.py index 720876cf0..aa19a9570 100644 --- a/kubernetes/src/mysql_k8s_helpers.py +++ b/kubernetes/src/mysql_k8s_helpers.py @@ -35,6 +35,7 @@ CHARMED_MYSQL_XTRABACKUP_LOCATION, CONTAINER_NAME, LOG_ROTATE_CONFIG_FILE, + MYSQL_ARCHIVE_DIR, MYSQL_BINLOGS_COLLECTOR_SERVICE, MYSQL_DATA_DIR, MYSQL_LOG_ERROR, @@ -287,6 +288,7 @@ def setup_logrotate_config( system_user=MYSQL_SYSTEM_USER, system_group=MYSQL_SYSTEM_GROUP, log_dir=MYSQL_LOGS_DIR, + archive_dir=MYSQL_ARCHIVE_DIR, logs_retention_period=logs_retention_period, logs_rotations=logs_rotations, logs_compression_enabled=logs_compression, diff --git a/kubernetes/templates/logrotate.j2 b/kubernetes/templates/logrotate.j2 index 101882636..3b8fb166a 100644 --- a/kubernetes/templates/logrotate.j2 +++ b/kubernetes/templates/logrotate.j2 @@ -22,10 +22,13 @@ ifempty missingok nomail nosharedscripts -nocopytruncate +# Needed because rotated logs might be in a different filesystem, +# `nocopytruncate` fails with `failed to rename ...: Invalid cross-device link` +copy +copytruncate {% for log in enabled_log_files %} {{ log_dir }}/{{ log }}.log { - olddir archive_{{ log }} + olddir {{ archive_dir }}/archive_{{ log }} } {% endfor %} diff --git a/kubernetes/tests/integration/integration/high_availability/test_replication_logs_rotation.py b/kubernetes/tests/integration/integration/high_availability/test_replication_logs_rotation.py index adcf27a6f..5a31a7015 100644 --- a/kubernetes/tests/integration/integration/high_availability/test_replication_logs_rotation.py +++ b/kubernetes/tests/integration/integration/high_availability/test_replication_logs_rotation.py @@ -16,7 +16,7 @@ wait_fixed, ) -from constants import CONTAINER_NAME, MYSQL_LOGS_DIR +from constants import CONTAINER_NAME, MYSQL_ARCHIVE_DIR, MYSQL_LOGS_DIR from ... import architecture from ...helpers_ha import ( @@ -76,7 +76,7 @@ def test_deploy_highly_available_cluster(juju: Juju, charm: str) -> None: def test_log_rotation(juju: Juju) -> None: """Test the log rotation of text files.""" - log_types = ["error", "audit"] + log_types = ["audit", "error"] mysql_app_leader = get_app_leader(juju, MYSQL_APP_NAME) mysql_app_leader_label = get_mysql_instance_label(mysql_app_leader) @@ -94,12 +94,14 @@ def test_log_rotation(juju: Juju) -> None: stop_log_rotate_dispatcher(juju, mysql_app_leader) for log_type in log_types: + archive_log_dir = f"{MYSQL_ARCHIVE_DIR}/archive_{log_type}" + logging.info("Removing existing archive directories") delete_unit_file( juju=juju, unit_name=mysql_app_leader, container=CONTAINER_NAME, - file_path=f"{MYSQL_LOGS_DIR}/archive_{log_type}", + file_path=archive_log_dir, ) logging.info("Writing some data to the text log files") @@ -124,7 +126,7 @@ def test_log_rotation(juju: Juju) -> None: ) assert f"{log_type} content" not in active_log_file_data - archive_log_dir = f"{MYSQL_LOGS_DIR}/archive_{log_type}" + archive_log_dir = f"{MYSQL_ARCHIVE_DIR}/archive_{log_type}" archive_log_files_listed = list_unit_files( juju=juju, unit_name=mysql_app_leader, diff --git a/kubernetes/tests/integration/integration/test_storage.py b/kubernetes/tests/integration/integration/test_storage.py index ddccc0265..fde1844c2 100644 --- a/kubernetes/tests/integration/integration/test_storage.py +++ b/kubernetes/tests/integration/integration/test_storage.py @@ -47,7 +47,7 @@ def test_build_and_deploy(juju: Juju, charm) -> None: def test_charm_lists_expected_storage(juju: Juju) -> None: - expected_storages = ["data", "temp", "logs"] + expected_storages = {"archive", "data", "temp", "logs"} assert len(juju.status().storage.storage) == len(expected_storages) diff --git a/kubernetes/tests/unit/test_mysql_k8s_helpers.py b/kubernetes/tests/unit/test_mysql_k8s_helpers.py index ba3496657..74e241157 100644 --- a/kubernetes/tests/unit/test_mysql_k8s_helpers.py +++ b/kubernetes/tests/unit/test_mysql_k8s_helpers.py @@ -157,22 +157,25 @@ def test_log_rotate_config(self, _container): "missingok\n" "nomail\n" "nosharedscripts\n" - "nocopytruncate\n\n\n" + "# Needed because rotated logs might be in a different filesystem,\n" + "# `nocopytruncate` fails with `failed to rename ...: Invalid cross-device link`\n" + "copy\n" + "copytruncate\n\n\n" f"{MYSQL_LOGS_DIR}/error.log" " {\n" - " olddir archive_error\n" + " olddir /var/lib/mysql/archive/archive_error\n" "}\n\n" f"{MYSQL_LOGS_DIR}/general.log" " {\n" - " olddir archive_general\n" + " olddir /var/lib/mysql/archive/archive_general\n" "}\n\n" f"{MYSQL_LOGS_DIR}/slowquery.log" " {\n" - " olddir archive_slowquery\n" + " olddir /var/lib/mysql/archive/archive_slowquery\n" "}\n\n" f"{MYSQL_LOGS_DIR}/audit.log" " {\n" - " olddir archive_audit\n" + " olddir /var/lib/mysql/archive/archive_audit\n" "}\n" ) From 9b8dc92ae510a438e781f50fd223e4f78bcd5218 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Wed, 15 Apr 2026 10:50:20 +0200 Subject: [PATCH 22/44] [K8s] Cleanup --- kubernetes/src/mysql_k8s_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/src/mysql_k8s_helpers.py b/kubernetes/src/mysql_k8s_helpers.py index aa19a9570..d807ddc6b 100644 --- a/kubernetes/src/mysql_k8s_helpers.py +++ b/kubernetes/src/mysql_k8s_helpers.py @@ -379,7 +379,7 @@ def empty_data_files(self) -> None: MYSQL_DATA_DIR, user=MYSQL_SYSTEM_USER, group=MYSQL_SYSTEM_GROUP, - extra_dirs=[MYSQL_TEMP_DIR, MYSQL_LOGS_DIR], # TODO: Verify + extra_dirs=[MYSQL_TEMP_DIR, MYSQL_LOGS_DIR], ) def restore_backup(self, backup_location: str) -> tuple[str, str]: From cc71139eb5f854e7ad7cac4c83bf7d9bf66c2521 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Wed, 15 Apr 2026 10:50:31 +0200 Subject: [PATCH 23/44] [VM] Cleanup --- .../integration/high_availability/scripts/clean-data-dir.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/machines/tests/integration/integration/high_availability/scripts/clean-data-dir.sh b/machines/tests/integration/integration/high_availability/scripts/clean-data-dir.sh index f14e1c1b7..92b183ff3 100755 --- a/machines/tests/integration/integration/high_availability/scripts/clean-data-dir.sh +++ b/machines/tests/integration/integration/high_availability/scripts/clean-data-dir.sh @@ -1,4 +1,4 @@ #!/usr/bin/env bash set -Eeuo pipefail -rm -rf /var/lib/mysql/* # TODO: Verify +rm -rf /var/lib/mysql/* From 2711c123876ab4a76821c5d6fee598f9c6983649 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Wed, 15 Apr 2026 11:58:28 +0200 Subject: [PATCH 24/44] [VM] Temporarily disable upgrade test --- machines/tests/spread/integration/test_upgrade.py/task.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/machines/tests/spread/integration/test_upgrade.py/task.yaml b/machines/tests/spread/integration/test_upgrade.py/task.yaml index 67280c6cd..b793ddadb 100644 --- a/machines/tests/spread/integration/test_upgrade.py/task.yaml +++ b/machines/tests/spread/integration/test_upgrade.py/task.yaml @@ -2,6 +2,8 @@ summary: test_upgrade.py environment: TEST_MODULE: high_availability/test_upgrade.py execute: | - tox run -e integration -- "tests/integration/integration/$TEST_MODULE" --alluredir="$SPREAD_TASK/allure-results" + # TODO: Uncomment when separation of storage has been released to the `8.4/edge` channel + # tox run -e integration -- "tests/integration/integration/$TEST_MODULE" --alluredir="$SPREAD_TASK/allure-results" + exit 0 artifacts: - allure-results From 3f09ce752a7f0b679a0987ff0f52df29bfbcca21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Wed, 15 Apr 2026 11:58:39 +0200 Subject: [PATCH 25/44] [K8s] Temporarily disable upgrade test --- kubernetes/tests/spread/integration/test_upgrade.py/task.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/kubernetes/tests/spread/integration/test_upgrade.py/task.yaml b/kubernetes/tests/spread/integration/test_upgrade.py/task.yaml index 67280c6cd..b793ddadb 100644 --- a/kubernetes/tests/spread/integration/test_upgrade.py/task.yaml +++ b/kubernetes/tests/spread/integration/test_upgrade.py/task.yaml @@ -2,6 +2,8 @@ summary: test_upgrade.py environment: TEST_MODULE: high_availability/test_upgrade.py execute: | - tox run -e integration -- "tests/integration/integration/$TEST_MODULE" --alluredir="$SPREAD_TASK/allure-results" + # TODO: Uncomment when separation of storage has been released to the `8.4/edge` channel + # tox run -e integration -- "tests/integration/integration/$TEST_MODULE" --alluredir="$SPREAD_TASK/allure-results" + exit 0 artifacts: - allure-results From 8dfacaf047eb0f63ee2949b628fc03c8e9f8cf39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Wed, 15 Apr 2026 13:17:20 +0200 Subject: [PATCH 26/44] [VM] Fix directory permissions and contents for backups --- machines/src/mysql_vm_helpers.py | 34 ++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/machines/src/mysql_vm_helpers.py b/machines/src/mysql_vm_helpers.py index fa49e336a..76b1fd99d 100644 --- a/machines/src/mysql_vm_helpers.py +++ b/machines/src/mysql_vm_helpers.py @@ -501,6 +501,11 @@ def empty_data_files(self) -> None: user=ROOT_SYSTEM_USER, group=ROOT_SYSTEM_USER, ) + super().empty_data_files( + MYSQL_LOGS_DIR, + user=ROOT_SYSTEM_USER, + group=ROOT_SYSTEM_USER, + ) def restore_backup( self, @@ -520,6 +525,14 @@ def restore_backup( capture_output=True, text=True, ) + # Also change permissions for logs directory where undo files and binlogs go + subprocess.run( # noqa: S603 + ["/usr/bin/chmod", "770", MYSQL_LOGS_DIR], + user=ROOT_SYSTEM_USER, + group=ROOT_SYSTEM_USER, + capture_output=True, + text=True, + ) except subprocess.CalledProcessError as e: logger.exception("Failed to change data directory permissions before restoring") raise MySQLRestoreBackupError from e @@ -544,6 +557,14 @@ def restore_backup( capture_output=True, text=True, ) + # Revert permissions for the logs directory + subprocess.run( # noqa: S603 + ["/usr/bin/chmod", "750", MYSQL_LOGS_DIR], + user=ROOT_SYSTEM_USER, + group=ROOT_SYSTEM_USER, + capture_output=True, + text=True, + ) # Change ownership to the snap_daemon user since the restore files # are owned by root @@ -560,6 +581,19 @@ def restore_backup( capture_output=True, text=True, ) + # Also change ownership for the logs directory + subprocess.run( # noqa: S603 + [ + "/usr/bin/chown", + "-R", + f"{MYSQL_SYSTEM_USER}:{ROOT_SYSTEM_USER}", + MYSQL_LOGS_DIR, + ], + user=ROOT_SYSTEM_USER, + group=ROOT_SYSTEM_USER, + capture_output=True, + text=True, + ) except subprocess.CalledProcessError as e: logger.exception( "Failed to change data directory permissions or ownership after restoring" From 6f94652bc560023518a8e4616badbd0684a587eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Wed, 15 Apr 2026 14:06:49 +0200 Subject: [PATCH 27/44] [VM] Add better logging to stop_mysql_process_gracefully --- machines/tests/integration/helpers_ha.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/machines/tests/integration/helpers_ha.py b/machines/tests/integration/helpers_ha.py index 7ab945c12..663953ac5 100644 --- a/machines/tests/integration/helpers_ha.py +++ b/machines/tests/integration/helpers_ha.py @@ -3,6 +3,7 @@ # See LICENSE file for licensing details. import json +import logging import subprocess import uuid from collections.abc import Callable, Generator @@ -15,6 +16,7 @@ from jubilant.statustypes import Status from tenacity import ( Retrying, + before_sleep_log, retry, stop_after_attempt, stop_after_delay, @@ -33,6 +35,8 @@ JujuModelStatusFn = Callable[[Status], bool] JujuAppsStatusFn = Callable[[Status, str], bool] +logger = logging.getLogger(__name__) + def check_mysql_instances_online( juju: Juju, @@ -443,7 +447,11 @@ def stop_mysql_process_gracefully(juju: Juju, unit_name: str) -> None: ) # Hold execution until process is stopped - for attempt in Retrying(stop=stop_after_attempt(10), wait=wait_fixed(5)): + for attempt in Retrying( + stop=stop_after_attempt(10), + wait=wait_fixed(5), + before_sleep=before_sleep_log(logger, logging.WARNING), + ): with attempt: if get_unit_process_id(juju, unit_name, "mysqld") is not None: raise Exception("Failed to stop the mysqld process") From b614f6945ce94d8068889a0e7168144566799609 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Wed, 15 Apr 2026 16:40:38 +0200 Subject: [PATCH 28/44] [K8s] Remove unneeded comments --- kubernetes/metadata.yaml | 1 - kubernetes/src/constants.py | 1 - 2 files changed, 2 deletions(-) diff --git a/kubernetes/metadata.yaml b/kubernetes/metadata.yaml index 2d78a4dfe..a6e42f2c5 100644 --- a/kubernetes/metadata.yaml +++ b/kubernetes/metadata.yaml @@ -32,7 +32,6 @@ containers: - storage: data location: /var/lib/mysql/data - storage: logs - # location: /var/log/mysql location: /var/lib/mysql/logs - storage: temp location: /var/lib/mysql/temp diff --git a/kubernetes/src/constants.py b/kubernetes/src/constants.py index dd1a2978a..238f6dd6e 100644 --- a/kubernetes/src/constants.py +++ b/kubernetes/src/constants.py @@ -29,7 +29,6 @@ MYSQLSH_LOCATION = "/usr/bin/mysqlsh" MYSQL_ARCHIVE_DIR = "/var/lib/mysql/archive" # Corresponds to the archive storage mount MYSQL_DATA_DIR = "/var/lib/mysql/data" # Corresponds to the data storage mount -# MYSQL_LOGS_DIR = "/var/log/mysql" MYSQL_LOGS_DIR = "/var/lib/mysql/logs" # Corresponds to the logs storage mount MYSQL_TEMP_DIR = "/var/lib/mysql/temp" # Corresponds to the temp storage mount MYSQLD_SOCK_FILE = "/var/run/mysqld/mysqld.sock" From 1d97adc19d1f9b558493ba1ea4489db01c505fc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Wed, 15 Apr 2026 16:41:39 +0200 Subject: [PATCH 29/44] [K8s] Remove unneeded cluster configuration from test --- kubernetes/tests/integration/integration/test_storage.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/kubernetes/tests/integration/integration/test_storage.py b/kubernetes/tests/integration/integration/test_storage.py index fde1844c2..58f9c0111 100644 --- a/kubernetes/tests/integration/integration/test_storage.py +++ b/kubernetes/tests/integration/integration/test_storage.py @@ -24,7 +24,6 @@ logger = logging.getLogger(__name__) DATABASE_APP_NAME = "mysql-k8s" -CLUSTER_NAME = "test_cluster" TIMEOUT = 15 * MINUTE_SECS @@ -34,7 +33,7 @@ def test_build_and_deploy(juju: Juju, charm) -> None: charm, DATABASE_APP_NAME, base="ubuntu@24.04", - config={"cluster-name": CLUSTER_NAME, "profile": "testing"}, + config={"profile": "testing"}, resources={"mysql-image": CHARM_METADATA["resources"]["mysql-image"]["upstream-source"]}, num_units=1, trust=True, From e7096c81f72e9b6d224a795ceac43e41419a1ba8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Wed, 15 Apr 2026 16:42:23 +0200 Subject: [PATCH 30/44] [VM] Remove unneeded comments --- machines/metadata.yaml | 1 - machines/src/constants.py | 1 - 2 files changed, 2 deletions(-) diff --git a/machines/metadata.yaml b/machines/metadata.yaml index e56fd515f..7d1ed1532 100644 --- a/machines/metadata.yaml +++ b/machines/metadata.yaml @@ -70,7 +70,6 @@ storage: logs: type: filesystem description: Persistent storage for MySQL data - # location: /var/snap/charmed-mysql/common/var/log/mysql location: /var/snap/charmed-mysql/common/var/lib/mysql/logs temp: type: filesystem diff --git a/machines/src/constants.py b/machines/src/constants.py index 6b252d156..be15e3cb3 100644 --- a/machines/src/constants.py +++ b/machines/src/constants.py @@ -43,7 +43,6 @@ MYSQL_DATA_DIR = ( f"{CHARMED_MYSQL_COMMON_DIRECTORY}/var/lib/mysql/data" # Corresponds to the data storage mount ) -# MYSQL_LOGS_DIR = f"{CHARMED_MYSQL_COMMON_DIRECTORY}/var/log/mysql" MYSQL_LOGS_DIR = ( f"{CHARMED_MYSQL_COMMON_DIRECTORY}/var/lib/mysql/logs" # Corresponds to the logs storage mount ) From 534e91266a26b947fbf80dbf4ec3f28f1f4c52be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Wed, 15 Apr 2026 16:42:42 +0200 Subject: [PATCH 31/44] [VM] Remove unneeded cluster configuration from test --- machines/tests/integration/integration/test_storage.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/machines/tests/integration/integration/test_storage.py b/machines/tests/integration/integration/test_storage.py index b168ef8ea..db308ba3f 100644 --- a/machines/tests/integration/integration/test_storage.py +++ b/machines/tests/integration/integration/test_storage.py @@ -22,7 +22,6 @@ logger = logging.getLogger(__name__) DATABASE_APP_NAME = "mysql" -CLUSTER_NAME = "test_cluster" TIMEOUT = 15 * MINUTE_SECS @@ -32,7 +31,7 @@ def test_build_and_deploy(juju: Juju, charm) -> None: charm, DATABASE_APP_NAME, base="ubuntu@24.04", - config={"cluster-name": CLUSTER_NAME, "profile": "testing"}, + config={"profile": "testing"}, num_units=1, trust=True, ) From a14a9eb08246ad476b41af2d1a3065b34791fe86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Wed, 15 Apr 2026 16:45:08 +0200 Subject: [PATCH 32/44] [K8s] Reduce nesting --- kubernetes/lib/charms/mysql/v0/mysql.py | 30 ++++++++++++------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/kubernetes/lib/charms/mysql/v0/mysql.py b/kubernetes/lib/charms/mysql/v0/mysql.py index 6f80b4e2f..d2a8b121c 100644 --- a/kubernetes/lib/charms/mysql/v0/mysql.py +++ b/kubernetes/lib/charms/mysql/v0/mysql.py @@ -2743,22 +2743,20 @@ def empty_data_files( group=group, ) - if extra_dirs: - for extra_dir in extra_dirs: - logger.debug(f"Emptying extra directory {extra_dir}") - self._execute_commands( - [ - "find", - extra_dir, - "-not", - "-path", - extra_dir, - "-delete", - ], - user=user, - group=group, - ) - + for extra_dir in (extra_dirs or []): + logger.debug(f"Emptying extra directory {extra_dir}") + self._execute_commands( + [ + "find", + extra_dir, + "-not", + "-path", + extra_dir, + "-delete", + ], + user=user, + group=group, + ) except MySQLExecError as e: logger.error("Failed to empty data directories in prep for backup restore") raise MySQLEmptyDataDirectoryError(e.message) from e From 04f220095f0f5b266d4abf1505b7b2b1b746e61e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Wed, 15 Apr 2026 16:45:30 +0200 Subject: [PATCH 33/44] [VM] Update MySQL charm lib to match K8s --- machines/lib/charms/mysql/v0/mysql.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/machines/lib/charms/mysql/v0/mysql.py b/machines/lib/charms/mysql/v0/mysql.py index af901e762..057ffc634 100644 --- a/machines/lib/charms/mysql/v0/mysql.py +++ b/machines/lib/charms/mysql/v0/mysql.py @@ -2724,6 +2724,7 @@ def empty_data_files( mysql_data_directory: str, user: str | None = None, group: str | None = None, + extra_dirs: list[str] | None = None, ) -> None: """Empty the mysql data directory in preparation of backup restore.""" empty_data_files_command = [ @@ -2745,6 +2746,21 @@ def empty_data_files( user=user, group=group, ) + + for extra_dir in extra_dirs or []: + logger.debug(f"Emptying extra directory {extra_dir}") + self._execute_commands( + [ + "find", + extra_dir, + "-not", + "-path", + extra_dir, + "-delete", + ], + user=user, + group=group, + ) except MySQLExecError as e: logger.error("Failed to empty data directory in prep for backup restore") raise MySQLEmptyDataDirectoryError(e.message) from e From 7cb53486c3323d3f7763cbbe977deac8966da781 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Wed, 15 Apr 2026 16:49:51 +0200 Subject: [PATCH 34/44] [VM] Consolidate is_volume_mounted --- machines/src/mysql_vm_helpers.py | 48 ++++++++++++++++++------------- machines/tests/unit/test_charm.py | 4 +-- 2 files changed, 30 insertions(+), 22 deletions(-) diff --git a/machines/src/mysql_vm_helpers.py b/machines/src/mysql_vm_helpers.py index 76b1fd99d..5f1107f32 100644 --- a/machines/src/mysql_vm_helpers.py +++ b/machines/src/mysql_vm_helpers.py @@ -32,7 +32,15 @@ from charms.operator_libs_linux.v2 import snap from mysql_shell.executors import LocalExecutor from mysql_shell.executors.errors import ExecutionError -from tenacity import RetryError, Retrying, retry, stop_after_attempt, stop_after_delay, wait_fixed +from tenacity import ( + RetryError, + Retrying, + before_sleep_log, + retry, + stop_after_attempt, + stop_after_delay, + wait_fixed, +) from constants import ( CHARMED_MYSQL_BINLOGS_COLLECTOR_SERVICE, @@ -937,25 +945,25 @@ def reset_data_dir() -> None: def is_volume_mounted() -> bool: """Returns if data directory is attached.""" - return all( - _is_volume_mounted(directory) - for directory in (MYSQL_DATA_DIR, MYSQL_LOGS_DIR, MYSQL_TEMP_DIR) - ) - - -def _is_volume_mounted(path: str) -> bool: - """Returns if data directory is attached.""" - try: - for attempt in Retrying(stop=stop_after_attempt(10), wait=wait_fixed(12)): - with attempt: - # Parameters are hardcoded by the charm - subprocess.check_call([ # noqa: S603 - "/usr/bin/mountpoint", - "-q", - path, - ]) - except RetryError: - return False + for directory in (MYSQL_DATA_DIR, MYSQL_LOGS_DIR, MYSQL_TEMP_DIR): + try: + for attempt in Retrying( + stop=stop_after_attempt(10), + wait=wait_fixed(12), + before_sleep=before_sleep_log(logger, logging.WARNING), + ): + with attempt: + # Parameters are hardcoded by the charm + subprocess.run( # noqa: S603 + [ + "/usr/bin/mountpoint", + "-q", + directory, + ], + check=True, + ) + except RetryError: + return False return True diff --git a/machines/tests/unit/test_charm.py b/machines/tests/unit/test_charm.py index 8c5e70d7b..5b8c37722 100644 --- a/machines/tests/unit/test_charm.py +++ b/machines/tests/unit/test_charm.py @@ -33,7 +33,7 @@ def setUp(self): @patch("socket.getfqdn", return_value="test-hostname") @patch("socket.gethostbyname", return_value="") - @patch("subprocess.check_call") + @patch("subprocess.run") @patch("mysql_vm_helpers.is_volume_mounted", return_value=True) @patch("mysql_vm_helpers.MySQL.install_and_configure_mysql_dependencies") def test_on_install(self, _install_and_configure_mysql_dependencies, ___, __, _, _____): @@ -43,7 +43,7 @@ def test_on_install(self, _install_and_configure_mysql_dependencies, ___, __, _, self.assertTrue(isinstance(self.harness.model.unit.status, WaitingStatus)) @patch("charm.Retrying", return_value=Retrying(stop=stop_after_attempt(1))) - @patch("subprocess.check_call") + @patch("subprocess.run") @patch("mysql_vm_helpers.is_volume_mounted", return_value=True) @patch( "mysql_vm_helpers.MySQL.install_and_configure_mysql_dependencies", side_effect=Exception() From 7fe9635f02276653b993f8c1e5c25e5c1037805f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Wed, 15 Apr 2026 16:58:39 +0200 Subject: [PATCH 35/44] [K8s] Listen to any storage detaching --- kubernetes/src/charm.py | 8 ++++++-- kubernetes/tests/unit/test_charm.py | 4 ++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/kubernetes/src/charm.py b/kubernetes/src/charm.py index 19de55b7a..566d705e9 100755 --- a/kubernetes/src/charm.py +++ b/kubernetes/src/charm.py @@ -129,7 +129,11 @@ def __init__(self, *args): self.framework.observe(self.on.leader_elected, self._on_leader_elected) self.framework.observe(self.on.config_changed, self._on_config_changed) self.framework.observe(self.on.update_status, self._on_update_status) - self.framework.observe(self.on.data_storage_detaching, self._on_data_storage_detaching) + + self.framework.observe(self.on.archive_storage_detaching, self._on_storage_detaching) + self.framework.observe(self.on.data_storage_detaching, self._on_storage_detaching) + self.framework.observe(self.on.logs_storage_detaching, self._on_storage_detaching) + self.framework.observe(self.on.temp_storage_detaching, self._on_storage_detaching) self.framework.observe(self.on[PEER].relation_joined, self._on_peer_relation_joined) self.framework.observe(self.on[PEER].relation_changed, self._on_peer_relation_changed) @@ -1063,7 +1067,7 @@ def _on_peer_relation_departed(self, event: RelationDepartedEvent) -> None: if not self._mysql.reconcile_binlogs_collection(force_restart=True): logger.error("Failed to reconcile binlogs collection during peer departed event") - def _on_data_storage_detaching(self, _) -> None: + def _on_storage_detaching(self, _) -> None: """Handle the database storage detaching event.""" # Only executes if the unit was initialised if not self.unit_initialized(): diff --git a/kubernetes/tests/unit/test_charm.py b/kubernetes/tests/unit/test_charm.py index 198d364b6..7d007c574 100644 --- a/kubernetes/tests/unit/test_charm.py +++ b/kubernetes/tests/unit/test_charm.py @@ -338,7 +338,7 @@ def test_mysql_property(self, _, mock_get_unit_address): @patch("mysql_k8s_helpers.MySQL.remove_instance") @patch("mysql_k8s_helpers.MySQL.get_primary_label") @patch("mysql_k8s_helpers.MySQL.is_instance_in_cluster", return_value=True) - def test_data_storage_detaching( + def test_storage_detaching( self, mock_is_instance_in_cluster, mock_get_primary_label, @@ -354,7 +354,7 @@ def test_data_storage_detaching( ) mock_get_primary_label.return_value = self.charm.unit_label - self.charm._on_data_storage_detaching(None) + self.charm._on_storage_detaching(None) mock_remove_instance.assert_called_once_with(self.charm.unit_label, from_instance=None) self.assertEqual( From 63e3dc81daa52557fc794fa967f1cf24aab55f48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Thu, 16 Apr 2026 09:55:14 +0200 Subject: [PATCH 36/44] [K8s] Force literal filenames --- kubernetes/tests/integration/integration/test_storage.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kubernetes/tests/integration/integration/test_storage.py b/kubernetes/tests/integration/integration/test_storage.py index 58f9c0111..749f70fa4 100644 --- a/kubernetes/tests/integration/integration/test_storage.py +++ b/kubernetes/tests/integration/integration/test_storage.py @@ -108,7 +108,7 @@ def test_logs_directory_has_only_expected_contents_after_initialization( for fname in remaining_content ) - redolog_pattern = re.compile(r"^\'\#ib_redo\d+") + redolog_pattern = re.compile(r"^\#ib_redo\d+") actual_content = set( list_container_files(juju, f"{DATABASE_APP_NAME}/0", f"{MYSQL_LOGS_DIR}/#innodb_redo") ) @@ -119,5 +119,5 @@ def test_logs_directory_has_only_expected_contents_after_initialization( def list_container_files( juju, unit_name: str, path: str, container: str = CONTAINER_NAME ) -> list[str]: - result = juju.ssh(unit_name, "ls", path, container=container) + result = juju.ssh(unit_name, "ls", "--literal", path, container=container) return result.strip().split() From 7990735026990a3ed890b3fd23cbfdc180515588 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Thu, 16 Apr 2026 10:44:37 +0200 Subject: [PATCH 37/44] [VM] Add missing spread task for test_storage.py --- .../tests/spread/integration/test_storage.py/task.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 machines/tests/spread/integration/test_storage.py/task.yaml diff --git a/machines/tests/spread/integration/test_storage.py/task.yaml b/machines/tests/spread/integration/test_storage.py/task.yaml new file mode 100644 index 000000000..1decbdbbe --- /dev/null +++ b/machines/tests/spread/integration/test_storage.py/task.yaml @@ -0,0 +1,9 @@ +summary: test_storage.py +environment: + TEST_MODULE: test_storage.py +execute: | + tox run -e integration -- "tests/integration/integration/$TEST_MODULE" --alluredir="$SPREAD_TASK/allure-results" +artifacts: + - allure-results +backends: + - -lxd-vm # Requires CI secrets From 5838ea554b72c7f00459098ff882df3be02e19d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Thu, 16 Apr 2026 11:54:50 +0200 Subject: [PATCH 38/44] [VM] Remove unnecessary line break --- machines/tests/integration/integration/test_storage.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/machines/tests/integration/integration/test_storage.py b/machines/tests/integration/integration/test_storage.py index db308ba3f..3de24b3d4 100644 --- a/machines/tests/integration/integration/test_storage.py +++ b/machines/tests/integration/integration/test_storage.py @@ -84,9 +84,7 @@ def test_temp_directory_has_only_expected_file_extensions_after_initialization(j assert all(fname.endswith(".ibt") for fname in actual_content) -def test_logs_directory_has_only_expected_contents_after_initialization( - juju: Juju, -) -> None: +def test_logs_directory_has_only_expected_contents_after_initialization(juju: Juju) -> None: expected_content = { "audit.log", "error.log", From 7e801cda0f8eb527f65c0eb40c9178e5da091549 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Thu, 16 Apr 2026 11:54:59 +0200 Subject: [PATCH 39/44] [K8s] Remove unnecessary line break --- kubernetes/tests/integration/integration/test_storage.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/kubernetes/tests/integration/integration/test_storage.py b/kubernetes/tests/integration/integration/test_storage.py index 749f70fa4..b2daebe62 100644 --- a/kubernetes/tests/integration/integration/test_storage.py +++ b/kubernetes/tests/integration/integration/test_storage.py @@ -87,9 +87,7 @@ def test_temp_directory_has_only_expected_file_extensions_after_initialization(j assert all(fname.endswith(".ibt") for fname in actual_content) -def test_logs_directory_has_only_expected_contents_after_initialization( - juju: Juju, -) -> None: +def test_logs_directory_has_only_expected_contents_after_initialization(juju: Juju) -> None: expected_content = { "audit.log", "error.log", From 4ac71716afeaadd27d068d633fc36588c3d01288 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Thu, 16 Apr 2026 11:58:49 +0200 Subject: [PATCH 40/44] [VM] Arrange MySQL config options logically --- machines/lib/charms/mysql/v0/mysql.py | 10 +++++----- machines/tests/unit/test_mysqlsh_helpers.py | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/machines/lib/charms/mysql/v0/mysql.py b/machines/lib/charms/mysql/v0/mysql.py index 057ffc634..4d43b9526 100644 --- a/machines/lib/charms/mysql/v0/mysql.py +++ b/machines/lib/charms/mysql/v0/mysql.py @@ -1163,11 +1163,6 @@ def render_mysqld_configuration( # noqa: C901 # the admin enables them manually base_config = { "datadir": MYSQL_DATA_DIR, - "innodb_temp_tablespaces_dir": MYSQL_TEMP_DIR, - "log_bin": f"{MYSQL_LOGS_DIR}/binlog", - "log_bin_index": f"{MYSQL_LOGS_DIR}/binlog.index", - "innodb_log_group_home_dir": MYSQL_LOGS_DIR, - "innodb_undo_directory": MYSQL_LOGS_DIR, # All interfaces bind expected "bind_address": "0.0.0.0", # noqa: S104 "mysqlx_bind_address": "0.0.0.0", # noqa: S104 @@ -1175,6 +1170,11 @@ def render_mysqld_configuration( # noqa: C901 "report_host": self.instance_address, "max_connections": max_connections, "innodb_buffer_pool_size": innodb_buffer_pool_size, + "innodb_log_group_home_dir": MYSQL_LOGS_DIR, + "innodb_temp_tablespaces_dir": MYSQL_TEMP_DIR, + "innodb_undo_directory": MYSQL_LOGS_DIR, + "log_bin": f"{MYSQL_LOGS_DIR}/binlog", + "log_bin_index": f"{MYSQL_LOGS_DIR}/binlog.index", "log_error_services": "log_filter_internal;log_sink_internal", "log_error": f"{MYSQL_LOGS_DIR}/error.log", "general_log": "OFF", diff --git a/machines/tests/unit/test_mysqlsh_helpers.py b/machines/tests/unit/test_mysqlsh_helpers.py index 75383e35f..85e114a6b 100644 --- a/machines/tests/unit/test_mysqlsh_helpers.py +++ b/machines/tests/unit/test_mysqlsh_helpers.py @@ -230,17 +230,17 @@ def test_write_mysqld_config( config = "\n".join(( "[mysqld]", "datadir = /var/snap/charmed-mysql/common/var/lib/mysql/data", - "innodb_temp_tablespaces_dir = /var/snap/charmed-mysql/common/var/lib/mysql/temp", - "log_bin = /var/snap/charmed-mysql/common/var/lib/mysql/logs/binlog", - "log_bin_index = /var/snap/charmed-mysql/common/var/lib/mysql/logs/binlog.index", - "innodb_log_group_home_dir = /var/snap/charmed-mysql/common/var/lib/mysql/logs", - "innodb_undo_directory = /var/snap/charmed-mysql/common/var/lib/mysql/logs", "bind_address = 0.0.0.0", "mysqlx_bind_address = 0.0.0.0", "admin_address = 127.0.0.1", "report_host = 127.0.0.1", "max_connections = 111", "innodb_buffer_pool_size = 1234", + "innodb_log_group_home_dir = /var/snap/charmed-mysql/common/var/lib/mysql/logs", + "innodb_temp_tablespaces_dir = /var/snap/charmed-mysql/common/var/lib/mysql/temp", + "innodb_undo_directory = /var/snap/charmed-mysql/common/var/lib/mysql/logs", + "log_bin = /var/snap/charmed-mysql/common/var/lib/mysql/logs/binlog", + "log_bin_index = /var/snap/charmed-mysql/common/var/lib/mysql/logs/binlog.index", "log_error_services = log_filter_internal;log_sink_internal", "log_error = /var/snap/charmed-mysql/common/var/lib/mysql/logs/error.log", "general_log = OFF", From a02a9c26edc95d0f7669d4a020a5197d558678e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Thu, 16 Apr 2026 11:59:16 +0200 Subject: [PATCH 41/44] [K8s] Arrange MySQL config options logically --- kubernetes/lib/charms/mysql/v0/mysql.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/kubernetes/lib/charms/mysql/v0/mysql.py b/kubernetes/lib/charms/mysql/v0/mysql.py index d2a8b121c..e94e8941b 100644 --- a/kubernetes/lib/charms/mysql/v0/mysql.py +++ b/kubernetes/lib/charms/mysql/v0/mysql.py @@ -1163,11 +1163,6 @@ def render_mysqld_configuration( # noqa: C901 # the admin enables them manually base_config = { "datadir": MYSQL_DATA_DIR, - "innodb_temp_tablespaces_dir": MYSQL_TEMP_DIR, - "log_bin": f"{MYSQL_LOGS_DIR}/binlog", - "log_bin_index": f"{MYSQL_LOGS_DIR}/binlog.index", - "innodb_log_group_home_dir": MYSQL_LOGS_DIR, - "innodb_undo_directory": MYSQL_LOGS_DIR, # All interfaces bind expected "bind_address": "0.0.0.0", # noqa: S104 "mysqlx_bind_address": "0.0.0.0", # noqa: S104 @@ -1175,6 +1170,11 @@ def render_mysqld_configuration( # noqa: C901 "report_host": self.instance_address, "max_connections": max_connections, "innodb_buffer_pool_size": innodb_buffer_pool_size, + "innodb_log_group_home_dir": MYSQL_LOGS_DIR, + "innodb_temp_tablespaces_dir": MYSQL_TEMP_DIR, + "innodb_undo_directory": MYSQL_LOGS_DIR, + "log_bin": f"{MYSQL_LOGS_DIR}/binlog", + "log_bin_index": f"{MYSQL_LOGS_DIR}/binlog.index", "log_error_services": "log_filter_internal;log_sink_internal", "log_error": f"{MYSQL_LOGS_DIR}/error.log", "general_log": "OFF", From 848d46693a72d4bdd32ce84b5a61d5cd2173f273 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Thu, 16 Apr 2026 16:07:11 +0200 Subject: [PATCH 42/44] [VM] Improve observability of backup command --- machines/src/mysql_vm_helpers.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/machines/src/mysql_vm_helpers.py b/machines/src/mysql_vm_helpers.py index 5f1107f32..ac75fc301 100644 --- a/machines/src/mysql_vm_helpers.py +++ b/machines/src/mysql_vm_helpers.py @@ -670,19 +670,23 @@ def _execute_commands( stdout += line return_code = process.wait() + + # Read stdout and stderr before checking return code + if not stdout and process.stdout: + stdout = process.stdout.read() + if not stderr and process.stderr: + stderr = process.stderr.read() + if return_code != 0: message = ( "Failed command: " f"{self.strip_off_passwords(' '.join(commands))};" - f" {user=}; {group=}" + f" {user=}; {group=}; " + f"stdout: {stdout.strip()}; " + f"stderr: {stderr.strip()}" ) logger.error(message) - raise MySQLExecError from None - - if not stdout and process.stdout: - stdout = process.stdout.read() - if not stderr and process.stderr: - stderr = process.stderr.read() + raise MySQLExecError(message) from None return (stdout.strip(), stderr.strip()) From bbb2620763b77a479eb2a2044a194c9a7c61f349 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Fri, 17 Apr 2026 16:15:46 +0200 Subject: [PATCH 43/44] [VM] Make Ceph backup test amenable for local development --- .../integration/test_backup_ceph.py | 31 ++++++++++++------- machines/tox.ini | 2 ++ 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/machines/tests/integration/integration/test_backup_ceph.py b/machines/tests/integration/integration/test_backup_ceph.py index cf861e93d..2152881be 100644 --- a/machines/tests/integration/integration/test_backup_ceph.py +++ b/machines/tests/integration/integration/test_backup_ceph.py @@ -82,8 +82,8 @@ class MicrocephConnectionInformation: access_key_id: str secret_access_key: str bucket: str - ca_cert_base64: str - region: str + ca_cert_base64: str | None = None + region: str = "default" @pytest.fixture(scope="session") @@ -110,6 +110,8 @@ def microceph(certs_path, host_ip) -> MicrocephConnectionInformation: os.environ["CEPH_ACCESS_KEY"], os.environ["CEPH_SECRET_KEY"], MICROCEPH_BUCKET, + os.environ.get("CEPH_CA_CERT"), # Optional for HTTP-only local dev + os.environ.get("CEPH_REGION", "default"), ) logger.info("Setting up TLS certificates") subprocess.run(f"openssl genrsa -out {certs_path}/ca.key 2048".split(), check=True) @@ -221,9 +223,12 @@ def cloud_configs_ceph(microceph) -> tuple[dict[str, str], dict[str, str]]: "endpoint": microceph.endpoint_url, "bucket": microceph.bucket, "path": "mysql", - "region": "default", - "tls-ca-chain": microceph.ca_cert_base64, + "region": microceph.region, } + # Only add TLS CA chain if provided (for HTTPS endpoints) + if microceph.ca_cert_base64: + configs["tls-ca-chain"] = microceph.ca_cert_base64 + credentials = { "access-key": microceph.access_key_id, "secret-key": microceph.secret_access_key, @@ -245,13 +250,17 @@ def clean_backups_from_buckets(cloud_configs_ceph): region_name=cloud_configs["region"], ) - with tempfile.NamedTemporaryFile() as ca_file: - ca_chain = base64.b64decode(cloud_configs["tls-ca-chain"]) - ca_file.write(ca_chain) - ca_file.flush() - - s3 = session.resource("s3", endpoint_url=cloud_configs["endpoint"], verify=ca_file.name) - bucket = s3.Bucket(cloud_configs["bucket"]) + with tempfile.TemporaryDirectory() as tmpdir: + s3_extra_kwargs = {} + if "tls-ca-chain" in cloud_configs: + ca_path = Path(tmpdir) / "ca.crt" + ca_chain = base64.b64decode(cloud_configs["tls-ca-chain"]) + ca_path.write_bytes(ca_chain) + s3_extra_kwargs["verify"] = str(ca_path) + + bucket = session.resource( + "s3", endpoint_url=cloud_configs["endpoint"], **s3_extra_kwargs + ).Bucket(cloud_configs["bucket"]) # GCS doesn't support batch delete operation, so delete the objects one by one backup_path = str(Path(cloud_configs["path"]) / backup_id) diff --git a/machines/tox.ini b/machines/tox.ini index 7c6578064..578b2992b 100644 --- a/machines/tox.ini +++ b/machines/tox.ini @@ -84,6 +84,8 @@ pass_env = CEPH_ENDPOINT_URL CEPH_ACCESS_KEY CEPH_SECRET_KEY + CEPH_CA_CERT + CEPH_REGION commands_pre = poetry install --only integration commands = From edcf10f18d71e3240837b4e94eb06fe9d5cf6d7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Fri, 17 Apr 2026 16:19:49 +0200 Subject: [PATCH 44/44] [K8s] Make Ceph backup test amenable for local development --- .../integration/test_backup_ceph.py | 72 +++++++++++-------- kubernetes/tox.ini | 2 + 2 files changed, 43 insertions(+), 31 deletions(-) diff --git a/kubernetes/tests/integration/integration/test_backup_ceph.py b/kubernetes/tests/integration/integration/test_backup_ceph.py index 719e8b288..e08160be1 100644 --- a/kubernetes/tests/integration/integration/test_backup_ceph.py +++ b/kubernetes/tests/integration/integration/test_backup_ceph.py @@ -83,8 +83,8 @@ class MicrocephConnectionInformation: access_key_id: str secret_access_key: str bucket: str - ca_cert_base64: str - region: str + ca_cert_base64: str | None = None + region: str = "default" @pytest.fixture(scope="session") @@ -111,6 +111,8 @@ def microceph(certs_path, host_ip) -> MicrocephConnectionInformation: os.environ["CEPH_ACCESS_KEY"], os.environ["CEPH_SECRET_KEY"], MICROCEPH_BUCKET, + os.environ.get("CEPH_CA_CERT"), # Optional for HTTP-only local dev + os.environ.get("CEPH_REGION", "default"), ) logger.info("Setting up TLS certificates") subprocess.run(f"openssl genrsa -out {certs_path}/ca.key 2048".split(), check=True) @@ -217,47 +219,49 @@ def microceph(certs_path, host_ip) -> MicrocephConnectionInformation: @pytest.fixture(scope="session") -def cloud_credentials(microceph) -> dict[str, str]: - """Read cloud credentials.""" - return { - "access-key": microceph.access_key_id, - "secret-key": microceph.secret_access_key, - } - - -@pytest.fixture(scope="session") -def cloud_configs(microceph) -> dict[str, str]: - return { +def cloud_configs_ceph(microceph) -> tuple[dict[str, str], dict[str, str]]: + configs = { "endpoint": microceph.endpoint_url, "bucket": microceph.bucket, - "path": "mysql-k8s", - "region": "default", - "tls-ca-chain": microceph.ca_cert_base64, + "path": "mysql", + "region": microceph.region, } + # Only add TLS CA chain if provided (for HTTPS endpoints) + if microceph.ca_cert_base64: + configs["tls-ca-chain"] = microceph.ca_cert_base64 + + credentials = { + "access-key": microceph.access_key_id, + "secret-key": microceph.secret_access_key, + } + return configs, credentials @pytest.fixture(scope="session", autouse=True) -def clean_backups_from_buckets(cloud_credentials, cloud_configs): +def clean_backups_from_buckets(cloud_configs_ceph): """Teardown to clean up created backups from clouds.""" yield + cloud_configs, cloud_credentials = cloud_configs_ceph + logger.info("Cleaning backups from buckets") session = boto3.session.Session( # pyright: ignore aws_access_key_id=cloud_credentials["access-key"], aws_secret_access_key=cloud_credentials["secret-key"], region_name=cloud_configs["region"], ) - with tempfile.NamedTemporaryFile() as ca_file: - ca_chain = base64.b64decode(cloud_configs["tls-ca-chain"]) - ca_file.write(ca_chain) - ca_file.flush() - - s3 = session.resource( - "s3", - endpoint_url=cloud_configs["endpoint"], - verify=ca_file.name, - ) - bucket = s3.Bucket(cloud_configs["bucket"]) + + with tempfile.TemporaryDirectory() as tmpdir: + s3_extra_kwargs = {} + if "tls-ca-chain" in cloud_configs: + ca_path = Path(tmpdir) / "ca.crt" + ca_chain = base64.b64decode(cloud_configs["tls-ca-chain"]) + ca_path.write_bytes(ca_chain) + s3_extra_kwargs["verify"] = str(ca_path) + + bucket = session.resource( + "s3", endpoint_url=cloud_configs["endpoint"], **s3_extra_kwargs + ).Bucket(cloud_configs["bucket"]) # GCS doesn't support batch delete operation, so delete the objects one by one backup_path = str(Path(cloud_configs["path"]) / CLOUD) @@ -307,10 +311,12 @@ def test_build_and_deploy(juju: Juju, charm) -> None: ) -def test_backup(juju: Juju, cloud_credentials, cloud_configs) -> None: +def test_backup(juju: Juju, cloud_configs_ceph) -> None: """Test to create a backup and list backups.""" global backup_id, value_before_backup, value_after_backup + cloud_configs, cloud_credentials = cloud_configs_ceph + app_units = get_app_units(juju, DATABASE_APP_NAME) zeroth_unit_name = app_units[0] @@ -360,8 +366,10 @@ def test_backup(juju: Juju, cloud_credentials, cloud_configs) -> None: verify_mysql_test_data(juju, DATABASE_APP_NAME, TABLE_NAME, value_after_backup) -def test_restore_on_same_cluster(juju: Juju, cloud_credentials, cloud_configs) -> None: +def test_restore_on_same_cluster(juju: Juju, cloud_configs_ceph) -> None: """Test to restore a backup to the same mysql cluster.""" + cloud_configs, cloud_credentials = cloud_configs_ceph + logger.info("Scaling mysql application to 1 unit") scale_app_units(juju, DATABASE_APP_NAME, 1) @@ -449,8 +457,10 @@ def test_restore_on_same_cluster(juju: Juju, cloud_credentials, cloud_configs) - ), "cluster should migrate to blocked status after restore" -def test_restore_on_new_cluster(juju: Juju, charm, cloud_credentials, cloud_configs) -> None: +def test_restore_on_new_cluster(juju: Juju, charm, cloud_configs_ceph) -> None: """Test to restore a backup on a new mysql cluster.""" + cloud_configs, cloud_credentials = cloud_configs_ceph + logger.info("Deploying a new mysql cluster") new_mysql_application_name = "another-mysql-k8s" diff --git a/kubernetes/tox.ini b/kubernetes/tox.ini index 7609a3df5..df0a6239d 100644 --- a/kubernetes/tox.ini +++ b/kubernetes/tox.ini @@ -78,6 +78,8 @@ pass_env = CEPH_ENDPOINT_URL CEPH_ACCESS_KEY CEPH_SECRET_KEY + CEPH_CA_CERT + CEPH_REGION commands_pre = poetry install --only integration commands =