[DPE-7908] Separation of storage (take 2)

kubernetes/lib/charms/mysql/v0/mysql.py

@@ -78,6 +78,9 @@ def __init__(
 import ops
 from charms.data_platform_libs.v0.data_interfaces import DataPeerData, DataPeerUnitData
 from constants import (
+    MYSQL_DATA_DIR,
+    MYSQL_LOGS_DIR,
+    MYSQL_TEMP_DIR,
     BACKUPS_PASSWORD_KEY,
     BACKUPS_USERNAME,
     CHARMED_MYSQL_PITR_HELPER,
@@ -1105,7 +1108,6 @@ def render_mysqld_configuration(  # noqa: C901
         memory_limit: int | None = None,
         experimental_max_connections: int | None = None,
         binlog_retention_days: int,
-        snap_common: str = "",
     ) -> tuple[str, dict]:
         """Render mysqld ini configuration file."""
         max_connections = None
@@ -1154,26 +1156,31 @@ def render_mysqld_configuration(  # noqa: C901
                 # disable memory instruments if we have less than 2GiB of RAM
                 performance_schema_instrument = "'memory/%=OFF'"
 
-        logging_path = f"{snap_common}/var/log/mysql"
         binlog_retention_seconds = binlog_retention_days * 24 * 60 * 60
         config = configparser.ConfigParser(interpolation=None)
 
         # do not enable slow query logs, but specify a log file path in case
         # the admin enables them manually
-        config["mysqld"] = {
+        base_config = {
+            "datadir": MYSQL_DATA_DIR,
             # All interfaces bind expected
             "bind_address": "0.0.0.0",  # noqa: S104
             "mysqlx_bind_address": "0.0.0.0",  # noqa: S104
             "admin_address": self.instance_address,
             "report_host": self.instance_address,
             "max_connections": max_connections,
             "innodb_buffer_pool_size": innodb_buffer_pool_size,
+            "innodb_log_group_home_dir": MYSQL_LOGS_DIR,
+            "innodb_temp_tablespaces_dir": MYSQL_TEMP_DIR,
+            "innodb_undo_directory": MYSQL_LOGS_DIR,
+            "log_bin": f"{MYSQL_LOGS_DIR}/binlog",
+            "log_bin_index": f"{MYSQL_LOGS_DIR}/binlog.index",
             "log_error_services": "log_filter_internal;log_sink_internal",
-            "log_error": f"{logging_path}/error.log",
+            "log_error": f"{MYSQL_LOGS_DIR}/error.log",
             "general_log": "OFF",
-            "general_log_file": f"{logging_path}/general.log",
+            "general_log_file": f"{MYSQL_LOGS_DIR}/general.log",
             "loose-group_replication_paxos_single_leader": "ON",
-            "slow_query_log_file": f"{logging_path}/slow.log",
+            "slow_query_log_file": f"{MYSQL_LOGS_DIR}/slow.log",
             "binlog_expire_logs_seconds": f"{binlog_retention_seconds}",
             "gtid_mode": "ON",
             "enforce_gtid_consistency": "ON",
@@ -1187,9 +1194,10 @@ def render_mysqld_configuration(  # noqa: C901
             "loose-validate_password.policy": "MEDIUM",
             "loose-validate_password.special_char_count": 0,
         }
+        config["mysqld"] = base_config  # ty:ignore[invalid-assignment]
 
         if audit_log_enabled:
-            config["mysqld"]["loose-audit_log_filter.file"] = f"{logging_path}/audit.log"
+            config["mysqld"]["loose-audit_log_filter.file"] = f"{MYSQL_LOGS_DIR}/audit.log"
             config["mysqld"]["loose-audit_log_filter.format"] = "JSON"
             config["mysqld"]["loose-audit_log_filter.policy"] = audit_log_policy.upper()
         if audit_log_strategy == "async":
@@ -2731,8 +2739,9 @@ def empty_data_files(
         mysql_data_directory: str,
         user: str | None = None,
         group: str | None = None,
+        extra_dirs: list[str] | None = None,
     ) -> None:
-        """Empty the mysql data directory in preparation of backup restore."""
+        """Empty the mysql data directories in preparation of backup restore."""
         empty_data_files_command = [
             "find",
             mysql_data_directory,
@@ -2752,11 +2761,26 @@ def empty_data_files(
                 user=user,
                 group=group,
             )
+
+            for extra_dir in (extra_dirs or []):
+                logger.debug(f"Emptying extra directory {extra_dir}")
+                self._execute_commands(
+                    [
+                        "find",
+                        extra_dir,
+                        "-not",
+                        "-path",
+                        extra_dir,
+                        "-delete",
+                    ],
+                    user=user,
+                    group=group,
+                )
         except MySQLExecError as e:
-            logger.error("Failed to empty data directory in prep for backup restore")
+            logger.error("Failed to empty data directories in prep for backup restore")
             raise MySQLEmptyDataDirectoryError(e.message) from e
         except Exception as e:
-            logger.error("Failed to empty data directory in prep for backup restore")
+            logger.error("Failed to empty data directories in prep for backup restore")
             raise MySQLEmptyDataDirectoryError from e
 
     def restore_backup(

kubernetes/metadata.yaml

@@ -27,8 +27,14 @@ containers:
     gid: 584788
     resource: mysql-image
     mounts:
-      - storage: database
-        location: /var/lib/mysql
+      - storage: archive
+        location: /var/lib/mysql/archive
+      - storage: data
+        location: /var/lib/mysql/data
+      - storage: logs
+        location: /var/lib/mysql/logs
+      - storage: temp
+        location: /var/lib/mysql/temp
 
 resources:
   mysql-image:
@@ -81,9 +87,18 @@ requires:
     optional: true
 
 storage:
-  database:
+  archive:
+    type: filesystem
+    description: Persistent storage for rotated logs and other archival purposes
+  data:
     type: filesystem
     description: Persistent storage for MySQL data
+  temp:
+    type: filesystem
+    description: Persistent storage for InnoDB temporary tablespaces
+  logs:
+    type: filesystem
+    description: Persistent storage for MySQL error logs, general query logs, slow query logs, binary logs, redo logs and undo logs
 
 assumes:
   - k8s-api

kubernetes/src/charm.py

@@ -75,6 +75,7 @@
     MONITORING_PASSWORD_KEY,
     MONITORING_USERNAME,
     MYSQL_BINLOGS_COLLECTOR_SERVICE,
+    MYSQL_DATA_DIR,
     MYSQL_LOG_ERROR,
     MYSQL_LOG_FILES,
     MYSQL_LOG_SERVICE,
@@ -128,9 +129,11 @@ def __init__(self, *args):
         self.framework.observe(self.on.leader_elected, self._on_leader_elected)
         self.framework.observe(self.on.config_changed, self._on_config_changed)
         self.framework.observe(self.on.update_status, self._on_update_status)
-        self.framework.observe(
-            self.on.database_storage_detaching, self._on_database_storage_detaching
-        )
+
+        self.framework.observe(self.on.archive_storage_detaching, self._on_storage_detaching)
+        self.framework.observe(self.on.data_storage_detaching, self._on_storage_detaching)
+        self.framework.observe(self.on.logs_storage_detaching, self._on_storage_detaching)
+        self.framework.observe(self.on.temp_storage_detaching, self._on_storage_detaching)
 
         self.framework.observe(self.on[PEER].relation_joined, self._on_peer_relation_joined)
         self.framework.observe(self.on[PEER].relation_changed, self._on_peer_relation_changed)
@@ -227,7 +230,7 @@ def _pebble_layer(self) -> Layer:
         mysqld_cmd = [
             MYSQLD_LOCATION,
             "--basedir=/usr",
-            "--datadir=/var/lib/mysql",
+            f"--datadir={MYSQL_DATA_DIR}",
             "--plugin-dir=/usr/lib/mysql/plugin",
             f"--log-error={MYSQL_LOG_ERROR}",
             f"--pid-file={self.unit_label}.pid",
@@ -1064,7 +1067,7 @@ def _on_peer_relation_departed(self, event: RelationDepartedEvent) -> None:
         if not self._mysql.reconcile_binlogs_collection(force_restart=True):
             logger.error("Failed to reconcile binlogs collection during peer departed event")
 
-    def _on_database_storage_detaching(self, _) -> None:
+    def _on_storage_detaching(self, _) -> None:
         """Handle the database storage detaching event."""
         # Only executes if the unit was initialised
         if not self.unit_initialized():

kubernetes/src/constants.py

@@ -27,16 +27,18 @@
 TLS_SSL_CERT_FILE = "custom-server-cert.pem"
 MYSQL_CLI_LOCATION = "/usr/bin/mysql"
 MYSQLSH_LOCATION = "/usr/bin/mysqlsh"
-MYSQL_DATA_DIR = "/var/lib/mysql"
+MYSQL_ARCHIVE_DIR = "/var/lib/mysql/archive"  # Corresponds to the archive storage mount
+MYSQL_DATA_DIR = "/var/lib/mysql/data"  # Corresponds to the data storage mount
+MYSQL_LOGS_DIR = "/var/lib/mysql/logs"  # Corresponds to the logs storage mount
+MYSQL_TEMP_DIR = "/var/lib/mysql/temp"  # Corresponds to the temp storage mount
 MYSQLD_SOCK_FILE = "/var/run/mysqld/mysqld.sock"
 MYSQLD_CONFIG_FILE = "/etc/mysql/mysql.conf.d/z-custom.cnf"
 MYSQLD_INIT_CONFIG_FILE = "/etc/mysql/mysql.conf.d/z-custom-init-file.cnf"
-MYSQL_LOG_DIR = "/var/log/mysql"
-MYSQL_LOG_ERROR = f"{MYSQL_LOG_DIR}/error.log"
+MYSQL_LOG_ERROR = f"{MYSQL_LOGS_DIR}/error.log"
 MYSQL_LOG_FILES = [
     MYSQL_LOG_ERROR,
-    f"{MYSQL_LOG_DIR}/audit.log",
-    f"{MYSQL_LOG_DIR}/general.log",
+    f"{MYSQL_LOGS_DIR}/audit.log",
+    f"{MYSQL_LOGS_DIR}/general.log",
 ]
 MYSQL_SYSTEM_USER = "mysql"
 MYSQL_SYSTEM_GROUP = "mysql"

kubernetes/src/mysql_k8s_helpers.py

@@ -35,12 +35,14 @@
     CHARMED_MYSQL_XTRABACKUP_LOCATION,
     CONTAINER_NAME,
     LOG_ROTATE_CONFIG_FILE,
+    MYSQL_ARCHIVE_DIR,
     MYSQL_BINLOGS_COLLECTOR_SERVICE,
     MYSQL_DATA_DIR,
-    MYSQL_LOG_DIR,
     MYSQL_LOG_ERROR,
+    MYSQL_LOGS_DIR,
     MYSQL_SYSTEM_GROUP,
     MYSQL_SYSTEM_USER,
+    MYSQL_TEMP_DIR,
     MYSQLD_DEFAULTS_CONFIG_FILE,
     MYSQLD_INIT_CONFIG_FILE,
     MYSQLD_LOCATION,
@@ -184,6 +186,14 @@ def initialise_mysqld(self) -> None:
             "--initialize",
             "-u",
             MYSQL_SYSTEM_USER,
+            "--datadir",
+            MYSQL_DATA_DIR,
+            "--innodb-log-group-home-dir",
+            MYSQL_LOGS_DIR,
+            "--innodb-undo-directory",
+            MYSQL_LOGS_DIR,
+            "--innodb-temp-tablespaces-dir",
+            MYSQL_TEMP_DIR,
         ]
 
         try:
@@ -277,7 +287,8 @@ def setup_logrotate_config(
         rendered = template.render(
             system_user=MYSQL_SYSTEM_USER,
             system_group=MYSQL_SYSTEM_GROUP,
-            log_dir=MYSQL_LOG_DIR,
+            log_dir=MYSQL_LOGS_DIR,
+            archive_dir=MYSQL_ARCHIVE_DIR,
             logs_retention_period=logs_retention_period,
             logs_rotations=logs_rotations,
             logs_compression_enabled=logs_compression,
@@ -300,7 +311,7 @@ def execute_backup_commands(
         xbcloud_location: str = CHARMED_MYSQL_XBCLOUD_LOCATION,
         xtrabackup_plugin_dir: str = XTRABACKUP_PLUGIN_DIR,
         mysqld_socket_file: str = MYSQLD_SOCK_FILE,
-        tmp_base_directory: str = MYSQL_DATA_DIR,
+        tmp_base_directory: str = MYSQL_TEMP_DIR,
         defaults_config_file: str = MYSQLD_DEFAULTS_CONFIG_FILE,
         user: str | None = MYSQL_SYSTEM_USER,
         group: str | None = MYSQL_SYSTEM_GROUP,
@@ -321,7 +332,7 @@ def execute_backup_commands(
 
     def delete_temp_backup_directory(
         self,
-        tmp_base_directory: str = MYSQL_DATA_DIR,
+        tmp_base_directory: str = MYSQL_TEMP_DIR,
         user=MYSQL_SYSTEM_USER,
         group=MYSQL_SYSTEM_GROUP,
     ) -> None:
@@ -336,7 +347,7 @@ def retrieve_backup_with_xbcloud(
         self,
         backup_id: str,
         s3_parameters: dict[str, str],
-        temp_restore_directory: str = MYSQL_DATA_DIR,
+        temp_restore_directory: str = MYSQL_TEMP_DIR,
         xbcloud_location: str = CHARMED_MYSQL_XBCLOUD_LOCATION,
         xbstream_location: str = CHARMED_MYSQL_XBSTREAM_LOCATION,
         user: str | None = MYSQL_SYSTEM_USER,
@@ -379,12 +390,17 @@ def empty_data_files(
         mysql_data_directory=MYSQL_DATA_DIR,
         user=MYSQL_SYSTEM_USER,
         group=MYSQL_SYSTEM_GROUP,
+        extra_dirs: list[str] | None = None,
     ) -> None:
         """Empty the mysql data directory in preparation of backup restore."""
+        if extra_dirs is None:
+            extra_dirs = [MYSQL_LOGS_DIR]
+
         super().empty_data_files(
             mysql_data_directory,
             user,
             group,
+            extra_dirs,
         )
 
     def restore_backup(
@@ -410,7 +426,7 @@ def restore_backup(
 
     def delete_temp_restore_directory(
         self,
-        temp_restore_directory: str = MYSQL_DATA_DIR,
+        temp_restore_directory: str = MYSQL_TEMP_DIR,
         user=MYSQL_SYSTEM_USER,
         group=MYSQL_SYSTEM_GROUP,
     ) -> None:
@@ -608,8 +624,6 @@ def is_data_dir_initialised(self) -> bool:
 
             # minimal expected content for an integral mysqld data-dir
             expected_content = {
-                "#innodb_redo",
-                "#innodb_temp",
                 "auto.cnf",
                 "ca-key.pem",
                 "ca.pem",
@@ -624,9 +638,8 @@ def is_data_dir_initialised(self) -> bool:
                 "server-cert.pem",
                 "server-key.pem",
                 "sys",
-                "undo_001",
-                "undo_002",
             }
+            logger.debug("mysql data dir contents: %s", content_set)
 
             return expected_content <= content_set
         except (ExecError, APIError):

kubernetes/templates/logrotate.j2

@@ -22,10 +22,13 @@ ifempty
 missingok
 nomail
 nosharedscripts
-nocopytruncate
+# Needed because rotated logs might be in a different filesystem,
+# `nocopytruncate` fails with `failed to rename ...: Invalid cross-device link`
+copy
+copytruncate
 
 {% for log in enabled_log_files %}
 {{ log_dir }}/{{ log }}.log {
-    olddir archive_{{ log }}
+    olddir {{ archive_dir }}/archive_{{ log }}
 }
 {% endfor %}

kubernetes/tests/integration/integration/high_availability/test_replication_logs_rotation.py

@@ -16,7 +16,7 @@
     wait_fixed,
 )
 
-from constants import CONTAINER_NAME, MYSQL_LOG_DIR
+from constants import CONTAINER_NAME, MYSQL_ARCHIVE_DIR, MYSQL_LOGS_DIR
 
 from ... import architecture
 from ...helpers_ha import (
@@ -76,7 +76,7 @@ def test_deploy_highly_available_cluster(juju: Juju, charm: str) -> None:
 
 def test_log_rotation(juju: Juju) -> None:
     """Test the log rotation of text files."""
-    log_types = ["error", "audit"]
+    log_types = ["audit", "error"]
 
     mysql_app_leader = get_app_leader(juju, MYSQL_APP_NAME)
     mysql_app_leader_label = get_mysql_instance_label(mysql_app_leader)
@@ -94,20 +94,22 @@ def test_log_rotation(juju: Juju) -> None:
     stop_log_rotate_dispatcher(juju, mysql_app_leader)
 
     for log_type in log_types:
+        archive_log_dir = f"{MYSQL_ARCHIVE_DIR}/archive_{log_type}"
+
         logging.info("Removing existing archive directories")
         delete_unit_file(
             juju=juju,
             unit_name=mysql_app_leader,
             container=CONTAINER_NAME,
-            file_path=f"{MYSQL_LOG_DIR}/archive_{log_type}",
+            file_path=archive_log_dir,
         )
 
         logging.info("Writing some data to the text log files")
         write_unit_file(
             juju=juju,
             unit_name=mysql_app_leader,
             container=CONTAINER_NAME,
-            file_path=f"{MYSQL_LOG_DIR}/{log_type}.log",
+            file_path=f"{MYSQL_LOGS_DIR}/{log_type}.log",
             file_data=f"{log_type} content",
         )
 
@@ -120,11 +122,11 @@ def test_log_rotation(juju: Juju) -> None:
             juju=juju,
             unit_name=mysql_app_leader,
             container=CONTAINER_NAME,
-            file_path=f"{MYSQL_LOG_DIR}/{log_type}.log",
+            file_path=f"{MYSQL_LOGS_DIR}/{log_type}.log",
         )
         assert f"{log_type} content" not in active_log_file_data
 
-        archive_log_dir = f"{MYSQL_LOG_DIR}/archive_{log_type}"
+        archive_log_dir = f"{MYSQL_ARCHIVE_DIR}/archive_{log_type}"
         archive_log_files_listed = list_unit_files(
             juju=juju,
             unit_name=mysql_app_leader,