create cluster set (#242)

lib/charms/mysql/v0/mysql.py

@@ -91,7 +91,7 @@ def wait_until_mysql_connection(self) -> None:
 
 # Increment this PATCH version before using `charmcraft publish-lib` or reset
 # to 0 if you are raising the major API version
-LIBPATCH = 34
+LIBPATCH = 35
 
 UNIT_TEARDOWN_LOCKNAME = "unit-teardown"
 UNIT_ADD_LOCKNAME = "unit-add"
@@ -159,6 +159,10 @@ class MySQLCreateClusterError(Error):
     """Exception raised when there is an issue creating an InnoDB cluster."""
 
 
+class MySQLCreateClusterSetError(Error):
+    """Exception raised when there is an issue creating an Cluster Set."""
+
+
 class MySQLAddInstanceToClusterError(Error):
     """Exception raised when there is an issue add an instance to the MySQL InnoDB cluster."""
 
@@ -232,7 +236,7 @@ class MySQLOfflineModeAndHiddenInstanceExistsError(Error):
     """
 
 
-class MySQLGetInnoDBBufferPoolParametersError(Error):
+class MySQLGetAutoTunningParametersError(Error):
     """Exception raised when there is an error computing the innodb buffer pool parameters."""
 
 
@@ -320,6 +324,7 @@ def __init__(
         self,
         instance_address: str,
         cluster_name: str,
+        cluster_set_name: str,
         root_password: str,
         server_config_user: str,
         server_config_password: str,
@@ -335,6 +340,7 @@ def __init__(
         Args:
             instance_address: address of the targeted instance
             cluster_name: cluster name
+            cluster_set_name: cluster set domain name
             root_password: password for the 'root' user
             server_config_user: user name for the server config user
             server_config_password: password for the server config user
@@ -347,6 +353,7 @@ def __init__(
         """
         self.instance_address = instance_address
         self.cluster_name = cluster_name
+        self.cluster_set_name = cluster_set_name
         self.root_password = root_password
         self.server_config_user = server_config_user
         self.server_config_password = server_config_password
@@ -731,6 +738,24 @@ def create_cluster(self, unit_label: str) -> None:
             )
             raise MySQLCreateClusterError(e.message)
 
+    def create_cluster_set(self) -> None:
+        """Create a cluster set for the cluster on cluster primary.
+
+        Raises MySQLCreateClusterSetError on cluster set creation failure.
+        """
+        commands = (
+            f"shell.connect_to_primary('{self.server_config_user}:{self.server_config_password}@{self.instance_address}')",
+            f"cluster = dba.get_cluster('{self.cluster_name}')",
+            f"cluster.create_cluster_set('{self.cluster_set_name}')",
+        )
+
+        try:
+            logger.debug(f"Creating cluster set name {self.cluster_set_name}")
+            self._run_mysqlsh_script("\n".join(commands))
+        except MySQLClientError:
+            logger.exception("Failed to add instance to cluster set on instance")
+            raise MySQLCreateClusterSetError
+
     def initialize_juju_units_operations_table(self) -> None:
         """Initialize the mysql.juju_units_operations table using the serverconfig user.
 
@@ -1018,6 +1043,7 @@ def get_cluster_endpoints(self, get_ips: bool = True) -> Tuple[str, str, str]:
 
         def _get_host_ip(host: str) -> str:
             try:
+                port = None
                 if ":" in host:
                     host, port = host.split(":")
 
@@ -1513,9 +1539,29 @@ def get_innodb_buffer_pool_parameters(self) -> Tuple[int, Optional[int]]:
                 innodb_buffer_pool_chunk_size = chunk_size
 
             return (pool_size, innodb_buffer_pool_chunk_size)
-        except Exception as e:
-            logger.exception("Failed to compute innodb buffer pool parameters", exc_info=e)
-            raise MySQLGetInnoDBBufferPoolParametersError("Error retrieving total free memory")
+        except Exception:
+            logger.exception("Failed to compute innodb buffer pool parameters")
+            raise MySQLGetAutoTunningParametersError("Error computing buffer pool parameters")
+
+    def get_max_connections(self) -> int:
+        """Calculate max_connections parameter for the instance."""
+        # Reference: based off xtradb-cluster-operator
+        # https://github.com/percona/percona-xtradb-cluster-operator/blob/main/pkg/pxc/app/config/autotune.go#L61-L70
+
+        bytes_per_connection = 12582912  # 12 Megabytes
+        total_memory = 0
+
+        try:
+            total_memory = self._get_total_memory()
+        except Exception:
+            logger.exception("Failed to retrieve total memory")
+            raise MySQLGetAutoTunningParametersError("Error retrieving total memory")
+
+        if total_memory < bytes_per_connection:
+            logger.error(f"Not enough memory for running MySQL: {total_memory=}")
+            raise MySQLGetAutoTunningParametersError("Not enough memory for running MySQL")
+
+        return total_memory // bytes_per_connection
 
     def _get_total_memory(self) -> int:
         """Retrieves the total memory of the server where mysql is running."""
@@ -1653,7 +1699,7 @@ def retrieve_backup_with_xbcloud(
         self,
         backup_id: str,
         s3_parameters: Dict[str, str],
-        mysql_data_directory: str,
+        temp_restore_directory: str,
         xbcloud_location: str,
         xbstream_location: str,
         user=None,
@@ -1662,11 +1708,12 @@ def retrieve_backup_with_xbcloud(
         """Retrieve the specified backup from S3.
 
         The backup is retrieved using xbcloud and stored in a temp dir in the
-        mysql container.
+        mysql container. This temp dir is supposed to be on the same volume as
+        the mysql data directory to reduce latency for IOPS.
         """
         nproc_command = "nproc".split()
         make_temp_dir_command = (
-            f"mktemp --directory {mysql_data_directory}/#mysql_sst_XXXX".split()
+            f"mktemp --directory {temp_restore_directory}/#mysql_sst_XXXX".split()
         )
 
         try:
@@ -1732,7 +1779,7 @@ def prepare_backup_for_restore(
         """Prepare the backup in the provided dir for restore."""
         try:
             innodb_buffer_pool_size, _ = self.get_innodb_buffer_pool_parameters()
-        except MySQLGetInnoDBBufferPoolParametersError as e:
+        except MySQLGetAutoTunningParametersError as e:
             raise MySQLPrepareBackupForRestoreError(e)
 
         prepare_backup_command = f"""
@@ -1823,13 +1870,13 @@ def restore_backup(
 
     def delete_temp_restore_directory(
         self,
-        mysql_data_directory: str,
+        temp_restore_directory: str,
         user=None,
         group=None,
     ) -> None:
         """Delete the temp restore directory from the mysql data directory."""
-        logger.info(f"Deleting temp restore directory in {mysql_data_directory}")
-        delete_temp_restore_directory_command = f"find {mysql_data_directory} -wholename {mysql_data_directory}/#mysql_sst_* -delete".split()
+        logger.info(f"Deleting temp restore directory in {temp_restore_directory}")
+        delete_temp_restore_directory_command = f"find {temp_restore_directory} -wholename {temp_restore_directory}/#mysql_sst_* -delete".split()
 
         try:
             logger.debug(

src/charm.py

@@ -152,6 +152,7 @@ def _mysql(self):
         return MySQL(
             self.get_unit_hostname(self.unit.name),
             self.app_peer_data["cluster-name"],
+            self.app_peer_data["cluster-set-domain-name"],
             self.get_secret("app", ROOT_PASSWORD_KEY),
             SERVER_CONFIG_USERNAME,
             self.get_secret("app", SERVER_CONFIG_PASSWORD_KEY),
@@ -470,11 +471,12 @@ def _on_config_changed(self, _) -> None:
         if not self.unit.is_leader():
             return
 
-        # Set the cluster name in the peer relation databag if it is not already set
-        if not self.app_peer_data.get("cluster-name"):
-            self.app_peer_data["cluster-name"] = (
-                self.config.get("cluster-name") or f"cluster_{generate_random_hash()}"
-            )
+        # Create and set cluster and cluster-set names in the peer relation databag
+        common_hash = generate_random_hash()
+        self.app_peer_data.setdefault(
+            "cluster-name", self.config.get("cluster-name", f"cluster-{common_hash}")
+        )
+        self.app_peer_data.setdefault("cluster-set-domain-name", f"cluster-set-{common_hash}")
 
     def _on_leader_elected(self, _: LeaderElectedEvent) -> None:
         """Handle the leader elected event.
@@ -596,6 +598,7 @@ def _on_mysql_pebble_ready(self, event) -> None:
             logger.info("Creating cluster on the leader unit")
             unit_label = self.unit.name.replace("/", "-")
             self._mysql.create_cluster(unit_label)
+            self._mysql.create_cluster_set()
 
             self._mysql.initialize_juju_units_operations_table()
             # Start control flag
@@ -608,13 +611,14 @@ def _on_mysql_pebble_ready(self, event) -> None:
             )
 
             self.unit.status = ActiveStatus(self.active_status_message)
-        except MySQLCreateClusterError as e:
-            self.unit.status = BlockedStatus("Unable to create cluster")
-            logger.debug("Unable to create cluster: {}".format(e))
-        except MySQLGetMemberStateError:
-            self.unit.status = BlockedStatus("Unable to query member state and role")
-        except MySQLInitializeJujuOperationsTableError:
-            self.unit.status = BlockedStatus("Failed to initialize juju operations table")
+        except (
+            MySQLCreateClusterError,
+            MySQLGetMemberStateError,
+            MySQLInitializeJujuOperationsTableError,
+            MySQLCreateClusterError,
+        ):
+            logger.exception("Failed to initialize primary")
+            raise
 
     def _handle_potential_cluster_crash_scenario(self) -> bool:
         """Handle potential full cluster crash scenarios.

src/mysql_k8s_helpers.py

@@ -134,6 +134,7 @@ def __init__(
         self,
         instance_address: str,
         cluster_name: str,
+        cluster_set_name: str,
         root_password: str,
         server_config_user: str,
         server_config_password: str,
@@ -151,6 +152,7 @@ def __init__(
         Args:
             instance_address: address of the targeted instance
             cluster_name: cluster name
+            cluster_set_name: cluster set name
             root_password: password for the 'root' user
             server_config_user: user name for the server config user
             server_config_password: password for the server config user
@@ -166,6 +168,7 @@ def __init__(
         super().__init__(
             instance_address=instance_address,
             cluster_name=cluster_name,
+            cluster_set_name=cluster_set_name,
             root_password=root_password,
             server_config_user=server_config_user,
             server_config_password=server_config_password,

tests/unit/test_charm.py

@@ -80,6 +80,7 @@ def test_on_leader_elected(self):
                 peer_data[password].isalnum() and len(peer_data[password]) == PASSWORD_LENGTH
             )
 
+    @patch("mysql_k8s_helpers.MySQL.create_cluster_set")
     @patch("mysql_k8s_helpers.MySQL.initialize_juju_units_operations_table")
     @patch("mysql_k8s_helpers.MySQL.safe_stop_mysqld_safe")
     @patch("mysql_k8s_helpers.MySQL.get_mysql_version", return_value="8.0.0")
@@ -110,6 +111,7 @@ def test_mysql_pebble_ready(
         _get_mysql_version,
         _safe_stop_mysqld_safe,
         _initialize_juju_units_operations_table,
+        _create_cluster_set,
     ):
         # Check if initial plan is empty
         self.harness.set_can_connect("mysql", True)

tests/unit/test_mysql_k8s_helpers.py

@@ -48,6 +48,7 @@ def setUp(self):
         self.mysql = MySQL(
             "127.0.0.1",
             "test_cluster",
+            "test_cluster_set",
             "password",
             "serverconfig",
             "serverconfigpassword",