Optimize NUMA Scheduling

config.h.cmake

@@ -363,6 +363,7 @@
 
 #define SO_EXT "@CMAKE_SHARED_MODULE_SUFFIX@"
 
+#cmakedefine GMOCK_FOUND 1
 
 /* From libmysql/CMakeLists.txt */
 #cmakedefine HAVE_UNIX_DNS_SRV @HAVE_UNIX_DNS_SRV@

share/messages_to_clients.txt

@@ -10800,6 +10800,15 @@ ER_EXTERNAL_TABLE_ENGINE_NOT_SPECIFIED
 ER_LH_NO_FILES_FOUND_NO_SECURE_FILE_PRIV
   eng "No files found or files provided not under secure-file-priv."
 
+ER_INVALID_CPU_STRING
+  eng "Invalid cpu string %s."
+
+ER_CANNOT_UPDATE_SCHED_AFFINITY_PARAMETER
+  eng "Cannot update %s successfully."
+
+ER_CANNOT_UPDATE_SCHED_AFFINITY_NUMA_AWARE
+  eng "Cannot update sched_affinity_numa_aware successfully."
+
 #
 #  End of 9.x error messages (server-to-client).
 #

share/messages_to_error_log.txt

@@ -12847,6 +12847,45 @@ ER_RPL_MTA_ALLOW_COMMIT_OUT_OF_ORDER
 ER_TEMPTABLE_ENGINE_ERROR
   eng "Temptable: %s"
 
+ER_CANT_PARSE_CPU_STRING
+  eng "Cannot parse cpu string '%s'."
+
+ER_LIBNUMA_TEST_FAIL
+  eng "libnuma test fail."
+
+ER_NUMA_AVAILABLE_TEST_FAIL
+  eng "numa_available test fail."
+
+ER_CANNOT_SET_THREAD_SCHED_AFFINIFY
+  eng "Cannot set thread %s sched affinity."
+
+ER_CANNOT_UNSET_THREAD_SCHED_AFFINIFY
+  eng "Cannot unset thread %s sched affinity."
+
+ER_CANNOT_REGISTER_THREAD_TO_SCHED_AFFINIFY_MANAGER
+  eng "Cannot register thread %s sched affinity manager."
+
+ER_CANNOT_UNREGISTER_THREAD_FROM_SCHED_AFFINIFY_MANAGER
+  eng "Cannot unregister thread %s sched affinity manager."
+
+ER_USE_DUMMY_SCHED_AFFINITY_MANAGER
+  eng "use dummy sched_affinity_manager."
+
+ER_SCHED_AFFINITY_THREAD_PROCESS_CONFLICT
+  eng "Found sched affinity conflict between threads and process."
+
+ER_SCHED_AFFINITY_FOREGROUND_BACKGROUND_CONFLICT
+  eng "Found sched affinity conflict between foreground threads and background threads."
+
+ER_CANNOT_CREATE_SCHED_AFFINITY_MANAGER
+  eng "Cannot create sched affinity manager."
+
+ER_SET_FALLBACK_MODE
+  eng "sched_affinity_manager is set to fallback mode."
+
+ER_FALLBACK_DELEGATE_SCHED_AFFINITY_MANAGER_IS_CALLED
+  eng "sched_affinity_manager is in fallback mode. A fallback version of sched_affinity_manager is called, which does nothing."
+
 # DO NOT add server-to-client messages here;
 # they go in messages_to_clients.txt
 # in the same directory as this file.

sql/CMakeLists.txt

@@ -20,6 +20,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301  USA
+INCLUDE_DIRECTORIES(${GMOCK_INCLUDE_DIRS})
 
 ADD_WSHADOW_WARNING()
 
@@ -530,6 +531,7 @@ SET(SQL_SHARED_SOURCES
   rpl_write_set_handler.cc
   rules_table_service.cc
   rwlock_scoped_lock.cc
+  sched_affinity_manager.cc
   sd_notify.cc
   sdi_utils.cc
   session_tracker.cc

sql/conn_handler/connection_handler_per_thread.cc

@@ -25,6 +25,7 @@
 
 #include <stddef.h>
 #include <sys/types.h>
+#include <unistd.h>
 #include <list>
 #include <new>
 
@@ -56,6 +57,7 @@
 #include "sql/mysqld.h"                                   // max_connections
 #include "sql/mysqld_thd_manager.h"                       // Global_THD_manager
 #include "sql/protocol_classic.h"
+#include "sql/sched_affinity_manager.h"
 #include "sql/sql_class.h"    // THD
 #include "sql/sql_connect.h"  // close_connection
 #include "sql/sql_error.h"
@@ -297,6 +299,18 @@ static void *handle_connection(void *arg) {
     mysql_socket_set_thread_owner(socket);
     thd_manager->add_thd(thd);
 
+    auto sched_affinity_manager =
+        sched_affinity::Sched_affinity_manager::get_instance();
+    bool is_registered_to_sched_affinity = false;
+    auto pid = sched_affinity::gettid();
+    if (sched_affinity_manager == nullptr ||
+        !(is_registered_to_sched_affinity =
+              sched_affinity_manager->register_thread(
+                  sched_affinity::Thread_type::FOREGROUND, pid))) {
+      LogErr(ERROR_LEVEL, ER_CANNOT_REGISTER_THREAD_TO_SCHED_AFFINIFY_MANAGER,
+             "foreground");
+    }
+
     if (thd_prepare_connection(thd))
       handler_manager->inc_aborted_connects();
     else {
@@ -307,6 +321,13 @@ static void *handle_connection(void *arg) {
     }
     close_connection(thd, 0, false, false);
 
+    if (is_registered_to_sched_affinity &&
+        !sched_affinity_manager->unregister_thread(pid)) {
+      LogErr(ERROR_LEVEL,
+             ER_CANNOT_UNREGISTER_THREAD_FROM_SCHED_AFFINIFY_MANAGER,
+             "foreground");
+    }
+
     thd->get_stmt_da()->reset_diagnostics_area();
     thd->release_resources();
 

sql/memory/aligned_atomic.h

@@ -75,7 +75,9 @@ static inline size_t _cache_line_size() {
   }
 
   free(buffer);
-  return line_size;
+  long size = sysconf(_SC_LEVEL1_DCACHE_LINESIZE);
+  if (size == -1 || size == 0) return 64;
+  return static_cast<size_t>(size);
 }
 
 #elif defined(__GLIBC__)

sql/mysqld.cc

@@ -857,6 +857,7 @@ MySQL clients support the protocol:
 #include "sql/rpl_rli.h"      // Relay_log_info
 #include "sql/rpl_source.h"   // max_binlog_dump_events
 #include "sql/rpl_trx_tracking.h"
+#include "sql/sched_affinity_manager.h"
 #include "sql/sd_notify.h"  // sd_notify_connect
 #include "sql/session_tracker.h"
 #include "sql/set_var.h"
@@ -1244,6 +1245,9 @@ bool opt_persist_sensitive_variables_in_plaintext{true};
 int argc_cached;
 char **argv_cached;
 
+extern std::map<sched_affinity::Thread_type, const char*> sched_affinity_parameter;
+extern bool sched_affinity_numa_aware;
+
 #if defined(_WIN32)
 /*
   Thread handle of shutdown event handler thread.
@@ -2897,6 +2901,7 @@ static void clean_up(bool print_message) {
   */
   sys_var_end();
   free_status_vars();
+  sched_affinity::Sched_affinity_manager::free_instance();
 
   finish_client_errs();
   deinit_errmessage();  // finish server errs
@@ -9829,6 +9834,11 @@ int mysqld_main(int argc, char **argv)
   /* Determine default TCP port and unix socket name */
   set_ports();
 
+  if (sched_affinity::Sched_affinity_manager::create_instance(sched_affinity_parameter, sched_affinity_numa_aware) == nullptr) {
+    LogErr(ERROR_LEVEL, ER_CANNOT_CREATE_SCHED_AFFINITY_MANAGER);
+    unireg_abort(MYSQLD_ABORT_EXIT);
+  }
+
   if (init_server_components()) unireg_abort(MYSQLD_ABORT_EXIT);
 
   if (!server_id_supplied)
@@ -11243,6 +11253,31 @@ static int show_queries(THD *thd, SHOW_VAR *var, char *) {
   return 0;
 }
 
+static int show_sched_affinity_status(THD *, SHOW_VAR *var, char *buff) {
+  var->type = SHOW_CHAR;
+  var->value = buff;
+  std::string group_snapshot = sched_affinity::Sched_affinity_manager::get_instance()->take_group_snapshot();
+  strncpy(buff, group_snapshot.c_str(), SHOW_VAR_FUNC_BUFF_SIZE);
+  buff[SHOW_VAR_FUNC_BUFF_SIZE]='\0';
+  return 0;
+}
+
+static int show_sched_affinity_group_number(THD *, SHOW_VAR *var, char *buff) {
+  var->type = SHOW_SIGNED_INT;
+  var->value = buff;
+  *(reinterpret_cast<int32 *>(buff)) = sched_affinity::Sched_affinity_manager::get_instance()
+                             ->get_total_node_number();                    
+  return 0;
+}
+
+static int show_sched_affinity_group_capacity(THD *, SHOW_VAR *var, char *buff) {
+  var->type = SHOW_SIGNED_INT;
+  var->value = buff;
+  *(reinterpret_cast<int32 *>(buff)) = sched_affinity::Sched_affinity_manager::get_instance()
+                             ->get_cpu_number_per_node();
+  return 0;
+}
+
 static int show_net_compression(THD *thd, SHOW_VAR *var, char *buff) {
   var->type = SHOW_MY_BOOL;
   var->value = buff;
@@ -11871,6 +11906,12 @@ SHOW_VAR status_vars[] = {
     {"Queries", (char *)&show_queries, SHOW_FUNC, SHOW_SCOPE_ALL},
     {"Questions", (char *)offsetof(System_status_var, questions),
      SHOW_LONGLONG_STATUS, SHOW_SCOPE_ALL},
+    {"Sched_affinity_status", 
+    (char *)&show_sched_affinity_status, SHOW_FUNC, SHOW_SCOPE_ALL},
+    {"Sched_affinity_group_number", 
+    (char *)&show_sched_affinity_group_number, SHOW_FUNC, SHOW_SCOPE_ALL},
+    {"Sched_affinity_group_capacity", 
+    (char *)&show_sched_affinity_group_capacity, SHOW_FUNC, SHOW_SCOPE_ALL},
     {"Secondary_engine_execution_count",
      (char *)offsetof(System_status_var, secondary_engine_execution_count),
      SHOW_LONGLONG_STATUS, SHOW_SCOPE_ALL},
@@ -13998,6 +14039,7 @@ PSI_mutex_key key_mutex_replica_worker_hash;
 PSI_mutex_key key_monitor_info_run_lock;
 PSI_mutex_key key_LOCK_delegate_connection_mutex;
 PSI_mutex_key key_LOCK_group_replication_connection_mutex;
+PSI_mutex_key key_sched_affinity_mutex;
 
 /* clang-format off */
 static PSI_mutex_info all_server_mutexes[]=
@@ -14094,6 +14136,7 @@ static PSI_mutex_info all_server_mutexes[]=
   { &key_LOCK_group_replication_connection_mutex, "LOCK_group_replication_connection_mutex", PSI_FLAG_SINGLETON, 0, PSI_DOCUMENT_ME},
   { &key_LOCK_authentication_policy, "LOCK_authentication_policy", PSI_FLAG_SINGLETON, 0, "A lock to ensure execution of CREATE USER or ALTER USER sql and SET @@global.authentication_policy variable are serialized"},
   { &key_LOCK_global_conn_mem_limit, "LOCK_global_conn_mem_limit", PSI_FLAG_SINGLETON, 0, PSI_DOCUMENT_ME},
+  { &key_sched_affinity_mutex, "Sched_affinity::m_mutex", 0, 0, PSI_DOCUMENT_ME}
 };
 /* clang-format on */
 

sql/mysqld.h

@@ -463,6 +463,8 @@ extern PSI_mutex_key key_LOCK_group_replication_connection_mutex;
 extern PSI_mutex_key key_commit_order_manager_mutex;
 extern PSI_mutex_key key_mutex_replica_worker_hash;
 
+extern PSI_mutex_key key_sched_affinity_mutex;
+
 extern PSI_rwlock_key key_rwlock_LOCK_logger;
 extern PSI_rwlock_key key_rwlock_channel_map_lock;
 extern PSI_rwlock_key key_rwlock_channel_lock;