PoC faster LSMs with static calls/keys: ask for feedbacks

include/linux/loop_unrolling.h

@@ -0,0 +1,116 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/*
+ * Copyright (C) 2020 Google LLC.
+ */
+
+#ifndef __LINUX_LOOP_UNROLLING_H
+#define __LINUX_LOOP_UNROLLING_H
+
+/*
+ * Call MACRO N times. 
+ * N must be an integer constant no greater than MAX_UNROLL_MACRO_LOOP
+ * MACRO should take as first argument the index and then
+ * the same __VA_ARGS__
+ * Essenially, this will expand to:
+ *	MACRO(0, ...)
+ *	MACRO(1, ...)
+ *	MACRO(2, ...)
+ *	...
+ */
+#define UNROLL_MACRO_LOOP(N, MACRO, ...)		\
+	_UNROLL_MACRO_LOOP(N, MACRO, __VA_ARGS__)
+
+#define MAX_UNROLL_MACRO_LOOP (20)
+
+// Intermediate macros to expand N if it is itself a macro
+#define _UNROLL_MACRO_LOOP(N, MACRO, ...)		\
+	__UNROLL_MACRO_LOOP(N, MACRO, __VA_ARGS__)
+
+#define __UNROLL_MACRO_LOOP(N, MACRO, ...)		\
+	__UNROLL_MACRO_LOOP_##N(MACRO, __VA_ARGS__)
+
+
+#define __UNROLL_MACRO_LOOP_0(MACRO, ...)
+
+#define __UNROLL_MACRO_LOOP_1(MACRO, ...)		\
+	__UNROLL_MACRO_LOOP_0(MACRO, __VA_ARGS__)	\
+	MACRO(0, __VA_ARGS__)
+
+#define __UNROLL_MACRO_LOOP_2(MACRO, ...)		\
+	__UNROLL_MACRO_LOOP_1(MACRO, __VA_ARGS__)	\
+	MACRO(1, __VA_ARGS__)
+
+#define __UNROLL_MACRO_LOOP_3(MACRO, ...)		\
+	__UNROLL_MACRO_LOOP_2(MACRO, __VA_ARGS__)	\
+	MACRO(2, __VA_ARGS__)
+
+#define __UNROLL_MACRO_LOOP_4(MACRO, ...)		\
+	__UNROLL_MACRO_LOOP_3(MACRO, __VA_ARGS__)	\
+	MACRO(3, __VA_ARGS__)
+
+#define __UNROLL_MACRO_LOOP_5(MACRO, ...)		\
+	__UNROLL_MACRO_LOOP_4(MACRO, __VA_ARGS__)	\
+	MACRO(4, __VA_ARGS__)
+
+#define __UNROLL_MACRO_LOOP_6(MACRO, ...)		\
+	__UNROLL_MACRO_LOOP_5(MACRO, __VA_ARGS__)	\
+	MACRO(5, __VA_ARGS__)
+
+#define __UNROLL_MACRO_LOOP_7(MACRO, ...)		\
+	__UNROLL_MACRO_LOOP_6(MACRO, __VA_ARGS__)	\
+	MACRO(6, __VA_ARGS__)
+
+#define __UNROLL_MACRO_LOOP_8(MACRO, ...)		\
+	__UNROLL_MACRO_LOOP_7(MACRO, __VA_ARGS__)	\
+	MACRO(7, __VA_ARGS__)
+
+#define __UNROLL_MACRO_LOOP_9(MACRO, ...)		\
+	__UNROLL_MACRO_LOOP_8(MACRO, __VA_ARGS__)	\
+	MACRO(8, __VA_ARGS__)
+
+#define __UNROLL_MACRO_LOOP_10(MACRO, ...)		\
+	__UNROLL_MACRO_LOOP_9(MACRO, __VA_ARGS__)	\
+	MACRO(9, __VA_ARGS__)
+
+#define __UNROLL_MACRO_LOOP_11(MACRO, ...)		\
+	__UNROLL_MACRO_LOOP_10(MACRO, __VA_ARGS__)	\
+	MACRO(10, __VA_ARGS__)
+
+#define __UNROLL_MACRO_LOOP_12(MACRO, ...)		\
+	__UNROLL_MACRO_LOOP_11(MACRO, __VA_ARGS__)	\
+	MACRO(11, __VA_ARGS__)
+
+#define __UNROLL_MACRO_LOOP_13(MACRO, ...)		\
+	__UNROLL_MACRO_LOOP_12(MACRO, __VA_ARGS__)	\
+	MACRO(12, __VA_ARGS__)
+
+#define __UNROLL_MACRO_LOOP_14(MACRO, ...)		\
+	__UNROLL_MACRO_LOOP_13(MACRO, __VA_ARGS__)	\
+	MACRO(13, __VA_ARGS__)
+
+#define __UNROLL_MACRO_LOOP_15(MACRO, ...)		\
+	__UNROLL_MACRO_LOOP_14(MACRO, __VA_ARGS__)	\
+	MACRO(14, __VA_ARGS__)
+
+#define __UNROLL_MACRO_LOOP_16(MACRO, ...)		\
+	__UNROLL_MACRO_LOOP_15(MACRO, __VA_ARGS__)	\
+	MACRO(15, __VA_ARGS__)
+
+#define __UNROLL_MACRO_LOOP_17(MACRO, ...)		\
+	__UNROLL_MACRO_LOOP_16(MACRO, __VA_ARGS__)	\
+	MACRO(16, __VA_ARGS__)
+
+#define __UNROLL_MACRO_LOOP_18(MACRO, ...)		\
+	__UNROLL_MACRO_LOOP_17(MACRO, __VA_ARGS__)	\
+	MACRO(17, __VA_ARGS__)
+
+#define __UNROLL_MACRO_LOOP_19(MACRO, ...)		\
+	__UNROLL_MACRO_LOOP_18(MACRO, __VA_ARGS__)	\
+	MACRO(18, __VA_ARGS__)
+
+#define __UNROLL_MACRO_LOOP_20(MACRO, ...)		\
+	__UNROLL_MACRO_LOOP_19(MACRO, __VA_ARGS__)	\
+	MACRO(19, __VA_ARGS__)
+
+#endif /* __LINUX_LOOP_UNROLLING_H */

include/linux/lsm_hooks.h

@@ -1524,6 +1524,7 @@ union security_list_options {
 	#define LSM_HOOK(RET, DEFAULT, NAME, ...) RET (*NAME)(__VA_ARGS__);
 	#include "lsm_hook_defs.h"
 	#undef LSM_HOOK
+	void *generic_func;
 };
 
 struct security_hook_heads {

security/security.c

@@ -28,12 +28,26 @@
 #include <linux/string.h>
 #include <linux/msg.h>
 #include <net/flow.h>
+#include <linux/static_call.h>
+#include <linux/loop_unrolling.h>
+#include <linux/build_bug.h>
 
 #define MAX_LSM_EVM_XATTR	2
 
 /* How many LSMs were built into the kernel? */
 #define LSM_COUNT (__end_lsm_info - __start_lsm_info)
 
+#define STATIC_SLOT(HOOK, NUM)	security_static_slot_##HOOK##_##NUM
+
+/*
+ * Static slots are placeholders for potential LSM hooks.
+ * Instead of a costly indirect call, they use static calls.
+ */
+#define SECURITY_STATIC_SLOT_COUNT 11
+static_assert(SECURITY_STATIC_SLOT_COUNT <= MAX_UNROLL_MACRO_LOOP);
+#define SECURITY_FOREACH_STATIC_SLOT(M, ...)				\
+	UNROLL_MACRO_LOOP(SECURITY_STATIC_SLOT_COUNT, M, __VA_ARGS__)
+
 /*
  * These are descriptions of the reasons that can be passed to the
  * security_locked_down() LSM hook. Placing this array here allows
@@ -86,6 +100,67 @@ static __initconst const char * const builtin_lsm_order = CONFIG_LSM;
 static __initdata struct lsm_info **ordered_lsms;
 static __initdata struct lsm_info *exclusive;
 
+/*
+ * Necessary information about a static
+ * slot to call __static_call_update
+ */
+struct security_static_slot {
+	/* static call key as defined by STATIC_CALL_KEY */
+	struct static_call_key *key;
+	/* static call tramp as defined by STATIC_CALL_TRAMP */
+	void *tramp;
+};
+
+/* Table of the static calls for each LSM hook */
+struct security_list_static_slots {
+	#define LSM_HOOK(RET, DEFAULT, NAME, ...) \
+		struct security_static_slot NAME[SECURITY_STATIC_SLOT_COUNT];
+	#include <linux/lsm_hook_defs.h>
+	#undef LSM_HOOK
+} __randomize_layout;
+
+/*
+ * Index of the first used static call for each LSM hook
+ * in the corresponding security_list_static_slots table.
+ * All calls with greater indices are used, INT_MAX if no call is used.
+ */
+struct security_first_static_slot_idx {
+	#define LSM_HOOK(RET, DEFAULT, NAME, ...) \
+		int NAME;
+	#include <linux/lsm_hook_defs.h>
+	#undef LSM_HOOK
+} __randomize_layout;
+
+/* Create the static slots for each LSM hook */
+#define CREATE_STATIC_SLOT(NUM, NAME, RET, ...)				\
+	DEFINE_STATIC_CALL_NULL(STATIC_SLOT(NAME, NUM),			\
+				*((RET(*)(__VA_ARGS__))NULL));
+
+#define LSM_HOOK(RET, DEFAULT, NAME, ...) 				\
+	SECURITY_FOREACH_STATIC_SLOT(CREATE_STATIC_SLOT, NAME, RET, __VA_ARGS__)
+#include <linux/lsm_hook_defs.h>
+#undef LSM_HOOK
+#undef CREATE_STATIC_SLOT
+
+static struct security_list_static_slots security_list_static_slots
+__initdata = {
+#define DEFINE_SLOT(NUM, NAME)						\
+	(struct security_static_slot) {					\
+		.key = &STATIC_CALL_KEY(STATIC_SLOT(NAME, NUM)),	\
+		.tramp = &STATIC_CALL_TRAMP(STATIC_SLOT(NAME, NUM))	\
+	},
+#define LSM_HOOK(RET, DEFAULT, NAME, ...) 				\
+	.NAME = {							\
+		SECURITY_FOREACH_STATIC_SLOT(DEFINE_SLOT, NAME)		\
+	},
+#include <linux/lsm_hook_defs.h>
+#undef LSM_HOOK
+#undef DEFINE_SLOT
+};
+
+static struct security_first_static_slot_idx security_first_static_slot_idx
+	__lsm_ro_after_init = {};
+
 static __initdata bool debug;
 #define init_debug(...)						\
 	do {							\
@@ -307,6 +382,50 @@ static void __init ordered_lsm_parse(const char *order, const char *origin)
 	kfree(sep);
 }
 
+static void __init lsm_init_hook_static_slot(struct security_static_slot *slots,
+					     struct hlist_head *head,
+					     int *first_slot_idx)
+{
+	struct security_hook_list *pos;
+	struct security_static_slot *slot;
+	int slot_cnt, first_slot;
+
+	slot_cnt = 0;
+	hlist_for_each_entry_rcu (pos, head, list) {
+		slot_cnt++;
+	}
+
+	first_slot = SECURITY_STATIC_SLOT_COUNT - slot_cnt;
+	if (first_slot < 0)
+		panic("%s - No static hook slot remaining to add LSM hook.\n",
+		      __func__);
+
+	if (slot_cnt == 0) {
+		*first_slot_idx = INT_MAX;
+		return;
+	}
+
+	slot = slots + first_slot;
+	hlist_for_each_entry_rcu (pos, head, list) {
+		__static_call_update(slot->key, slot->tramp,
+				     pos->hook.generic_func);
+		slot++;
+	}
+
+	// write only after the static calls are filled
+	*first_slot_idx = first_slot;
+}
+
+static void __init lsm_init_static_slots(void)
+{
+#define LSM_HOOK(RET, DEFAULT, NAME, ...)				\
+	lsm_init_hook_static_slot(security_list_static_slots.NAME,	\
+				  &security_hook_heads.NAME,		\
+				  &security_first_static_slot_idx.NAME);
+#include <linux/lsm_hook_defs.h>
+#undef LSM_HOOK
+}
+
 static void __init lsm_early_cred(struct cred *cred);
 static void __init lsm_early_task(struct task_struct *task);
 
@@ -354,6 +473,7 @@ static void __init ordered_lsm_init(void)
 	lsm_early_task(current);
 	for (lsm = ordered_lsms; *lsm; lsm++)
 		initialize_lsm(*lsm);
+	lsm_init_static_slots();
 
 	kfree(ordered_lsms);
 }
@@ -374,6 +494,7 @@ int __init early_security_init(void)
 		prepare_lsm(lsm);
 		initialize_lsm(lsm);
 	}
+	lsm_init_static_slots();
 
 	return 0;
 }
@@ -696,27 +817,33 @@ static void __init lsm_early_task(struct task_struct *task)
  * call_int_hook:
  *	This is a hook that returns a value.
  */
-
-#define call_void_hook(FUNC, ...)				\
-	do {							\
-		struct security_hook_list *P;			\
-								\
-		hlist_for_each_entry(P, &security_hook_heads.FUNC, list) \
-			P->hook.FUNC(__VA_ARGS__);		\
-	} while (0)
-
-#define call_int_hook(FUNC, IRC, ...) ({			\
-	int RC = IRC;						\
-	do {							\
-		struct security_hook_list *P;			\
-								\
-		hlist_for_each_entry(P, &security_hook_heads.FUNC, list) { \
-			RC = P->hook.FUNC(__VA_ARGS__);		\
-			if (RC != 0)				\
-				break;				\
-		}						\
-	} while (0);						\
-	RC;							\
+#define __CASE_CALL_STATIC_VOID(NUM, HOOK, ...)				\
+	case NUM:							\
+		static_call_cond(STATIC_SLOT(HOOK, NUM))(__VA_ARGS__);	\
+		fallthrough;
+
+#define call_void_hook(FUNC, ...) do {					\
+	switch (security_first_static_slot_idx.FUNC) {			\
+		SECURITY_FOREACH_STATIC_SLOT(__CASE_CALL_STATIC_VOID,	\
+		FUNC, __VA_ARGS__)					\
+		default: break;						\
+	}								\
+} while (0)
+
+#define __CASE_CALL_STATIC_INT(NUM, R, HOOK, ...)			\
+	case NUM:							\
+		R = static_call(STATIC_SLOT(HOOK, NUM))(__VA_ARGS__);	\
+		if (R != 0) break;					\
+		fallthrough;
+
+#define call_int_hook(FUNC, IRC, ...) ({				\
+	int RC = IRC;							\
+	switch (security_first_static_slot_idx.FUNC) {			\
+		SECURITY_FOREACH_STATIC_SLOT(__CASE_CALL_STATIC_INT,	\
+					     RC, FUNC, __VA_ARGS__)	\
+		default: break;						\
+	}								\
+	RC;								\
 })
 
 /* Security operations */