diff --git a/include/linux/loop_unrolling.h b/include/linux/loop_unrolling.h new file mode 100644 index 00000000000000..dca96140f2230f --- /dev/null +++ b/include/linux/loop_unrolling.h @@ -0,0 +1,116 @@ +// SPDX-License-Identifier: GPL-2.0 + +/* + * Copyright (C) 2020 Google LLC. + */ + +#ifndef __LINUX_LOOP_UNROLLING_H +#define __LINUX_LOOP_UNROLLING_H + +/* + * Call MACRO N times. + * N must be an integer constant no greater than MAX_UNROLL_MACRO_LOOP + * MACRO should take as first argument the index and then + * the same __VA_ARGS__ + * Essenially, this will expand to: + * MACRO(0, ...) + * MACRO(1, ...) + * MACRO(2, ...) + * ... + */ +#define UNROLL_MACRO_LOOP(N, MACRO, ...) \ + _UNROLL_MACRO_LOOP(N, MACRO, __VA_ARGS__) + +#define MAX_UNROLL_MACRO_LOOP (20) + +// Intermediate macros to expand N if it is itself a macro +#define _UNROLL_MACRO_LOOP(N, MACRO, ...) \ + __UNROLL_MACRO_LOOP(N, MACRO, __VA_ARGS__) + +#define __UNROLL_MACRO_LOOP(N, MACRO, ...) \ + __UNROLL_MACRO_LOOP_##N(MACRO, __VA_ARGS__) + + +#define __UNROLL_MACRO_LOOP_0(MACRO, ...) + +#define __UNROLL_MACRO_LOOP_1(MACRO, ...) \ + __UNROLL_MACRO_LOOP_0(MACRO, __VA_ARGS__) \ + MACRO(0, __VA_ARGS__) + +#define __UNROLL_MACRO_LOOP_2(MACRO, ...) \ + __UNROLL_MACRO_LOOP_1(MACRO, __VA_ARGS__) \ + MACRO(1, __VA_ARGS__) + +#define __UNROLL_MACRO_LOOP_3(MACRO, ...) \ + __UNROLL_MACRO_LOOP_2(MACRO, __VA_ARGS__) \ + MACRO(2, __VA_ARGS__) + +#define __UNROLL_MACRO_LOOP_4(MACRO, ...) \ + __UNROLL_MACRO_LOOP_3(MACRO, __VA_ARGS__) \ + MACRO(3, __VA_ARGS__) + +#define __UNROLL_MACRO_LOOP_5(MACRO, ...) \ + __UNROLL_MACRO_LOOP_4(MACRO, __VA_ARGS__) \ + MACRO(4, __VA_ARGS__) + +#define __UNROLL_MACRO_LOOP_6(MACRO, ...) \ + __UNROLL_MACRO_LOOP_5(MACRO, __VA_ARGS__) \ + MACRO(5, __VA_ARGS__) + +#define __UNROLL_MACRO_LOOP_7(MACRO, ...) \ + __UNROLL_MACRO_LOOP_6(MACRO, __VA_ARGS__) \ + MACRO(6, __VA_ARGS__) + +#define __UNROLL_MACRO_LOOP_8(MACRO, ...) \ + __UNROLL_MACRO_LOOP_7(MACRO, __VA_ARGS__) \ + MACRO(7, __VA_ARGS__) + +#define __UNROLL_MACRO_LOOP_9(MACRO, ...) \ + __UNROLL_MACRO_LOOP_8(MACRO, __VA_ARGS__) \ + MACRO(8, __VA_ARGS__) + +#define __UNROLL_MACRO_LOOP_10(MACRO, ...) \ + __UNROLL_MACRO_LOOP_9(MACRO, __VA_ARGS__) \ + MACRO(9, __VA_ARGS__) + +#define __UNROLL_MACRO_LOOP_11(MACRO, ...) \ + __UNROLL_MACRO_LOOP_10(MACRO, __VA_ARGS__) \ + MACRO(10, __VA_ARGS__) + +#define __UNROLL_MACRO_LOOP_12(MACRO, ...) \ + __UNROLL_MACRO_LOOP_11(MACRO, __VA_ARGS__) \ + MACRO(11, __VA_ARGS__) + +#define __UNROLL_MACRO_LOOP_13(MACRO, ...) \ + __UNROLL_MACRO_LOOP_12(MACRO, __VA_ARGS__) \ + MACRO(12, __VA_ARGS__) + +#define __UNROLL_MACRO_LOOP_14(MACRO, ...) \ + __UNROLL_MACRO_LOOP_13(MACRO, __VA_ARGS__) \ + MACRO(13, __VA_ARGS__) + +#define __UNROLL_MACRO_LOOP_15(MACRO, ...) \ + __UNROLL_MACRO_LOOP_14(MACRO, __VA_ARGS__) \ + MACRO(14, __VA_ARGS__) + +#define __UNROLL_MACRO_LOOP_16(MACRO, ...) \ + __UNROLL_MACRO_LOOP_15(MACRO, __VA_ARGS__) \ + MACRO(15, __VA_ARGS__) + +#define __UNROLL_MACRO_LOOP_17(MACRO, ...) \ + __UNROLL_MACRO_LOOP_16(MACRO, __VA_ARGS__) \ + MACRO(16, __VA_ARGS__) + +#define __UNROLL_MACRO_LOOP_18(MACRO, ...) \ + __UNROLL_MACRO_LOOP_17(MACRO, __VA_ARGS__) \ + MACRO(17, __VA_ARGS__) + +#define __UNROLL_MACRO_LOOP_19(MACRO, ...) \ + __UNROLL_MACRO_LOOP_18(MACRO, __VA_ARGS__) \ + MACRO(18, __VA_ARGS__) + +#define __UNROLL_MACRO_LOOP_20(MACRO, ...) \ + __UNROLL_MACRO_LOOP_19(MACRO, __VA_ARGS__) \ + MACRO(19, __VA_ARGS__) + +#endif /* __LINUX_LOOP_UNROLLING_H */ \ No newline at end of file diff --git a/include/linux/lsm_hooks.h b/include/linux/lsm_hooks.h index 95b7c1d3206218..d11e116b588e7c 100644 --- a/include/linux/lsm_hooks.h +++ b/include/linux/lsm_hooks.h @@ -1524,6 +1524,7 @@ union security_list_options { #define LSM_HOOK(RET, DEFAULT, NAME, ...) RET (*NAME)(__VA_ARGS__); #include "lsm_hook_defs.h" #undef LSM_HOOK + void *generic_func; }; struct security_hook_heads { diff --git a/security/security.c b/security/security.c index 70a7ad357bc6ab..a53e57b26e8788 100644 --- a/security/security.c +++ b/security/security.c @@ -28,12 +28,26 @@ #include #include #include +#include +#include +#include #define MAX_LSM_EVM_XATTR 2 /* How many LSMs were built into the kernel? */ #define LSM_COUNT (__end_lsm_info - __start_lsm_info) +#define STATIC_SLOT(HOOK, NUM) security_static_slot_##HOOK##_##NUM + +/* + * Static slots are placeholders for potential LSM hooks. + * Instead of a costly indirect call, they use static calls. + */ +#define SECURITY_STATIC_SLOT_COUNT 11 +static_assert(SECURITY_STATIC_SLOT_COUNT <= MAX_UNROLL_MACRO_LOOP); +#define SECURITY_FOREACH_STATIC_SLOT(M, ...) \ + UNROLL_MACRO_LOOP(SECURITY_STATIC_SLOT_COUNT, M, __VA_ARGS__) + /* * These are descriptions of the reasons that can be passed to the * security_locked_down() LSM hook. Placing this array here allows @@ -86,6 +100,67 @@ static __initconst const char * const builtin_lsm_order = CONFIG_LSM; static __initdata struct lsm_info **ordered_lsms; static __initdata struct lsm_info *exclusive; +/* + * Necessary information about a static + * slot to call __static_call_update + */ +struct security_static_slot { + /* static call key as defined by STATIC_CALL_KEY */ + struct static_call_key *key; + /* static call tramp as defined by STATIC_CALL_TRAMP */ + void *tramp; +}; + +/* Table of the static calls for each LSM hook */ +struct security_list_static_slots { + #define LSM_HOOK(RET, DEFAULT, NAME, ...) \ + struct security_static_slot NAME[SECURITY_STATIC_SLOT_COUNT]; + #include + #undef LSM_HOOK +} __randomize_layout; + +/* + * Index of the first used static call for each LSM hook + * in the corresponding security_list_static_slots table. + * All calls with greater indices are used, INT_MAX if no call is used. + */ +struct security_first_static_slot_idx { + #define LSM_HOOK(RET, DEFAULT, NAME, ...) \ + int NAME; + #include + #undef LSM_HOOK +} __randomize_layout; + +/* Create the static slots for each LSM hook */ +#define CREATE_STATIC_SLOT(NUM, NAME, RET, ...) \ + DEFINE_STATIC_CALL_NULL(STATIC_SLOT(NAME, NUM), \ + *((RET(*)(__VA_ARGS__))NULL)); + +#define LSM_HOOK(RET, DEFAULT, NAME, ...) \ + SECURITY_FOREACH_STATIC_SLOT(CREATE_STATIC_SLOT, NAME, RET, __VA_ARGS__) +#include +#undef LSM_HOOK +#undef CREATE_STATIC_SLOT + +static struct security_list_static_slots security_list_static_slots +__initdata = { +#define DEFINE_SLOT(NUM, NAME) \ + (struct security_static_slot) { \ + .key = &STATIC_CALL_KEY(STATIC_SLOT(NAME, NUM)), \ + .tramp = &STATIC_CALL_TRAMP(STATIC_SLOT(NAME, NUM)) \ + }, +#define LSM_HOOK(RET, DEFAULT, NAME, ...) \ + .NAME = { \ + SECURITY_FOREACH_STATIC_SLOT(DEFINE_SLOT, NAME) \ + }, +#include +#undef LSM_HOOK +#undef DEFINE_SLOT +}; + +static struct security_first_static_slot_idx security_first_static_slot_idx + __lsm_ro_after_init = {}; + static __initdata bool debug; #define init_debug(...) \ do { \ @@ -307,6 +382,50 @@ static void __init ordered_lsm_parse(const char *order, const char *origin) kfree(sep); } +static void __init lsm_init_hook_static_slot(struct security_static_slot *slots, + struct hlist_head *head, + int *first_slot_idx) +{ + struct security_hook_list *pos; + struct security_static_slot *slot; + int slot_cnt, first_slot; + + slot_cnt = 0; + hlist_for_each_entry_rcu (pos, head, list) { + slot_cnt++; + } + + first_slot = SECURITY_STATIC_SLOT_COUNT - slot_cnt; + if (first_slot < 0) + panic("%s - No static hook slot remaining to add LSM hook.\n", + __func__); + + if (slot_cnt == 0) { + *first_slot_idx = INT_MAX; + return; + } + + slot = slots + first_slot; + hlist_for_each_entry_rcu (pos, head, list) { + __static_call_update(slot->key, slot->tramp, + pos->hook.generic_func); + slot++; + } + + // write only after the static calls are filled + *first_slot_idx = first_slot; +} + +static void __init lsm_init_static_slots(void) +{ +#define LSM_HOOK(RET, DEFAULT, NAME, ...) \ + lsm_init_hook_static_slot(security_list_static_slots.NAME, \ + &security_hook_heads.NAME, \ + &security_first_static_slot_idx.NAME); +#include +#undef LSM_HOOK +} + static void __init lsm_early_cred(struct cred *cred); static void __init lsm_early_task(struct task_struct *task); @@ -354,6 +473,7 @@ static void __init ordered_lsm_init(void) lsm_early_task(current); for (lsm = ordered_lsms; *lsm; lsm++) initialize_lsm(*lsm); + lsm_init_static_slots(); kfree(ordered_lsms); } @@ -374,6 +494,7 @@ int __init early_security_init(void) prepare_lsm(lsm); initialize_lsm(lsm); } + lsm_init_static_slots(); return 0; } @@ -696,27 +817,33 @@ static void __init lsm_early_task(struct task_struct *task) * call_int_hook: * This is a hook that returns a value. */ - -#define call_void_hook(FUNC, ...) \ - do { \ - struct security_hook_list *P; \ - \ - hlist_for_each_entry(P, &security_hook_heads.FUNC, list) \ - P->hook.FUNC(__VA_ARGS__); \ - } while (0) - -#define call_int_hook(FUNC, IRC, ...) ({ \ - int RC = IRC; \ - do { \ - struct security_hook_list *P; \ - \ - hlist_for_each_entry(P, &security_hook_heads.FUNC, list) { \ - RC = P->hook.FUNC(__VA_ARGS__); \ - if (RC != 0) \ - break; \ - } \ - } while (0); \ - RC; \ +#define __CASE_CALL_STATIC_VOID(NUM, HOOK, ...) \ + case NUM: \ + static_call_cond(STATIC_SLOT(HOOK, NUM))(__VA_ARGS__); \ + fallthrough; + +#define call_void_hook(FUNC, ...) do { \ + switch (security_first_static_slot_idx.FUNC) { \ + SECURITY_FOREACH_STATIC_SLOT(__CASE_CALL_STATIC_VOID, \ + FUNC, __VA_ARGS__) \ + default: break; \ + } \ +} while (0) + +#define __CASE_CALL_STATIC_INT(NUM, R, HOOK, ...) \ + case NUM: \ + R = static_call(STATIC_SLOT(HOOK, NUM))(__VA_ARGS__); \ + if (R != 0) break; \ + fallthrough; + +#define call_int_hook(FUNC, IRC, ...) ({ \ + int RC = IRC; \ + switch (security_first_static_slot_idx.FUNC) { \ + SECURITY_FOREACH_STATIC_SLOT(__CASE_CALL_STATIC_INT, \ + RC, FUNC, __VA_ARGS__) \ + default: break; \ + } \ + RC; \ }) /* Security operations */