1 /* SPDX-License-Identifier: GPL-2.0 */
3 #ifndef __NOSPEC_BRANCH_H__
4 #define __NOSPEC_BRANCH_H__
6 #include <asm/alternative.h>
7 #include <asm/alternative-asm.h>
8 #include <asm/cpufeature.h>
11 * Fill the CPU return stack buffer.
13 * Each entry in the RSB, if used for a speculative 'ret', contains an
14 * infinite 'pause; lfence; jmp' loop to capture speculative execution.
16 * This is required in various cases for retpoline and IBRS-based
17 * mitigations for the Spectre variant 2 vulnerability. Sometimes to
18 * eliminate potentially bogus entries from the RSB, and sometimes
19 * purely to ensure that it doesn't get empty, which on some CPUs would
20 * allow predictions from other (unwanted!) sources to be used.
22 * We define a CPP macro such that it can be used from both .S files and
23 * inline assembly. It's possible to do a .macro and then include that
24 * from C via asm(".include <asm/nospec-branch.h>") but let's not go there.
27 #define RSB_CLEAR_LOOPS 32 /* To forcibly overwrite all entries */
28 #define RSB_FILL_LOOPS 16 /* To avoid underflow */
31 * Google experimented with loop-unrolling and this turned out to be
32 * the optimal version — two calls, each with their own speculation
33 * trap should their return address end up getting used, in a loop.
35 #define __FILL_RETURN_BUFFER(reg, nr, sp) \
39 773: /* speculation trap */ \
45 775: /* speculation trap */ \
52 add $(BITS_PER_LONG/8) * nr, sp;
57 * These are the bare retpoline primitives for indirect jmp and call.
58 * Do not use these directly; they only exist to make the ALTERNATIVE
59 * invocation below less ugly.
61 .macro RETPOLINE_JMP reg:req
73 * This is a wrapper around RETPOLINE_JMP so the called function in reg
74 * returns to the instruction after the macro.
76 .macro RETPOLINE_CALL reg:req
78 .Ldo_retpoline_jmp_\@:
81 call .Ldo_retpoline_jmp_\@
85 * JMP_NOSPEC and CALL_NOSPEC macros can be used instead of a simple
86 * indirect jmp/call which may be susceptible to the Spectre variant 2
89 .macro JMP_NOSPEC reg:req
90 #ifdef CONFIG_RETPOLINE
91 ALTERNATIVE_2 __stringify(jmp *\reg), \
92 __stringify(RETPOLINE_JMP \reg), X86_FEATURE_RETPOLINE, \
93 __stringify(lfence; jmp *\reg), X86_FEATURE_RETPOLINE_AMD
99 .macro CALL_NOSPEC reg:req
100 #ifdef CONFIG_RETPOLINE
101 ALTERNATIVE_2 __stringify(call *\reg), \
102 __stringify(RETPOLINE_CALL \reg), X86_FEATURE_RETPOLINE,\
103 __stringify(lfence; call *\reg), X86_FEATURE_RETPOLINE_AMD
110 * A simpler FILL_RETURN_BUFFER macro. Don't make people use the CPP
111 * monstrosity above, manually.
113 .macro FILL_RETURN_BUFFER reg:req nr:req ftr:req
114 #ifdef CONFIG_RETPOLINE
115 ALTERNATIVE "jmp .Lskip_rsb_\@", \
116 __stringify(__FILL_RETURN_BUFFER(\reg,\nr,%_ASM_SP)) \
122 #else /* __ASSEMBLY__ */
124 #if defined(CONFIG_X86_64) && defined(RETPOLINE)
127 * Since the inline asm uses the %V modifier which is only in newer GCC,
128 * the 64-bit one is dependent on RETPOLINE not CONFIG_RETPOLINE.
130 # define CALL_NOSPEC \
132 "call *%[thunk_target]\n", \
133 "call __x86_indirect_thunk_%V[thunk_target]\n", \
134 X86_FEATURE_RETPOLINE)
135 # define THUNK_TARGET(addr) [thunk_target] "r" (addr)
137 #elif defined(CONFIG_X86_32) && defined(CONFIG_RETPOLINE)
139 * For i386 we use the original ret-equivalent retpoline, because
140 * otherwise we'll run out of registers. We don't care about CET
143 # define CALL_NOSPEC ALTERNATIVE("call *%[thunk_target]\n", \
146 "901: call 903f;\n" \
151 "903: addl $4, %%esp;\n" \
152 " pushl %[thunk_target];\n" \
155 "904: call 901b;\n", \
156 X86_FEATURE_RETPOLINE)
158 # define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
159 #else /* No retpoline for C / inline asm */
160 # define CALL_NOSPEC "call *%[thunk_target]\n"
161 # define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
164 /* The Spectre V2 mitigation variants */
165 enum spectre_v2_mitigation {
167 SPECTRE_V2_RETPOLINE_MINIMAL,
168 SPECTRE_V2_RETPOLINE_MINIMAL_AMD,
169 SPECTRE_V2_RETPOLINE_GENERIC,
170 SPECTRE_V2_RETPOLINE_AMD,
174 extern char __indirect_thunk_start[];
175 extern char __indirect_thunk_end[];
176 extern char __indirect_thunk_size[];
179 * On VMEXIT we must ensure that no RSB predictions learned in the guest
180 * can be followed in the host, by overwriting the RSB completely. Both
181 * retpoline and IBRS mitigations for Spectre v2 need this; only on future
182 * CPUs with IBRS_ATT *might* it be avoided.
184 static inline void vmexit_fill_RSB(void)
186 #ifdef CONFIG_RETPOLINE
189 asm volatile (ALTERNATIVE("jmp 910f",
190 __stringify(__FILL_RETURN_BUFFER(%0, RSB_CLEAR_LOOPS, %1)),
191 X86_FEATURE_RETPOLINE)
193 : "=r" (loops), ASM_CALL_CONSTRAINT
198 #endif /* __ASSEMBLY__ */
199 #endif /* __NOSPEC_BRANCH_H__ */