From b97b40ebe1fd9999f0c066ab04afa4dec822c67d Mon Sep 17 00:00:00 2001 From: Brett Mastbergen Date: Wed, 21 Oct 2026 15:32:55 -0400 Subject: [PATCH] x86/bugs: Make Safe-RET robust against interrupt injection jira VULN-196245 cve CVE-2026-68480 commit-author Borislav Petkov (AMD) commit 7e7f81cf6f5ca3311e526308f55d7c54d3ba71f9 upstream-diff Two adjustments: (1) __HANDLE_INTR_SAFERET() is inserted before this tree's #ifdef __ASSEMBLY__ boundary where upstream adds it before its #ifdef __ASSEMBLER__ boundary - the same shared macro zone, so only the surrounding context differs; (2) upstream's new include in nospec-branch.h defines RIP as 128, which the preprocessor substitutes into the two uppercase ONE(%RIP) operands in arch/x86/crypto/aesni-intel_asm.S (lines 799 and 871, the only uppercase register-name operands in any .S file), so those are lowercased to ONE(%rip); register names are case-insensitive in gas and the assembled objects are byte-identical. An attacker injecting interrupts while the Safe-RET mitigation executes on machines affected by SRSO can neutralize the safe return sequence, potentially leading to data leakage through speculative execution. Fixup register state as if the Safe-RET sequence executed successfully by "emulating" it, in a manner of speaking, and avoid executing a RET instruction after returning from the interrupt. Co-developed-by: David Kaplan Signed-off-by: David Kaplan Signed-off-by: Borislav Petkov (AMD) (cherry picked from commit 7e7f81cf6f5ca3311e526308f55d7c54d3ba71f9) Signed-off-by: Brett Mastbergen --- arch/x86/crypto/aesni-intel_asm.S | 4 +- arch/x86/entry/entry_64.S | 8 +++- arch/x86/include/asm/nospec-branch.h | 57 ++++++++++++++++++++++++++++ arch/x86/kernel/cpu/bugs.c | 39 +++++++++++++++++++ arch/x86/lib/retpoline.S | 20 ++++++++++ 5 files changed, 125 insertions(+), 3 deletions(-) diff --git a/arch/x86/crypto/aesni-intel_asm.S b/arch/x86/crypto/aesni-intel_asm.S index f53f996b27f93..809c206ab7efc 100644 --- a/arch/x86/crypto/aesni-intel_asm.S +++ b/arch/x86/crypto/aesni-intel_asm.S @@ -796,7 +796,7 @@ _partial_block_done_\@: .if (\i == 5) || (\i == 6) || (\i == 7) - MOVADQ ONE(%RIP),\TMP1 + MOVADQ ONE(%rip),\TMP1 MOVADQ 0(%arg1),\TMP2 .irpc index, \i_seq paddd \TMP1, \XMM0 # INCR Y0 @@ -868,7 +868,7 @@ aes_loop_initial_\@: * Precomputations for HashKey parallel with encryption of first 4 blocks. * Haskey_i_k holds XORed values of the low and high parts of the Haskey_i */ - MOVADQ ONE(%RIP),\TMP1 + MOVADQ ONE(%rip),\TMP1 paddd \TMP1, \XMM0 # INCR Y0 MOVADQ \XMM0, \XMM1 pshufb %xmm14, \XMM1 # perform a 16 byte swap diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S index 4295e299e41d6..73b8a8a9f783d 100644 --- a/arch/x86/entry/entry_64.S +++ b/arch/x86/entry/entry_64.S @@ -941,6 +941,8 @@ SYM_CODE_START(paranoid_entry) IBRS_ENTER save_reg=%r15 UNTRAIN_RET_FROM_CALL + HANDLE_INTR_SAFERET 8(%rsp) + RET SYM_CODE_END(paranoid_entry) @@ -1043,6 +1045,11 @@ SYM_CODE_START(error_entry) movl %ecx, %eax /* zero extend */ cmpq %rax, RIP+8(%rsp) je .Lbstep_iret + + VALIDATE_UNRET_END + + HANDLE_INTR_SAFERET 8(%rsp) + cmpq $.Lgs_change, RIP+8(%rsp) jne .Lerror_entry_done_lfence @@ -1061,7 +1068,6 @@ SYM_CODE_START(error_entry) FENCE_SWAPGS_KERNEL_ENTRY CALL_DEPTH_ACCOUNT leaq 8(%rsp), %rax /* return pt_regs pointer */ - VALIDATE_UNRET_END RET .Lbstep_iret: diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h index ea955c20c96e6..8f547cd9ea1ec 100644 --- a/arch/x86/include/asm/nospec-branch.h +++ b/arch/x86/include/asm/nospec-branch.h @@ -12,6 +12,7 @@ #include #include #include +#include #include /* @@ -186,6 +187,50 @@ add $(BITS_PER_LONG/8), %_ASM_SP; \ lfence; +/* + * Helper for detecting if an interrupt occurred at an unsafe location within + * Safe-RET. If Safe-RET is interrupted after the CALL or LEA the RSB may get + * poisoned by the interrupt handler. + * + * The Safe-RET sequence is: + * + * CALL + * LEA 8(%RSP), %RSP + * RET + * + * The two CMPs below check whether RIP points to after the CALL or after the + * LEA. + * + * The LFENCE below is to address this particular speculation case: + * + * 1. Userspace runs and poisons the BTB around the safe-RET routine + * + * 2. Userspace triggers some kind of exception + * + * 3. Kernel executes error_entry() and mis-speculates the branch into thinking + * it actually came from kernel space + * + * 4. The kernel then further mis-speculates that the exception occurred due + * to an interrupted safe-RET + * + * 5. The handle_interrupted_saferet() routine speculatively executes and + * speculatively does a safe-RET. But this is unsafe since it was never + * untrained. + * + * The LFENCE fixes this by ensuring step 5 is never reached speculatively. + * Note that this LFENCE only occurs if safe-RET was actually interrupted (so + * it's outside of the normal path). + */ +#define __HANDLE_INTR_SAFERET(name, pt_regs) \ + cmpq $(name), RIP+pt_regs; \ + jb 1f; \ + cmpq $(name)+5, RIP+pt_regs; \ + ja 1f; \ + lfence; \ + leaq pt_regs, %rdi; \ + call handle_interrupted_saferet; \ + 1: + #ifdef __ASSEMBLY__ /* @@ -316,6 +361,14 @@ #define UNTRAIN_RET_FROM_CALL \ __UNTRAIN_RET X86_FEATURE_ENTRY_IBPB, __stringify(RESET_CALL_DEPTH_FROM_CALL) +.macro HANDLE_INTR_SAFERET pt_regs +#ifdef CONFIG_MITIGATION_SRSO + ALTERNATIVE_2 "", \ + __stringify(__HANDLE_INTR_SAFERET(srso_safe_ret, \pt_regs)), X86_FEATURE_SRSO, \ + __stringify(__HANDLE_INTR_SAFERET(srso_alias_safe_ret, \pt_regs)), X86_FEATURE_SRSO_ALIAS + +#endif +.endm .macro CALL_DEPTH_ACCOUNT #ifdef CONFIG_MITIGATION_CALL_DEPTH_TRACKING @@ -624,6 +677,10 @@ static __always_inline void mds_idle_clear_cpu_buffers(void) mds_clear_cpu_buffers(); } +void srso_safe_ret(void); +void srso_alias_safe_ret(void); +void handle_interrupted_saferet(struct pt_regs *regs); + #endif /* __ASSEMBLY__ */ #endif /* _ASM_X86_NOSPEC_BRANCH_H_ */ diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c index 4af535c5968d2..0f038bc781142 100644 --- a/arch/x86/kernel/cpu/bugs.c +++ b/arch/x86/kernel/cpu/bugs.c @@ -3275,3 +3275,42 @@ void __warn_thunk(void) { WARN_ONCE(1, "Unpatched return thunk in use. This should not happen!\n"); } + +#ifdef CONFIG_MITIGATION_SRSO +/* + * Called during exception/interrupt entry if interrupted during the + * safe-RET sequence. The safe-RET sequence consists of 3 instructions: + * + * CALL + * LEA 8(%RSP), %RSP + * RET + * + * An interrupt after the CALL or after the LEA could potentially lead + * to branch predictor poisoning and results in the sequence not being + * able to be safely resumed. + * + * Therefore, modify the regs state as if the remaining part of the + * safe-RET sequence executed so the interrupt returns back to the + * desired return target, instead of the to the safe-RET sequence. + */ +void noinstr handle_interrupted_saferet(struct pt_regs *regs) +{ + unsigned long rip = regs->ip; + + if (rip == (unsigned long) srso_safe_ret || + rip == (unsigned long) srso_alias_safe_ret) { + /* Modify stack pointer as if LEA executed: */ + regs->sp += 8; + } + + /* + * Adjust registers as if RET executed: + * + * 1. Read the return address off the stack and into rIP: + */ + regs->ip = *(unsigned long *)(regs->sp); + + /* 2. Pop rIP off the stack: */ + regs->sp += 8; +} +#endif /* CONFIG_MITIGATION_SRSO */ diff --git a/arch/x86/lib/retpoline.S b/arch/x86/lib/retpoline.S index 57733a94afbde..875fc3a690db6 100644 --- a/arch/x86/lib/retpoline.S +++ b/arch/x86/lib/retpoline.S @@ -168,10 +168,24 @@ __EXPORT_THUNK(srso_alias_untrain_ret) .pushsection .text..__x86.rethunk_safe SYM_CODE_START_NOALIGN(srso_alias_safe_ret) + + /* + * Tell objtool that those are not function pointers referenced by + * __HANDLE_INTR_SAFERET(). Below too. + */ + ANNOTATE_NOENDBR + + /* + * Safe-RET sequence. If you need to change it, adjust + * handle_interrupted_saferet() too. + */ lea 8(%_ASM_SP), %_ASM_SP UNWIND_HINT_FUNC + + ANNOTATE_NOENDBR ANNOTATE_UNRET_SAFE ret + /* End of Safe-RET sequence */ int3 SYM_FUNC_END(srso_alias_safe_ret) @@ -206,8 +220,14 @@ SYM_CODE_START_LOCAL_NOALIGN(srso_untrain_ret) * the stack. */ SYM_INNER_LABEL(srso_safe_ret, SYM_L_GLOBAL) + /* + * Safe-RET sequence. If you need to change it, adjust + * handle_interrupted_saferet() too. + */ lea 8(%_ASM_SP), %_ASM_SP ret + /* End of Safe-RET sequence */ + int3 int3 /* end of movabs */