From 7c508612f7a5096b4819d4ef2ce566e01bd66c0c Mon Sep 17 00:00:00 2001 From: Andrew Cooper Date: Mon, 18 Dec 2017 13:54:25 +0000 Subject: [PATCH] x86: Support indirect thunks from assembly code Introduce INDIRECT_CALL and INDIRECT_JMP which either degrade to a normal indirect branch, or dispatch to the __x86_indirect_thunk_* symbols. Update all the manual indirect branches in to use the new thunks. The indirect branches in the early boot and kexec path are left intact as we can't use the compiled-in thunks at those points. This is part of XSA-254. Signed-off-by: Andrew Cooper Reviewed-by: Jan Beulich --- xen/Rules.mk | 4 +-- xen/arch/x86/Rules.mk | 6 ++++ xen/arch/x86/boot/trampoline.S | 24 ++++++++++++-- xen/arch/x86/extable.c | 4 +-- xen/arch/x86/pv/emul-priv-op.c | 36 ++++++++++++++------- xen/arch/x86/x86_64/entry.S | 6 ++-- xen/arch/x86/x86_emulate/x86_emulate.c | 4 +-- xen/common/wait.c | 8 +++-- xen/include/asm-x86/asm_defns.h | 8 +++++ xen/include/asm-x86/indirect_thunk_asm.h | 41 ++++++++++++++++++++++++ 10 files changed, 116 insertions(+), 25 deletions(-) create mode 100644 xen/include/asm-x86/indirect_thunk_asm.h diff --git a/xen/Rules.mk b/xen/Rules.mk index 2659f8a4d1..3cf40754a6 100644 --- a/xen/Rules.mk +++ b/xen/Rules.mk @@ -66,8 +66,8 @@ endif AFLAGS-y += -D__ASSEMBLY__ -# Clang's built-in assembler can't handle .code16/.code32/.code64 yet -AFLAGS-$(clang) += -no-integrated-as +# Clang's built-in assembler can't handle embedded .include's +CFLAGS-$(clang) += -no-integrated-as ALL_OBJS := $(ALL_OBJS-y) diff --git a/xen/arch/x86/Rules.mk b/xen/arch/x86/Rules.mk index 54662077d7..56b2ea8356 100644 --- a/xen/arch/x86/Rules.mk +++ b/xen/arch/x86/Rules.mk @@ -41,3 +41,9 @@ CFLAGS += -mindirect-branch=thunk-extern -mindirect-branch-register CFLAGS += -DCONFIG_INDIRECT_THUNK export CONFIG_INDIRECT_THUNK=y endif + +# Set up the assembler include path properly for older GCC toolchains. Clang +# objects to the agument being passed however. +ifneq ($(clang),y) +CFLAGS += -Wa,-I$(BASEDIR)/include +endif diff --git a/xen/arch/x86/boot/trampoline.S b/xen/arch/x86/boot/trampoline.S index 4d640f3fcd..f70d913544 100644 --- a/xen/arch/x86/boot/trampoline.S +++ b/xen/arch/x86/boot/trampoline.S @@ -153,8 +153,28 @@ trampoline_protmode_entry: .code64 start64: /* Jump to high mappings. */ - movabs $__high_start,%rax - jmpq *%rax + movabs $__high_start, %rdi + +#ifdef CONFIG_INDIRECT_THUNK + /* + * If booting virtualised, or hot-onlining a CPU, sibling threads can + * attempt Branch Target Injection against this jmp. + * + * We've got no usable stack so can't use a RETPOLINE thunk, and are + * further than disp32 from the high mappings so couldn't use + * JUMP_THUNK even if it was a non-RETPOLINE thunk. Furthermore, an + * LFENCE isn't necessarily safe to use at this point. + * + * As this isn't a hotpath, use a fully serialising event to reduce + * the speculation window as much as possible. %ebx needs preserving + * for __high_start. + */ + mov %ebx, %esi + cpuid + mov %esi, %ebx +#endif + + jmpq *%rdi #include "wakeup.S" diff --git a/xen/arch/x86/extable.c b/xen/arch/x86/extable.c index 6fffe057c6..72f30d9060 100644 --- a/xen/arch/x86/extable.c +++ b/xen/arch/x86/extable.c @@ -158,7 +158,7 @@ static int __init stub_selftest(void) memcpy(ptr, tests[i].opc, ARRAY_SIZE(tests[i].opc)); unmap_domain_page(ptr); - asm volatile ( "call *%[stb]\n" + asm volatile ( "INDIRECT_CALL %[stb]\n" ".Lret%=:\n\t" ".pushsection .fixup,\"ax\"\n" ".Lfix%=:\n\t" @@ -167,7 +167,7 @@ static int __init stub_selftest(void) ".popsection\n\t" _ASM_EXTABLE(.Lret%=, .Lfix%=) : [exn] "+m" (res) - : [stb] "rm" (addr), "a" (tests[i].rax)); + : [stb] "r" (addr), "a" (tests[i].rax)); ASSERT(res == tests[i].res.raw); } diff --git a/xen/arch/x86/pv/emul-priv-op.c b/xen/arch/x86/pv/emul-priv-op.c index ebd6dc19b0..21ecfc8e76 100644 --- a/xen/arch/x86/pv/emul-priv-op.c +++ b/xen/arch/x86/pv/emul-priv-op.c @@ -73,46 +73,58 @@ void (*pv_post_outb_hook)(unsigned int port, u8 value); typedef void io_emul_stub_t(struct cpu_user_regs *); +void __x86_indirect_thunk_rcx(void); + static io_emul_stub_t *io_emul_stub_setup(struct priv_op_ctxt *ctxt, u8 opcode, unsigned int port, unsigned int bytes) { + struct stubs *this_stubs = &this_cpu(stubs); + unsigned long stub_va = this_stubs->addr + STUB_BUF_SIZE / 2; bool use_quirk_stub = false; if ( !ctxt->io_emul_stub ) - ctxt->io_emul_stub = map_domain_page(_mfn(this_cpu(stubs.mfn))) + - (this_cpu(stubs.addr) & - ~PAGE_MASK) + - STUB_BUF_SIZE / 2; + ctxt->io_emul_stub = + map_domain_page(_mfn(this_stubs->mfn)) + (stub_va & ~PAGE_MASK); /* movq $host_to_guest_gpr_switch,%rcx */ ctxt->io_emul_stub[0] = 0x48; ctxt->io_emul_stub[1] = 0xb9; *(void **)&ctxt->io_emul_stub[2] = (void *)host_to_guest_gpr_switch; + +#ifdef CONFIG_INDIRECT_THUNK + /* callq __x86_indirect_thunk_rcx */ + ctxt->io_emul_stub[10] = 0xe8; + *(int32_t *)&ctxt->io_emul_stub[11] = + (long)__x86_indirect_thunk_rcx - (stub_va + 11 + 4); +#else /* callq *%rcx */ ctxt->io_emul_stub[10] = 0xff; ctxt->io_emul_stub[11] = 0xd1; + /* TODO: untangle ideal_nops from init/livepatch Kconfig options. */ + memcpy(&ctxt->io_emul_stub[12], "\x0f\x1f\x00", 3); /* P6_NOP3 */ +#endif if ( unlikely(ioemul_handle_quirk) ) - use_quirk_stub = ioemul_handle_quirk(opcode, &ctxt->io_emul_stub[12], + use_quirk_stub = ioemul_handle_quirk(opcode, &ctxt->io_emul_stub[15], ctxt->ctxt.regs); if ( !use_quirk_stub ) { /* data16 or nop */ - ctxt->io_emul_stub[12] = (bytes != 2) ? 0x90 : 0x66; + ctxt->io_emul_stub[15] = (bytes != 2) ? 0x90 : 0x66; /* */ - ctxt->io_emul_stub[13] = opcode; + ctxt->io_emul_stub[16] = opcode; /* imm8 or nop */ - ctxt->io_emul_stub[14] = !(opcode & 8) ? port : 0x90; + ctxt->io_emul_stub[17] = !(opcode & 8) ? port : 0x90; /* ret (jumps to guest_to_host_gpr_switch) */ - ctxt->io_emul_stub[15] = 0xc3; + ctxt->io_emul_stub[18] = 0xc3; } - BUILD_BUG_ON(STUB_BUF_SIZE / 2 < MAX(16, /* Default emul stub */ - 12 + IOEMUL_QUIRK_STUB_BYTES)); + BUILD_BUG_ON(STUB_BUF_SIZE / 2 < MAX(19, /* Default emul stub */ + 15 + IOEMUL_QUIRK_STUB_BYTES)); /* Handy function-typed pointer to the stub. */ - return (void *)(this_cpu(stubs.addr) + STUB_BUF_SIZE / 2); + return (void *)stub_va; } diff --git a/xen/arch/x86/x86_64/entry.S b/xen/arch/x86/x86_64/entry.S index a8825c89df..710c0616ba 100644 --- a/xen/arch/x86/x86_64/entry.S +++ b/xen/arch/x86/x86_64/entry.S @@ -585,7 +585,8 @@ handle_exception_saved: movzbl UREGS_entry_vector(%rsp),%eax leaq exception_table(%rip),%rdx PERFC_INCR(exceptions, %rax, %rbx) - callq *(%rdx,%rax,8) + mov (%rdx, %rax, 8), %rdx + INDIRECT_CALL %rdx mov %r15, STACK_CPUINFO_FIELD(xen_cr3)(%r14) testb $3,UREGS_cs(%rsp) jz restore_all_xen @@ -757,7 +758,8 @@ handle_ist_exception: 1: movq %rsp,%rdi movzbl UREGS_entry_vector(%rsp),%eax leaq exception_table(%rip),%rdx - callq *(%rdx,%rax,8) + mov (%rdx, %rax, 8), %rdx + INDIRECT_CALL %rdx mov %r15, STACK_CPUINFO_FIELD(xen_cr3)(%r14) cmpb $TRAP_nmi,UREGS_entry_vector(%rsp) jne ret_from_intr diff --git a/xen/arch/x86/x86_emulate/x86_emulate.c b/xen/arch/x86/x86_emulate/x86_emulate.c index 820495fb9c..ff0a003902 100644 --- a/xen/arch/x86/x86_emulate/x86_emulate.c +++ b/xen/arch/x86/x86_emulate/x86_emulate.c @@ -867,7 +867,7 @@ static inline int mkec(uint8_t e, int32_t ec, ...) #ifdef __XEN__ # define invoke_stub(pre, post, constraints...) do { \ union stub_exception_token res_ = { .raw = ~0 }; \ - asm volatile ( pre "\n\tcall *%[stub]\n\t" post "\n" \ + asm volatile ( pre "\n\tINDIRECT_CALL %[stub]\n\t" post "\n" \ ".Lret%=:\n\t" \ ".pushsection .fixup,\"ax\"\n" \ ".Lfix%=:\n\t" \ @@ -876,7 +876,7 @@ static inline int mkec(uint8_t e, int32_t ec, ...) ".popsection\n\t" \ _ASM_EXTABLE(.Lret%=, .Lfix%=) \ : [exn] "+g" (res_), constraints, \ - [stub] "rm" (stub.func), \ + [stub] "r" (stub.func), \ "m" (*(uint8_t(*)[MAX_INST_LEN + 1])stub.ptr) ); \ if ( unlikely(~res_.raw) ) \ { \ diff --git a/xen/common/wait.c b/xen/common/wait.c index 3d3d9fe7a2..a57bc10d61 100644 --- a/xen/common/wait.c +++ b/xen/common/wait.c @@ -203,12 +203,14 @@ void check_wakeup_from_wait(void) /* * Hand-rolled longjmp(). Returns to the pointer on the top of - * wqv->stack, and lands on a `rep movs` instruction. + * wqv->stack, and lands on a `rep movs` instruction. All other GPRs are + * restored from the stack, so are available for use here. */ asm volatile ( - "mov %1,%%"__OP"sp; jmp *(%0)" + "mov %1,%%"__OP"sp; INDIRECT_JMP %[ip]" : : "S" (wqv->stack), "D" (wqv->esp), - "c" ((char *)get_cpu_info() - (char *)wqv->esp) + "c" ((char *)get_cpu_info() - (char *)wqv->esp), + [ip] "r" (*(unsigned long *)wqv->stack) : "memory" ); unreachable(); } diff --git a/xen/include/asm-x86/asm_defns.h b/xen/include/asm-x86/asm_defns.h index 73d96227f4..9cfd79f0c6 100644 --- a/xen/include/asm-x86/asm_defns.h +++ b/xen/include/asm-x86/asm_defns.h @@ -13,6 +13,14 @@ #include #include +#ifdef __ASSEMBLY__ +# include +#else +asm ( "\t.equ CONFIG_INDIRECT_THUNK, " + __stringify(IS_ENABLED(CONFIG_INDIRECT_THUNK)) ); +asm ( "\t.include \"asm/indirect_thunk_asm.h\"" ); +#endif + #ifndef __ASSEMBLY__ void ret_from_intr(void); #endif diff --git a/xen/include/asm-x86/indirect_thunk_asm.h b/xen/include/asm-x86/indirect_thunk_asm.h new file mode 100644 index 0000000000..96bcc25497 --- /dev/null +++ b/xen/include/asm-x86/indirect_thunk_asm.h @@ -0,0 +1,41 @@ +/* + * Warning! This file is included at an assembler level for .c files, causing + * usual #ifdef'ary to turn into comments. + */ + +.macro INDIRECT_BRANCH insn:req arg:req +/* + * Create an indirect branch. insn is one of call/jmp, arg is a single + * register. + * + * With no compiler support, this degrades into a plain indirect call/jmp. + * With compiler support, dispatch to the correct __x86_indirect_thunk_* + */ + .if CONFIG_INDIRECT_THUNK == 1 + + $done = 0 + .irp reg, ax, cx, dx, bx, bp, si, di, 8, 9, 10, 11, 12, 13, 14, 15 + .ifeqs "\arg", "%r\reg" + \insn __x86_indirect_thunk_r\reg + $done = 1 + .exitm + .endif + .endr + + .if $done != 1 + .error "Bad register arg \arg" + .endif + + .else + \insn *\arg + .endif +.endm + +/* Convenience wrappers. */ +.macro INDIRECT_CALL arg:req + INDIRECT_BRANCH call \arg +.endm + +.macro INDIRECT_JMP arg:req + INDIRECT_BRANCH jmp \arg +.endm -- 2.30.2