From 383c80899960cf41bcdd63f200b9cf5c12005b06 Mon Sep 17 00:00:00 2001 From: Dimitris Papastamos Date: Wed, 24 Jan 2018 16:41:14 +0000 Subject: [PATCH 1/6] Disable workaround for CVE-2017-5715 on unaffected platforms Change-Id: Ib67b841ab621ca1ace3280e44cf3e1d83052cb73 Signed-off-by: Dimitris Papastamos --- plat/hisilicon/hikey/platform.mk | 2 ++ plat/hisilicon/poplar/platform.mk | 2 ++ plat/mediatek/mt6795/platform.mk | 2 ++ plat/rockchip/rk3328/platform.mk | 2 ++ plat/rockchip/rk3368/platform.mk | 2 ++ plat/rpi3/platform.mk | 2 ++ plat/xilinx/zynqmp/platform.mk | 2 ++ 7 files changed, 14 insertions(+) diff --git a/plat/hisilicon/hikey/platform.mk b/plat/hisilicon/hikey/platform.mk index c8a29924a..b11d2085e 100644 --- a/plat/hisilicon/hikey/platform.mk +++ b/plat/hisilicon/hikey/platform.mk @@ -122,4 +122,6 @@ ERRATA_A53_836870 := 1 ERRATA_A53_843419 := 1 ERRATA_A53_855873 := 1 +WORKAROUND_CVE_2017_5715 := 0 + FIP_ALIGN := 512 diff --git a/plat/hisilicon/poplar/platform.mk b/plat/hisilicon/poplar/platform.mk index 2dbbac6e7..d53e06274 100644 --- a/plat/hisilicon/poplar/platform.mk +++ b/plat/hisilicon/poplar/platform.mk @@ -29,6 +29,8 @@ ERRATA_A53_835769 := 1 ERRATA_A53_843419 := 1 ENABLE_SVE_FOR_NS := 0 +WORKAROUND_CVE_2017_5715 := 0 + ARM_GIC_ARCH := 2 $(eval $(call add_define,ARM_GIC_ARCH)) diff --git a/plat/mediatek/mt6795/platform.mk b/plat/mediatek/mt6795/platform.mk index 82300674d..1bdf30af3 100644 --- a/plat/mediatek/mt6795/platform.mk +++ b/plat/mediatek/mt6795/platform.mk @@ -61,6 +61,8 @@ $(eval $(call add_define,ARM_GIC_ARCH)) ERRATA_A53_826319 := 1 ERRATA_A53_836870 := 1 +WORKAROUND_CVE_2017_5715 := 0 + # indicate the reset vector address can be programmed PROGRAMMABLE_RESET_ADDRESS := 1 diff --git a/plat/rockchip/rk3328/platform.mk b/plat/rockchip/rk3328/platform.mk index 6e4d5b4dc..f0fd36f9b 100644 --- a/plat/rockchip/rk3328/platform.mk +++ b/plat/rockchip/rk3328/platform.mk @@ -58,3 +58,5 @@ $(eval $(call add_define,PLAT_SKIP_OPTEE_S_EL1_INT_REGISTER)) # Do not enable SVE ENABLE_SVE_FOR_NS := 0 + +WORKAROUND_CVE_2017_5715 := 0 diff --git a/plat/rockchip/rk3368/platform.mk b/plat/rockchip/rk3368/platform.mk index ad204e9eb..7ecb21ad5 100644 --- a/plat/rockchip/rk3368/platform.mk +++ b/plat/rockchip/rk3368/platform.mk @@ -57,3 +57,5 @@ $(eval $(call add_define,PLAT_EXTRA_LD_SCRIPT)) # Do not enable SVE ENABLE_SVE_FOR_NS := 0 + +WORKAROUND_CVE_2017_5715 := 0 diff --git a/plat/rpi3/platform.mk b/plat/rpi3/platform.mk index 821f80153..e201ceed0 100644 --- a/plat/rpi3/platform.mk +++ b/plat/rpi3/platform.mk @@ -64,6 +64,8 @@ ERRATA_A53_836870 := 1 ERRATA_A53_843419 := 1 ERRATA_A53_855873 := 1 +WORKAROUND_CVE_2017_5715 := 0 + # Disable the PSCI platform compatibility layer by default ENABLE_PLAT_COMPAT := 0 diff --git a/plat/xilinx/zynqmp/platform.mk b/plat/xilinx/zynqmp/platform.mk index bdd194bda..bddf30562 100644 --- a/plat/xilinx/zynqmp/platform.mk +++ b/plat/xilinx/zynqmp/platform.mk @@ -14,6 +14,8 @@ override RESET_TO_BL31 := 1 # Do not enable SVE ENABLE_SVE_FOR_NS := 0 +WORKAROUND_CVE_2017_5715 := 0 + ifdef ZYNQMP_ATF_MEM_BASE $(eval $(call add_define,ZYNQMP_ATF_MEM_BASE)) From 201ca5b6b49197a2c22b3bb0b519d63758fff0b7 Mon Sep 17 00:00:00 2001 From: Dimitris Papastamos Date: Mon, 22 Jan 2018 11:53:04 +0000 Subject: [PATCH 2/6] runtime_exceptions: Save x4-x29 unconditionally In preparation for SMCCC v1.1 support, save x4 to x29 unconditionally. Previously we expected callers coming from AArch64 mode to preserve x8-x17. This is no longer the case with SMCCC v1.1 as AArch64 callers only need to save x0-x3. Change-Id: Ie62d620776533969ff4a02c635422f1b9208be9c Signed-off-by: Dimitris Papastamos --- bl31/aarch64/runtime_exceptions.S | 35 +++++++++++-------------------- 1 file changed, 12 insertions(+), 23 deletions(-) diff --git a/bl31/aarch64/runtime_exceptions.S b/bl31/aarch64/runtime_exceptions.S index 9b7735f1e..57c065c6a 100644 --- a/bl31/aarch64/runtime_exceptions.S +++ b/bl31/aarch64/runtime_exceptions.S @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2016, ARM Limited and Contributors. All rights reserved. + * Copyright (c) 2013-2018, ARM Limited and Contributors. All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ @@ -153,7 +153,14 @@ interrupt_exit_\label: .endm - .macro save_x18_to_x29_sp_el0 + .macro save_x4_to_x29_sp_el0 + stp x4, x5, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X4] + stp x6, x7, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X6] + stp x8, x9, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X8] + stp x10, x11, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X10] + stp x12, x13, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X12] + stp x14, x15, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X14] + stp x16, x17, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X16] stp x18, x19, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X18] stp x20, x21, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X20] stp x22, x23, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X22] @@ -297,34 +304,16 @@ smc_handler32: /* Check whether aarch32 issued an SMC64 */ tbnz x0, #FUNCID_CC_SHIFT, smc_prohibited - /* - * Since we're are coming from aarch32, x8-x18 need to be saved as per - * SMC32 calling convention. If a lower EL in aarch64 is making an - * SMC32 call then it must have saved x8-x17 already therein. - */ - stp x8, x9, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X8] - stp x10, x11, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X10] - stp x12, x13, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X12] - stp x14, x15, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X14] - stp x16, x17, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X16] - - /* x4-x7, x18, sp_el0 are saved below */ - smc_handler64: /* * Populate the parameters for the SMC handler. * We already have x0-x4 in place. x5 will point to a cookie (not used * now). x6 will point to the context structure (SP_EL3) and x7 will - * contain flags we need to pass to the handler Hence save x5-x7. + * contain flags we need to pass to the handler. * - * Note: x4 only needs to be preserved for AArch32 callers but we do it - * for AArch64 callers as well for convenience + * Save x4-x29 and sp_el0. Refer to SMCCC v1.1. */ - stp x4, x5, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X4] - stp x6, x7, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X6] - - /* Save rest of the gpregs and sp_el0*/ - save_x18_to_x29_sp_el0 + save_x4_to_x29_sp_el0 mov x5, xzr mov x6, sp From 3a1b0676c72989d75276da7f11e89d00933eccdd Mon Sep 17 00:00:00 2001 From: Dimitris Papastamos Date: Fri, 19 Jan 2018 16:58:29 +0000 Subject: [PATCH 3/6] Implement support for SMCCC v1.1 SMCCC v1.1 comes with a relaxed calling convention for AArch64 callers. The caller only needs to save x0-x3 before doing an SMC call. This patch adds support for SMCCC_VERSION and SMCCC_ARCH_FEATURES. Refer to "Firmware Interfaces for mitigating CVE_2017_5715 System Software on Arm Systems"[0] for more information. [0] https://developer.arm.com/-/media/developer/pdf/ARM%20DEN%200070A%20Firmware%20interfaces%20for%20mitigating%20CVE-2017-5715_V1.0.pdf Change-Id: If5b1c55c17d6c5c7cb9c2c3ed355d3a91cdad0a9 Signed-off-by: Dimitris Papastamos --- bl31/bl31.mk | 3 +- include/lib/smcc.h | 11 +++- include/services/arm_arch_svc.h | 14 ++++ services/arm_arch_svc/arm_arch_svc_setup.c | 74 ++++++++++++++++++++++ 4 files changed, 100 insertions(+), 2 deletions(-) create mode 100644 include/services/arm_arch_svc.h create mode 100644 services/arm_arch_svc/arm_arch_svc_setup.c diff --git a/bl31/bl31.mk b/bl31/bl31.mk index 2db48564e..886d3016f 100644 --- a/bl31/bl31.mk +++ b/bl31/bl31.mk @@ -1,5 +1,5 @@ # -# Copyright (c) 2013-2017, ARM Limited and Contributors. All rights reserved. +# Copyright (c) 2013-2018, ARM Limited and Contributors. All rights reserved. # # SPDX-License-Identifier: BSD-3-Clause # @@ -23,6 +23,7 @@ BL31_SOURCES += bl31/bl31_main.c \ bl31/bl31_context_mgmt.c \ common/runtime_svc.c \ plat/common/aarch64/platform_mp_stack.S \ + services/arm_arch_svc/arm_arch_svc_setup.c \ services/std_svc/std_svc_setup.c \ ${PSCI_LIB_SOURCES} \ ${SPM_SOURCES} \ diff --git a/include/lib/smcc.h b/include/lib/smcc.h index 13b1e7ac4..a273b3af4 100644 --- a/include/lib/smcc.h +++ b/include/lib/smcc.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2017, ARM Limited and Contributors. All rights reserved. + * Copyright (c) 2016-2018, ARM Limited and Contributors. All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ @@ -67,6 +67,11 @@ #include #include +#define SMCCC_MAJOR_VERSION U(1) +#define SMCCC_MINOR_VERSION U(1) + +#define MAKE_SMCCC_VERSION(_major, _minor) (((_major) << 16) | (_minor)) + /* Various flags passed to SMC handlers */ #define SMC_FROM_SECURE (U(0) << 0) #define SMC_FROM_NON_SECURE (U(1) << 0) @@ -78,6 +83,10 @@ #define is_std_svc_call(_fid) ((((_fid) >> FUNCID_OEN_SHIFT) & \ FUNCID_OEN_MASK) == OEN_STD_START) +/* The macro below is used to identify a Arm Architectural Service SMC call */ +#define is_arm_arch_svc_call(_fid) ((((_fid) >> FUNCID_OEN_SHIFT) & \ + FUNCID_OEN_MASK) == OEN_ARM_START) + /* The macro below is used to identify a valid Fast SMC call */ #define is_valid_fast_smc(_fid) ((!(((_fid) >> 16) & U(0xff))) && \ (GET_SMC_TYPE(_fid) == SMC_TYPE_FAST)) diff --git a/include/services/arm_arch_svc.h b/include/services/arm_arch_svc.h new file mode 100644 index 000000000..29616013a --- /dev/null +++ b/include/services/arm_arch_svc.h @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2018, ARM Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef __ARM_ARCH_SVC_H__ +#define __ARM_ARCH_SVC_H__ + +#define SMCCC_VERSION U(0x80000000) +#define SMCCC_ARCH_FEATURES U(0x80000001) +#define SMCCC_ARCH_WORKAROUND_1 U(0x80008000) + +#endif /* __ARM_ARCH_SVC_H__ */ diff --git a/services/arm_arch_svc/arm_arch_svc_setup.c b/services/arm_arch_svc/arm_arch_svc_setup.c new file mode 100644 index 000000000..eedac8694 --- /dev/null +++ b/services/arm_arch_svc/arm_arch_svc_setup.c @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2018, ARM Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include +#include +#include +#include +#include + +static int32_t smccc_version(void) +{ + return MAKE_SMCCC_VERSION(SMCCC_MAJOR_VERSION, SMCCC_MINOR_VERSION); +} + +static int32_t smccc_arch_features(u_register_t arg) +{ + switch (arg) { + case SMCCC_VERSION: + case SMCCC_ARCH_FEATURES: + return SMC_OK; +#if WORKAROUND_CVE_2017_5715 + case SMCCC_ARCH_WORKAROUND_1: + return SMC_OK; +#endif + default: + return SMC_UNK; + } +} + +/* + * Top-level Arm Architectural Service SMC handler. + */ +uintptr_t arm_arch_svc_smc_handler(uint32_t smc_fid, + u_register_t x1, + u_register_t x2, + u_register_t x3, + u_register_t x4, + void *cookie, + void *handle, + u_register_t flags) +{ + switch (smc_fid) { + case SMCCC_VERSION: + SMC_RET1(handle, smccc_version()); + case SMCCC_ARCH_FEATURES: + SMC_RET1(handle, smccc_arch_features(x1)); +#if WORKAROUND_CVE_2017_5715 + case SMCCC_ARCH_WORKAROUND_1: + /* + * The workaround has already been applied on affected PEs + * during entry to EL3. On unaffected PEs, this function + * has no effect. + */ + SMC_RET0(handle); +#endif + default: + WARN("Unimplemented Arm Architecture Service Call: 0x%x \n", + smc_fid); + SMC_RET1(handle, SMC_UNK); + } +} + +/* Register Standard Service Calls as runtime service */ +DECLARE_RT_SVC( + arm_arch_svc, + OEN_ARM_START, + OEN_ARM_END, + SMC_TYPE_FAST, + NULL, + arm_arch_svc_smc_handler +); From 6eabbb07d7ee2aac3a8e8e734649c8eaa8385af6 Mon Sep 17 00:00:00 2001 From: Dimitris Papastamos Date: Mon, 22 Jan 2018 12:58:52 +0000 Subject: [PATCH 4/6] Add support for SMCCC_VERSION in PSCI features On some platforms it may be necessary to discover the SMCCC version via a PSCI features call. Change-Id: I95281ac2263ca9aefda1809eb03464fbdb8ac24d Signed-off-by: Dimitris Papastamos --- lib/psci/psci_main.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/psci/psci_main.c b/lib/psci/psci_main.c index 8e41cf026..88cf5cbe4 100644 --- a/lib/psci/psci_main.c +++ b/lib/psci/psci_main.c @@ -1,11 +1,12 @@ /* - * Copyright (c) 2013-2017, ARM Limited and Contributors. All rights reserved. + * Copyright (c) 2013-2018, ARM Limited and Contributors. All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ #include #include +#include #include #include #include @@ -322,6 +323,9 @@ int psci_features(unsigned int psci_fid) { unsigned int local_caps = psci_caps; + if (psci_fid == SMCCC_VERSION) + return PSCI_E_SUCCESS; + /* Check if it is a 64 bit function */ if (((psci_fid >> FUNCID_CC_SHIFT) & FUNCID_CC_MASK) == SMC_64) local_caps &= PSCI_CAP_64BIT_MASK; From d9bd656cf5a4d0c21597a956c3711d08626ecfd0 Mon Sep 17 00:00:00 2001 From: Dimitris Papastamos Date: Thu, 11 Jan 2018 15:29:36 +0000 Subject: [PATCH 5/6] Optimize/cleanup BPIALL workaround In the initial implementation of this workaround we used a dedicated workaround context to save/restore state. This patch reduces the footprint as no additional context is needed. Additionally, this patch reduces the memory loads and stores by 20%, reduces the instruction count and exploits static branch prediction to optimize the SMC path. Change-Id: Ia9f6bf06fbf8a9037cfe7f1f1fb32e8aec38ec7d Signed-off-by: Dimitris Papastamos --- include/lib/el3_runtime/aarch64/context.h | 28 +-- .../aarch64/workaround_cve_2017_5715_bpiall.S | 210 ++++++++---------- 2 files changed, 90 insertions(+), 148 deletions(-) diff --git a/include/lib/el3_runtime/aarch64/context.h b/include/lib/el3_runtime/aarch64/context.h index 5e212ec3f..5f6bdc97a 100644 --- a/include/lib/el3_runtime/aarch64/context.h +++ b/include/lib/el3_runtime/aarch64/context.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2017, ARM Limited and Contributors. All rights reserved. + * Copyright (c) 2013-2018, ARM Limited and Contributors. All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ @@ -46,26 +46,12 @@ #define CTX_GPREG_SP_EL0 U(0xf8) #define CTX_GPREGS_END U(0x100) -#if WORKAROUND_CVE_2017_5715 -#define CTX_CVE_2017_5715_OFFSET (CTX_GPREGS_OFFSET + CTX_GPREGS_END) -#define CTX_CVE_2017_5715_QUAD0 U(0x0) -#define CTX_CVE_2017_5715_QUAD1 U(0x8) -#define CTX_CVE_2017_5715_QUAD2 U(0x10) -#define CTX_CVE_2017_5715_QUAD3 U(0x18) -#define CTX_CVE_2017_5715_QUAD4 U(0x20) -#define CTX_CVE_2017_5715_QUAD5 U(0x28) -#define CTX_CVE_2017_5715_END U(0x30) -#else -#define CTX_CVE_2017_5715_OFFSET CTX_GPREGS_OFFSET -#define CTX_CVE_2017_5715_END CTX_GPREGS_END -#endif - /******************************************************************************* * Constants that allow assembler code to access members of and the 'el3_state' * structure at their correct offsets. Note that some of the registers are only * 32-bits wide but are stored as 64-bit values for convenience ******************************************************************************/ -#define CTX_EL3STATE_OFFSET (CTX_CVE_2017_5715_OFFSET + CTX_CVE_2017_5715_END) +#define CTX_EL3STATE_OFFSET (CTX_GPREGS_OFFSET + CTX_GPREGS_END) #define CTX_SCR_EL3 U(0x0) #define CTX_RUNTIME_SP U(0x8) #define CTX_SPSR_EL3 U(0x10) @@ -200,9 +186,6 @@ /* Constants to determine the size of individual context structures */ #define CTX_GPREG_ALL (CTX_GPREGS_END >> DWORD_SHIFT) -#if WORKAROUND_CVE_2017_5715 -#define CTX_CVE_2017_5715_ALL (CTX_CVE_2017_5715_END >> DWORD_SHIFT) -#endif #define CTX_SYSREG_ALL (CTX_SYSREGS_END >> DWORD_SHIFT) #if CTX_INCLUDE_FPREGS #define CTX_FPREG_ALL (CTX_FPREGS_END >> DWORD_SHIFT) @@ -218,10 +201,6 @@ */ DEFINE_REG_STRUCT(gp_regs, CTX_GPREG_ALL); -#if WORKAROUND_CVE_2017_5715 -DEFINE_REG_STRUCT(cve_2017_5715_regs, CTX_CVE_2017_5715_ALL); -#endif - /* * AArch64 EL1 system register context structure for preserving the * architectural state during switches from one security state to @@ -263,9 +242,6 @@ DEFINE_REG_STRUCT(el3_state, CTX_EL3STATE_ALL); */ typedef struct cpu_context { gp_regs_t gpregs_ctx; -#if WORKAROUND_CVE_2017_5715 - cve_2017_5715_regs_t cve_2017_5715_regs_ctx; -#endif el3_state_t el3state_ctx; el1_sys_regs_t sysregs_ctx; #if CTX_INCLUDE_FPREGS diff --git a/lib/cpus/aarch64/workaround_cve_2017_5715_bpiall.S b/lib/cpus/aarch64/workaround_cve_2017_5715_bpiall.S index cd29266ed..9677e2e05 100644 --- a/lib/cpus/aarch64/workaround_cve_2017_5715_bpiall.S +++ b/lib/cpus/aarch64/workaround_cve_2017_5715_bpiall.S @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, ARM Limited and Contributors. All rights reserved. + * Copyright (c) 2017-2018, ARM Limited and Contributors. All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ @@ -11,10 +11,15 @@ .globl workaround_bpiall_vbar0_runtime_exceptions #define EMIT_BPIALL 0xee070fd5 -#define EMIT_MOV_R0_IMM(v) 0xe3a0000##v #define EMIT_SMC 0xe1600070 - .macro enter_workaround _stub_name + .macro enter_workaround _from_vector + /* + * Save register state to enable a call to AArch32 S-EL1 and return + * Identify the original calling vector in w2 (==_from_vector) + * Use w3-w6 for additional register state preservation while in S-EL1 + */ + /* Save GP regs */ stp x0, x1, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X0] stp x2, x3, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X2] @@ -32,47 +37,50 @@ stp x26, x27, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X26] stp x28, x29, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X28] - adr x4, \_stub_name + /* Identify the original exception vector */ + mov w2, \_from_vector + + /* Preserve 32-bit system registers in GP registers through the workaround */ + mrs x3, esr_el3 + mrs x4, spsr_el3 + mrs x5, scr_el3 + mrs x6, sctlr_el1 /* - * Load SPSR_EL3 and VBAR_EL3. SPSR_EL3 is set up to have - * all interrupts masked in preparation to running the workaround - * stub in S-EL1. VBAR_EL3 points to the vector table that - * will handle the SMC back from the workaround stub. + * Preserve LR and ELR_EL3 registers in the GP regs context. + * Temporarily use the CTX_GPREG_SP_EL0 slot to preserve ELR_EL3 + * through the workaround. This is OK because at this point the + * current state for this context's SP_EL0 is in the live system + * register, which is unmodified by the workaround. */ - ldp x0, x1, [x4, #0] + mrs x7, elr_el3 + stp x30, x7, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_LR] /* - * Load SCTLR_EL1 and ELR_EL3. SCTLR_EL1 is configured to disable - * the MMU in S-EL1. ELR_EL3 points to the appropriate stub in S-EL1. + * Load system registers for entry to S-EL1. */ - ldp x2, x3, [x4, #16] - mrs x4, scr_el3 - mrs x5, spsr_el3 - mrs x6, elr_el3 - mrs x7, sctlr_el1 - mrs x8, esr_el3 + /* Mask all interrupts and set AArch32 Supervisor mode */ + movz w8, SPSR_MODE32(MODE32_svc, SPSR_T_ARM, SPSR_E_LITTLE, SPSR_AIF_MASK) - /* Preserve system registers in the workaround context */ - stp x4, x5, [sp, #CTX_CVE_2017_5715_OFFSET + CTX_CVE_2017_5715_QUAD0] - stp x6, x7, [sp, #CTX_CVE_2017_5715_OFFSET + CTX_CVE_2017_5715_QUAD2] - stp x8, x30, [sp, #CTX_CVE_2017_5715_OFFSET + CTX_CVE_2017_5715_QUAD4] + /* Switch EL3 exception vectors while the workaround is executing. */ + adr x9, workaround_bpiall_vbar1_runtime_exceptions + + /* Setup SCTLR_EL1 with MMU off and I$ on */ + ldr x10, stub_sel1_sctlr + + /* Land at the S-EL1 workaround stub */ + adr x11, aarch32_stub /* * Setting SCR_EL3 to all zeroes means that the NS, RW * and SMD bits are configured as expected. */ msr scr_el3, xzr - - /* - * Reload system registers with the crafted values - * in preparation for entry in S-EL1. - */ - msr spsr_el3, x0 - msr vbar_el3, x1 - msr sctlr_el1, x2 - msr elr_el3, x3 + msr spsr_el3, x8 + msr vbar_el3, x9 + msr sctlr_el1, x10 + msr elr_el3, x11 eret .endm @@ -91,76 +99,31 @@ vector_base workaround_bpiall_vbar0_runtime_exceptions */ vector_entry workaround_bpiall_vbar0_sync_exception_sp_el0 b sync_exception_sp_el0 + nop /* to force 8 byte alignment for the following stub */ + /* * Since each vector table entry is 128 bytes, we can store the * stub context in the unused space to minimize memory footprint. */ -aarch32_stub_smc: - .word EMIT_BPIALL - .word EMIT_MOV_R0_IMM(1) - .word EMIT_SMC -aarch32_stub_ctx_smc: - /* Mask all interrupts and set AArch32 Supervisor mode */ - .quad (SPSR_AIF_MASK << SPSR_AIF_SHIFT | \ - SPSR_M_AARCH32 << SPSR_M_SHIFT | \ - MODE32_svc << MODE32_SHIFT) - - /* - * VBAR_EL3 points to vbar1 which is the vector table - * used while the workaround is executing. - */ - .quad workaround_bpiall_vbar1_runtime_exceptions - - /* Setup SCTLR_EL1 with MMU off and I$ on */ +stub_sel1_sctlr: .quad SCTLR_AARCH32_EL1_RES1 | SCTLR_I_BIT - /* ELR_EL3 is setup to point to the sync exception stub in AArch32 */ - .quad aarch32_stub_smc +aarch32_stub: + .word EMIT_BPIALL + .word EMIT_SMC + check_vector_size workaround_bpiall_vbar0_sync_exception_sp_el0 vector_entry workaround_bpiall_vbar0_irq_sp_el0 b irq_sp_el0 -aarch32_stub_irq: - .word EMIT_BPIALL - .word EMIT_MOV_R0_IMM(2) - .word EMIT_SMC -aarch32_stub_ctx_irq: - .quad (SPSR_AIF_MASK << SPSR_AIF_SHIFT | \ - SPSR_M_AARCH32 << SPSR_M_SHIFT | \ - MODE32_svc << MODE32_SHIFT) - .quad workaround_bpiall_vbar1_runtime_exceptions - .quad SCTLR_AARCH32_EL1_RES1 | SCTLR_I_BIT - .quad aarch32_stub_irq check_vector_size workaround_bpiall_vbar0_irq_sp_el0 vector_entry workaround_bpiall_vbar0_fiq_sp_el0 b fiq_sp_el0 -aarch32_stub_fiq: - .word EMIT_BPIALL - .word EMIT_MOV_R0_IMM(4) - .word EMIT_SMC -aarch32_stub_ctx_fiq: - .quad (SPSR_AIF_MASK << SPSR_AIF_SHIFT | \ - SPSR_M_AARCH32 << SPSR_M_SHIFT | \ - MODE32_svc << MODE32_SHIFT) - .quad workaround_bpiall_vbar1_runtime_exceptions - .quad SCTLR_AARCH32_EL1_RES1 | SCTLR_I_BIT - .quad aarch32_stub_fiq check_vector_size workaround_bpiall_vbar0_fiq_sp_el0 vector_entry workaround_bpiall_vbar0_serror_sp_el0 b serror_sp_el0 -aarch32_stub_serror: - .word EMIT_BPIALL - .word EMIT_MOV_R0_IMM(8) - .word EMIT_SMC -aarch32_stub_ctx_serror: - .quad (SPSR_AIF_MASK << SPSR_AIF_SHIFT | \ - SPSR_M_AARCH32 << SPSR_M_SHIFT | \ - MODE32_svc << MODE32_SHIFT) - .quad workaround_bpiall_vbar1_runtime_exceptions - .quad SCTLR_AARCH32_EL1_RES1 | SCTLR_I_BIT - .quad aarch32_stub_serror check_vector_size workaround_bpiall_vbar0_serror_sp_el0 /* --------------------------------------------------------------------- @@ -188,19 +151,19 @@ vector_entry workaround_bpiall_vbar0_serror_sp_elx * --------------------------------------------------------------------- */ vector_entry workaround_bpiall_vbar0_sync_exception_aarch64 - enter_workaround aarch32_stub_ctx_smc + enter_workaround 1 check_vector_size workaround_bpiall_vbar0_sync_exception_aarch64 vector_entry workaround_bpiall_vbar0_irq_aarch64 - enter_workaround aarch32_stub_ctx_irq + enter_workaround 2 check_vector_size workaround_bpiall_vbar0_irq_aarch64 vector_entry workaround_bpiall_vbar0_fiq_aarch64 - enter_workaround aarch32_stub_ctx_fiq + enter_workaround 4 check_vector_size workaround_bpiall_vbar0_fiq_aarch64 vector_entry workaround_bpiall_vbar0_serror_aarch64 - enter_workaround aarch32_stub_ctx_serror + enter_workaround 8 check_vector_size workaround_bpiall_vbar0_serror_aarch64 /* --------------------------------------------------------------------- @@ -208,19 +171,19 @@ vector_entry workaround_bpiall_vbar0_serror_aarch64 * --------------------------------------------------------------------- */ vector_entry workaround_bpiall_vbar0_sync_exception_aarch32 - enter_workaround aarch32_stub_ctx_smc + enter_workaround 1 check_vector_size workaround_bpiall_vbar0_sync_exception_aarch32 vector_entry workaround_bpiall_vbar0_irq_aarch32 - enter_workaround aarch32_stub_ctx_irq + enter_workaround 2 check_vector_size workaround_bpiall_vbar0_irq_aarch32 vector_entry workaround_bpiall_vbar0_fiq_aarch32 - enter_workaround aarch32_stub_ctx_fiq + enter_workaround 4 check_vector_size workaround_bpiall_vbar0_fiq_aarch32 vector_entry workaround_bpiall_vbar0_serror_aarch32 - enter_workaround aarch32_stub_ctx_serror + enter_workaround 8 check_vector_size workaround_bpiall_vbar0_serror_aarch32 /* --------------------------------------------------------------------- @@ -297,31 +260,33 @@ vector_entry workaround_bpiall_vbar1_serror_aarch64 * --------------------------------------------------------------------- */ vector_entry workaround_bpiall_vbar1_sync_exception_aarch32 - /* Restore register state from the workaround context */ - ldp x2, x3, [sp, #CTX_CVE_2017_5715_OFFSET + CTX_CVE_2017_5715_QUAD0] - ldp x4, x5, [sp, #CTX_CVE_2017_5715_OFFSET + CTX_CVE_2017_5715_QUAD2] - ldp x6, x30, [sp, #CTX_CVE_2017_5715_OFFSET + CTX_CVE_2017_5715_QUAD4] + /* + * w2 indicates which SEL1 stub was run and thus which original vector was used + * w3-w6 contain saved system register state (esr_el3 in w3) + * Restore LR and ELR_EL3 register state from the GP regs context + */ + ldp x30, x7, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_LR] /* Apply the restored system register state */ - msr scr_el3, x2 - msr spsr_el3, x3 - msr elr_el3, x4 - msr sctlr_el1, x5 - msr esr_el3, x6 + msr esr_el3, x3 + msr spsr_el3, x4 + msr scr_el3, x5 + msr sctlr_el1, x6 + msr elr_el3, x7 /* * Workaround is complete, so swap VBAR_EL3 to point * to workaround entry table in preparation for subsequent * Sync/IRQ/FIQ/SError exceptions. */ - adr x2, workaround_bpiall_vbar0_runtime_exceptions - msr vbar_el3, x2 + adr x0, workaround_bpiall_vbar0_runtime_exceptions + msr vbar_el3, x0 /* - * Restore all GP regs except x0 and x1. The value in x0 + * Restore all GP regs except x2 and x3 (esr). The value in x2 * indicates the type of the original exception. */ - ldp x2, x3, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X2] + ldp x0, x1, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X0] ldp x4, x5, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X4] ldp x6, x7, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X6] ldp x8, x9, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X8] @@ -336,37 +301,38 @@ vector_entry workaround_bpiall_vbar1_sync_exception_aarch32 ldp x26, x27, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X26] ldp x28, x29, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X28] - /* - * Each of these handlers will first restore x0 and x1 from - * the context and the branch to the common implementation for - * each of the exception types. - */ - tbnz x0, #1, workaround_bpiall_vbar1_irq - tbnz x0, #2, workaround_bpiall_vbar1_fiq - tbnz x0, #3, workaround_bpiall_vbar1_serror - - /* Fallthrough case for Sync exception */ - ldp x0, x1, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X0] + /* Fast path Sync exceptions. Static predictor will fall through. */ + tbz w2, #0, workaround_not_sync + ldp x2, x3, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X2] b sync_exception_aarch64 check_vector_size workaround_bpiall_vbar1_sync_exception_aarch32 vector_entry workaround_bpiall_vbar1_irq_aarch32 b report_unhandled_interrupt -workaround_bpiall_vbar1_irq: - ldp x0, x1, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X0] + + /* + * Post-workaround fan-out for non-sync exceptions + */ +workaround_not_sync: + tbnz w2, #3, workaround_bpiall_vbar1_serror + tbnz w2, #2, workaround_bpiall_vbar1_fiq + /* IRQ */ + ldp x2, x3, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X2] b irq_aarch64 + +workaround_bpiall_vbar1_fiq: + ldp x2, x3, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X2] + b fiq_aarch64 + +workaround_bpiall_vbar1_serror: + ldp x2, x3, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X2] + b serror_aarch64 check_vector_size workaround_bpiall_vbar1_irq_aarch32 vector_entry workaround_bpiall_vbar1_fiq_aarch32 b report_unhandled_interrupt -workaround_bpiall_vbar1_fiq: - ldp x0, x1, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X0] - b fiq_aarch64 check_vector_size workaround_bpiall_vbar1_fiq_aarch32 vector_entry workaround_bpiall_vbar1_serror_aarch32 b report_unhandled_exception -workaround_bpiall_vbar1_serror: - ldp x0, x1, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X0] - b serror_aarch64 check_vector_size workaround_bpiall_vbar1_serror_aarch32 From 1d6d47a82a9aafc17d084738f79dc0c8d40dff45 Mon Sep 17 00:00:00 2001 From: Dimitris Papastamos Date: Mon, 8 Jan 2018 13:57:39 +0000 Subject: [PATCH 6/6] Optimize SMCCC_ARCH_WORKAROUND_1 on Cortex A57/A72/A73 and A75 This patch implements a fast path for this SMC call on affected PEs by detecting and returning immediately after executing the workaround. NOTE: The MMU disable/enable workaround now assumes that the MMU was enabled on entry to EL3. This is a valid assumption as the code turns on the MMU after reset and leaves it on until the core powers off. Change-Id: I13c336d06a52297620a9760fb2461b4d606a30b3 Signed-off-by: Dimitris Papastamos --- .../aarch64/workaround_cve_2017_5715_bpiall.S | 19 ++++++ .../aarch64/workaround_cve_2017_5715_mmu.S | 62 ++++++++++++++----- 2 files changed, 67 insertions(+), 14 deletions(-) diff --git a/lib/cpus/aarch64/workaround_cve_2017_5715_bpiall.S b/lib/cpus/aarch64/workaround_cve_2017_5715_bpiall.S index 9677e2e05..cd8249732 100644 --- a/lib/cpus/aarch64/workaround_cve_2017_5715_bpiall.S +++ b/lib/cpus/aarch64/workaround_cve_2017_5715_bpiall.S @@ -5,6 +5,7 @@ */ #include +#include #include #include @@ -12,6 +13,7 @@ #define EMIT_BPIALL 0xee070fd5 #define EMIT_SMC 0xe1600070 +#define ESR_EL3_A64_SMC0 0x5e000000 .macro enter_workaround _from_vector /* @@ -303,6 +305,23 @@ vector_entry workaround_bpiall_vbar1_sync_exception_aarch32 /* Fast path Sync exceptions. Static predictor will fall through. */ tbz w2, #0, workaround_not_sync + + /* + * Check if SMC is coming from A64 state on #0 + * with W0 = SMCCC_ARCH_WORKAROUND_1 + * + * This sequence evaluates as: + * (W0==SMCCC_ARCH_WORKAROUND_1) ? (ESR_EL3==SMC#0) : (NE) + * allowing use of a single branch operation + */ + orr w2, wzr, #SMCCC_ARCH_WORKAROUND_1 + cmp w0, w2 + mov_imm w2, ESR_EL3_A64_SMC0 + ccmp w3, w2, #0, eq + /* Static predictor will predict a fall through */ + bne 1f + eret +1: ldp x2, x3, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X2] b sync_exception_aarch64 check_vector_size workaround_bpiall_vbar1_sync_exception_aarch32 diff --git a/lib/cpus/aarch64/workaround_cve_2017_5715_mmu.S b/lib/cpus/aarch64/workaround_cve_2017_5715_mmu.S index f4781484c..b24b620c8 100644 --- a/lib/cpus/aarch64/workaround_cve_2017_5715_mmu.S +++ b/lib/cpus/aarch64/workaround_cve_2017_5715_mmu.S @@ -1,26 +1,60 @@ /* - * Copyright (c) 2017, ARM Limited and Contributors. All rights reserved. + * Copyright (c) 2017-2018, ARM Limited and Contributors. All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ #include +#include #include #include .globl workaround_mmu_runtime_exceptions +#define ESR_EL3_A64_SMC0 0x5e000000 + vector_base workaround_mmu_runtime_exceptions - .macro apply_workaround + .macro apply_workaround _is_sync_exception stp x0, x1, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X0] - mrs x0, sctlr_el3 + mrs x1, sctlr_el3 /* Disable MMU */ - bic x1, x0, #SCTLR_M_BIT + bic x1, x1, #SCTLR_M_BIT msr sctlr_el3, x1 isb - /* Restore MMU config */ - msr sctlr_el3, x0 + /* Enable MMU */ + orr x1, x1, #SCTLR_M_BIT + msr sctlr_el3, x1 + /* + * Defer ISB to avoid synchronizing twice in case we hit + * the workaround SMC call which will implicitly synchronize + * because of the ERET instruction. + */ + + /* + * Ensure SMC is coming from A64 state on #0 + * with W0 = SMCCC_ARCH_WORKAROUND_1 + * + * This sequence evaluates as: + * (W0==SMCCC_ARCH_WORKAROUND_1) ? (ESR_EL3==SMC#0) : (NE) + * allowing use of a single branch operation + */ + .if \_is_sync_exception + orr w1, wzr, #SMCCC_ARCH_WORKAROUND_1 + cmp w0, w1 + mrs x0, esr_el3 + mov_imm w1, ESR_EL3_A64_SMC0 + ccmp w0, w1, #0, eq + /* Static predictor will predict a fall through */ + bne 1f + eret +1: + .endif + + /* + * Synchronize now to enable the MMU. This is required + * to ensure the load pair below reads the data stored earlier. + */ isb ldp x0, x1, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X0] .endm @@ -70,22 +104,22 @@ vector_entry workaround_mmu_serror_sp_elx * --------------------------------------------------------------------- */ vector_entry workaround_mmu_sync_exception_aarch64 - apply_workaround + apply_workaround _is_sync_exception=1 b sync_exception_aarch64 check_vector_size workaround_mmu_sync_exception_aarch64 vector_entry workaround_mmu_irq_aarch64 - apply_workaround + apply_workaround _is_sync_exception=0 b irq_aarch64 check_vector_size workaround_mmu_irq_aarch64 vector_entry workaround_mmu_fiq_aarch64 - apply_workaround + apply_workaround _is_sync_exception=0 b fiq_aarch64 check_vector_size workaround_mmu_fiq_aarch64 vector_entry workaround_mmu_serror_aarch64 - apply_workaround + apply_workaround _is_sync_exception=0 b serror_aarch64 check_vector_size workaround_mmu_serror_aarch64 @@ -94,21 +128,21 @@ vector_entry workaround_mmu_serror_aarch64 * --------------------------------------------------------------------- */ vector_entry workaround_mmu_sync_exception_aarch32 - apply_workaround + apply_workaround _is_sync_exception=1 b sync_exception_aarch32 check_vector_size workaround_mmu_sync_exception_aarch32 vector_entry workaround_mmu_irq_aarch32 - apply_workaround + apply_workaround _is_sync_exception=0 b irq_aarch32 check_vector_size workaround_mmu_irq_aarch32 vector_entry workaround_mmu_fiq_aarch32 - apply_workaround + apply_workaround _is_sync_exception=0 b fiq_aarch32 check_vector_size workaround_mmu_fiq_aarch32 vector_entry workaround_mmu_serror_aarch32 - apply_workaround + apply_workaround _is_sync_exception=0 b serror_aarch32 check_vector_size workaround_mmu_serror_aarch32