From 1a853370ff8e9ff3b147297d1fc7942e7db43dd9 Mon Sep 17 00:00:00 2001 From: David Cunado Date: Fri, 20 Oct 2017 11:30:57 +0100 Subject: [PATCH] Enable SVE for Non-secure world This patch adds a new build option, ENABLE_SVE_FOR_NS, which when set to one EL3 will check to see if the Scalable Vector Extension (SVE) is implemented when entering and exiting the Non-secure world. If SVE is implemented, EL3 will do the following: - Entry to Non-secure world: SIMD, FP and SVE functionality is enabled. - Exit from Non-secure world: SIMD, FP and SVE functionality is disabled. As SIMD and FP registers are part of the SVE Z-registers then any use of SIMD / FP functionality would corrupt the SVE registers. The build option default is 1. The SVE functionality is only supported on AArch64 and so the build option is set to zero when the target archiecture is AArch32. This build option is not compatible with the CTX_INCLUDE_FPREGS - an assert will be raised on platforms where SVE is implemented and both ENABLE_SVE_FOR_NS and CTX_INCLUDE_FPREGS are set to 1. Also note this change prevents secure world use of FP&SIMD registers on SVE-enabled platforms. Existing Secure-EL1 Payloads will not work on such platforms unless ENABLE_SVE_FOR_NS is set to 0. Additionally, on the first entry into the Non-secure world the SVE functionality is enabled and the SVE Z-register length is set to the maximum size allowed by the architecture. This includes the use case where EL2 is implemented but not used. Change-Id: Ie2d733ddaba0b9bef1d7c9765503155188fe7dae Signed-off-by: David Cunado --- Makefile | 2 + bl31/bl31.mk | 4 + docs/user-guide.rst | 11 ++ include/common/aarch64/el3_common_macros.S | 6 +- include/lib/aarch64/arch.h | 17 +++ include/lib/aarch64/arch_helpers.h | 3 + include/lib/extensions/sve.h | 12 ++ lib/el3_runtime/aarch64/context_mgmt.c | 5 + lib/extensions/sve/sve.c | 126 +++++++++++++++++++++ make_helpers/defaults.mk | 9 ++ 10 files changed, 192 insertions(+), 3 deletions(-) create mode 100644 include/lib/extensions/sve.h create mode 100644 lib/extensions/sve/sve.c diff --git a/Makefile b/Makefile index fe18bc0ea..324eb8a18 100644 --- a/Makefile +++ b/Makefile @@ -463,6 +463,7 @@ $(eval $(call assert_boolean,ENABLE_PMF)) $(eval $(call assert_boolean,ENABLE_PSCI_STAT)) $(eval $(call assert_boolean,ENABLE_RUNTIME_INSTRUMENTATION)) $(eval $(call assert_boolean,ENABLE_SPE_FOR_LOWER_ELS)) +$(eval $(call assert_boolean,ENABLE_SVE_FOR_NS)) $(eval $(call assert_boolean,ERROR_DEPRECATED)) $(eval $(call assert_boolean,GENERATE_COT)) $(eval $(call assert_boolean,GICV2_G0_FOR_EL3)) @@ -503,6 +504,7 @@ $(eval $(call add_define,ENABLE_PMF)) $(eval $(call add_define,ENABLE_PSCI_STAT)) $(eval $(call add_define,ENABLE_RUNTIME_INSTRUMENTATION)) $(eval $(call add_define,ENABLE_SPE_FOR_LOWER_ELS)) +$(eval $(call add_define,ENABLE_SVE_FOR_NS)) $(eval $(call add_define,ERROR_DEPRECATED)) $(eval $(call add_define,GICV2_G0_FOR_EL3)) $(eval $(call add_define,HW_ASSISTED_COHERENCY)) diff --git a/bl31/bl31.mk b/bl31/bl31.mk index ebd0e71a9..fdcc93139 100644 --- a/bl31/bl31.mk +++ b/bl31/bl31.mk @@ -54,6 +54,10 @@ ifeq (${ENABLE_AMU},1) BL31_SOURCES += lib/extensions/amu/aarch64/amu.c endif +ifeq (${ENABLE_SVE_FOR_NS},1) +BL31_SOURCES += lib/extensions/sve/sve.c +endif + BL31_LINKERFILE := bl31/bl31.ld.S # Flag used to indicate if Crash reporting via console should be included diff --git a/docs/user-guide.rst b/docs/user-guide.rst index 79d660d02..04acbe7f3 100644 --- a/docs/user-guide.rst +++ b/docs/user-guide.rst @@ -354,6 +354,17 @@ Common build options The default is 1 but is automatically disabled when the target architecture is AArch32. +- ``ENABLE_SVE_FOR_NS``: Boolean option to enable Scalable Vector Extension + (SVE) for the Non-secure world only. SVE is an optional architectural feature + for AArch64. Note that when SVE is enabled for the Non-secure world, access + to SIMD and floating-point functionality from the Secure world is disabled. + This is to avoid corruption of the Non-secure world data in the Z-registers + which are aliased by the SIMD and FP registers. The build option is not + compatible with the ``CTX_INCLUDE_FPREGS`` build option, and will raise an + assert on platforms where SVE is implemented and ``ENABLE_SVE_FOR_NS`` set to + 1. The default is 1 but is automatically disabled when the target + architecture is AArch32. + - ``ENABLE_STACK_PROTECTOR``: String option to enable the stack protection checks in GCC. Allowed values are "all", "strong" and "0" (default). "strong" is the recommended stack protection level if this feature is diff --git a/include/common/aarch64/el3_common_macros.S b/include/common/aarch64/el3_common_macros.S index ed35df82e..63a0fa770 100644 --- a/include/common/aarch64/el3_common_macros.S +++ b/include/common/aarch64/el3_common_macros.S @@ -127,9 +127,9 @@ * CPTR_EL3.TTA: Set to zero so that System register accesses to the * trace registers do not trap to EL3. * - * CPTR_EL3.TFP: Set to zero so that accesses to Advanced SIMD and - * floating-point functionality do not trap to EL3. - * --------------------------------------------------------------------- + * CPTR_EL3.TFP: Set to zero so that accesses to the V- or Z- registers + * by Advanced SIMD, floating-point or SVE instructions (if implemented) + * do not trap to EL3. */ mov_imm x0, (CPTR_EL3_RESET_VAL & ~(TCPAC_BIT | TTA_BIT | TFP_BIT)) msr cptr_el3, x0 diff --git a/include/lib/aarch64/arch.h b/include/lib/aarch64/arch.h index 65e9fc1be..96e2d5fe2 100644 --- a/include/lib/aarch64/arch.h +++ b/include/lib/aarch64/arch.h @@ -114,6 +114,9 @@ #define ID_AA64PFR0_AMU_LENGTH U(4) #define ID_AA64PFR0_AMU_MASK U(0xf) #define ID_AA64PFR0_ELX_MASK U(0xf) +#define ID_AA64PFR0_SVE_SHIFT U(32) +#define ID_AA64PFR0_SVE_MASK U(0xf) +#define ID_AA64PFR0_SVE_LENGTH U(4) /* ID_AA64DFR0_EL1.PMS definitions (for ARMv8.2+) */ #define ID_AA64DFR0_PMS_SHIFT U(32) @@ -301,6 +304,7 @@ #define TAM_BIT (U(1) << 30) #define TTA_BIT (U(1) << 20) #define TFP_BIT (U(1) << 10) +#define CPTR_EZ_BIT (U(1) << 8) #define CPTR_EL3_RESET_VAL U(0x0) /* CPTR_EL2 definitions */ @@ -309,6 +313,7 @@ #define CPTR_EL2_TAM_BIT (U(1) << 30) #define CPTR_EL2_TTA_BIT (U(1) << 20) #define CPTR_EL2_TFP_BIT (U(1) << 10) +#define CPTR_EL2_TZ_BIT (U(1) << 8) #define CPTR_EL2_RESET_VAL CPTR_EL2_RES1 /* CPSR/SPSR definitions */ @@ -555,6 +560,18 @@ #define PMCR_EL0_X_BIT (U(1) << 4) #define PMCR_EL0_D_BIT (U(1) << 3) +/******************************************************************************* + * Definitions for system register interface to SVE + ******************************************************************************/ +#define ZCR_EL3 S3_6_C1_C2_0 +#define ZCR_EL2 S3_4_C1_C2_0 + +/* ZCR_EL3 definitions */ +#define ZCR_EL3_LEN_MASK U(0xf) + +/* ZCR_EL2 definitions */ +#define ZCR_EL2_LEN_MASK U(0xf) + /******************************************************************************* * Definitions of MAIR encodings for device and normal memory ******************************************************************************/ diff --git a/include/lib/aarch64/arch_helpers.h b/include/lib/aarch64/arch_helpers.h index b6be16759..831dfb067 100644 --- a/include/lib/aarch64/arch_helpers.h +++ b/include/lib/aarch64/arch_helpers.h @@ -329,6 +329,9 @@ DEFINE_RENAME_SYSREG_RW_FUNCS(amcntenset1_el0, AMCNTENSET1_EL0) DEFINE_RENAME_SYSREG_RW_FUNCS(pmblimitr_el1, PMBLIMITR_EL1) +DEFINE_RENAME_SYSREG_WRITE_FUNC(zcr_el3, ZCR_EL3) +DEFINE_RENAME_SYSREG_WRITE_FUNC(zcr_el2, ZCR_EL2) + #define IS_IN_EL(x) \ (GET_EL(read_CurrentEl()) == MODE_EL##x) diff --git a/include/lib/extensions/sve.h b/include/lib/extensions/sve.h new file mode 100644 index 000000000..28923e3f3 --- /dev/null +++ b/include/lib/extensions/sve.h @@ -0,0 +1,12 @@ +/* + * Copyright (c) 2017, ARM Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef __SVE_H__ +#define __SVE_H__ + +void sve_enable(int el2_unused); + +#endif /* __SVE_H__ */ diff --git a/lib/el3_runtime/aarch64/context_mgmt.c b/lib/el3_runtime/aarch64/context_mgmt.c index b892729ef..c6c2249a4 100644 --- a/lib/el3_runtime/aarch64/context_mgmt.c +++ b/lib/el3_runtime/aarch64/context_mgmt.c @@ -18,6 +18,7 @@ #include #include #include +#include #include @@ -225,6 +226,10 @@ static void enable_extensions_nonsecure(int el2_unused) #if ENABLE_AMU amu_enable(el2_unused); #endif + +#if ENABLE_SVE_FOR_NS + sve_enable(el2_unused); +#endif #endif } diff --git a/lib/extensions/sve/sve.c b/lib/extensions/sve/sve.c new file mode 100644 index 000000000..14e51bd86 --- /dev/null +++ b/lib/extensions/sve/sve.c @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2017, ARM Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include +#include +#include +#include + +static void *disable_sve_hook(const void *arg) +{ + uint64_t features; + + features = read_id_aa64pfr0_el1() >> ID_AA64PFR0_SVE_SHIFT; + if ((features & ID_AA64PFR0_SVE_MASK) == 1) { + uint64_t cptr; + + /* + * Disable SVE, SIMD and FP access for the Secure world. + * As the SIMD/FP registers are part of the SVE Z-registers, any + * use of SIMD/FP functionality will corrupt the SVE registers. + * Therefore it is necessary to prevent use of SIMD/FP support + * in the Secure world as well as SVE functionality. + */ + cptr = read_cptr_el3(); + cptr = (cptr | TFP_BIT) & ~(CPTR_EZ_BIT); + write_cptr_el3(cptr); + + /* + * No explicit ISB required here as ERET to switch to Secure + * world covers it + */ + } + return 0; +} + +static void *enable_sve_hook(const void *arg) +{ + uint64_t features; + + features = read_id_aa64pfr0_el1() >> ID_AA64PFR0_SVE_SHIFT; + if ((features & ID_AA64PFR0_SVE_MASK) == 1) { + uint64_t cptr; + + /* + * Enable SVE, SIMD and FP access for the Non-secure world. + */ + cptr = read_cptr_el3(); + cptr = (cptr | CPTR_EZ_BIT) & ~(TFP_BIT); + write_cptr_el3(cptr); + + /* + * No explicit ISB required here as ERET to switch to Non-secure + * world covers it + */ + } + return 0; +} + +void sve_enable(int el2_unused) +{ + uint64_t features; + + features = read_id_aa64pfr0_el1() >> ID_AA64PFR0_SVE_SHIFT; + if ((features & ID_AA64PFR0_SVE_MASK) == 1) { + uint64_t cptr; +#if CTX_INCLUDE_FPREGS + /* + * CTX_INCLUDE_FPREGS is not supported on SVE enabled systems. + */ + assert(0); +#endif + /* + * Update CPTR_EL3 to enable access to SVE functionality for the + * Non-secure world. + * NOTE - assumed that CPTR_EL3.TFP is set to allow access to + * the SIMD, floating-point and SVE support. + * + * CPTR_EL3.EZ: Set to 1 to enable access to SVE functionality + * in the Non-secure world. + */ + cptr = read_cptr_el3(); + cptr |= CPTR_EZ_BIT; + write_cptr_el3(cptr); + + /* + * Need explicit ISB here to guarantee that update to ZCR_ELx + * and CPTR_EL2.TZ do not result in trap to EL3. + */ + isb(); + + /* + * Ensure lower ELs have access to full vector length. + */ + write_zcr_el3(ZCR_EL3_LEN_MASK); + + if (el2_unused) { + /* + * Update CPTR_EL2 to enable access to SVE functionality + * for Non-secure world, EL2 and Non-secure EL1 and EL0. + * NOTE - assumed that CPTR_EL2.TFP is set to allow + * access to the SIMD, floating-point and SVE support. + * + * CPTR_EL2.TZ: Set to 0 to enable access to SVE support + * for EL2 and Non-secure EL1 and EL0. + */ + cptr = read_cptr_el2(); + cptr &= ~(CPTR_EL2_TZ_BIT); + write_cptr_el2(cptr); + + /* + * Ensure lower ELs have access to full vector length. + */ + write_zcr_el2(ZCR_EL2_LEN_MASK); + } + /* + * No explicit ISB required here as ERET to switch to + * Non-secure world covers it. + */ + } +} + +SUBSCRIBE_TO_EVENT(cm_exited_normal_world, disable_sve_hook); +SUBSCRIBE_TO_EVENT(cm_entering_normal_world, enable_sve_hook); diff --git a/make_helpers/defaults.mk b/make_helpers/defaults.mk index 9f7abed1e..1d20080eb 100644 --- a/make_helpers/defaults.mk +++ b/make_helpers/defaults.mk @@ -158,3 +158,12 @@ ifeq (${ARCH},aarch32) endif ENABLE_AMU := 0 + +# By default, enable Scalable Vector Extension if implemented for Non-secure +# lower ELs +# Note SVE is only supported on AArch64 - therefore do not enable in AArch32 +ifneq (${ARCH},aarch32) + ENABLE_SVE_FOR_NS := 1 +else + override ENABLE_SVE_FOR_NS := 0 +endif