Optimize SMCCC_ARCH_WORKAROUND_1 on Cortex A57/A72/A73 and A75
This patch implements a fast path for this SMC call on affected PEs by detecting and returning immediately after executing the workaround. NOTE: The MMU disable/enable workaround now assumes that the MMU was enabled on entry to EL3. This is a valid assumption as the code turns on the MMU after reset and leaves it on until the core powers off. Change-Id: I13c336d06a52297620a9760fb2461b4d606a30b3 Signed-off-by: Dimitris Papastamos <dimitris.papastamos@arm.com>
This commit is contained in:
Dimitris Papastamos
parent
d9bd656cf5
commit
1d6d47a82a
|
|
@ -5,6 +5,7 @@
|
|||
*/
|
||||
|
||||
#include <arch.h>
|
||||
#include <arm_arch_svc.h>
|
||||
#include <asm_macros.S>
|
||||
#include <context.h>
|
||||
|
||||
|
|
@ -12,6 +13,7 @@
|
|||
|
||||
#define EMIT_BPIALL 0xee070fd5
|
||||
#define EMIT_SMC 0xe1600070
|
||||
#define ESR_EL3_A64_SMC0 0x5e000000
|
||||
|
||||
.macro enter_workaround _from_vector
|
||||
/*
|
||||
|
|
@ -303,6 +305,23 @@ vector_entry workaround_bpiall_vbar1_sync_exception_aarch32
|
|||
|
||||
/* Fast path Sync exceptions. Static predictor will fall through. */
|
||||
tbz w2, #0, workaround_not_sync
|
||||
|
||||
/*
|
||||
* Check if SMC is coming from A64 state on #0
|
||||
* with W0 = SMCCC_ARCH_WORKAROUND_1
|
||||
*
|
||||
* This sequence evaluates as:
|
||||
* (W0==SMCCC_ARCH_WORKAROUND_1) ? (ESR_EL3==SMC#0) : (NE)
|
||||
* allowing use of a single branch operation
|
||||
*/
|
||||
orr w2, wzr, #SMCCC_ARCH_WORKAROUND_1
|
||||
cmp w0, w2
|
||||
mov_imm w2, ESR_EL3_A64_SMC0
|
||||
ccmp w3, w2, #0, eq
|
||||
/* Static predictor will predict a fall through */
|
||||
bne 1f
|
||||
eret
|
||||
1:
|
||||
ldp x2, x3, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X2]
|
||||
b sync_exception_aarch64
|
||||
check_vector_size workaround_bpiall_vbar1_sync_exception_aarch32
|
||||
|
|
|
|||
|
|
@ -1,26 +1,60 @@
|
|||
/*
|
||||
* Copyright (c) 2017, ARM Limited and Contributors. All rights reserved.
|
||||
* Copyright (c) 2017-2018, ARM Limited and Contributors. All rights reserved.
|
||||
*
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*/
|
||||
|
||||
#include <arch.h>
|
||||
#include <arm_arch_svc.h>
|
||||
#include <asm_macros.S>
|
||||
#include <context.h>
|
||||
|
||||
.globl workaround_mmu_runtime_exceptions
|
||||
|
||||
#define ESR_EL3_A64_SMC0 0x5e000000
|
||||
|
||||
vector_base workaround_mmu_runtime_exceptions
|
||||
|
||||
.macro apply_workaround
|
||||
.macro apply_workaround _is_sync_exception
|
||||
stp x0, x1, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X0]
|
||||
mrs x0, sctlr_el3
|
||||
mrs x1, sctlr_el3
|
||||
/* Disable MMU */
|
||||
bic x1, x0, #SCTLR_M_BIT
|
||||
bic x1, x1, #SCTLR_M_BIT
|
||||
msr sctlr_el3, x1
|
||||
isb
|
||||
/* Restore MMU config */
|
||||
msr sctlr_el3, x0
|
||||
/* Enable MMU */
|
||||
orr x1, x1, #SCTLR_M_BIT
|
||||
msr sctlr_el3, x1
|
||||
/*
|
||||
* Defer ISB to avoid synchronizing twice in case we hit
|
||||
* the workaround SMC call which will implicitly synchronize
|
||||
* because of the ERET instruction.
|
||||
*/
|
||||
|
||||
/*
|
||||
* Ensure SMC is coming from A64 state on #0
|
||||
* with W0 = SMCCC_ARCH_WORKAROUND_1
|
||||
*
|
||||
* This sequence evaluates as:
|
||||
* (W0==SMCCC_ARCH_WORKAROUND_1) ? (ESR_EL3==SMC#0) : (NE)
|
||||
* allowing use of a single branch operation
|
||||
*/
|
||||
.if \_is_sync_exception
|
||||
orr w1, wzr, #SMCCC_ARCH_WORKAROUND_1
|
||||
cmp w0, w1
|
||||
mrs x0, esr_el3
|
||||
mov_imm w1, ESR_EL3_A64_SMC0
|
||||
ccmp w0, w1, #0, eq
|
||||
/* Static predictor will predict a fall through */
|
||||
bne 1f
|
||||
eret
|
||||
1:
|
||||
.endif
|
||||
|
||||
/*
|
||||
* Synchronize now to enable the MMU. This is required
|
||||
* to ensure the load pair below reads the data stored earlier.
|
||||
*/
|
||||
isb
|
||||
ldp x0, x1, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X0]
|
||||
.endm
|
||||
|
|
@ -70,22 +104,22 @@ vector_entry workaround_mmu_serror_sp_elx
|
|||
* ---------------------------------------------------------------------
|
||||
*/
|
||||
vector_entry workaround_mmu_sync_exception_aarch64
|
||||
apply_workaround
|
||||
apply_workaround _is_sync_exception=1
|
||||
b sync_exception_aarch64
|
||||
check_vector_size workaround_mmu_sync_exception_aarch64
|
||||
|
||||
vector_entry workaround_mmu_irq_aarch64
|
||||
apply_workaround
|
||||
apply_workaround _is_sync_exception=0
|
||||
b irq_aarch64
|
||||
check_vector_size workaround_mmu_irq_aarch64
|
||||
|
||||
vector_entry workaround_mmu_fiq_aarch64
|
||||
apply_workaround
|
||||
apply_workaround _is_sync_exception=0
|
||||
b fiq_aarch64
|
||||
check_vector_size workaround_mmu_fiq_aarch64
|
||||
|
||||
vector_entry workaround_mmu_serror_aarch64
|
||||
apply_workaround
|
||||
apply_workaround _is_sync_exception=0
|
||||
b serror_aarch64
|
||||
check_vector_size workaround_mmu_serror_aarch64
|
||||
|
||||
|
|
@ -94,21 +128,21 @@ vector_entry workaround_mmu_serror_aarch64
|
|||
* ---------------------------------------------------------------------
|
||||
*/
|
||||
vector_entry workaround_mmu_sync_exception_aarch32
|
||||
apply_workaround
|
||||
apply_workaround _is_sync_exception=1
|
||||
b sync_exception_aarch32
|
||||
check_vector_size workaround_mmu_sync_exception_aarch32
|
||||
|
||||
vector_entry workaround_mmu_irq_aarch32
|
||||
apply_workaround
|
||||
apply_workaround _is_sync_exception=0
|
||||
b irq_aarch32
|
||||
check_vector_size workaround_mmu_irq_aarch32
|
||||
|
||||
vector_entry workaround_mmu_fiq_aarch32
|
||||
apply_workaround
|
||||
apply_workaround _is_sync_exception=0
|
||||
b fiq_aarch32
|
||||
check_vector_size workaround_mmu_fiq_aarch32
|
||||
|
||||
vector_entry workaround_mmu_serror_aarch32
|
||||
apply_workaround
|
||||
apply_workaround _is_sync_exception=0
|
||||
b serror_aarch32
|
||||
check_vector_size workaround_mmu_serror_aarch32
|
||||
|
|
|
|||
Loading…
Reference in New Issue