From 883d1b5d4a1a4da956310a951df9eb2056c84597 Mon Sep 17 00:00:00 2001 From: Antonio Nino Diaz Date: Fri, 23 Feb 2018 15:07:54 +0000 Subject: [PATCH 1/2] Add comments about mismatched TCR_ELx and xlat tables When the MMU is enabled and the translation tables are mapped, data read/writes to the translation tables are made using the attributes specified in the translation tables themselves. However, the MMU performs table walks with the attributes specified in TCR_ELx. They are completely independent, so special care has to be taken to make sure that they are the same. This has to be done manually because it is not practical to have a test in the code. Such a test would need to know the virtual memory region that contains the translation tables and check that for all of the tables the attributes match the ones in TCR_ELx. As the tables may not even be mapped at all, this isn't a test that can be made generic. The flags used by enable_mmu_xxx() have been moved to the same header where the functions are. Also, some comments in the linker scripts related to the translation tables have been fixed. Change-Id: I1754768bffdae75f53561b1c4a5baf043b45a304 Signed-off-by: Antonio Nino Diaz --- bl1/bl1.ld.S | 5 +-- bl2/bl2.ld.S | 5 +-- bl2/bl2_el3.ld.S | 5 +-- bl2u/bl2u.ld.S | 5 +-- bl31/aarch64/bl31_entrypoint.S | 4 +-- bl31/bl31.ld.S | 5 +-- bl32/sp_min/sp_min.ld.S | 5 +-- bl32/tsp/tsp.ld.S | 5 +-- include/lib/xlat_tables/xlat_mmu_helpers.h | 42 ++++++++++++++++++++-- include/lib/xlat_tables/xlat_tables_defs.h | 15 ++------ plat/mediatek/mt6795/bl31.ld.S | 5 +-- 11 files changed, 68 insertions(+), 33 deletions(-) diff --git a/bl1/bl1.ld.S b/bl1/bl1.ld.S index e4c454b31..26c0ae4b3 100644 --- a/bl1/bl1.ld.S +++ b/bl1/bl1.ld.S @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2017, ARM Limited and Contributors. All rights reserved. + * Copyright (c) 2013-2018, ARM Limited and Contributors. All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ @@ -129,7 +129,8 @@ SECTIONS /* * The xlat_table section is for full, aligned page tables (4K). * Removing them from .bss avoids forcing 4K alignment on - * the .bss section and eliminates the unecessary zero init + * the .bss section. The tables are initialized to zero by the translation + * tables library. */ xlat_table (NOLOAD) : { *(xlat_table) diff --git a/bl2/bl2.ld.S b/bl2/bl2.ld.S index 4fe78f9e9..69c22eb32 100644 --- a/bl2/bl2.ld.S +++ b/bl2/bl2.ld.S @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2017, ARM Limited and Contributors. All rights reserved. + * Copyright (c) 2013-2018, ARM Limited and Contributors. All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ @@ -108,7 +108,8 @@ SECTIONS /* * The xlat_table section is for full, aligned page tables (4K). * Removing them from .bss avoids forcing 4K alignment on - * the .bss section and eliminates the unecessary zero init + * the .bss section. The tables are initialized to zero by the translation + * tables library. */ xlat_table (NOLOAD) : { *(xlat_table) diff --git a/bl2/bl2_el3.ld.S b/bl2/bl2_el3.ld.S index 57709e352..3728643ca 100644 --- a/bl2/bl2_el3.ld.S +++ b/bl2/bl2_el3.ld.S @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, ARM Limited and Contributors. All rights reserved. + * Copyright (c) 2017-2018, ARM Limited and Contributors. All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ @@ -139,7 +139,8 @@ SECTIONS /* * The xlat_table section is for full, aligned page tables (4K). * Removing them from .bss avoids forcing 4K alignment on - * the .bss section and eliminates the unnecessary zero init + * the .bss section. The tables are initialized to zero by the translation + * tables library. */ xlat_table (NOLOAD) : { *(xlat_table) diff --git a/bl2u/bl2u.ld.S b/bl2u/bl2u.ld.S index da587172e..7b97758d9 100644 --- a/bl2u/bl2u.ld.S +++ b/bl2u/bl2u.ld.S @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2017, ARM Limited and Contributors. All rights reserved. + * Copyright (c) 2015-2018, ARM Limited and Contributors. All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ @@ -95,7 +95,8 @@ SECTIONS /* * The xlat_table section is for full, aligned page tables (4K). * Removing them from .bss avoids forcing 4K alignment on - * the .bss section and eliminates the unecessary zero init + * the .bss section. The tables are initialized to zero by the translation + * tables library. */ xlat_table (NOLOAD) : { *(xlat_table) diff --git a/bl31/aarch64/bl31_entrypoint.S b/bl31/aarch64/bl31_entrypoint.S index 419927d8e..e7d94b509 100644 --- a/bl31/aarch64/bl31_entrypoint.S +++ b/bl31/aarch64/bl31_entrypoint.S @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2017, ARM Limited and Contributors. All rights reserved. + * Copyright (c) 2013-2018, ARM Limited and Contributors. All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ @@ -9,7 +9,7 @@ #include #include #include -#include +#include .globl bl31_entrypoint .globl bl31_warm_entrypoint diff --git a/bl31/bl31.ld.S b/bl31/bl31.ld.S index dd046c431..c6a4fe499 100644 --- a/bl31/bl31.ld.S +++ b/bl31/bl31.ld.S @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2017, ARM Limited and Contributors. All rights reserved. + * Copyright (c) 2013-2018, ARM Limited and Contributors. All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ @@ -217,7 +217,8 @@ SECTIONS /* * The xlat_table section is for full, aligned page tables (4K). * Removing them from .bss avoids forcing 4K alignment on - * the .bss section and eliminates the unecessary zero init + * the .bss section. The tables are initialized to zero by the translation + * tables library. */ xlat_table (NOLOAD) : { #if ENABLE_SPM diff --git a/bl32/sp_min/sp_min.ld.S b/bl32/sp_min/sp_min.ld.S index e798a0d38..71de88397 100644 --- a/bl32/sp_min/sp_min.ld.S +++ b/bl32/sp_min/sp_min.ld.S @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2017, ARM Limited and Contributors. All rights reserved. + * Copyright (c) 2016-2018, ARM Limited and Contributors. All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ @@ -176,7 +176,8 @@ SECTIONS /* * The xlat_table section is for full, aligned page tables (4K). * Removing them from .bss avoids forcing 4K alignment on - * the .bss section and eliminates the unecessary zero init + * the .bss section. The tables are initialized to zero by the translation + * tables library. */ xlat_table (NOLOAD) : { *(xlat_table) diff --git a/bl32/tsp/tsp.ld.S b/bl32/tsp/tsp.ld.S index d256b46c8..31c5a67e0 100644 --- a/bl32/tsp/tsp.ld.S +++ b/bl32/tsp/tsp.ld.S @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2017, ARM Limited and Contributors. All rights reserved. + * Copyright (c) 2013-2018, ARM Limited and Contributors. All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ @@ -94,7 +94,8 @@ SECTIONS /* * The xlat_table section is for full, aligned page tables (4K). * Removing them from .bss avoids forcing 4K alignment on - * the .bss section and eliminates the unecessary zero init + * the .bss section. The tables are initialized to zero by the translation + * tables library. */ xlat_table (NOLOAD) : { *(xlat_table) diff --git a/include/lib/xlat_tables/xlat_mmu_helpers.h b/include/lib/xlat_tables/xlat_mmu_helpers.h index fd3efc3f1..d83d7640f 100644 --- a/include/lib/xlat_tables/xlat_mmu_helpers.h +++ b/include/lib/xlat_tables/xlat_mmu_helpers.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014-2017, ARM Limited and Contributors. All rights reserved. + * Copyright (c) 2014-2018, ARM Limited and Contributors. All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ @@ -7,13 +7,51 @@ #ifndef __XLAT_MMU_HELPERS_H__ #define __XLAT_MMU_HELPERS_H__ +/* + * The following flags are passed to enable_mmu_xxx() to override the default + * values used to program system registers while enabling the MMU. + */ + +/* + * When this flag is used, all data access to Normal memory from this EL and all + * Normal memory accesses to the translation tables of this EL are non-cacheable + * for all levels of data and unified cache until the caches are enabled by + * setting the bit SCTLR_ELx.C. + */ +#define DISABLE_DCACHE (U(1) << 0) + +/* + * Mark the translation tables as non-cacheable for the MMU table walker, which + * is a different observer from the PE/CPU. If the flag is not specified, the + * tables are cacheable for the MMU table walker. + * + * Note that, as far as the PE/CPU observer is concerned, the attributes used + * are the ones specified in the translation tables themselves. The MAIR + * register specifies the cacheability through the field AttrIndx of the lower + * attributes of the translation tables. The shareability is specified in the SH + * field of the lower attributes. + * + * The MMU table walker uses the attributes specified in the fields ORGNn, IRGNn + * and SHn of the TCR register to access the translation tables. + * + * The attributes specified in the TCR register and the tables can be different + * as there are no checks to prevent that. Special care must be taken to ensure + * that there aren't mismatches. The behaviour in that case is described in the + * sections 'Mismatched memory attributes' in the ARMv8 ARM. + */ +#define XLAT_TABLE_NC (U(1) << 1) + +#ifndef __ASSEMBLY__ + #ifdef AARCH32 /* AArch32 specific translation table API */ -void enable_mmu_secure(uint32_t flags); +void enable_mmu_secure(unsigned int flags); #else /* AArch64 specific translation table APIs */ void enable_mmu_el1(unsigned int flags); void enable_mmu_el3(unsigned int flags); #endif /* AARCH32 */ +#endif /* __ASSEMBLY__ */ + #endif /* __XLAT_MMU_HELPERS_H__ */ diff --git a/include/lib/xlat_tables/xlat_tables_defs.h b/include/lib/xlat_tables/xlat_tables_defs.h index 3a7f2456b..1c84fe07e 100644 --- a/include/lib/xlat_tables/xlat_tables_defs.h +++ b/include/lib/xlat_tables/xlat_tables_defs.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, ARM Limited and Contributors. All rights reserved. + * Copyright (c) 2017-2018, ARM Limited and Contributors. All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ @@ -9,6 +9,7 @@ #include #include +#include /* Miscellaneous MMU related constants */ #define NUM_2MB_IN_GB (U(1) << 9) @@ -165,16 +166,4 @@ #define XN_SHIFT 54 #define UXN_SHIFT XN_SHIFT -/* - * Flags to override default values used to program system registers while - * enabling the MMU. - */ -#define DISABLE_DCACHE (U(1) << 0) - -/* - * This flag marks the translation tables are Non-cacheable for MMU accesses. - * If the flag is not specified, by default the tables are cacheable. - */ -#define XLAT_TABLE_NC (U(1) << 1) - #endif /* __XLAT_TABLES_DEFS_H__ */ diff --git a/plat/mediatek/mt6795/bl31.ld.S b/plat/mediatek/mt6795/bl31.ld.S index eacb1b27f..0fbd3f785 100644 --- a/plat/mediatek/mt6795/bl31.ld.S +++ b/plat/mediatek/mt6795/bl31.ld.S @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2017, ARM Limited and Contributors. All rights reserved. + * Copyright (c) 2016-2018, ARM Limited and Contributors. All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ @@ -132,7 +132,8 @@ SECTIONS /* * The xlat_table section is for full, aligned page tables (4K). * Removing them from .bss avoids forcing 4K alignment on - * the .bss section and eliminates the unecessary zero init + * the .bss section. The tables are initialized to zero by the translation + * tables library. */ xlat_table (NOLOAD) : { *(xlat_table) From 264410306381d4edceeb03b3a0e8db66605427be Mon Sep 17 00:00:00 2001 From: Antonio Nino Diaz Date: Mon, 19 Feb 2018 16:27:06 +0000 Subject: [PATCH 2/2] Invalidate TLB entries during warm boot During the warm boot sequence: 1. The MMU is enabled with the data cache disabled. The MMU table walker is set up to access the translation tables as in cacheable memory, but its accesses are non-cacheable because SCTLR_EL3.C controls them as well. 2. The interconnect is set up and the CPU enters coherency with the rest of the system. 3. The data cache is enabled. If the support for dynamic translation tables is enabled and another CPU makes changes to a region, the changes may only be present in the data cache, not in RAM. The CPU that is booting isn't in coherency with the rest of the system, so the table walker of that CPU isn't either. This means that it may read old entries from RAM and it may have invalid TLB entries corresponding to the dynamic mappings. This is not a problem for the boot code because the mapping is 1:1 and the regions are static. However, the code that runs after the boot sequence may need to access the dynamically mapped regions. This patch invalidates all TLBs during warm boot when the dynamic translation tables support is enabled to prevent this problem. Change-Id: I80264802dc0aa1cb3edd77d0b66b91db6961af3d Signed-off-by: Antonio Nino Diaz --- lib/psci/aarch32/psci_helpers.S | 24 +++++++++++++++++++++++- lib/psci/aarch64/psci_helpers.S | 24 +++++++++++++++++++++++- 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/lib/psci/aarch32/psci_helpers.S b/lib/psci/aarch32/psci_helpers.S index 9373d4f16..a29a29c49 100644 --- a/lib/psci/aarch32/psci_helpers.S +++ b/lib/psci/aarch32/psci_helpers.S @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, ARM Limited and Contributors. All rights reserved. + * Copyright (c) 2016-2018, ARM Limited and Contributors. All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ @@ -91,6 +91,28 @@ func psci_do_pwrup_cache_maintenance stcopr r0, SCTLR isb +#if PLAT_XLAT_TABLES_DYNAMIC + /* --------------------------------------------- + * During warm boot the MMU is enabled with data + * cache disabled, then the interconnect is set + * up and finally the data cache is enabled. + * + * During this period, if another CPU modifies + * the translation tables, the MMU table walker + * may read the old entries. This is only a + * problem for dynamic regions, the warm boot + * code isn't affected because it is static. + * + * Invalidate all TLB entries loaded while the + * CPU wasn't coherent with the rest of the + * system. + * --------------------------------------------- + */ + stcopr r0, TLBIALL + dsb ish + isb +#endif + pop {r12, pc} endfunc psci_do_pwrup_cache_maintenance diff --git a/lib/psci/aarch64/psci_helpers.S b/lib/psci/aarch64/psci_helpers.S index afe21ebe4..d37ca764a 100644 --- a/lib/psci/aarch64/psci_helpers.S +++ b/lib/psci/aarch64/psci_helpers.S @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014-2016, ARM Limited and Contributors. All rights reserved. + * Copyright (c) 2014-2018, ARM Limited and Contributors. All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ @@ -115,6 +115,28 @@ func psci_do_pwrup_cache_maintenance msr sctlr_el3, x0 isb +#if PLAT_XLAT_TABLES_DYNAMIC + /* --------------------------------------------- + * During warm boot the MMU is enabled with data + * cache disabled, then the interconnect is set + * up and finally the data cache is enabled. + * + * During this period, if another CPU modifies + * the translation tables, the MMU table walker + * may read the old entries. This is only a + * problem for dynamic regions, the warm boot + * code isn't affected because it is static. + * + * Invalidate all TLB entries loaded while the + * CPU wasn't coherent with the rest of the + * system. + * --------------------------------------------- + */ + tlbi alle3 + dsb ish + isb +#endif + ldp x29, x30, [sp], #16 ret endfunc psci_do_pwrup_cache_maintenance