From ec834925f3cb5cb3934010bbc8077293e610d2ac Mon Sep 17 00:00:00 2001 From: Madhukar Pappireddy Date: Wed, 15 May 2019 18:25:41 -0500 Subject: [PATCH 1/3] GICv3: Enable multi socket GIC redistributor frame discovery This patch provides declaration and definition of new GICv3 driver API: gicv3_rdistif_probe().This function delegates the responsibility of discovering the corresponding Redistributor base frame to each CPU itself. It is a modified version of gicv3_rdistif_base_addrs_probe() and is executed by each CPU in the platform unlike the previous approach in which only the Primary CPU did the discovery of all the Redistributor frames for every CPU. The flush operations as part of gicv3_driver_init() function are made necessary even for platforms with WARMBOOT_ENABLE_DCACHE_EARLY because the GICv3 driver data structure contents are accessed by CPU with D-Cache turned off during power down operations. Change-Id: I1833e81d3974b32a3e4a3df4766a33d070982268 Signed-off-by: Madhukar Pappireddy --- drivers/arm/gic/v3/gicv3_main.c | 145 ++++++++++++++++++++++++-------- include/drivers/arm/gicv3.h | 3 +- 2 files changed, 114 insertions(+), 34 deletions(-) diff --git a/drivers/arm/gic/v3/gicv3_main.c b/drivers/arm/gic/v3/gicv3_main.c index 94a20ba07..f37133096 100644 --- a/drivers/arm/gic/v3/gicv3_main.c +++ b/drivers/arm/gic/v3/gicv3_main.c @@ -16,7 +16,6 @@ #include "gicv3_private.h" const gicv3_driver_data_t *gicv3_driver_data; -static unsigned int gicv2_compat; /* * Spinlock to guard registers needing read-modify-write. APIs protected by this @@ -60,51 +59,61 @@ static spinlock_t gic_lock; void __init gicv3_driver_init(const gicv3_driver_data_t *plat_driver_data) { unsigned int gic_version; + unsigned int gicv2_compat; assert(plat_driver_data != NULL); assert(plat_driver_data->gicd_base != 0U); - assert(plat_driver_data->gicr_base != 0U); assert(plat_driver_data->rdistif_num != 0U); assert(plat_driver_data->rdistif_base_addrs != NULL); assert(IS_IN_EL3()); - assert(plat_driver_data->interrupt_props_num > 0 ? - plat_driver_data->interrupt_props != NULL : 1); + assert((plat_driver_data->interrupt_props_num != 0U) ? + (plat_driver_data->interrupt_props != NULL) : 1); /* Check for system register support */ -#ifdef __aarch64__ +#ifndef __aarch64__ + assert((read_id_pfr1() & + (ID_PFR1_GIC_MASK << ID_PFR1_GIC_SHIFT)) != 0U); +#else assert((read_id_aa64pfr0_el1() & (ID_AA64PFR0_GIC_MASK << ID_AA64PFR0_GIC_SHIFT)) != 0U); -#else - assert((read_id_pfr1() & (ID_PFR1_GIC_MASK << ID_PFR1_GIC_SHIFT)) != 0U); -#endif /* __aarch64__ */ +#endif /* !__aarch64__ */ /* The GIC version should be 3.0 */ gic_version = gicd_read_pidr2(plat_driver_data->gicd_base); - gic_version >>= PIDR2_ARCH_REV_SHIFT; + gic_version >>= PIDR2_ARCH_REV_SHIFT; gic_version &= PIDR2_ARCH_REV_MASK; assert(gic_version == ARCH_REV_GICV3); /* - * Find out whether the GIC supports the GICv2 compatibility mode. The - * ARE_S bit resets to 0 if supported + * Find out whether the GIC supports the GICv2 compatibility mode. + * The ARE_S bit resets to 0 if supported */ gicv2_compat = gicd_read_ctlr(plat_driver_data->gicd_base); gicv2_compat >>= CTLR_ARE_S_SHIFT; - gicv2_compat = !(gicv2_compat & CTLR_ARE_S_MASK); - - /* - * Find the base address of each implemented Redistributor interface. - * The number of interfaces should be equal to the number of CPUs in the - * system. The memory for saving these addresses has to be allocated by - * the platform port - */ - gicv3_rdistif_base_addrs_probe(plat_driver_data->rdistif_base_addrs, - plat_driver_data->rdistif_num, - plat_driver_data->gicr_base, - plat_driver_data->mpidr_to_core_pos); + gicv2_compat = gicv2_compat & CTLR_ARE_S_MASK; + if (plat_driver_data->gicr_base != 0U) { + /* + * Find the base address of each implemented Redistributor interface. + * The number of interfaces should be equal to the number of CPUs in the + * system. The memory for saving these addresses has to be allocated by + * the platform port + */ + gicv3_rdistif_base_addrs_probe(plat_driver_data->rdistif_base_addrs, + plat_driver_data->rdistif_num, + plat_driver_data->gicr_base, + plat_driver_data->mpidr_to_core_pos); +#if !HW_ASSISTED_COHERENCY + /* + * Flush the rdistif_base_addrs[] contents linked to the GICv3 driver. + */ + flush_dcache_range((uintptr_t)(plat_driver_data->rdistif_base_addrs), + plat_driver_data->rdistif_num * + sizeof(*(plat_driver_data->rdistif_base_addrs))); +#endif + } gicv3_driver_data = plat_driver_data; /* @@ -112,19 +121,19 @@ void __init gicv3_driver_init(const gicv3_driver_data_t *plat_driver_data) * enabled. When the secondary CPU boots up, it initializes the * GICC/GICR interface with the caches disabled. Hence flush the * driver data to ensure coherency. This is not required if the - * platform has HW_ASSISTED_COHERENCY or WARMBOOT_ENABLE_DCACHE_EARLY - * enabled. + * platform has HW_ASSISTED_COHERENCY enabled. */ -#if !(HW_ASSISTED_COHERENCY || WARMBOOT_ENABLE_DCACHE_EARLY) - flush_dcache_range((uintptr_t) &gicv3_driver_data, - sizeof(gicv3_driver_data)); - flush_dcache_range((uintptr_t) gicv3_driver_data, - sizeof(*gicv3_driver_data)); +#if !HW_ASSISTED_COHERENCY + flush_dcache_range((uintptr_t)&gicv3_driver_data, + sizeof(gicv3_driver_data)); + flush_dcache_range((uintptr_t)gicv3_driver_data, + sizeof(*gicv3_driver_data)); #endif - INFO("GICv3 %s legacy support detected." - " ARM GICV3 driver initialized in EL3\n", - gicv2_compat ? "with" : "without"); + INFO("GICv3 with%s legacy support detected." + " ARM GICv3 driver initialized in EL3\n", + (gicv2_compat == 0U) ? "" : "out"); + } /******************************************************************************* @@ -192,6 +201,7 @@ void gicv3_rdistif_init(unsigned int proc_num) gicv3_rdistif_on(proc_num); gicr_base = gicv3_driver_data->rdistif_base_addrs[proc_num]; + assert(gicr_base != 0U); /* Set the default attribute of all SGIs and PPIs */ gicv3_ppi_sgi_config_defaults(gicr_base); @@ -313,6 +323,7 @@ void gicv3_cpuif_disable(unsigned int proc_num) /* Mark the connected core as asleep */ gicr_base = gicv3_driver_data->rdistif_base_addrs[proc_num]; + assert(gicr_base != 0U); gicv3_rdistif_mark_core_asleep(gicr_base); } @@ -1081,3 +1092,71 @@ unsigned int gicv3_set_pmr(unsigned int mask) return old_mask; } + +/******************************************************************************* + * This function delegates the responsibility of discovering the corresponding + * Redistributor frames to each CPU itself. It is a modified version of + * gicv3_rdistif_base_addrs_probe() and is executed by each CPU in the platform + * unlike the previous way in which only the Primary CPU did the discovery of + * all the Redistributor frames for every CPU. It also handles the scenario in + * which the frames of various CPUs are not contiguous in physical memory. + ******************************************************************************/ +int gicv3_rdistif_probe(const uintptr_t gicr_frame) +{ + u_register_t mpidr; + unsigned int proc_num, proc_self; + uint64_t typer_val; + uintptr_t rdistif_base; + bool gicr_frame_found = false; + + assert(gicv3_driver_data->gicr_base == 0U); + + /* Ensure this function is called with Data Cache enabled */ +#ifndef __aarch64__ + assert((read_sctlr() & SCTLR_C_BIT) != 0U); +#else + assert((read_sctlr_el3() & SCTLR_C_BIT) != 0U); +#endif /* !__aarch64__ */ + + proc_self = gicv3_driver_data->mpidr_to_core_pos(read_mpidr_el1()); + rdistif_base = gicr_frame; + do { + typer_val = gicr_read_typer(rdistif_base); + if (gicv3_driver_data->mpidr_to_core_pos != NULL) { + mpidr = mpidr_from_gicr_typer(typer_val); + proc_num = gicv3_driver_data->mpidr_to_core_pos(mpidr); + } else { + proc_num = (unsigned int)(typer_val >> TYPER_PROC_NUM_SHIFT) & + TYPER_PROC_NUM_MASK; + } + if (proc_num == proc_self) { + /* The base address doesn't need to be initialized on + * every warm boot. + */ + if (gicv3_driver_data->rdistif_base_addrs[proc_num] != 0U) + return 0; + gicv3_driver_data->rdistif_base_addrs[proc_num] = + rdistif_base; + gicr_frame_found = true; + break; + } + rdistif_base += (uintptr_t)(ULL(1) << GICR_PCPUBASE_SHIFT); + } while ((typer_val & TYPER_LAST_BIT) == 0U); + + if (!gicr_frame_found) + return -1; + + /* + * Flush the driver data to ensure coherency. This is + * not required if platform has HW_ASSISTED_COHERENCY + * enabled. + */ +#if !HW_ASSISTED_COHERENCY + /* + * Flush the rdistif_base_addrs[] contents linked to the GICv3 driver. + */ + flush_dcache_range((uintptr_t)&(gicv3_driver_data->rdistif_base_addrs[proc_num]), + sizeof(*(gicv3_driver_data->rdistif_base_addrs))); +#endif + return 0; /* Found matching GICR frame */ +} diff --git a/include/drivers/arm/gicv3.h b/include/drivers/arm/gicv3.h index 9c72d4dff..c4f42d04d 100644 --- a/include/drivers/arm/gicv3.h +++ b/include/drivers/arm/gicv3.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2018, ARM Limited and Contributors. All rights reserved. + * Copyright (c) 2015-2019, ARM Limited and Contributors. All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ @@ -366,6 +366,7 @@ typedef struct gicv3_its_ctx { * GICv3 EL3 driver API ******************************************************************************/ void gicv3_driver_init(const gicv3_driver_data_t *plat_driver_data); +int gicv3_rdistif_probe(const uintptr_t gicr_frame); void gicv3_distif_init(void); void gicv3_rdistif_init(unsigned int proc_num); void gicv3_rdistif_on(unsigned int proc_num); From 10107707196d67731de57126b846169c5b29aac0 Mon Sep 17 00:00:00 2001 From: Madhukar Pappireddy Date: Mon, 12 Aug 2019 18:31:33 -0500 Subject: [PATCH 2/3] Adding new optional PSCI hook pwr_domain_on_finish_late This PSCI hook is similar to pwr_domain_on_finish but is guaranteed to be invoked with the respective core and cluster are participating in coherency. This will be necessary to safely invoke the new GICv3 API which modifies shared GIC data structures concurrently. Change-Id: I8e54f05c9d4ef5712184c9c18ba45ac97a29eb7a Signed-off-by: Madhukar Pappireddy --- docs/getting_started/porting-guide.rst | 13 +++++++++++++ include/lib/psci/psci.h | 4 +++- lib/psci/psci_on.c | 10 +++++++++- 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/docs/getting_started/porting-guide.rst b/docs/getting_started/porting-guide.rst index 5786dd384..af36b6a5f 100644 --- a/docs/getting_started/porting-guide.rst +++ b/docs/getting_started/porting-guide.rst @@ -2202,6 +2202,19 @@ immediately before the CPU was turned on. It indicates which power domains above the CPU might require initialization due to having previously been in low power states. The generic code expects the handler to succeed. +plat_psci_ops.pwr_domain_on_finish_late() [optional] +........................................................... + +This optional function is called by the PSCI implementation after the calling +CPU is fully powered on with respective data caches enabled. The calling CPU and +the associated cluster are guaranteed to be participating in coherency. This +function gives the flexibility to perform any platform-specific actions safely, +such as initialization or modification of shared data structures, without the +overhead of explicit cache maintainace operations. + +The ``target_state`` has a similar meaning as described in the ``pwr_domain_on_finish()`` +operation. The generic code expects the handler to succeed. + plat_psci_ops.pwr_domain_suspend_finish() ......................................... diff --git a/include/lib/psci/psci.h b/include/lib/psci/psci.h index 04e5e3d72..7f7b7e3ff 100644 --- a/include/lib/psci/psci.h +++ b/include/lib/psci/psci.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2018, ARM Limited and Contributors. All rights reserved. + * Copyright (c) 2013-2019, ARM Limited and Contributors. All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ @@ -301,6 +301,8 @@ typedef struct plat_psci_ops { const psci_power_state_t *target_state); void (*pwr_domain_suspend)(const psci_power_state_t *target_state); void (*pwr_domain_on_finish)(const psci_power_state_t *target_state); + void (*pwr_domain_on_finish_late)( + const psci_power_state_t *target_state); void (*pwr_domain_suspend_finish)( const psci_power_state_t *target_state); void __dead2 (*pwr_domain_pwr_down_wfi)( diff --git a/lib/psci/psci_on.c b/lib/psci/psci_on.c index aa6b324ed..470b4f33e 100644 --- a/lib/psci/psci_on.c +++ b/lib/psci/psci_on.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2018, ARM Limited and Contributors. All rights reserved. + * Copyright (c) 2013-2019, ARM Limited and Contributors. All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ @@ -181,6 +181,14 @@ void psci_cpu_on_finish(int cpu_idx, const psci_power_state_t *state_info) psci_do_pwrup_cache_maintenance(); #endif + /* + * Plat. management: Perform any platform specific actions which + * can only be done with the cpu and the cluster guaranteed to + * be coherent. + */ + if (psci_plat_pm_ops->pwr_domain_on_finish_late != NULL) + psci_plat_pm_ops->pwr_domain_on_finish_late(state_info); + /* * All the platform specific actions for turning this cpu * on have completed. Perform enough arch.initialization From 6806cd2381901d424b40ba3f17d23f5ffa4ca57e Mon Sep 17 00:00:00 2001 From: Madhukar Pappireddy Date: Mon, 10 Jun 2019 16:54:36 -0500 Subject: [PATCH 3/3] Migrate ARM platforms to use the new GICv3 API This patch invokes the new function gicv3_rdistif_probe() in the ARM platform specific gicv3 driver. Since this API modifies the shared GIC related data structure, it must be invoked coherently by using the platform specific pwr_domain_on_finish_late hook. Change-Id: I6efb17d5da61545a1c5a6641b8f58472b31e62a8 Signed-off-by: Madhukar Pappireddy --- include/plat/arm/css/common/css_pm.h | 1 + plat/arm/board/fvp/fvp_pm.c | 18 ++++++++++++++---- plat/arm/common/arm_gicv3.c | 20 ++++++++++++++++++-- plat/arm/css/common/css_pm.c | 22 +++++++++++++++++----- 4 files changed, 50 insertions(+), 11 deletions(-) diff --git a/include/plat/arm/css/common/css_pm.h b/include/plat/arm/css/common/css_pm.h index b82ff47e7..93f86162e 100644 --- a/include/plat/arm/css/common/css_pm.h +++ b/include/plat/arm/css/common/css_pm.h @@ -27,6 +27,7 @@ static inline unsigned int css_system_pwr_state(const psci_power_state_t *state) int css_pwr_domain_on(u_register_t mpidr); void css_pwr_domain_on_finish(const psci_power_state_t *target_state); +void css_pwr_domain_on_finish_late(const psci_power_state_t *target_state); void css_pwr_domain_off(const psci_power_state_t *target_state); void css_pwr_domain_suspend(const psci_power_state_t *target_state); void css_pwr_domain_suspend_finish( diff --git a/plat/arm/board/fvp/fvp_pm.c b/plat/arm/board/fvp/fvp_pm.c index 42dec8dfc..0a62543fa 100644 --- a/plat/arm/board/fvp/fvp_pm.c +++ b/plat/arm/board/fvp/fvp_pm.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2018, ARM Limited and Contributors. All rights reserved. + * Copyright (c) 2013-2019, ARM Limited and Contributors. All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ @@ -247,10 +247,19 @@ static void fvp_pwr_domain_on_finish(const psci_power_state_t *target_state) { fvp_power_domain_on_finish_common(target_state); - /* Enable the gic cpu interface */ +} + +/******************************************************************************* + * FVP handler called when a power domain has just been powered on and the cpu + * and its cluster are fully participating in coherent transaction on the + * interconnect. Data cache must be enabled for CPU at this point. + ******************************************************************************/ +static void fvp_pwr_domain_on_finish_late(const psci_power_state_t *target_state) +{ + /* Program GIC per-cpu distributor or re-distributor interface */ plat_arm_gic_pcpu_init(); - /* Program the gic per-cpu distributor or re-distributor interface */ + /* Enable GIC CPU interface */ plat_arm_gic_cpuif_enable(); } @@ -272,7 +281,7 @@ static void fvp_pwr_domain_suspend_finish(const psci_power_state_t *target_state fvp_power_domain_on_finish_common(target_state); - /* Enable the gic cpu interface */ + /* Enable GIC CPU interface */ plat_arm_gic_cpuif_enable(); } @@ -397,6 +406,7 @@ plat_psci_ops_t plat_arm_psci_pm_ops = { .pwr_domain_off = fvp_pwr_domain_off, .pwr_domain_suspend = fvp_pwr_domain_suspend, .pwr_domain_on_finish = fvp_pwr_domain_on_finish, + .pwr_domain_on_finish_late = fvp_pwr_domain_on_finish_late, .pwr_domain_suspend_finish = fvp_pwr_domain_suspend_finish, .system_off = fvp_system_off, .system_reset = fvp_system_reset, diff --git a/plat/arm/common/arm_gicv3.c b/plat/arm/common/arm_gicv3.c index 7f4957fa9..fef53761c 100644 --- a/plat/arm/common/arm_gicv3.c +++ b/plat/arm/common/arm_gicv3.c @@ -4,6 +4,7 @@ * SPDX-License-Identifier: BSD-3-Clause */ +#include #include #include @@ -67,7 +68,7 @@ static unsigned int arm_gicv3_mpidr_hash(u_register_t mpidr) static const gicv3_driver_data_t arm_gic_data __unused = { .gicd_base = PLAT_ARM_GICD_BASE, - .gicr_base = PLAT_ARM_GICR_BASE, + .gicr_base = 0U, .interrupt_props = arm_interrupt_props, .interrupt_props_num = ARRAY_SIZE(arm_interrupt_props), .rdistif_num = PLATFORM_CORE_COUNT, @@ -86,6 +87,11 @@ void __init plat_arm_gic_driver_init(void) #if (!defined(__aarch64__) && defined(IMAGE_BL32)) || \ (defined(__aarch64__) && defined(IMAGE_BL31)) gicv3_driver_init(&arm_gic_data); + + if (gicv3_rdistif_probe(PLAT_ARM_GICR_BASE) == -1) { + ERROR("No GICR base frame found for Primary CPU\n"); + panic(); + } #endif } @@ -116,10 +122,20 @@ void plat_arm_gic_cpuif_disable(void) } /****************************************************************************** - * ARM common helper to initialize the per-cpu redistributor interface in GICv3 + * ARM common helper function to iterate over all GICR frames and discover the + * corresponding per-cpu redistributor frame as well as initialize the + * corresponding interface in GICv3. At the moment, Arm platforms do not have + * non-contiguous GICR frames. *****************************************************************************/ void plat_arm_gic_pcpu_init(void) { + int result; + + result = gicv3_rdistif_probe(PLAT_ARM_GICR_BASE); + if (result == -1) { + ERROR("No GICR base frame found for CPU 0x%lx\n", read_mpidr()); + panic(); + } gicv3_rdistif_init(plat_my_core_pos()); } diff --git a/plat/arm/css/common/css_pm.c b/plat/arm/css/common/css_pm.c index f6fc6aa7a..01c674f82 100644 --- a/plat/arm/css/common/css_pm.c +++ b/plat/arm/css/common/css_pm.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2018, ARM Limited and Contributors. All rights reserved. + * Copyright (c) 2015-2019, ARM Limited and Contributors. All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ @@ -76,9 +76,6 @@ static void css_pwr_domain_on_finisher_common( { assert(CSS_CORE_PWR_STATE(target_state) == ARM_LOCAL_STATE_OFF); - /* Enable the gic cpu interface */ - plat_arm_gic_cpuif_enable(); - /* * Perform the common cluster specific operations i.e enable coherency * if this cluster was off. @@ -100,10 +97,21 @@ void css_pwr_domain_on_finish(const psci_power_state_t *target_state) /* Assert that the system power domain need not be initialized */ assert(css_system_pwr_state(target_state) == ARM_LOCAL_STATE_RUN); + css_pwr_domain_on_finisher_common(target_state); +} + +/******************************************************************************* + * Handler called when a power domain has just been powered on and the cpu + * and its cluster are fully participating in coherent transaction on the + * interconnect. Data cache must be enabled for CPU at this point. + ******************************************************************************/ +void css_pwr_domain_on_finish_late(const psci_power_state_t *target_state) +{ /* Program the gic per-cpu distributor or re-distributor interface */ plat_arm_gic_pcpu_init(); - css_pwr_domain_on_finisher_common(target_state); + /* Enable the gic cpu interface */ + plat_arm_gic_cpuif_enable(); } /******************************************************************************* @@ -185,6 +193,9 @@ void css_pwr_domain_suspend_finish( arm_system_pwr_domain_resume(); css_pwr_domain_on_finisher_common(target_state); + + /* Enable the gic cpu interface */ + plat_arm_gic_cpuif_enable(); } /******************************************************************************* @@ -306,6 +317,7 @@ static int css_translate_power_state_by_mpidr(u_register_t mpidr, plat_psci_ops_t plat_arm_psci_pm_ops = { .pwr_domain_on = css_pwr_domain_on, .pwr_domain_on_finish = css_pwr_domain_on_finish, + .pwr_domain_on_finish_late = css_pwr_domain_on_finish_late, .pwr_domain_off = css_pwr_domain_off, .cpu_standby = css_cpu_standby, .pwr_domain_suspend = css_pwr_domain_suspend,