2017-10-17 14:03:14 +01:00
|
|
|
/*
|
2020-10-02 19:41:11 +01:00
|
|
|
* Copyright (c) 2017-2021, ARM Limited and Contributors. All rights reserved.
|
2017-10-17 14:03:14 +01:00
|
|
|
*
|
|
|
|
* SPDX-License-Identifier: BSD-3-Clause
|
|
|
|
*/
|
|
|
|
|
2020-07-14 08:17:56 +01:00
|
|
|
#include <assert.h>
|
2021-05-26 11:58:23 +01:00
|
|
|
#include <cdefs.h>
|
2018-12-14 00:18:21 +00:00
|
|
|
#include <stdbool.h>
|
|
|
|
|
2017-10-17 14:03:14 +01:00
|
|
|
#include <arch.h>
|
|
|
|
#include <arch_helpers.h>
|
2020-07-14 08:17:56 +01:00
|
|
|
|
2018-12-14 00:18:21 +00:00
|
|
|
#include <lib/el3_runtime/pubsub_events.h>
|
|
|
|
#include <lib/extensions/amu.h>
|
|
|
|
#include <lib/extensions/amu_private.h>
|
2017-11-28 13:47:06 +00:00
|
|
|
|
2020-07-14 08:17:56 +01:00
|
|
|
#include <plat/common/platform.h>
|
2017-11-28 13:47:06 +00:00
|
|
|
|
|
|
|
static struct amu_ctx amu_ctxs[PLATFORM_CORE_COUNT];
|
2017-10-17 14:03:14 +01:00
|
|
|
|
2021-05-26 11:58:23 +01:00
|
|
|
static inline __unused uint32_t read_id_pfr0_amu(void)
|
2017-10-17 14:03:14 +01:00
|
|
|
{
|
2021-05-26 11:58:23 +01:00
|
|
|
return (read_id_pfr0() >> ID_PFR0_AMU_SHIFT) &
|
2020-10-02 19:41:11 +01:00
|
|
|
ID_PFR0_AMU_MASK;
|
2017-12-21 15:21:20 +00:00
|
|
|
}
|
|
|
|
|
2021-05-26 11:58:23 +01:00
|
|
|
static inline __unused void write_hcptr_tam(uint32_t value)
|
2020-07-14 08:17:56 +01:00
|
|
|
{
|
2021-05-26 11:58:23 +01:00
|
|
|
write_hcptr((read_hcptr() & ~TAM_BIT) |
|
|
|
|
((value << TAM_SHIFT) & TAM_BIT));
|
|
|
|
}
|
2020-07-14 08:17:56 +01:00
|
|
|
|
2021-05-26 11:58:23 +01:00
|
|
|
static inline __unused void write_amcr_cg1rz(uint32_t value)
|
|
|
|
{
|
|
|
|
write_amcr((read_amcr() & ~AMCR_CG1RZ_BIT) |
|
|
|
|
((value << AMCR_CG1RZ_SHIFT) & AMCR_CG1RZ_BIT));
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline __unused uint32_t read_amcfgr_ncg(void)
|
|
|
|
{
|
|
|
|
return (read_amcfgr() >> AMCFGR_NCG_SHIFT) &
|
|
|
|
AMCFGR_NCG_MASK;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline __unused uint32_t read_amcgcr_cg1nc(void)
|
|
|
|
{
|
|
|
|
return (read_amcgcr() >> AMCGCR_CG1NC_SHIFT) &
|
|
|
|
AMCGCR_CG1NC_MASK;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline __unused uint32_t read_amcntenset0_px(void)
|
|
|
|
{
|
|
|
|
return (read_amcntenset0() >> AMCNTENSET0_Pn_SHIFT) &
|
|
|
|
AMCNTENSET0_Pn_MASK;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline __unused uint32_t read_amcntenset1_px(void)
|
|
|
|
{
|
|
|
|
return (read_amcntenset1() >> AMCNTENSET1_Pn_SHIFT) &
|
|
|
|
AMCNTENSET1_Pn_MASK;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline __unused void write_amcntenset0_px(uint32_t px)
|
|
|
|
{
|
|
|
|
uint32_t value = read_amcntenset0();
|
|
|
|
|
|
|
|
value &= ~AMCNTENSET0_Pn_MASK;
|
|
|
|
value |= (px << AMCNTENSET0_Pn_SHIFT) &
|
|
|
|
AMCNTENSET0_Pn_MASK;
|
|
|
|
|
|
|
|
write_amcntenset0(value);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline __unused void write_amcntenset1_px(uint32_t px)
|
|
|
|
{
|
|
|
|
uint32_t value = read_amcntenset1();
|
|
|
|
|
|
|
|
value &= ~AMCNTENSET1_Pn_MASK;
|
|
|
|
value |= (px << AMCNTENSET1_Pn_SHIFT) &
|
|
|
|
AMCNTENSET1_Pn_MASK;
|
|
|
|
|
|
|
|
write_amcntenset1(value);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline __unused void write_amcntenclr0_px(uint32_t px)
|
|
|
|
{
|
|
|
|
uint32_t value = read_amcntenclr0();
|
|
|
|
|
|
|
|
value &= ~AMCNTENCLR0_Pn_MASK;
|
|
|
|
value |= (px << AMCNTENCLR0_Pn_SHIFT) & AMCNTENCLR0_Pn_MASK;
|
|
|
|
|
|
|
|
write_amcntenclr0(value);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline __unused void write_amcntenclr1_px(uint32_t px)
|
|
|
|
{
|
|
|
|
uint32_t value = read_amcntenclr1();
|
|
|
|
|
|
|
|
value &= ~AMCNTENCLR1_Pn_MASK;
|
|
|
|
value |= (px << AMCNTENCLR1_Pn_SHIFT) & AMCNTENCLR1_Pn_MASK;
|
|
|
|
|
|
|
|
write_amcntenclr1(value);
|
|
|
|
}
|
|
|
|
|
|
|
|
static bool amu_supported(void)
|
|
|
|
{
|
|
|
|
return read_id_pfr0_amu() >= ID_PFR0_AMU_V1;
|
|
|
|
}
|
|
|
|
|
|
|
|
static bool amu_v1p1_supported(void)
|
|
|
|
{
|
|
|
|
return read_id_pfr0_amu() >= ID_PFR0_AMU_V1P1;
|
|
|
|
}
|
|
|
|
|
|
|
|
#if ENABLE_AMU_AUXILIARY_COUNTERS
|
|
|
|
static bool amu_group1_supported(void)
|
|
|
|
{
|
|
|
|
return read_amcfgr_ncg() > 0U;
|
2020-07-14 08:17:56 +01:00
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Enable counters. This function is meant to be invoked
|
|
|
|
* by the context management library before exiting from EL3.
|
|
|
|
*/
|
2018-10-25 16:52:26 +01:00
|
|
|
void amu_enable(bool el2_unused)
|
2017-12-21 15:21:20 +00:00
|
|
|
{
|
2021-05-26 11:58:23 +01:00
|
|
|
if (!amu_supported()) {
|
2017-11-13 09:49:45 +00:00
|
|
|
return;
|
2020-07-14 08:17:56 +01:00
|
|
|
}
|
|
|
|
|
2021-05-25 10:42:56 +01:00
|
|
|
#if ENABLE_AMU_AUXILIARY_COUNTERS
|
|
|
|
if (AMU_GROUP1_NR_COUNTERS > 0U) {
|
|
|
|
/* Check and set presence of group 1 counters */
|
|
|
|
if (!amu_group1_supported()) {
|
|
|
|
ERROR("AMU Counter Group 1 is not implemented\n");
|
|
|
|
panic();
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Check number of group 1 counters */
|
|
|
|
uint32_t cnt_num = read_amcgcr_cg1nc();
|
2020-07-14 08:17:56 +01:00
|
|
|
|
2021-05-25 10:42:56 +01:00
|
|
|
VERBOSE("%s%u. %s%u\n",
|
|
|
|
"Number of AMU Group 1 Counters ", cnt_num,
|
|
|
|
"Requested number ", AMU_GROUP1_NR_COUNTERS);
|
|
|
|
|
|
|
|
if (cnt_num < AMU_GROUP1_NR_COUNTERS) {
|
|
|
|
ERROR("%s%u is less than %s%u\n",
|
|
|
|
"Number of AMU Group 1 Counters ", cnt_num,
|
|
|
|
"Requested number ", AMU_GROUP1_NR_COUNTERS);
|
|
|
|
panic();
|
|
|
|
}
|
2020-07-14 08:17:56 +01:00
|
|
|
}
|
|
|
|
#endif
|
2017-10-17 14:03:14 +01:00
|
|
|
|
2017-11-13 09:49:45 +00:00
|
|
|
if (el2_unused) {
|
|
|
|
/*
|
|
|
|
* Non-secure access from EL0 or EL1 to the Activity Monitor
|
|
|
|
* registers do not trap to EL2.
|
|
|
|
*/
|
2021-05-26 11:58:23 +01:00
|
|
|
write_hcptr_tam(0U);
|
2017-10-17 14:03:14 +01:00
|
|
|
}
|
2017-11-13 09:49:45 +00:00
|
|
|
|
|
|
|
/* Enable group 0 counters */
|
2021-05-26 11:58:23 +01:00
|
|
|
write_amcntenset0_px(AMU_GROUP0_COUNTERS_MASK);
|
2017-12-21 15:21:20 +00:00
|
|
|
|
2021-05-25 10:42:56 +01:00
|
|
|
#if ENABLE_AMU_AUXILIARY_COUNTERS
|
|
|
|
if (AMU_GROUP1_NR_COUNTERS > 0U) {
|
|
|
|
/* Enable group 1 counters */
|
|
|
|
write_amcntenset1_px(AMU_GROUP1_COUNTERS_MASK);
|
|
|
|
}
|
2020-07-14 08:17:56 +01:00
|
|
|
#endif
|
2020-10-02 19:41:11 +01:00
|
|
|
|
|
|
|
/* Initialize FEAT_AMUv1p1 features if present. */
|
2021-05-26 11:58:23 +01:00
|
|
|
if (!amu_v1p1_supported()) {
|
2020-10-02 19:41:11 +01:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
#if AMU_RESTRICT_COUNTERS
|
|
|
|
/*
|
|
|
|
* FEAT_AMUv1p1 adds a register field to restrict access to group 1
|
|
|
|
* counters at all but the highest implemented EL. This is controlled
|
|
|
|
* with the AMU_RESTRICT_COUNTERS compile time flag, when set, system
|
|
|
|
* register reads at lower ELs return zero. Reads from the memory
|
|
|
|
* mapped view are unaffected.
|
|
|
|
*/
|
|
|
|
VERBOSE("AMU group 1 counter access restricted.\n");
|
2021-05-26 11:58:23 +01:00
|
|
|
write_amcr_cg1rz(1U);
|
2020-10-02 19:41:11 +01:00
|
|
|
#else
|
2021-05-26 11:58:23 +01:00
|
|
|
write_amcr_cg1rz(0U);
|
2020-10-02 19:41:11 +01:00
|
|
|
#endif
|
2017-12-21 15:21:20 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/* Read the group 0 counter identified by the given `idx`. */
|
2021-05-24 21:00:07 +01:00
|
|
|
static uint64_t amu_group0_cnt_read(unsigned int idx)
|
2017-12-21 15:21:20 +00:00
|
|
|
{
|
2021-05-26 11:58:23 +01:00
|
|
|
assert(amu_supported());
|
2020-07-14 08:17:56 +01:00
|
|
|
assert(idx < AMU_GROUP0_NR_COUNTERS);
|
2017-12-21 15:21:20 +00:00
|
|
|
|
|
|
|
return amu_group0_cnt_read_internal(idx);
|
|
|
|
}
|
|
|
|
|
2020-07-14 08:17:56 +01:00
|
|
|
/* Write the group 0 counter identified by the given `idx` with `val` */
|
2021-05-24 21:00:07 +01:00
|
|
|
static void amu_group0_cnt_write(unsigned int idx, uint64_t val)
|
2017-12-21 15:21:20 +00:00
|
|
|
{
|
2021-05-26 11:58:23 +01:00
|
|
|
assert(amu_supported());
|
2020-07-14 08:17:56 +01:00
|
|
|
assert(idx < AMU_GROUP0_NR_COUNTERS);
|
2017-12-21 15:21:20 +00:00
|
|
|
|
|
|
|
amu_group0_cnt_write_internal(idx, val);
|
|
|
|
isb();
|
|
|
|
}
|
|
|
|
|
2021-05-25 10:42:56 +01:00
|
|
|
#if ENABLE_AMU_AUXILIARY_COUNTERS
|
2020-07-14 08:17:56 +01:00
|
|
|
/* Read the group 1 counter identified by the given `idx` */
|
2021-05-24 21:00:07 +01:00
|
|
|
static uint64_t amu_group1_cnt_read(unsigned int idx)
|
2017-12-21 15:21:20 +00:00
|
|
|
{
|
2021-05-26 11:58:23 +01:00
|
|
|
assert(amu_supported());
|
2020-07-14 08:17:56 +01:00
|
|
|
assert(amu_group1_supported());
|
|
|
|
assert(idx < AMU_GROUP1_NR_COUNTERS);
|
2017-12-21 15:21:20 +00:00
|
|
|
|
|
|
|
return amu_group1_cnt_read_internal(idx);
|
|
|
|
}
|
|
|
|
|
2020-07-14 08:17:56 +01:00
|
|
|
/* Write the group 1 counter identified by the given `idx` with `val` */
|
2021-05-24 21:00:07 +01:00
|
|
|
static void amu_group1_cnt_write(unsigned int idx, uint64_t val)
|
2017-12-21 15:21:20 +00:00
|
|
|
{
|
2021-05-26 11:58:23 +01:00
|
|
|
assert(amu_supported());
|
2020-07-14 08:17:56 +01:00
|
|
|
assert(amu_group1_supported());
|
|
|
|
assert(idx < AMU_GROUP1_NR_COUNTERS);
|
2017-12-21 15:21:20 +00:00
|
|
|
|
|
|
|
amu_group1_cnt_write_internal(idx, val);
|
|
|
|
isb();
|
|
|
|
}
|
2021-05-25 10:42:56 +01:00
|
|
|
#endif
|
2017-11-28 13:47:06 +00:00
|
|
|
|
|
|
|
static void *amu_context_save(const void *arg)
|
|
|
|
{
|
2020-07-14 08:17:56 +01:00
|
|
|
struct amu_ctx *ctx = &amu_ctxs[plat_my_core_pos()];
|
|
|
|
unsigned int i;
|
2017-11-28 13:47:06 +00:00
|
|
|
|
2021-05-26 11:58:23 +01:00
|
|
|
if (!amu_supported()) {
|
2017-11-28 13:47:06 +00:00
|
|
|
return (void *)-1;
|
2020-07-14 08:17:56 +01:00
|
|
|
}
|
2017-11-28 13:47:06 +00:00
|
|
|
|
2021-05-25 10:42:56 +01:00
|
|
|
#if ENABLE_AMU_AUXILIARY_COUNTERS
|
|
|
|
if (AMU_GROUP1_NR_COUNTERS > 0U) {
|
|
|
|
if (!amu_group1_supported()) {
|
|
|
|
return (void *)-1;
|
|
|
|
}
|
2020-07-14 08:17:56 +01:00
|
|
|
}
|
|
|
|
#endif
|
2021-05-25 10:42:56 +01:00
|
|
|
|
2020-07-14 08:17:56 +01:00
|
|
|
/* Assert that group 0/1 counter configuration is what we expect */
|
2021-05-26 11:58:23 +01:00
|
|
|
assert(read_amcntenset0_px() == AMU_GROUP0_COUNTERS_MASK);
|
2017-11-28 13:47:06 +00:00
|
|
|
|
2021-05-25 10:42:56 +01:00
|
|
|
#if ENABLE_AMU_AUXILIARY_COUNTERS
|
|
|
|
if (AMU_GROUP1_NR_COUNTERS > 0U) {
|
|
|
|
assert(read_amcntenset1_px() == AMU_GROUP1_COUNTERS_MASK);
|
|
|
|
}
|
2020-07-14 08:17:56 +01:00
|
|
|
#endif
|
2017-11-28 13:47:06 +00:00
|
|
|
/*
|
2020-07-14 08:17:56 +01:00
|
|
|
* Disable group 0/1 counters to avoid other observers like SCP sampling
|
2017-11-28 13:47:06 +00:00
|
|
|
* counter values from the future via the memory mapped view.
|
|
|
|
*/
|
2021-05-26 11:58:23 +01:00
|
|
|
write_amcntenclr0_px(AMU_GROUP0_COUNTERS_MASK);
|
2020-07-14 08:17:56 +01:00
|
|
|
|
2021-05-25 10:42:56 +01:00
|
|
|
#if ENABLE_AMU_AUXILIARY_COUNTERS
|
|
|
|
if (AMU_GROUP1_NR_COUNTERS > 0U) {
|
|
|
|
write_amcntenclr1_px(AMU_GROUP1_COUNTERS_MASK);
|
|
|
|
}
|
2020-07-14 08:17:56 +01:00
|
|
|
#endif
|
2021-05-25 10:42:56 +01:00
|
|
|
|
2017-11-28 13:47:06 +00:00
|
|
|
isb();
|
|
|
|
|
2020-07-14 08:17:56 +01:00
|
|
|
/* Save all group 0 counters */
|
|
|
|
for (i = 0U; i < AMU_GROUP0_NR_COUNTERS; i++) {
|
2017-12-21 15:21:20 +00:00
|
|
|
ctx->group0_cnts[i] = amu_group0_cnt_read(i);
|
2020-07-14 08:17:56 +01:00
|
|
|
}
|
2017-12-21 15:21:20 +00:00
|
|
|
|
2021-05-25 10:42:56 +01:00
|
|
|
#if ENABLE_AMU_AUXILIARY_COUNTERS
|
|
|
|
if (AMU_GROUP1_NR_COUNTERS > 0U) {
|
|
|
|
/* Save group 1 counters */
|
|
|
|
for (i = 0U; i < AMU_GROUP1_NR_COUNTERS; i++) {
|
|
|
|
if ((AMU_GROUP1_COUNTERS_MASK & (1U << i)) != 0U) {
|
|
|
|
ctx->group1_cnts[i] = amu_group1_cnt_read(i);
|
|
|
|
}
|
2020-07-14 08:17:56 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
#endif
|
2021-05-25 10:42:56 +01:00
|
|
|
|
2018-10-25 16:52:26 +01:00
|
|
|
return (void *)0;
|
2017-11-28 13:47:06 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void *amu_context_restore(const void *arg)
|
|
|
|
{
|
2020-07-14 08:17:56 +01:00
|
|
|
struct amu_ctx *ctx = &amu_ctxs[plat_my_core_pos()];
|
|
|
|
unsigned int i;
|
2017-11-28 13:47:06 +00:00
|
|
|
|
2021-05-26 11:58:23 +01:00
|
|
|
if (!amu_supported()) {
|
2017-11-28 13:47:06 +00:00
|
|
|
return (void *)-1;
|
2020-07-14 08:17:56 +01:00
|
|
|
}
|
2017-11-28 13:47:06 +00:00
|
|
|
|
2021-05-25 10:42:56 +01:00
|
|
|
#if ENABLE_AMU_AUXILIARY_COUNTERS
|
|
|
|
if (AMU_GROUP1_NR_COUNTERS > 0U) {
|
|
|
|
if (!amu_group1_supported()) {
|
|
|
|
return (void *)-1;
|
|
|
|
}
|
2020-07-14 08:17:56 +01:00
|
|
|
}
|
|
|
|
#endif
|
2021-05-25 10:42:56 +01:00
|
|
|
|
2017-11-28 13:47:06 +00:00
|
|
|
/* Counters were disabled in `amu_context_save()` */
|
2021-05-26 11:58:23 +01:00
|
|
|
assert(read_amcntenset0_px() == 0U);
|
2020-07-14 08:17:56 +01:00
|
|
|
|
2021-05-25 10:42:56 +01:00
|
|
|
#if ENABLE_AMU_AUXILIARY_COUNTERS
|
|
|
|
if (AMU_GROUP1_NR_COUNTERS > 0U) {
|
|
|
|
assert(read_amcntenset1_px() == 0U);
|
|
|
|
}
|
2020-07-14 08:17:56 +01:00
|
|
|
#endif
|
2017-11-28 13:47:06 +00:00
|
|
|
|
2020-07-14 08:17:56 +01:00
|
|
|
/* Restore all group 0 counters */
|
|
|
|
for (i = 0U; i < AMU_GROUP0_NR_COUNTERS; i++) {
|
2017-12-21 15:21:20 +00:00
|
|
|
amu_group0_cnt_write(i, ctx->group0_cnts[i]);
|
2020-07-14 08:17:56 +01:00
|
|
|
}
|
2017-11-28 13:47:06 +00:00
|
|
|
|
2020-07-14 08:17:56 +01:00
|
|
|
/* Restore group 0 counter configuration */
|
2021-05-26 11:58:23 +01:00
|
|
|
write_amcntenset0_px(AMU_GROUP0_COUNTERS_MASK);
|
2017-11-28 13:47:06 +00:00
|
|
|
|
2021-05-25 10:42:56 +01:00
|
|
|
#if ENABLE_AMU_AUXILIARY_COUNTERS
|
|
|
|
if (AMU_GROUP1_NR_COUNTERS > 0U) {
|
|
|
|
/* Restore group 1 counters */
|
|
|
|
for (i = 0U; i < AMU_GROUP1_NR_COUNTERS; i++) {
|
|
|
|
if ((AMU_GROUP1_COUNTERS_MASK & (1U << i)) != 0U) {
|
|
|
|
amu_group1_cnt_write(i, ctx->group1_cnts[i]);
|
|
|
|
}
|
2020-07-14 08:17:56 +01:00
|
|
|
}
|
|
|
|
|
2021-05-25 10:42:56 +01:00
|
|
|
/* Restore group 1 counter configuration */
|
|
|
|
write_amcntenset1_px(AMU_GROUP1_COUNTERS_MASK);
|
|
|
|
}
|
2020-07-14 08:17:56 +01:00
|
|
|
#endif
|
|
|
|
|
2018-10-25 16:52:26 +01:00
|
|
|
return (void *)0;
|
2017-11-28 13:47:06 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
SUBSCRIBE_TO_EVENT(psci_suspend_pwrdown_start, amu_context_save);
|
|
|
|
SUBSCRIBE_TO_EVENT(psci_suspend_pwrdown_finish, amu_context_restore);
|