diff options
Diffstat (limited to 'lib/extensions/amu/aarch64/amu.c')
-rw-r--r-- | lib/extensions/amu/aarch64/amu.c | 656 |
1 files changed, 451 insertions, 205 deletions
diff --git a/lib/extensions/amu/aarch64/amu.c b/lib/extensions/amu/aarch64/amu.c index 295c0d569..d329c3d33 100644 --- a/lib/extensions/amu/aarch64/amu.c +++ b/lib/extensions/amu/aarch64/amu.c @@ -5,86 +5,220 @@ */ #include <assert.h> +#include <cdefs.h> +#include <inttypes.h> #include <stdbool.h> +#include <stdint.h> +#include "../amu_private.h" #include <arch.h> #include <arch_features.h> #include <arch_helpers.h> - +#include <common/debug.h> #include <lib/el3_runtime/pubsub_events.h> #include <lib/extensions/amu.h> -#include <lib/extensions/amu_private.h> #include <plat/common/platform.h> -static struct amu_ctx amu_ctxs[PLATFORM_CORE_COUNT]; +#if ENABLE_AMU_FCONF +# include <lib/fconf/fconf.h> +# include <lib/fconf/fconf_amu_getter.h> +#endif -/* - * Get AMU version value from aa64pfr0. - * Return values - * ID_AA64PFR0_AMU_V1: FEAT_AMUv1 supported (introduced in ARM v8.4) - * ID_AA64PFR0_AMU_V1P1: FEAT_AMUv1p1 supported (introduced in ARM v8.6) - * ID_AA64PFR0_AMU_NOT_SUPPORTED: not supported - */ -unsigned int amu_get_version(void) +#if ENABLE_MPMM +# include <lib/mpmm/mpmm.h> +#endif + +struct amu_ctx { + uint64_t group0_cnts[AMU_GROUP0_MAX_COUNTERS]; +#if ENABLE_AMU_AUXILIARY_COUNTERS + uint64_t group1_cnts[AMU_GROUP1_MAX_COUNTERS]; +#endif + + /* Architected event counter 1 does not have an offset register */ + uint64_t group0_voffsets[AMU_GROUP0_MAX_COUNTERS - 1U]; +#if ENABLE_AMU_AUXILIARY_COUNTERS + uint64_t group1_voffsets[AMU_GROUP1_MAX_COUNTERS]; +#endif + + uint16_t group0_enable; +#if ENABLE_AMU_AUXILIARY_COUNTERS + uint16_t group1_enable; +#endif +}; + +static struct amu_ctx amu_ctxs_[PLATFORM_CORE_COUNT]; + +CASSERT((sizeof(amu_ctxs_[0].group0_enable) * CHAR_BIT) <= AMU_GROUP0_MAX_COUNTERS, + amu_ctx_group0_enable_cannot_represent_all_group0_counters); + +#if ENABLE_AMU_AUXILIARY_COUNTERS +CASSERT((sizeof(amu_ctxs_[0].group1_enable) * CHAR_BIT) <= AMU_GROUP1_MAX_COUNTERS, + amu_ctx_group1_enable_cannot_represent_all_group1_counters); +#endif + +static inline __unused uint64_t read_id_aa64pfr0_el1_amu(void) { - return (unsigned int)(read_id_aa64pfr0_el1() >> ID_AA64PFR0_AMU_SHIFT) & + return (read_id_aa64pfr0_el1() >> ID_AA64PFR0_AMU_SHIFT) & ID_AA64PFR0_AMU_MASK; } -#if AMU_GROUP1_NR_COUNTERS -/* Check if group 1 counters is implemented */ -bool amu_group1_supported(void) +static inline __unused uint64_t read_hcr_el2_amvoffen(void) +{ + return (read_hcr_el2() & HCR_AMVOFFEN_BIT) >> + HCR_AMVOFFEN_SHIFT; +} + +static inline __unused void write_cptr_el2_tam(uint64_t value) { - uint64_t features = read_amcfgr_el0() >> AMCFGR_EL0_NCG_SHIFT; + write_cptr_el2((read_cptr_el2() & ~CPTR_EL2_TAM_BIT) | + ((value << CPTR_EL2_TAM_SHIFT) & CPTR_EL2_TAM_BIT)); +} + +static inline __unused void write_cptr_el3_tam(cpu_context_t *ctx, uint64_t tam) +{ + uint64_t value = read_ctx_reg(get_el3state_ctx(ctx), CTX_CPTR_EL3); - return (features & AMCFGR_EL0_NCG_MASK) == 1U; + value &= ~TAM_BIT; + value |= (tam << TAM_SHIFT) & TAM_BIT; + + write_ctx_reg(get_el3state_ctx(ctx), CTX_CPTR_EL3, value); +} + +static inline __unused void write_hcr_el2_amvoffen(uint64_t value) +{ + write_hcr_el2((read_hcr_el2() & ~HCR_AMVOFFEN_BIT) | + ((value << HCR_AMVOFFEN_SHIFT) & HCR_AMVOFFEN_BIT)); +} + +static inline __unused void write_amcr_el0_cg1rz(uint64_t value) +{ + write_amcr_el0((read_amcr_el0() & ~AMCR_CG1RZ_BIT) | + ((value << AMCR_CG1RZ_SHIFT) & AMCR_CG1RZ_BIT)); +} + +static inline __unused uint64_t read_amcfgr_el0_ncg(void) +{ + return (read_amcfgr_el0() >> AMCFGR_EL0_NCG_SHIFT) & + AMCFGR_EL0_NCG_MASK; +} + +static inline __unused uint64_t read_amcgcr_el0_cg0nc(void) +{ + return (read_amcgcr_el0() >> AMCGCR_EL0_CG0NC_SHIFT) & + AMCGCR_EL0_CG0NC_MASK; +} + +static inline __unused uint64_t read_amcg1idr_el0_voff(void) +{ + return (read_amcg1idr_el0() >> AMCG1IDR_VOFF_SHIFT) & + AMCG1IDR_VOFF_MASK; +} + +static inline __unused uint64_t read_amcgcr_el0_cg1nc(void) +{ + return (read_amcgcr_el0() >> AMCGCR_EL0_CG1NC_SHIFT) & + AMCGCR_EL0_CG1NC_MASK; +} + +static inline __unused uint64_t read_amcntenset0_el0_px(void) +{ + return (read_amcntenset0_el0() >> AMCNTENSET0_EL0_Pn_SHIFT) & + AMCNTENSET0_EL0_Pn_MASK; +} + +static inline __unused uint64_t read_amcntenset1_el0_px(void) +{ + return (read_amcntenset1_el0() >> AMCNTENSET1_EL0_Pn_SHIFT) & + AMCNTENSET1_EL0_Pn_MASK; +} + +static inline __unused void write_amcntenset0_el0_px(uint64_t px) +{ + uint64_t value = read_amcntenset0_el0(); + + value &= ~AMCNTENSET0_EL0_Pn_MASK; + value |= (px << AMCNTENSET0_EL0_Pn_SHIFT) & AMCNTENSET0_EL0_Pn_MASK; + + write_amcntenset0_el0(value); +} + +static inline __unused void write_amcntenset1_el0_px(uint64_t px) +{ + uint64_t value = read_amcntenset1_el0(); + + value &= ~AMCNTENSET1_EL0_Pn_MASK; + value |= (px << AMCNTENSET1_EL0_Pn_SHIFT) & AMCNTENSET1_EL0_Pn_MASK; + + write_amcntenset1_el0(value); +} + +static inline __unused void write_amcntenclr0_el0_px(uint64_t px) +{ + uint64_t value = read_amcntenclr0_el0(); + + value &= ~AMCNTENCLR0_EL0_Pn_MASK; + value |= (px << AMCNTENCLR0_EL0_Pn_SHIFT) & AMCNTENCLR0_EL0_Pn_MASK; + + write_amcntenclr0_el0(value); +} + +static inline __unused void write_amcntenclr1_el0_px(uint64_t px) +{ + uint64_t value = read_amcntenclr1_el0(); + + value &= ~AMCNTENCLR1_EL0_Pn_MASK; + value |= (px << AMCNTENCLR1_EL0_Pn_SHIFT) & AMCNTENCLR1_EL0_Pn_MASK; + + write_amcntenclr1_el0(value); +} + +static __unused bool amu_supported(void) +{ + return read_id_aa64pfr0_el1_amu() >= ID_AA64PFR0_AMU_V1; +} + +static __unused bool amu_v1p1_supported(void) +{ + return read_id_aa64pfr0_el1_amu() >= ID_AA64PFR0_AMU_V1P1; +} + +#if ENABLE_AMU_AUXILIARY_COUNTERS +static __unused bool amu_group1_supported(void) +{ + return read_amcfgr_el0_ncg() > 0U; } #endif /* - * Enable counters. This function is meant to be invoked - * by the context management library before exiting from EL3. + * Enable counters. This function is meant to be invoked by the context + * management library before exiting from EL3. */ void amu_enable(bool el2_unused, cpu_context_t *ctx) { - uint64_t v; - unsigned int amu_version = amu_get_version(); + uint64_t id_aa64pfr0_el1_amu; /* AMU version */ - if (amu_version == ID_AA64PFR0_AMU_NOT_SUPPORTED) { - return; - } + uint64_t amcfgr_el0_ncg; /* Number of counter groups */ + uint64_t amcgcr_el0_cg0nc; /* Number of group 0 counters */ -#if AMU_GROUP1_NR_COUNTERS - /* Check and set presence of group 1 counters */ - if (!amu_group1_supported()) { - ERROR("AMU Counter Group 1 is not implemented\n"); - panic(); - } + uint64_t amcntenset0_el0_px = 0x0; /* Group 0 enable mask */ + uint64_t amcntenset1_el0_px = 0x0; /* Group 1 enable mask */ - /* Check number of group 1 counters */ - uint64_t cnt_num = (read_amcgcr_el0() >> AMCGCR_EL0_CG1NC_SHIFT) & - AMCGCR_EL0_CG1NC_MASK; - VERBOSE("%s%llu. %s%u\n", - "Number of AMU Group 1 Counters ", cnt_num, - "Requested number ", AMU_GROUP1_NR_COUNTERS); - - if (cnt_num < AMU_GROUP1_NR_COUNTERS) { - ERROR("%s%llu is less than %s%u\n", - "Number of AMU Group 1 Counters ", cnt_num, - "Requested number ", AMU_GROUP1_NR_COUNTERS); - panic(); + id_aa64pfr0_el1_amu = read_id_aa64pfr0_el1_amu(); + if (id_aa64pfr0_el1_amu == ID_AA64PFR0_AMU_NOT_SUPPORTED) { + /* + * If the AMU is unsupported, nothing needs to be done. + */ + + return; } -#endif if (el2_unused) { /* - * CPTR_EL2.TAM: Set to zero so any accesses to - * the Activity Monitor registers do not trap to EL2. + * CPTR_EL2.TAM: Set to zero so any accesses to the Activity + * Monitor registers do not trap to EL2. */ - v = read_cptr_el2(); - v &= ~CPTR_EL2_TAM_BIT; - write_cptr_el2(v); + write_cptr_el2_tam(0U); } /* @@ -92,72 +226,141 @@ void amu_enable(bool el2_unused, cpu_context_t *ctx) * in 'ctx'. Set CPTR_EL3.TAM to zero so that any accesses to * the Activity Monitor registers do not trap to EL3. */ - v = read_ctx_reg(get_el3state_ctx(ctx), CTX_CPTR_EL3); - v &= ~TAM_BIT; - write_ctx_reg(get_el3state_ctx(ctx), CTX_CPTR_EL3, v); + write_cptr_el3_tam(ctx, 0U); - /* Enable group 0 counters */ - write_amcntenset0_el0(AMU_GROUP0_COUNTERS_MASK); + /* + * Retrieve the number of architected counters. All of these counters + * are enabled by default. + */ -#if AMU_GROUP1_NR_COUNTERS - /* Enable group 1 counters */ - write_amcntenset1_el0(AMU_GROUP1_COUNTERS_MASK); -#endif + amcgcr_el0_cg0nc = read_amcgcr_el0_cg0nc(); + amcntenset0_el0_px = (UINT64_C(1) << (amcgcr_el0_cg0nc)) - 1U; - /* Initialize FEAT_AMUv1p1 features if present. */ - if (amu_version < ID_AA64PFR0_AMU_V1P1) { - return; - } + assert(amcgcr_el0_cg0nc <= AMU_AMCGCR_CG0NC_MAX); - if (el2_unused) { - /* Make sure virtual offsets are disabled if EL2 not used. */ - write_hcr_el2(read_hcr_el2() & ~HCR_AMVOFFEN_BIT); + /* + * The platform may opt to enable specific auxiliary counters. This can + * be done via the common FCONF getter, or via the platform-implemented + * function. + */ + +#if ENABLE_AMU_AUXILIARY_COUNTERS + const struct amu_topology *topology; + +#if ENABLE_AMU_FCONF + topology = FCONF_GET_PROPERTY(amu, config, topology); +#else + topology = plat_amu_topology(); +#endif /* ENABLE_AMU_FCONF */ + + if (topology != NULL) { + unsigned int core_pos = plat_my_core_pos(); + + amcntenset1_el0_px = topology->cores[core_pos].enable; + } else { + ERROR("AMU: failed to generate AMU topology\n"); } +#endif /* ENABLE_AMU_AUXILIARY_COUNTERS */ -#if AMU_RESTRICT_COUNTERS /* - * FEAT_AMUv1p1 adds a register field to restrict access to group 1 - * counters at all but the highest implemented EL. This is controlled - * with the AMU_RESTRICT_COUNTERS compile time flag, when set, system - * register reads at lower ELs return zero. Reads from the memory - * mapped view are unaffected. + * Enable the requested counters. */ - VERBOSE("AMU group 1 counter access restricted.\n"); - write_amcr_el0(read_amcr_el0() | AMCR_CG1RZ_BIT); + + write_amcntenset0_el0_px(amcntenset0_el0_px); + + amcfgr_el0_ncg = read_amcfgr_el0_ncg(); + if (amcfgr_el0_ncg > 0U) { + write_amcntenset1_el0_px(amcntenset1_el0_px); + +#if !ENABLE_AMU_AUXILIARY_COUNTERS + VERBOSE("AMU: auxiliary counters detected but support is disabled\n"); +#endif + } + + /* Initialize FEAT_AMUv1p1 features if present. */ + if (id_aa64pfr0_el1_amu >= ID_AA64PFR0_AMU_V1P1) { + if (el2_unused) { + /* + * Make sure virtual offsets are disabled if EL2 not + * used. + */ + write_hcr_el2_amvoffen(0U); + } + +#if AMU_RESTRICT_COUNTERS + /* + * FEAT_AMUv1p1 adds a register field to restrict access to + * group 1 counters at all but the highest implemented EL. This + * is controlled with the `AMU_RESTRICT_COUNTERS` compile time + * flag, when set, system register reads at lower ELs return + * zero. Reads from the memory mapped view are unaffected. + */ + VERBOSE("AMU group 1 counter access restricted.\n"); + write_amcr_el0_cg1rz(1U); #else - write_amcr_el0(read_amcr_el0() & ~AMCR_CG1RZ_BIT); + write_amcr_el0_cg1rz(0U); +#endif + } + +#if ENABLE_MPMM + mpmm_enable(); #endif } /* Read the group 0 counter identified by the given `idx`. */ -uint64_t amu_group0_cnt_read(unsigned int idx) +static uint64_t amu_group0_cnt_read(unsigned int idx) { - assert(amu_get_version() != ID_AA64PFR0_AMU_NOT_SUPPORTED); - assert(idx < AMU_GROUP0_NR_COUNTERS); + assert(amu_supported()); + assert(idx < read_amcgcr_el0_cg0nc()); return amu_group0_cnt_read_internal(idx); } /* Write the group 0 counter identified by the given `idx` with `val` */ -void amu_group0_cnt_write(unsigned int idx, uint64_t val) +static void amu_group0_cnt_write(unsigned int idx, uint64_t val) { - assert(amu_get_version() != ID_AA64PFR0_AMU_NOT_SUPPORTED); - assert(idx < AMU_GROUP0_NR_COUNTERS); + assert(amu_supported()); + assert(idx < read_amcgcr_el0_cg0nc()); amu_group0_cnt_write_internal(idx, val); isb(); } /* + * Unlike with auxiliary counters, we cannot detect at runtime whether an + * architected counter supports a virtual offset. These are instead fixed + * according to FEAT_AMUv1p1, but this switch will need to be updated if later + * revisions of FEAT_AMU add additional architected counters. + */ +static bool amu_group0_voffset_supported(uint64_t idx) +{ + switch (idx) { + case 0U: + case 2U: + case 3U: + return true; + + case 1U: + return false; + + default: + ERROR("AMU: can't set up virtual offset for unknown " + "architected counter %" PRIu64 "!\n", idx); + + panic(); + } +} + +/* * Read the group 0 offset register for a given index. Index must be 0, 2, * or 3, the register for 1 does not exist. * * Using this function requires FEAT_AMUv1p1 support. */ -uint64_t amu_group0_voffset_read(unsigned int idx) +static uint64_t amu_group0_voffset_read(unsigned int idx) { - assert(amu_get_version() >= ID_AA64PFR0_AMU_V1P1); - assert(idx < AMU_GROUP0_NR_COUNTERS); + assert(amu_v1p1_supported()); + assert(idx < read_amcgcr_el0_cg0nc()); assert(idx != 1U); return amu_group0_voffset_read_internal(idx); @@ -169,33 +372,33 @@ uint64_t amu_group0_voffset_read(unsigned int idx) * * Using this function requires FEAT_AMUv1p1 support. */ -void amu_group0_voffset_write(unsigned int idx, uint64_t val) +static void amu_group0_voffset_write(unsigned int idx, uint64_t val) { - assert(amu_get_version() >= ID_AA64PFR0_AMU_V1P1); - assert(idx < AMU_GROUP0_NR_COUNTERS); + assert(amu_v1p1_supported()); + assert(idx < read_amcgcr_el0_cg0nc()); assert(idx != 1U); amu_group0_voffset_write_internal(idx, val); isb(); } -#if AMU_GROUP1_NR_COUNTERS +#if ENABLE_AMU_AUXILIARY_COUNTERS /* Read the group 1 counter identified by the given `idx` */ -uint64_t amu_group1_cnt_read(unsigned int idx) +static uint64_t amu_group1_cnt_read(unsigned int idx) { - assert(amu_get_version() != ID_AA64PFR0_AMU_NOT_SUPPORTED); + assert(amu_supported()); assert(amu_group1_supported()); - assert(idx < AMU_GROUP1_NR_COUNTERS); + assert(idx < read_amcgcr_el0_cg1nc()); return amu_group1_cnt_read_internal(idx); } /* Write the group 1 counter identified by the given `idx` with `val` */ -void amu_group1_cnt_write(unsigned int idx, uint64_t val) +static void amu_group1_cnt_write(unsigned int idx, uint64_t val) { - assert(amu_get_version() != ID_AA64PFR0_AMU_NOT_SUPPORTED); + assert(amu_supported()); assert(amu_group1_supported()); - assert(idx < AMU_GROUP1_NR_COUNTERS); + assert(idx < read_amcgcr_el0_cg1nc()); amu_group1_cnt_write_internal(idx, val); isb(); @@ -206,13 +409,12 @@ void amu_group1_cnt_write(unsigned int idx, uint64_t val) * * Using this function requires FEAT_AMUv1p1 support. */ -uint64_t amu_group1_voffset_read(unsigned int idx) +static uint64_t amu_group1_voffset_read(unsigned int idx) { - assert(amu_get_version() >= ID_AA64PFR0_AMU_V1P1); + assert(amu_v1p1_supported()); assert(amu_group1_supported()); - assert(idx < AMU_GROUP1_NR_COUNTERS); - assert(((read_amcg1idr_el0() >> AMCG1IDR_VOFF_SHIFT) & - (1ULL << idx)) != 0ULL); + assert(idx < read_amcgcr_el0_cg1nc()); + assert((read_amcg1idr_el0_voff() & (UINT64_C(1) << idx)) != 0U); return amu_group1_voffset_read_internal(idx); } @@ -222,167 +424,211 @@ uint64_t amu_group1_voffset_read(unsigned int idx) * * Using this function requires FEAT_AMUv1p1 support. */ -void amu_group1_voffset_write(unsigned int idx, uint64_t val) +static void amu_group1_voffset_write(unsigned int idx, uint64_t val) { - assert(amu_get_version() >= ID_AA64PFR0_AMU_V1P1); + assert(amu_v1p1_supported()); assert(amu_group1_supported()); - assert(idx < AMU_GROUP1_NR_COUNTERS); - assert(((read_amcg1idr_el0() >> AMCG1IDR_VOFF_SHIFT) & - (1ULL << idx)) != 0ULL); + assert(idx < read_amcgcr_el0_cg1nc()); + assert((read_amcg1idr_el0_voff() & (UINT64_C(1) << idx)) != 0U); amu_group1_voffset_write_internal(idx, val); isb(); } +#endif -/* - * Program the event type register for the given `idx` with - * the event number `val` - */ -void amu_group1_set_evtype(unsigned int idx, unsigned int val) +static void *amu_context_save(const void *arg) { - assert(amu_get_version() != ID_AA64PFR0_AMU_NOT_SUPPORTED); - assert(amu_group1_supported()); - assert(idx < AMU_GROUP1_NR_COUNTERS); + uint64_t i, j; - amu_group1_set_evtype_internal(idx, val); - isb(); -} -#endif /* AMU_GROUP1_NR_COUNTERS */ + unsigned int core_pos; + struct amu_ctx *ctx; -static void *amu_context_save(const void *arg) -{ - struct amu_ctx *ctx = &amu_ctxs[plat_my_core_pos()]; - unsigned int i; + uint64_t id_aa64pfr0_el1_amu; /* AMU version */ + uint64_t hcr_el2_amvoffen; /* AMU virtual offsets enabled */ + uint64_t amcgcr_el0_cg0nc; /* Number of group 0 counters */ - if (amu_get_version() == ID_AA64PFR0_AMU_NOT_SUPPORTED) { - return (void *)-1; - } +#if ENABLE_AMU_AUXILIARY_COUNTERS + uint64_t amcg1idr_el0_voff; /* Auxiliary counters with virtual offsets */ + uint64_t amcfgr_el0_ncg; /* Number of counter groups */ + uint64_t amcgcr_el0_cg1nc; /* Number of group 1 counters */ +#endif -#if AMU_GROUP1_NR_COUNTERS - if (!amu_group1_supported()) { - return (void *)-1; + id_aa64pfr0_el1_amu = read_id_aa64pfr0_el1_amu(); + if (id_aa64pfr0_el1_amu == ID_AA64PFR0_AMU_NOT_SUPPORTED) { + return (void *)0; } -#endif - /* Assert that group 0/1 counter configuration is what we expect */ - assert(read_amcntenset0_el0() == AMU_GROUP0_COUNTERS_MASK); -#if AMU_GROUP1_NR_COUNTERS - assert(read_amcntenset1_el0() == AMU_GROUP1_COUNTERS_MASK); + core_pos = plat_my_core_pos(); + ctx = &amu_ctxs_[core_pos]; + + amcgcr_el0_cg0nc = read_amcgcr_el0_cg0nc(); + hcr_el2_amvoffen = (id_aa64pfr0_el1_amu >= ID_AA64PFR0_AMU_V1P1) ? + read_hcr_el2_amvoffen() : 0U; + +#if ENABLE_AMU_AUXILIARY_COUNTERS + amcfgr_el0_ncg = read_amcfgr_el0_ncg(); + amcgcr_el0_cg1nc = (amcfgr_el0_ncg > 0U) ? read_amcgcr_el0_cg1nc() : 0U; + amcg1idr_el0_voff = (hcr_el2_amvoffen != 0U) ? read_amcg1idr_el0_voff() : 0U; #endif + /* - * Disable group 0/1 counters to avoid other observers like SCP sampling - * counter values from the future via the memory mapped view. + * Disable all AMU counters. */ - write_amcntenclr0_el0(AMU_GROUP0_COUNTERS_MASK); -#if AMU_GROUP1_NR_COUNTERS - write_amcntenclr1_el0(AMU_GROUP1_COUNTERS_MASK); + ctx->group0_enable = read_amcntenset0_el0_px(); + write_amcntenclr0_el0_px(ctx->group0_enable); + +#if ENABLE_AMU_AUXILIARY_COUNTERS + if (amcfgr_el0_ncg > 0U) { + ctx->group1_enable = read_amcntenset1_el0_px(); + write_amcntenclr1_el0_px(ctx->group1_enable); + } #endif - isb(); - /* Save all group 0 counters */ - for (i = 0U; i < AMU_GROUP0_NR_COUNTERS; i++) { + /* + * Save the counters to the local context. + */ + + isb(); /* Ensure counters have been stopped */ + + for (i = 0U; i < amcgcr_el0_cg0nc; i++) { ctx->group0_cnts[i] = amu_group0_cnt_read(i); } - /* Save group 0 virtual offsets if supported and enabled. */ - if ((amu_get_version() >= ID_AA64PFR0_AMU_V1P1) && - ((read_hcr_el2() & HCR_AMVOFFEN_BIT) != 0ULL)) { - /* Not using a loop because count is fixed and index 1 DNE. */ - ctx->group0_voffsets[0U] = amu_group0_voffset_read(0U); - ctx->group0_voffsets[1U] = amu_group0_voffset_read(2U); - ctx->group0_voffsets[2U] = amu_group0_voffset_read(3U); +#if ENABLE_AMU_AUXILIARY_COUNTERS + for (i = 0U; i < amcgcr_el0_cg1nc; i++) { + ctx->group1_cnts[i] = amu_group1_cnt_read(i); } +#endif + + /* + * Save virtual offsets for counters that offer them. + */ + + if (hcr_el2_amvoffen != 0U) { + for (i = 0U, j = 0U; i < amcgcr_el0_cg0nc; i++) { + if (!amu_group0_voffset_supported(i)) { + continue; /* No virtual offset */ + } -#if AMU_GROUP1_NR_COUNTERS - /* Save group 1 counters */ - for (i = 0U; i < AMU_GROUP1_NR_COUNTERS; i++) { - if ((AMU_GROUP1_COUNTERS_MASK & (1UL << i)) != 0U) { - ctx->group1_cnts[i] = amu_group1_cnt_read(i); + ctx->group0_voffsets[j++] = amu_group0_voffset_read(i); } - } - /* Save group 1 virtual offsets if supported and enabled. */ - if ((amu_get_version() >= ID_AA64PFR0_AMU_V1P1) && - ((read_hcr_el2() & HCR_AMVOFFEN_BIT) != 0ULL)) { - u_register_t amcg1idr = read_amcg1idr_el0() >> - AMCG1IDR_VOFF_SHIFT; - amcg1idr = amcg1idr & AMU_GROUP1_COUNTERS_MASK; - - for (i = 0U; i < AMU_GROUP1_NR_COUNTERS; i++) { - if (((amcg1idr >> i) & 1ULL) != 0ULL) { - ctx->group1_voffsets[i] = - amu_group1_voffset_read(i); +#if ENABLE_AMU_AUXILIARY_COUNTERS + for (i = 0U, j = 0U; i < amcgcr_el0_cg1nc; i++) { + if ((amcg1idr_el0_voff >> i) & 1U) { + continue; /* No virtual offset */ } + + ctx->group1_voffsets[j++] = amu_group1_voffset_read(i); } - } #endif + } + return (void *)0; } static void *amu_context_restore(const void *arg) { - struct amu_ctx *ctx = &amu_ctxs[plat_my_core_pos()]; - unsigned int i; + uint64_t i, j; - if (amu_get_version() == ID_AA64PFR0_AMU_NOT_SUPPORTED) { - return (void *)-1; - } + unsigned int core_pos; + struct amu_ctx *ctx; -#if AMU_GROUP1_NR_COUNTERS - if (!amu_group1_supported()) { - return (void *)-1; - } + uint64_t id_aa64pfr0_el1_amu; /* AMU version */ + + uint64_t hcr_el2_amvoffen; /* AMU virtual offsets enabled */ + + uint64_t amcfgr_el0_ncg; /* Number of counter groups */ + uint64_t amcgcr_el0_cg0nc; /* Number of group 0 counters */ + +#if ENABLE_AMU_AUXILIARY_COUNTERS + uint64_t amcgcr_el0_cg1nc; /* Number of group 1 counters */ + uint64_t amcg1idr_el0_voff; /* Auxiliary counters with virtual offsets */ #endif - /* Counters were disabled in `amu_context_save()` */ - assert(read_amcntenset0_el0() == 0U); -#if AMU_GROUP1_NR_COUNTERS - assert(read_amcntenset1_el0() == 0U); + id_aa64pfr0_el1_amu = read_id_aa64pfr0_el1_amu(); + if (id_aa64pfr0_el1_amu == ID_AA64PFR0_AMU_NOT_SUPPORTED) { + return (void *)0; + } + + core_pos = plat_my_core_pos(); + ctx = &amu_ctxs_[core_pos]; + + amcfgr_el0_ncg = read_amcfgr_el0_ncg(); + amcgcr_el0_cg0nc = read_amcgcr_el0_cg0nc(); + + hcr_el2_amvoffen = (id_aa64pfr0_el1_amu >= ID_AA64PFR0_AMU_V1P1) ? + read_hcr_el2_amvoffen() : 0U; + +#if ENABLE_AMU_AUXILIARY_COUNTERS + amcgcr_el0_cg1nc = (amcfgr_el0_ncg > 0U) ? read_amcgcr_el0_cg1nc() : 0U; + amcg1idr_el0_voff = (hcr_el2_amvoffen != 0U) ? read_amcg1idr_el0_voff() : 0U; #endif - /* Restore all group 0 counters */ - for (i = 0U; i < AMU_GROUP0_NR_COUNTERS; i++) { + /* + * Sanity check that all counters were disabled when the context was + * previously saved. + */ + + assert(read_amcntenset0_el0_px() == 0U); + + if (amcfgr_el0_ncg > 0U) { + assert(read_amcntenset1_el0_px() == 0U); + } + + /* + * Restore the counter values from the local context. + */ + + for (i = 0U; i < amcgcr_el0_cg0nc; i++) { amu_group0_cnt_write(i, ctx->group0_cnts[i]); } - /* Restore group 0 virtual offsets if supported and enabled. */ - if ((amu_get_version() >= ID_AA64PFR0_AMU_V1P1) && - ((read_hcr_el2() & HCR_AMVOFFEN_BIT) != 0ULL)) { - /* Not using a loop because count is fixed and index 1 DNE. */ - amu_group0_voffset_write(0U, ctx->group0_voffsets[0U]); - amu_group0_voffset_write(2U, ctx->group0_voffsets[1U]); - amu_group0_voffset_write(3U, ctx->group0_voffsets[2U]); +#if ENABLE_AMU_AUXILIARY_COUNTERS + for (i = 0U; i < amcgcr_el0_cg1nc; i++) { + amu_group1_cnt_write(i, ctx->group1_cnts[i]); } +#endif + + /* + * Restore virtual offsets for counters that offer them. + */ - /* Restore group 0 counter configuration */ - write_amcntenset0_el0(AMU_GROUP0_COUNTERS_MASK); + if (hcr_el2_amvoffen != 0U) { + for (i = 0U, j = 0U; i < amcgcr_el0_cg0nc; i++) { + if (!amu_group0_voffset_supported(i)) { + continue; /* No virtual offset */ + } -#if AMU_GROUP1_NR_COUNTERS - /* Restore group 1 counters */ - for (i = 0U; i < AMU_GROUP1_NR_COUNTERS; i++) { - if ((AMU_GROUP1_COUNTERS_MASK & (1UL << i)) != 0U) { - amu_group1_cnt_write(i, ctx->group1_cnts[i]); + amu_group0_voffset_write(i, ctx->group0_voffsets[j++]); } - } - /* Restore group 1 virtual offsets if supported and enabled. */ - if ((amu_get_version() >= ID_AA64PFR0_AMU_V1P1) && - ((read_hcr_el2() & HCR_AMVOFFEN_BIT) != 0ULL)) { - u_register_t amcg1idr = read_amcg1idr_el0() >> - AMCG1IDR_VOFF_SHIFT; - amcg1idr = amcg1idr & AMU_GROUP1_COUNTERS_MASK; - - for (i = 0U; i < AMU_GROUP1_NR_COUNTERS; i++) { - if (((amcg1idr >> i) & 1ULL) != 0ULL) { - amu_group1_voffset_write(i, - ctx->group1_voffsets[i]); +#if ENABLE_AMU_AUXILIARY_COUNTERS + for (i = 0U, j = 0U; i < amcgcr_el0_cg1nc; i++) { + if ((amcg1idr_el0_voff >> i) & 1U) { + continue; /* No virtual offset */ } + + amu_group1_voffset_write(i, ctx->group1_voffsets[j++]); } +#endif + } + + /* + * Re-enable counters that were disabled during context save. + */ + + write_amcntenset0_el0_px(ctx->group0_enable); + +#if ENABLE_AMU_AUXILIARY_COUNTERS + if (amcfgr_el0_ncg > 0) { + write_amcntenset1_el0_px(ctx->group1_enable); } +#endif - /* Restore group 1 counter configuration */ - write_amcntenset1_el0(AMU_GROUP1_COUNTERS_MASK); +#if ENABLE_MPMM + mpmm_enable(); #endif return (void *)0; |