BL31: Introduce Exception Handling Framework

EHF is a framework that allows dispatching of EL3 interrupts to their
respective handlers in EL3.

This framework facilitates the firmware-first error handling policy in
which asynchronous exceptions may be routed to EL3. Such exceptions may
be handed over to respective exception handlers. Individual handlers
might further delegate exception handling to lower ELs.

The framework associates the delegated execution to lower ELs with a
priority value. For interrupts, this corresponds to the priorities
programmed in GIC; for other types of exceptions, viz. SErrors or
Synchronous External Aborts, individual dispatchers shall explicitly
associate delegation to a secure priority. In order to prevent lower
priority interrupts from preempting higher priority execution, the
framework provides helpers to control preemption by virtue of
programming Priority Mask register in the interrupt controller.

This commit allows for handling interrupts targeted at EL3. Exception
handlers own interrupts by assigning them a range of secure priorities,
and registering handlers for each priority range it owns.

Support for exception handling in BL31 image is enabled by setting the
build option EL3_EXCEPTION_HANDLING=1.

Documentation to follow.

NOTE: The framework assumes the priority scheme supported by platform
interrupt controller is compliant with that of ARM GIC architecture (v2
or later).

Change-Id: I7224337e4cea47c6ca7d7a4ca22a3716939f7e42
Signed-off-by: Jeenu Viswambharan <jeenu.viswambharan@arm.com>

bl31/bl31.mk

@@ -32,6 +32,10 @@ ifeq (${ENABLE_PMF}, 1)
 BL31_SOURCES		+=	lib/pmf/pmf_main.c
 endif
 
+ifeq (${EL3_EXCEPTION_HANDLING},1)
+BL31_SOURCES		+=	bl31/ehf.c
+endif
+
 BL31_LINKERFILE		:=	bl31/bl31.ld.S
 
 # Flag used to indicate if Crash reporting via console should be included
@@ -41,4 +45,7 @@ CRASH_REPORTING		:=	$(DEBUG)
 endif
 
 $(eval $(call assert_boolean,CRASH_REPORTING))
+$(eval $(call assert_boolean,EL3_EXCEPTION_HANDLING))
+
 $(eval $(call add_define,CRASH_REPORTING))
+$(eval $(call add_define,EL3_EXCEPTION_HANDLING))

bl31/bl31_main.c

@@ -12,6 +12,7 @@
 #include <console.h>
 #include <context_mgmt.h>
 #include <debug.h>
+#include <ehf.h>
 #include <platform.h>
 #include <pmf.h>
 #include <runtime_instr.h>
@@ -79,6 +80,11 @@ void bl31_main(void)
 	/* Initialise helper libraries */
 	bl31_lib_init();
 
+#if EL3_EXCEPTION_HANDLING
+	INFO("BL31: Initialising Exception Handling Framework\n");
+	ehf_init();
+#endif
+
 	/* Initialize the runtime services e.g. psci. */
 	INFO("BL31: Initializing runtime services\n");
 	runtime_svc_init();

bl31/ehf.c

@@ -0,0 +1,340 @@
+/*
+ * Copyright (c) 2017, ARM Limited and Contributors. All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+/*
+ * Exception handlers at EL3, their priority levels, and management.
+ */
+
+#include <assert.h>
+#include <cpu_data.h>
+#include <debug.h>
+#include <ehf.h>
+#include <interrupt_mgmt.h>
+#include <platform.h>
+
+/* Output EHF logs as verbose */
+#define EHF_LOG(...)	VERBOSE("EHF: " __VA_ARGS__)
+
+#define EHF_INVALID_IDX	(-1)
+
+/* For a valid handler, return the actual function pointer; otherwise, 0. */
+#define RAW_HANDLER(h) \
+	((ehf_handler_t) ((h & _EHF_PRI_VALID) ? (h & ~_EHF_PRI_VALID) : 0))
+
+#define PRI_BIT(idx)	(((ehf_pri_bits_t) 1) << idx)
+
+/*
+ * Convert index into secure priority using the platform-defined priority bits
+ * field.
+ */
+#define IDX_TO_PRI(idx) \
+	((idx << (7 - exception_data.pri_bits)) & 0x7f)
+
+/* Check whether a given index is valid */
+#define IS_IDX_VALID(idx) \
+	((exception_data.ehf_priorities[idx].ehf_handler & _EHF_PRI_VALID) != 0)
+
+/* Returns whether given priority is in secure priority range */
+#define IS_PRI_SECURE(pri)	((pri & 0x80) == 0)
+
+/* To be defined by the platform */
+extern const ehf_priorities_t exception_data;
+
+/* Translate priority to the index in the priority array */
+static int pri_to_idx(unsigned int priority)
+{
+	int idx;
+
+	idx = EHF_PRI_TO_IDX(priority, exception_data.pri_bits);
+	assert((idx >= 0) && (idx < exception_data.num_priorities));
+	assert(IS_IDX_VALID(idx));
+
+	return idx;
+}
+
+/* Return whether there are outstanding priority activation */
+static int has_valid_pri_activations(pe_exc_data_t *pe_data)
+{
+	return pe_data->active_pri_bits != 0;
+}
+
+static pe_exc_data_t *this_cpu_data(void)
+{
+	return &get_cpu_data(ehf_data);
+}
+
+/*
+ * Return the current priority index of this CPU. If no priority is active,
+ * return EHF_INVALID_IDX.
+ */
+static int get_pe_highest_active_idx(pe_exc_data_t *pe_data)
+{
+	if (!has_valid_pri_activations(pe_data))
+		return EHF_INVALID_IDX;
+
+	/* Current priority is the right-most bit */
+	return __builtin_ctz(pe_data->active_pri_bits);
+}
+
+/*
+ * Mark priority active by setting the corresponding bit in active_pri_bits and
+ * programming the priority mask.
+ *
+ * This API is to be used as part of delegating to lower ELs other than for
+ * interrupts; e.g. while handling synchronous exceptions.
+ *
+ * This API is expected to be invoked before restoring context (Secure or
+ * Non-secure) in preparation for the respective dispatch.
+ */
+void ehf_activate_priority(unsigned int priority)
+{
+	int idx, cur_pri_idx;
+	unsigned int old_mask, run_pri;
+	pe_exc_data_t *pe_data = this_cpu_data();
+
+	/*
+	 * Query interrupt controller for the running priority, or idle priority
+	 * if no interrupts are being handled. The requested priority must be
+	 * less (higher priority) than the active running priority.
+	 */
+	run_pri = plat_ic_get_running_priority();
+	if (priority >= run_pri) {
+		ERROR("Running priority higher (0x%x) than requested (0x%x)\n",
+				run_pri, priority);
+		panic();
+	}
+
+	/*
+	 * If there were priority activations already, the requested priority
+	 * must be less (higher priority) than the current highest priority
+	 * activation so far.
+	 */
+	cur_pri_idx = get_pe_highest_active_idx(pe_data);
+	idx = pri_to_idx(priority);
+	if ((cur_pri_idx != EHF_INVALID_IDX) && (idx >= cur_pri_idx)) {
+		ERROR("Activation priority mismatch: req=0x%x current=0x%x\n",
+				priority, IDX_TO_PRI(cur_pri_idx));
+		panic();
+	}
+
+	/* Set the bit corresponding to the requested priority */
+	pe_data->active_pri_bits |= PRI_BIT(idx);
+
+	/*
+	 * Program priority mask for the activated level. Check that the new
+	 * priority mask is setting a higher priority level than the existing
+	 * mask.
+	 */
+	old_mask = plat_ic_set_priority_mask(priority);
+	if (priority >= old_mask) {
+		ERROR("Requested priority (0x%x) lower than Priority Mask (0x%x)\n",
+				priority, old_mask);
+		panic();
+	}
+
+	/*
+	 * If this is the first activation, save the priority mask. This will be
+	 * restored after the last deactivation.
+	 */
+	if (cur_pri_idx == EHF_INVALID_IDX)
+		pe_data->init_pri_mask = old_mask;
+
+	EHF_LOG("activate prio=%d\n", get_pe_highest_active_idx(pe_data));
+}
+
+/*
+ * Mark priority inactive by clearing the corresponding bit in active_pri_bits,
+ * and programming the priority mask.
+ *
+ * This API is expected to be used as part of delegating to to lower ELs other
+ * than for interrupts; e.g. while handling synchronous exceptions.
+ *
+ * This API is expected to be invoked after saving context (Secure or
+ * Non-secure), having concluded the respective dispatch.
+ */
+void ehf_deactivate_priority(unsigned int priority)
+{
+	int idx, cur_pri_idx;
+	pe_exc_data_t *pe_data = this_cpu_data();
+	unsigned int old_mask, run_pri;
+
+	/*
+	 * Query interrupt controller for the running priority, or idle priority
+	 * if no interrupts are being handled. The requested priority must be
+	 * less (higher priority) than the active running priority.
+	 */
+	run_pri = plat_ic_get_running_priority();
+	if (priority >= run_pri) {
+		ERROR("Running priority higher (0x%x) than requested (0x%x)\n",
+				run_pri, priority);
+		panic();
+	}
+
+	/*
+	 * Deactivation is allowed only when there are priority activations, and
+	 * the deactivation priority level must match the current activated
+	 * priority.
+	 */
+	cur_pri_idx = get_pe_highest_active_idx(pe_data);
+	idx = pri_to_idx(priority);
+	if ((cur_pri_idx == EHF_INVALID_IDX) || (idx != cur_pri_idx)) {
+		ERROR("Deactivation priority mismatch: req=0x%x current=0x%x\n",
+				priority, IDX_TO_PRI(cur_pri_idx));
+		panic();
+	}
+
+	/* Clear bit corresponding to highest priority */
+	pe_data->active_pri_bits &= (pe_data->active_pri_bits - 1);
+
+	/*
+	 * Restore priority mask corresponding to the next priority, or the
+	 * one stashed earlier if there are no more to deactivate.
+	 */
+	idx = get_pe_highest_active_idx(pe_data);
+	if (idx == EHF_INVALID_IDX)
+		old_mask = plat_ic_set_priority_mask(pe_data->init_pri_mask);
+	else
+		old_mask = plat_ic_set_priority_mask(priority);
+
+	if (old_mask >= priority) {
+		ERROR("Deactivation priority (0x%x) lower than Priority Mask (0x%x)\n",
+				priority, old_mask);
+		panic();
+	}
+
+	EHF_LOG("deactivate prio=%d\n", get_pe_highest_active_idx(pe_data));
+}
+
+/*
+ * Top-level EL3 interrupt handler.
+ */
+static uint64_t ehf_el3_interrupt_handler(uint32_t id, uint32_t flags,
+		void *handle, void *cookie)
+{
+	int pri, idx, intr, intr_raw, ret = 0;
+	ehf_handler_t handler;
+
+	/*
+	 * Top-level interrupt type handler from Interrupt Management Framework
+	 * doesn't acknowledge the interrupt; so the interrupt ID must be
+	 * invalid.
+	 */
+	assert(id == INTR_ID_UNAVAILABLE);
+
+	/*
+	 * Acknowledge interrupt. Proceed with handling only for valid interrupt
+	 * IDs. This situation may arise because of Interrupt Management
+	 * Framework identifying an EL3 interrupt, but before it's been
+	 * acknowledged here, the interrupt was either deasserted, or there was
+	 * a higher-priority interrupt of another type.
+	 */
+	intr_raw = plat_ic_acknowledge_interrupt();
+	intr = plat_ic_get_interrupt_id(intr_raw);
+	if (intr == INTR_ID_UNAVAILABLE)
+		return 0;
+
+	/* Having acknowledged the interrupt, get the running priority */
+	pri = plat_ic_get_running_priority();
+
+	/* Check EL3 interrupt priority is in secure range */
+	assert(IS_PRI_SECURE(pri));
+
+	/*
+	 * Translate the priority to a descriptor index. We do this by masking
+	 * and shifting the running priority value (platform-supplied).
+	 */
+	idx = pri_to_idx(pri);
+
+	/* Validate priority */
+	assert(pri == IDX_TO_PRI(idx));
+
+	handler = RAW_HANDLER(exception_data.ehf_priorities[idx].ehf_handler);
+	if (!handler) {
+		ERROR("No EL3 exception handler for priority 0x%x\n",
+				IDX_TO_PRI(idx));
+		panic();
+	}
+
+	/*
+	 * Call registered handler. Pass the raw interrupt value to registered
+	 * handlers.
+	 */
+	ret = handler(intr_raw, flags, handle, cookie);
+
+	return ret;
+}
+
+/*
+ * Initialize the EL3 exception handling.
+ */
+void ehf_init(void)
+{
+	unsigned int flags = 0;
+	int ret __unused;
+
+	/* Ensure EL3 interrupts are supported */
+	assert(plat_ic_has_interrupt_type(INTR_TYPE_EL3));
+
+	/*
+	 * Make sure that priority water mark has enough bits to represent the
+	 * whole priority array.
+	 */
+	assert(exception_data.num_priorities <= (sizeof(ehf_pri_bits_t) * 8));
+
+	assert(exception_data.ehf_priorities);
+
+	/*
+	 * Bit 7 of GIC priority must be 0 for secure interrupts. This means
+	 * platforms must use at least 1 of the remaining 7 bits.
+	 */
+	assert((exception_data.pri_bits >= 1) || (exception_data.pri_bits < 8));
+
+	/* Route EL3 interrupts when in Secure and Non-secure. */
+	set_interrupt_rm_flag(flags, NON_SECURE);
+	set_interrupt_rm_flag(flags, SECURE);
+
+	/* Register handler for EL3 interrupts */
+	ret = register_interrupt_type_handler(INTR_TYPE_EL3,
+			ehf_el3_interrupt_handler, flags);
+	assert(ret == 0);
+}
+
+/*
+ * Register a handler at the supplied priority. Registration is allowed only if
+ * a handler hasn't been registered before, or one wasn't provided at build
+ * time. The priority for which the handler is being registered must also accord
+ * with the platform-supplied data.
+ */
+void ehf_register_priority_handler(unsigned int pri, ehf_handler_t handler)
+{
+	int idx;
+
+	/* Sanity check for handler */
+	assert(handler != NULL);
+
+	/* Handler ought to be 4-byte aligned */
+	assert((((uintptr_t) handler) & 3) == 0);
+
+	/* Ensure we register for valid priority */
+	idx = pri_to_idx(pri);
+	assert(idx < exception_data.num_priorities);
+	assert(IDX_TO_PRI(idx) == pri);
+
+	/* Return failure if a handler was already registered */
+	if (exception_data.ehf_priorities[idx].ehf_handler != _EHF_NO_HANDLER) {
+		ERROR("Handler already registered for priority 0x%x\n", pri);
+		panic();
+	}
+
+	/*
+	 * Install handler, and retain the valid bit. We assume that the handler
+	 * is 4-byte aligned, which is usually the case.
+	 */
+	exception_data.ehf_priorities[idx].ehf_handler =
+		(((uintptr_t) handler) | _EHF_PRI_VALID);
+
+	EHF_LOG("register pri=0x%x handler=%p\n", pri, handler);
+}

docs/user-guide.rst

@@ -361,6 +361,11 @@ Common build options
    Firmware as error. It can take the value 1 (flag the use of deprecated
    APIs as error) or 0. The default is 0.
 
+-  ``EL3_EXCEPTION_HANDLING``: When set to ``1``, enable handling of exceptions
+   targeted at EL3. When set ``0`` (default), no exceptions are expected or
+   handled at EL3, and a panic will result. This is supported only for AArch64
+   builds.
+
 -  ``FIP_NAME``: This is an optional build option which specifies the FIP
    filename for the ``fip`` target. Default is ``fip.bin``.
 

include/bl31/ehf.h

@@ -0,0 +1,85 @@
+/*
+ * Copyright (c) 2017, ARM Limited and Contributors. All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#ifndef __EHF_H__
+#define __EHF_H__
+
+#ifndef __ASSEMBLY__
+
+#include <stdint.h>
+#include <utils_def.h>
+
+/* Valid priorities set bit 0 of the priority handler. */
+#define _EHF_PRI_VALID	(((uintptr_t) 1) << 0)
+
+/* Marker for no handler registered for a valid priority */
+#define _EHF_NO_HANDLER	(0 | _EHF_PRI_VALID)
+
+/* Extract the specified number of top bits from 7 lower bits of priority */
+#define EHF_PRI_TO_IDX(pri, plat_bits) \
+	((pri & 0x7f) >> (7 - plat_bits))
+
+/* Install exception priority descriptor at a suitable index */
+#define EHF_PRI_DESC(plat_bits, priority) \
+	[EHF_PRI_TO_IDX(priority, plat_bits)] = { \
+		.ehf_handler = _EHF_NO_HANDLER, \
+	}
+
+/* Macro for platforms to regiter its exception priorities */
+#define EHF_REGISTER_PRIORITIES(priorities, num, bits) \
+	const ehf_priorities_t exception_data = { \
+		.num_priorities = num, \
+		.ehf_priorities = priorities, \
+		.pri_bits = bits, \
+	}
+
+/*
+ * Priority stack, managed as a bitmap.
+ *
+ * Currently only supports 32 priority levels, allowing platforms to use up to 5
+ * top bits of priority. But the type can be changed to uint64_t should need
+ * arise to support 64 priority levels, allowing platforms to use up to 6 top
+ * bits of priority.
+ */
+typedef uint32_t ehf_pri_bits_t;
+
+/*
+ * Per-PE exception data. The data for each PE is kept as a per-CPU data field.
+ * See cpu_data.h.
+ */
+typedef struct {
+	ehf_pri_bits_t active_pri_bits;
+
+	/* Priority mask value before any priority levels were active */
+	uint8_t init_pri_mask;
+} __aligned(sizeof(uint64_t)) pe_exc_data_t;
+
+typedef int (*ehf_handler_t)(uint32_t intr_raw, uint32_t flags, void *handle,
+		void *cookie);
+
+typedef struct ehf_pri_desc {
+	/*
+	 * 4-byte-aligned exception handler. Bit 0 indicates the corresponding
+	 * priority level is valid. This is effectively of ehf_handler_t type,
+	 * but left as uintptr_t in order to make pointer arithmetic convenient.
+	 */
+	uintptr_t ehf_handler;
+} ehf_pri_desc_t;
+
+typedef struct ehf_priorities {
+	ehf_pri_desc_t *ehf_priorities;
+	unsigned int num_priorities;
+	int pri_bits;
+} ehf_priorities_t;
+
+void ehf_init(void);
+void ehf_activate_priority(unsigned int priority);
+void ehf_deactivate_priority(unsigned int priority);
+void ehf_register_priority_handler(unsigned int pri, ehf_handler_t handler);
+
+#endif /* __ASSEMBLY__ */
+
+#endif /* __EHF_H__ */

include/lib/el3_runtime/cpu_data.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014-2016, ARM Limited and Contributors. All rights reserved.
+ * Copyright (c) 2014-2017, ARM Limited and Contributors. All rights reserved.
  *
  * SPDX-License-Identifier: BSD-3-Clause
  */
@@ -7,6 +7,7 @@
 #ifndef __CPU_DATA_H__
 #define __CPU_DATA_H__
 
+#include <ehf.h>
 #include <platform_def.h>	/* CACHE_WRITEBACK_GRANULE required */
 
 #ifdef AARCH32
@@ -96,6 +97,9 @@ typedef struct cpu_data {
 #if PLAT_PCPU_DATA_SIZE
 	uint8_t platform_cpu_data[PLAT_PCPU_DATA_SIZE];
 #endif
+#if defined(IMAGE_BL31) && EL3_EXCEPTION_HANDLING
+	pe_exc_data_t ehf_data;
+#endif
 } __aligned(CACHE_WRITEBACK_GRANULE) cpu_data_t;
 
 #if CRASH_REPORTING

make_helpers/defaults.mk

@@ -62,6 +62,9 @@ ENABLE_RUNTIME_INSTRUMENTATION	:= 0
 # Flag to enable stack corruption protection
 ENABLE_STACK_PROTECTOR		:= 0
 
+# Flag to enable exception handling in EL3
+EL3_EXCEPTION_HANDLING		:= 0
+
 # Build flag to treat usage of deprecated platform and framework APIs as error.
 ERROR_DEPRECATED		:= 0