cortex-m0: Add deferred scheduler

If 2 interrupts happen at the same time, there is a chance that the nested
interrupt will not call svc_handler when it needs to. In extreme cases this
could lead to tasks not getting woken up when they're supposed to and watchdog
resetting.

The reason stuff worked was because there were enough other interrupts
around to eventually call the scheduler and switch to the ready task.

This change modifies the interrupt calls to not call the scheduler directly
(because in nested interrupt situation this causes problems), but defer the
call to scheduling until after the irq finishes by triggering a low priority
interrupt which will for sure call svc_host at the end. The PendSV irq was
used for this purpose.

BUG=chrome-os-partner:36193
TEST=No more SPI errors caused by scheduler problems
TEST=usleeps now are more accurate, they're guaranteed to not take forever now
BRANCH=veyron

Change-Id: I42acde6b3eb7be2540a0de9a8562dee2ea2be7ab
Signed-off-by: Alexandru M Stan <amstan@chromium.org>
Signed-off-by: Vincent Palatin <vpalatin@chromium.org>
Reviewed-on: https://chromium-review.googlesource.com/248902
Tested-by: Alec Berg <alecaberg@chromium.org>
Reviewed-by: Alec Berg <alecaberg@chromium.org>
Commit-Queue: Alec Berg <alecaberg@chromium.org>

7 files changed, 72 insertions, 80 deletions

diff --git a/chip/stm32/gpio-stm32f0.c b/chip/stm32/gpio-stm32f0.c
index 5418399595..fcead72d4f 100644
--- a/chip/stm32/gpio-stm32f0.c
+++ b/chip/stm32/gpio-stm32f0.c
@@ -41,8 +41,4 @@ static void gpio_init(void)
 }
 DECLARE_HOOK(HOOK_INIT, gpio_init, HOOK_PRIO_DEFAULT);
 
-DECLARE_IRQ(STM32_IRQ_EXTI0_1, gpio_interrupt, 1);
-DECLARE_IRQ(STM32_IRQ_EXTI2_3, gpio_interrupt, 1);
-DECLARE_IRQ(STM32_IRQ_EXTI4_15, gpio_interrupt, 1);
-
 #include "gpio-f0-l.c"
diff --git a/chip/stm32/gpio.c b/chip/stm32/gpio.c
index 6715e540b0..ef7fc818ba 100644
--- a/chip/stm32/gpio.c
+++ b/chip/stm32/gpio.c
@@ -128,3 +128,8 @@ void gpio_interrupt(void)
 			g->irq_handler(g - gpio_list);
 	}
 }
+#ifdef CHIP_FAMILY_STM32F0
+DECLARE_IRQ(STM32_IRQ_EXTI0_1, gpio_interrupt, 1);
+DECLARE_IRQ(STM32_IRQ_EXTI2_3, gpio_interrupt, 1);
+DECLARE_IRQ(STM32_IRQ_EXTI4_15, gpio_interrupt, 1);
+#endif
diff --git a/core/cortex-m0/cpu.c b/core/cortex-m0/cpu.c
index 5dd758ac5f..540777d664 100644
--- a/core/cortex-m0/cpu.c
+++ b/core/cortex-m0/cpu.c
@@ -14,4 +14,7 @@ void cpu_init(void)
 
 	/* Set supervisor call (SVC) to priority 0 */
 	CPU_NVIC_SHCSR2 = 0;
+
+	/* Set lowest priority for PendSV */
+	CPU_NVIC_SHCSR3 = (0xff << 16);
 }
diff --git a/core/cortex-m0/cpu.h b/core/cortex-m0/cpu.h
index 52e46551b0..18686dee1c 100644
--- a/core/cortex-m0/cpu.h
+++ b/core/cortex-m0/cpu.h
@@ -21,6 +21,7 @@
 #define CPU_NVIC_PRI(x)        CPUREG(0xe000e400 + 4 * (x))
 
 /* System Control Block */
+#define CPU_SCB_ICSR           CPUREG(0xe000ed04)
 
 /* SCB AIRCR : Application interrupt and reset control register */
 #define CPU_NVIC_APINT         CPUREG(0xe000ed0c)
diff --git a/core/cortex-m0/irq_handler.h b/core/cortex-m0/irq_handler.h
index 442c20ac00..f6742f5c82 100644
--- a/core/cortex-m0/irq_handler.h
+++ b/core/cortex-m0/irq_handler.h
@@ -8,60 +8,35 @@
 #ifndef __IRQ_HANDLER_H
 #define __IRQ_HANDLER_H
 
-#ifdef CONFIG_TASK_PROFILING
-#define bl_task_start_irq_handler "bl task_start_irq_handler\n"
-#else
-#define bl_task_start_irq_handler ""
-#endif
+#include "cpu.h"
 
 /* Helper macros to build the IRQ handler and priority struct names */
 #define IRQ_HANDLER(irqname) CONCAT3(irq_, irqname, _handler)
 #define IRQ_PRIORITY(irqname) CONCAT2(prio_, irqname)
 
-/* re-scheduling flag */
-extern int need_resched_or_profiling;
-
 /*
  * Macro to connect the interrupt handler "routine" to the irq number "irq" and
  * ensure it is enabled in the interrupt controller with the right priority.
  */
 #define DECLARE_IRQ(irq, routine, priority) DECLARE_IRQ_(irq, routine, priority)
+#ifdef CONFIG_TASK_PROFILING
 #define DECLARE_IRQ_(irq, routine, priority)                    \
-	void IRQ_HANDLER(irq)(void) __attribute__((naked));	\
 	void IRQ_HANDLER(irq)(void)				\
 	{							\
-		asm volatile("mov r0, lr\n"			\
-	/* Must push registers in pairs to keep 64-bit aligned*/\
-	/* stack for ARM EABI. */				\
-			     "push {r0, %0}\n"			\
-			     bl_task_start_irq_handler		\
-			     "bl "#routine"\n"			\
-			     "pop {r2, r3}\n"			\
-	/* read need_resched_or_profiling result after IRQ */   \
-			     "ldr r0, [r3]\n"			\
-			     "mov r1, #8\n"			\
-			     "cmp r0, #0\n"			\
-	/* if we need to go through the re-scheduling, go on */ \
-			     "bne 2f\n"				\
-	/* else return from exception */			\
-			  "1: bx r2\n"				\
-	/* check if that's a nested exception */		\
-			  "2: tst r1, r2\n"			\
-	/* if yes return immediatly */				\
-			     "beq 1b\n"				\
-			     "push {r0, r2}\n"			\
-			     "mov r0, #0\n"			\
-			     "mov r1, #0\n"			\
-	/* ensure we have priority 0 during re-scheduling */	\
-			     "cpsid i\n isb\n"			\
-	/* re-schedule the highest priority task */		\
-			     "bl svc_handler\n"			\
-	/* enable interrupts and return from exception */	\
-			     "cpsie i\n"			\
-			     "pop {r0,pc}\n"			\
-			: : "r"(&need_resched_or_profiling));	\
+		void *ret = __builtin_return_address(0);	\
+		task_start_irq_handler(ret);			\
+		routine();					\
+		task_end_irq_handler(ret);			\
 	}							\
 	const struct irq_priority IRQ_PRIORITY(irq)		\
 	__attribute__((section(".rodata.irqprio")))		\
 			= {irq, priority}
+#else /* CONFIG_TASK_PROFILING */
+/* No Profiling : connect directly the IRQ vector */
+#define DECLARE_IRQ_(irq, routine, priority)                    \
+	void IRQ_HANDLER(irq)(void) __attribute__((alias(STRINGIFY(routine))));\
+	const struct irq_priority IRQ_PRIORITY(irq)		\
+	__attribute__((section(".rodata.irqprio")))		\
+			= {irq, priority}
+#endif /* CONFIG_TASK_PROFILING */
 #endif  /* __IRQ_HANDLER_H */
diff --git a/core/cortex-m0/task.c b/core/cortex-m0/task.c
index 4e9c66ee95..e51621b41e 100644
--- a/core/cortex-m0/task.c
+++ b/core/cortex-m0/task.c
@@ -123,19 +123,6 @@ uint32_t scratchpad[17];
 static task_ *current_task = (task_ *)scratchpad;
 
 /*
- * Should IRQs chain to svc_handler()?  This should be set if either of the
- * following is true:
- *
- * 1) Task scheduling has started, and task profiling is enabled.  Task
- * profiling does its tracking in svc_handler().
- *
- * 2) An event was set by an interrupt; this could result in a higher-priority
- * task unblocking.  After checking for a task switch, svc_handler() will clear
- * the flag (unless profiling is also enabled; then the flag remains set).
- */
-int need_resched_or_profiling;
-
-/*
  * Bitmap of all tasks ready to be run.
  *
  * Currently all tasks are enabled at startup.
@@ -209,7 +196,9 @@ task_ *__svc_handler(int desched, task_id_t resched)
 	 * start time explicitly.
 	 */
 	if (exc == 0xb) {
-		exc_start_time = get_time().val;
+		t = get_time().val;
+		current_task->runtime += (t - exc_end_time);
+		exc_end_time = t;
 		svc_calls++;
 	}
 #endif
@@ -237,22 +226,11 @@ task_ *__svc_handler(int desched, task_id_t resched)
 	next = __task_id_to_ptr(31 - __builtin_clz(tasks_ready));
 
 #ifdef CONFIG_TASK_PROFILING
-	/* Track time in interrupts */
+	/* Track additional time in re-sched exception context */
 	t = get_time().val;
-	exc_total_time += (t - exc_start_time);
+	exc_total_time += (t - exc_end_time);
 
-	/*
-	 * Bill the current task for time between the end of the last interrupt
-	 * and the start of this one.
-	 */
-	current->runtime += (exc_start_time - exc_end_time);
 	exc_end_time = t;
-#else
-	/*
-	 * Don't chain here from interrupts until the next time an interrupt
-	 * sets an event.
-	 */
-	need_resched_or_profiling = 0;
 #endif
 
 	/* Switch to new task */
@@ -284,6 +262,18 @@ void __schedule(int desched, int resched)
 	asm("svc 0" : : "r"(p0), "r"(p1));
 }
 
+void pendsv_handler(void)
+{
+	/* Clear pending flag */
+	CPU_SCB_ICSR = (1 << 27);
+
+	/* ensure we have priority 0 during re-scheduling */
+	__asm__ __volatile__("cpsid i");
+	/* re-schedule the highest priority task */
+	svc_handler(0, 0);
+	__asm__ __volatile__("cpsie i");
+}
+
 #ifdef CONFIG_TASK_PROFILING
 void task_start_irq_handler(void *excep_return)
 {
@@ -302,14 +292,33 @@ void task_start_irq_handler(void *excep_return)
 		irq_dist[irq]++;
 
 	/*
-	 * Continue iff a rescheduling event happened or profiling is active,
-	 * and we are not called from another exception (this must match the
-	 * logic for when we chain to svc_handler() below).
+	 * Continue iff the tasks are ready and we are not called from another
+	 * exception (as the time accouting is done in the outer irq).
 	 */
-	if (!need_resched_or_profiling || (((uint32_t)excep_return & 0xf) == 1))
+	if (!start_called || ((uint32_t)excep_return & 0xf) == 1)
 		return;
 
 	exc_start_time = t;
+	/*
+	 * Bill the current task for time between the end of the last interrupt
+	 * and the start of this one.
+	 */
+	current_task->runtime += (exc_start_time - exc_end_time);
+}
+
+void task_end_irq_handler(void *excep_return)
+{
+	uint64_t t = get_time().val;
+	/*
+	 * Continue iff the tasks are ready and we are not called from another
+	 * exception (as the time accouting is done in the outer irq).
+	 */
+	if (!start_called || ((uint32_t)excep_return & 0xf) == 1)
+		return;
+
+	/* Track time in interrupts */
+	exc_total_time += (t - exc_start_time);
+	exc_end_time = t;
 }
 #endif
 
@@ -357,10 +366,13 @@ uint32_t task_set_event(task_id_t tskid, uint32_t event, int wait)
 	if (in_interrupt_context()) {
 		/* The receiver might run again */
 		atomic_or(&tasks_ready, 1 << tskid);
-#ifndef CONFIG_TASK_PROFILING
-		if (start_called)
-			need_resched_or_profiling = 1;
-#endif
+		if (start_called) {
+			/*
+			 * Trigger the scheduler when there's
+			 * no other irqs happening.
+			 */
+			CPU_SCB_ICSR = (1 << 28);
+		}
 	} else {
 		if (wait) {
 			return __wait_evt(-1, tskid);
@@ -646,7 +658,6 @@ int task_start(void)
 #ifdef CONFIG_TASK_PROFILING
 	task_start_time = exc_end_time = get_time().val;
 #endif
-	start_called = 1;
 
-	return __task_start(&need_resched_or_profiling);
+	return __task_start(&start_called);
 }
diff --git a/include/task.h b/include/task.h
index 0c71e804ba..06cef86946 100644
--- a/include/task.h
+++ b/include/task.h
@@ -141,6 +141,7 @@ const char *task_get_name(task_id_t tskid);
  * is called.
  */
 void task_start_irq_handler(void *excep_return);
+void task_end_irq_handler(void *excep_return);
 #else
 #define task_start_irq_handler(excep_return)
 #endif