zephyr: ap_pwrseq: Add S0IX error recovery support

This CL supports s0ix error recovery to ensure the AP doesn't get itself
stuck in a state where it's no longer in a sleep state (S0ix/S3), but
from the Linux perspective is still suspended.

BUG=none
BRANCH=none
TEST=zmake build nivviks
     Tested using 'powerd_dbus_suspend' on AP and observed timeout and
     hang detected through EC logs.

Signed-off-by: Rajesh Kumar <rajesh3.kumar@intel.com>
Change-Id: Iaf37b2afc7e1bdec251b9c21f6a3f131acab6a50
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/platform/ec/+/3760670
Reviewed-by: Andrew McRae <amcrae@google.com>

7 files changed, 286 insertions, 3 deletions

diff --git a/zephyr/subsys/ap_pwrseq/CMakeLists.txt b/zephyr/subsys/ap_pwrseq/CMakeLists.txt
index 30edfae4dd..0a77bc821d 100644
--- a/zephyr/subsys/ap_pwrseq/CMakeLists.txt
+++ b/zephyr/subsys/ap_pwrseq/CMakeLists.txt
@@ -13,9 +13,10 @@ zephyr_library_sources_ifdef(CONFIG_AP_PWRSEQ
 	signal_adc.c
 	)
 zephyr_library_sources_ifdef(CONFIG_X86_NON_DSX_PWRSEQ
-	x86_non_dsx_common_pwrseq_sm_handler.c)
-zephyr_library_sources_ifdef(CONFIG_X86_NON_DSX_PWRSEQ
+	x86_non_dsx_common_pwrseq_sm_handler.c
 	x86_non_dsx_chipset_power_state.c)
+zephyr_library_sources_ifdef(CONFIG_AP_PWRSEQ_S0IX_ERROR_RECOVERY
+	x86_non_dsx_common_pwrseq_host_sleep.c)
 zephyr_library_sources_ifdef(CONFIG_X86_NON_DSX_PWRSEQ_CONSOLE
 	x86_non_dsx_common_pwrseq_console.c)
 zephyr_library_sources_ifdef(CONFIG_X86_NON_DSX_PWRSEQ_HOST_CMD
diff --git a/zephyr/subsys/ap_pwrseq/Kconfig b/zephyr/subsys/ap_pwrseq/Kconfig
index 1845baa2bd..093543d86b 100644
--- a/zephyr/subsys/ap_pwrseq/Kconfig
+++ b/zephyr/subsys/ap_pwrseq/Kconfig
@@ -107,4 +107,14 @@ config AP_PWRSEQ_S0IX
 	  required, AP_PWRSEQ_HOST_SLEEP for host sleep event handling is
 	  enabled.
 
+config AP_PWRSEQ_S0IX_ERROR_RECOVERY
+	bool "Detect failure to enter or exit Sleep state"
+	depends on AP_PWRSEQ_HOST_SLEEP
+	help
+	  Enables detection of the AP failing to go to sleep, perhaps due to a
+	  bug in the internal SoC periodic housekeeping code.
+
+	  Failure information is reported via the EC_CMD_HOST_SLEEP_EVENT host
+	  command.
+
 endif
diff --git a/zephyr/subsys/ap_pwrseq/include/ap_power_host_sleep.h b/zephyr/subsys/ap_pwrseq/include/ap_power_host_sleep.h
index 85b4ffe2ff..287d438a63 100644
--- a/zephyr/subsys/ap_pwrseq/include/ap_power_host_sleep.h
+++ b/zephyr/subsys/ap_pwrseq/include/ap_power_host_sleep.h
@@ -56,4 +56,17 @@ enum ap_power_sleep_type ap_power_sleep_get_notify(void);
 void ap_power_sleep_notify_transition(enum ap_power_sleep_type check_state);
 #endif /* CONFIG_AP_PWRSEQ_S0IX */
 
+/*
+ * Get sleep timeout from host command context
+ */
+uint16_t host_get_sleep_timeout(void);
+
+/*
+ * Set sleep transitions for host command response
+ *
+ * @param val sleep transitions
+ *
+ */
+void host_set_sleep_transitions(uint32_t val);
+
 #endif /* __AP_PWRSEQ_HOST_SLEEP_H */
diff --git a/zephyr/subsys/ap_pwrseq/power_host_sleep.c b/zephyr/subsys/ap_pwrseq/power_host_sleep.c
index c3ed1e9aff..35aada1022 100644
--- a/zephyr/subsys/ap_pwrseq/power_host_sleep.c
+++ b/zephyr/subsys/ap_pwrseq/power_host_sleep.c
@@ -184,9 +184,12 @@ void ap_power_sleep_notify_transition(enum ap_power_sleep_type check_state)
 #if CONFIG_AP_PWRSEQ_HOST_SLEEP
 #define HOST_SLEEP_EVENT_DEFAULT_RESET 0
 
+static struct host_sleep_event_context *g_ctx;
+
 void ap_power_reset_host_sleep_state(void)
 {
 	power_set_host_sleep_state(HOST_SLEEP_EVENT_DEFAULT_RESET);
+	ap_power_ev_send_callbacks(AP_POWER_S0IX_RESET_TRACKING);
 	ap_power_chipset_handle_host_sleep_event(HOST_SLEEP_EVENT_DEFAULT_RESET,
 						 NULL);
 }
@@ -202,6 +205,9 @@ void ap_power_chipset_handle_host_sleep_event(
 	enum host_sleep_event state, struct host_sleep_event_context *ctx)
 {
 	LOG_DBG("host sleep event = %d!", state);
+
+	g_ctx = ctx;
+
 #if CONFIG_AP_PWRSEQ_S0IX
 	if (state == HOST_SLEEP_EVENT_S0IX_SUSPEND) {
 		/*
@@ -210,6 +216,7 @@ void ap_power_chipset_handle_host_sleep_event(
 		 * notification needs to be sent to listeners.
 		 */
 		ap_power_sleep_set_notify(AP_POWER_SLEEP_SUSPEND);
+		ap_power_ev_send_callbacks(AP_POWER_S0IX_SUSPEND_START);
 		power_signal_enable(PWR_SLP_S0);
 
 	} else if (state == HOST_SLEEP_EVENT_S0IX_RESUME) {
@@ -220,6 +227,7 @@ void ap_power_chipset_handle_host_sleep_event(
 		ap_power_sleep_set_notify(AP_POWER_SLEEP_RESUME);
 		power_s0ix_resume_restore_masks();
 		power_signal_disable(PWR_SLP_S0);
+		ap_power_ev_send_callbacks(AP_POWER_S0IX_RESUME_COMPLETE);
 
 		/*
 		 * If the sleep signal timed out and never transitioned, then
@@ -236,4 +244,14 @@ void ap_power_chipset_handle_host_sleep_event(
 	ap_pwrseq_wake();
 }
 
+uint16_t host_get_sleep_timeout(void)
+{
+	return g_ctx->sleep_timeout_ms;
+}
+
+void host_set_sleep_transitions(uint32_t val)
+{
+	g_ctx->sleep_transitions = val;
+}
+
 #endif /* CONFIG_AP_PWRSEQ_HOST_SLEEP */
diff --git a/zephyr/subsys/ap_pwrseq/x86_non_dsx_common_pwrseq_host_command.c b/zephyr/subsys/ap_pwrseq/x86_non_dsx_common_pwrseq_host_command.c
index 10c6bdf207..90aa78ecf1 100644
--- a/zephyr/subsys/ap_pwrseq/x86_non_dsx_common_pwrseq_host_command.c
+++ b/zephyr/subsys/ap_pwrseq/x86_non_dsx_common_pwrseq_host_command.c
@@ -30,4 +30,63 @@ host_command_reboot_ap_on_g3(struct host_cmd_handler_args *args)
 DECLARE_HOST_COMMAND(EC_CMD_REBOOT_AP_ON_G3, host_command_reboot_ap_on_g3,
 		     EC_VER_MASK(0) | EC_VER_MASK(1));
 
+#if CONFIG_AP_PWRSEQ_HOST_SLEEP
+/* Track last reported sleep event */
+static enum host_sleep_event host_sleep_state;
+
+static enum ec_status
+host_command_host_sleep_event(struct host_cmd_handler_args *args)
+{
+	const struct ec_params_host_sleep_event_v1 *p = args->params;
+	struct ec_response_host_sleep_event_v1 *r = args->response;
+	struct host_sleep_event_context ctx;
+	enum host_sleep_event state = p->sleep_event;
+
+	host_sleep_state = state;
+	ctx.sleep_transitions = 0;
+	switch (state) {
+	case HOST_SLEEP_EVENT_S0IX_SUSPEND:
+	case HOST_SLEEP_EVENT_S3_SUSPEND:
+	case HOST_SLEEP_EVENT_S3_WAKEABLE_SUSPEND:
+		ctx.sleep_timeout_ms = EC_HOST_SLEEP_TIMEOUT_DEFAULT;
+
+		/* The original version contained only state. */
+		if (args->version >= 1)
+			ctx.sleep_timeout_ms =
+				p->suspend_params.sleep_timeout_ms;
+
+		break;
+
+	default:
+		break;
+	}
+
+	ap_power_chipset_handle_host_sleep_event(host_sleep_state, &ctx);
+	switch (state) {
+	case HOST_SLEEP_EVENT_S0IX_RESUME:
+	case HOST_SLEEP_EVENT_S3_RESUME:
+		if (args->version >= 1) {
+			r->resume_response.sleep_transitions =
+				ctx.sleep_transitions;
+
+			args->response_size = sizeof(*r);
+		}
+
+		break;
+
+	default:
+		break;
+	}
+
+	return EC_RES_SUCCESS;
+}
+DECLARE_HOST_COMMAND(EC_CMD_HOST_SLEEP_EVENT, host_command_host_sleep_event,
+		     EC_VER_MASK(0) | EC_VER_MASK(1));
+
+void power_set_host_sleep_state(enum host_sleep_event state)
+{
+	host_sleep_state = state;
+}
+#endif /* CONFIG_AP_PWRSEQ_HOST_SLEEP */
+
 /* End of host commands */
diff --git a/zephyr/subsys/ap_pwrseq/x86_non_dsx_common_pwrseq_host_sleep.c b/zephyr/subsys/ap_pwrseq/x86_non_dsx_common_pwrseq_host_sleep.c
new file mode 100644
index 0000000000..e13be78dad
--- /dev/null
+++ b/zephyr/subsys/ap_pwrseq/x86_non_dsx_common_pwrseq_host_sleep.c
@@ -0,0 +1,179 @@
+/* Copyright 2022 The ChromiumOS Authors.
+ * Use of this source code is governed by a BSD-style license that can be
+ * found in the LICENSE file.
+ */
+
+#include <ap_power_host_sleep.h>
+#include <x86_non_dsx_common_pwrseq_sm_handler.h>
+
+static uint16_t sleep_signal_timeout;
+static uint16_t host_sleep_timeout_default = CONFIG_SLEEP_TIMEOUT_MS;
+static uint32_t sleep_signal_transitions;
+static enum sleep_hang_type timeout_hang_type;
+
+static void sleep_transition_timeout(struct k_work *work);
+
+static K_WORK_DELAYABLE_DEFINE(sleep_transition_timeout_data,
+			       sleep_transition_timeout);
+
+/**
+ * Type of sleep hang detected
+ */
+enum sleep_hang_type {
+	SLEEP_HANG_NONE,
+	SLEEP_HANG_S0IX_SUSPEND,
+	SLEEP_HANG_S0IX_RESUME
+};
+
+void power_chipset_handle_sleep_hang(enum sleep_hang_type hang_type)
+{
+	/*
+	 * Wake up the AP so they don't just chill in a non-suspended state and
+	 * burn power. Overload a vaguely related event bit since event bits are
+	 * at a premium. If the system never entered S0ix, then manually set the
+	 * wake mask to pretend it did, so that the hang detect event wakes the
+	 * system.
+	 */
+	if (pwr_sm_get_state() == SYS_POWER_STATE_S0) {
+		host_event_t sleep_wake_mask;
+
+		ap_power_get_lazy_wake_mask(SYS_POWER_STATE_S0ix,
+					    &sleep_wake_mask);
+		lpc_set_host_event_mask(LPC_HOST_EVENT_WAKE, sleep_wake_mask);
+	}
+
+	ccprintf("Warning: Detected sleep hang! Waking host up!");
+	host_set_single_event(EC_HOST_EVENT_HANG_DETECT);
+}
+
+static void sleep_transition_timeout(struct k_work *work)
+{
+	/* Mark the timeout. */
+	sleep_signal_transitions |= EC_HOST_RESUME_SLEEP_TIMEOUT;
+	k_work_cancel_delayable(&sleep_transition_timeout_data);
+
+	if (timeout_hang_type != SLEEP_HANG_NONE) {
+		power_chipset_handle_sleep_hang(timeout_hang_type);
+	}
+}
+
+static void sleep_increment_transition(void)
+{
+	if ((sleep_signal_transitions & EC_HOST_RESUME_SLEEP_TRANSITIONS_MASK) <
+	    EC_HOST_RESUME_SLEEP_TRANSITIONS_MASK)
+		sleep_signal_transitions += 1;
+}
+
+void sleep_suspend_transition(void)
+{
+	sleep_increment_transition();
+	k_work_cancel_delayable(&sleep_transition_timeout_data);
+}
+
+void sleep_resume_transition(void)
+{
+	sleep_increment_transition();
+
+	/*
+	 * Start the timer again to ensure the AP doesn't get itself stuck in
+	 * a state where it's no longer in a sleep state (S0ix/S3), but from
+	 * the Linux perspective is still suspended. Perhaps a bug in the SoC-
+	 * internal periodic housekeeping code might result in a situation
+	 * like this.
+	 */
+	if (sleep_signal_timeout) {
+		timeout_hang_type = SLEEP_HANG_S0IX_RESUME;
+		k_work_schedule(&sleep_transition_timeout_data,
+				K_MSEC(sleep_signal_timeout));
+	}
+}
+
+void sleep_start_suspend(void)
+{
+	uint16_t timeout = host_get_sleep_timeout();
+
+	sleep_signal_transitions = 0;
+
+	/* Use 0xFFFF to disable the timeout */
+	if (timeout == EC_HOST_SLEEP_TIMEOUT_INFINITE) {
+		sleep_signal_timeout = 0;
+		return;
+	}
+
+	/* Use zero internally to indicate host doesn't set timeout value;
+	 * we will use default timeout.
+	 */
+	if (timeout == EC_HOST_SLEEP_TIMEOUT_DEFAULT) {
+		timeout = host_sleep_timeout_default;
+	}
+
+	sleep_signal_timeout = timeout;
+	timeout_hang_type = SLEEP_HANG_S0IX_SUSPEND;
+	k_work_schedule(&sleep_transition_timeout_data, K_MSEC(timeout));
+}
+
+void sleep_complete_resume(void)
+{
+	/*
+	 * Ensure we don't schedule another sleep_transition_timeout
+	 * if the the HOST_SLEEP_EVENT_S0IX_RESUME message arrives before
+	 * the CHIPSET task transitions to the POWER_S0ixS0 state.
+	 */
+	sleep_signal_timeout = 0;
+	k_work_cancel_delayable(&sleep_transition_timeout_data);
+	host_set_sleep_transitions(sleep_signal_transitions);
+}
+
+void sleep_reset_tracking(void)
+{
+	sleep_signal_transitions = 0;
+	sleep_signal_timeout = 0;
+	timeout_hang_type = SLEEP_HANG_NONE;
+}
+
+/*
+ * s0ix event handler.
+ */
+static void ap_power_sleep_event_handler(struct ap_power_ev_callback *cb,
+					 struct ap_power_ev_data data)
+{
+	switch (data.event) {
+	case AP_POWER_S0IX_SUSPEND_START:
+		sleep_start_suspend();
+		break;
+	case AP_POWER_S0IX_SUSPEND:
+		sleep_suspend_transition();
+		break;
+	case AP_POWER_S0IX_RESUME:
+		sleep_resume_transition();
+		break;
+	case AP_POWER_S0IX_RESUME_COMPLETE:
+		sleep_complete_resume();
+		break;
+	case AP_POWER_S0IX_RESET_TRACKING:
+		sleep_reset_tracking();
+		break;
+	default:
+		break;
+	}
+}
+
+/*
+ * Registers callback for s0ix events.
+ */
+static int ap_power_sleep_s0ix_event(const struct device *unused)
+{
+	static struct ap_power_ev_callback cb;
+
+	/*
+	 * Register for all events.
+	 */
+	ap_power_ev_init_callback(
+		&cb, ap_power_sleep_event_handler,
+		AP_POWER_S0IX_SUSPEND_START | AP_POWER_S0IX_SUSPEND |
+			AP_POWER_S0IX_RESUME | AP_POWER_S0IX_RESUME_COMPLETE |
+			AP_POWER_S0IX_RESET_TRACKING);
+	ap_power_ev_add_callback(&cb);
+	return 0;
+}
+SYS_INIT(ap_power_sleep_s0ix_event, APPLICATION, 1);
diff --git a/zephyr/subsys/ap_pwrseq/x86_non_dsx_common_pwrseq_sm_handler.c b/zephyr/subsys/ap_pwrseq/x86_non_dsx_common_pwrseq_sm_handler.c
index 4fcd52f2d4..f812338b93 100644
--- a/zephyr/subsys/ap_pwrseq/x86_non_dsx_common_pwrseq_sm_handler.c
+++ b/zephyr/subsys/ap_pwrseq/x86_non_dsx_common_pwrseq_sm_handler.c
@@ -410,6 +410,7 @@ static int common_pwr_sm_run(int state)
 		 * HC already set sleep suspend state.
 		 */
 		ap_power_sleep_notify_transition(AP_POWER_SLEEP_SUSPEND);
+		ap_power_ev_send_callbacks(AP_POWER_S0IX_SUSPEND);
 
 		/*
 		 * Enable idle task deep sleep. Allow the low power idle task
@@ -430,9 +431,11 @@ static int common_pwr_sm_run(int state)
 		 */
 		disable_sleep(SLEEP_MASK_AP_RUN);
 
+		ap_power_ev_send_callbacks(AP_POWER_S0IX_RESUME
 #if CONFIG_PLATFORM_EC_CHIPSET_RESUME_INIT_HOOK
-		ap_power_ev_send_callbacks(AP_POWER_RESUME_INIT);
+					   | AP_POWER_RESUME_INIT
 #endif
+		);
 
 		return SYS_POWER_STATE_S0;