[IA64] utilize notify_die() for XPC disengage

XPC (as in arch/ia64/sn/kernel/xp*) has a need to notify other partitions
(SGI Altix) whenever a partition is going down in order to get them to
disengage from accessing the halting partition's memory. If this is not
done before the reset of the hardware, the other partitions can find
themselves encountering MCAs that bring them down.

Signed-off-by: Dean Nelson <dcn@sgi.com>
Signed-off-by: Tony Luck <tony.luck@intel.com>

3 files changed, 107 insertions, 5 deletions

diff --git a/arch/ia64/sn/kernel/xpc.h b/arch/ia64/sn/kernel/xpc.h
index fbcedc7c27fa..5483a9f227d4 100644
--- a/arch/ia64/sn/kernel/xpc.h
+++ b/arch/ia64/sn/kernel/xpc.h
@@ -163,7 +163,7 @@ struct xpc_vars {
 	u8 version;
 	u64 heartbeat;
 	u64 heartbeating_to_mask;
-	u64 kdb_status;		/* 0 = machine running */
+	u64 heartbeat_offline;	/* if 0, heartbeat should be changing */
 	int act_nasid;
 	int act_phys_cpuid;
 	u64 vars_part_pa;
diff --git a/arch/ia64/sn/kernel/xpc_main.c b/arch/ia64/sn/kernel/xpc_main.c
index cece3c7c69be..b617236524c6 100644
--- a/arch/ia64/sn/kernel/xpc_main.c
+++ b/arch/ia64/sn/kernel/xpc_main.c
@@ -57,6 +57,7 @@
 #include <linux/reboot.h>
 #include <asm/sn/intr.h>
 #include <asm/sn/sn_sal.h>
+#include <asm/kdebug.h>
 #include <asm/uaccess.h>
 #include "xpc.h"
 
@@ -188,6 +189,11 @@ static struct notifier_block xpc_reboot_notifier = {
 	.notifier_call = xpc_system_reboot,
 };
 
+static int xpc_system_die(struct notifier_block *, unsigned long, void *);
+static struct notifier_block xpc_die_notifier = {
+	.notifier_call = xpc_system_die,
+};
+
 
 /*
  * Timer function to enforce the timelimit on the partition disengage request.
@@ -997,6 +1003,9 @@ xpc_do_exit(enum xpc_retval reason)
 	/* take ourselves off of the reboot_notifier_list */
 	(void) unregister_reboot_notifier(&xpc_reboot_notifier);
 
+	/* take ourselves off of the die_notifier list */
+	(void) unregister_die_notifier(&xpc_die_notifier);
+
 	/* close down protections for IPI operations */
 	xpc_restrict_IPI_ops();
 
@@ -1011,6 +1020,63 @@ xpc_do_exit(enum xpc_retval reason)
 
 
 /*
+ * Called when the system is about to be either restarted or halted.
+ */
+static void
+xpc_die_disengage(void)
+{
+	struct xpc_partition *part;
+	partid_t partid;
+	unsigned long engaged;
+	long time, print_time, disengage_request_timeout;
+
+
+	/* keep xpc_hb_checker thread from doing anything (just in case) */
+	xpc_exiting = 1;
+
+	xpc_vars->heartbeating_to_mask = 0;  /* indicate we're deactivated */
+
+	for (partid = 1; partid < XP_MAX_PARTITIONS; partid++) {
+		part = &xpc_partitions[partid];
+
+		if (!XPC_SUPPORTS_DISENGAGE_REQUEST(part->
+							remote_vars_version)) {
+
+			/* just in case it was left set by an earlier XPC */
+			xpc_clear_partition_engaged(1UL << partid);
+			continue;
+		}
+
+		if (xpc_partition_engaged(1UL << partid) ||
+					part->act_state != XPC_P_INACTIVE) {
+			xpc_request_partition_disengage(part);
+			xpc_mark_partition_disengaged(part);
+			xpc_IPI_send_disengage(part);
+		}
+	}
+
+	print_time = rtc_time();
+	disengage_request_timeout = print_time +
+		(xpc_disengage_request_timelimit * sn_rtc_cycles_per_second);
+
+	/* wait for all other partitions to disengage from us */
+
+	while ((engaged = xpc_partition_engaged(-1UL)) &&
+			(time = rtc_time()) < disengage_request_timeout) {
+
+		if (time >= print_time) {
+			dev_info(xpc_part, "waiting for remote partitions to "
+				"disengage, engaged=0x%lx\n", engaged);
+			print_time = time + (XPC_DISENGAGE_PRINTMSG_INTERVAL *
+						sn_rtc_cycles_per_second);
+		}
+	}
+	dev_info(xpc_part, "finished waiting for remote partitions to "
+				"disengage, engaged=0x%lx\n", engaged);
+}
+
+
+/*
  * This function is called when the system is being rebooted.
  */
 static int
@@ -1038,6 +1104,33 @@ xpc_system_reboot(struct notifier_block *nb, unsigned long event, void *unused)
 }
 
 
+/*
+ * This function is called when the system is being rebooted.
+ */
+static int
+xpc_system_die(struct notifier_block *nb, unsigned long event, void *unused)
+{
+	switch (event) {
+	case DIE_MACHINE_RESTART:
+	case DIE_MACHINE_HALT:
+		xpc_die_disengage();
+		break;
+	case DIE_MCA_MONARCH_ENTER:
+	case DIE_INIT_MONARCH_ENTER:
+		xpc_vars->heartbeat++;
+		xpc_vars->heartbeat_offline = 1;
+		break;
+	case DIE_MCA_MONARCH_LEAVE:
+	case DIE_INIT_MONARCH_LEAVE:
+		xpc_vars->heartbeat++;
+		xpc_vars->heartbeat_offline = 0;
+		break;
+	}
+
+	return NOTIFY_DONE;
+}
+
+
 int __init
 xpc_init(void)
 {
@@ -1154,6 +1247,12 @@ xpc_init(void)
 		dev_warn(xpc_part, "can't register reboot notifier\n");
 	}
 
+	/* add ourselves to the die_notifier list (i.e., ia64die_chain) */
+	ret = register_die_notifier(&xpc_die_notifier);
+	if (ret != 0) {
+		dev_warn(xpc_part, "can't register die notifier\n");
+	}
+
 
 	/*
 	 * Set the beating to other partitions into motion.  This is
@@ -1179,6 +1278,9 @@ xpc_init(void)
 		/* take ourselves off of the reboot_notifier_list */
 		(void) unregister_reboot_notifier(&xpc_reboot_notifier);
 
+		/* take ourselves off of the die_notifier list */
+		(void) unregister_die_notifier(&xpc_die_notifier);
+
 		del_timer_sync(&xpc_hb_timer);
 		free_irq(SGI_XPC_ACTIVATE, NULL);
 		xpc_restrict_IPI_ops();
diff --git a/arch/ia64/sn/kernel/xpc_partition.c b/arch/ia64/sn/kernel/xpc_partition.c
index 581e113d2d37..cdd6431853a1 100644
--- a/arch/ia64/sn/kernel/xpc_partition.c
+++ b/arch/ia64/sn/kernel/xpc_partition.c
@@ -436,13 +436,13 @@ xpc_check_remote_hb(void)
 		}
 
 		dev_dbg(xpc_part, "partid = %d, heartbeat = %ld, last_heartbeat"
-			" = %ld, kdb_status = %ld, HB_mask = 0x%lx\n", partid,
-			remote_vars->heartbeat, part->last_heartbeat,
-			remote_vars->kdb_status,
+			" = %ld, heartbeat_offline = %ld, HB_mask = 0x%lx\n",
+			partid, remote_vars->heartbeat, part->last_heartbeat,
+			remote_vars->heartbeat_offline,
 			remote_vars->heartbeating_to_mask);
 
 		if (((remote_vars->heartbeat == part->last_heartbeat) &&
-			(remote_vars->kdb_status == 0)) ||
+			(remote_vars->heartbeat_offline == 0)) ||
 			     !xpc_hb_allowed(sn_partition_id, remote_vars)) {
 
 			XPC_DEACTIVATE_PARTITION(part, xpcNoHeartbeat);