dcb: separate DCB enable/disable and wait for carrier changes (rh #799241) (rh #1081991)

Non-git-master versions of lldpad refuse to touch a device that doesn't
have a carrier.  And when enabling/disabling DCB, the kernel driver will
reconfigure itself and may turn carrier off for a few seconds.  So we
must ensure that before enabling/disabling DCB, the carrier is already
on.  Next we must ensure that *after* enabling/disabling DCB, the
carrier is back on before doing further DCB setup.

There's a race condition between enabling/disabling DCB and receiving
the carrier event in NetworkManager that has to be handled carefully.
Because the carrier may not yet be down after the dcbtool call to
enable/disable DCB returns, we need to wait for a couple seconds for
the carrier to go down, and then again for it to come back up.
Otherwise we might see the still-on carrier, proceed with DCB setup,
and the carrier finally goes down halfway through the setup, which
will fail the operations with "DCB not enabled, link down, or DCB
not supported" errors from lldpad.

4 files changed, 300 insertions, 42 deletions

diff --git a/src/devices/nm-device-ethernet.c b/src/devices/nm-device-ethernet.c
index a6279a4cb5..e6f091d62c 100644
--- a/src/devices/nm-device-ethernet.c
+++ b/src/devices/nm-device-ethernet.c
@@ -86,6 +86,20 @@ typedef struct Supplicant {
 	guint con_timeout_id;
 } Supplicant;
 
+typedef enum {
+	DCB_WAIT_UNKNOWN = 0,
+	/* Ensure carrier is up before enabling DCB */
+	DCB_WAIT_CARRIER_PREENABLE_UP,
+	/* Wait for carrier down when device starts enabling */
+	DCB_WAIT_CARRIER_PRECONFIG_DOWN,
+	/* Wait for carrier up when device has finished enabling */
+	DCB_WAIT_CARRIER_PRECONFIG_UP,
+	/* Wait carrier down when device starts configuring */
+	DCB_WAIT_CARRIER_POSTCONFIG_DOWN,
+	/* Wait carrier up when device has finished configuring */
+	DCB_WAIT_CARRIER_POSTCONFIG_UP,
+} DcbWait;
+
 typedef struct {
 	guint8              perm_hw_addr[ETH_ALEN];    /* Permanent MAC address */
 	guint8              initial_hw_addr[ETH_ALEN]; /* Initial MAC address (as seen when NM starts) */
@@ -106,6 +120,11 @@ typedef struct {
 	NMIP4Config  *pending_ip4_config;
 	gint32        last_pppoe_time;
 	guint         pppoe_wait_id;
+
+	/* DCB */
+	DcbWait       dcb_wait;
+	guint         dcb_timeout_id;
+	guint         dcb_carrier_id;
 } NMDeviceEthernetPrivate;
 
 enum {
@@ -1094,33 +1113,217 @@ pppoe_stage3_ip4_config_start (NMDeviceEthernet *self, NMDeviceStateReason *reas
 	return ret;
 }
 
+/****************************************************************/
+
+static void
+dcb_timeout_cleanup (NMDevice *device)
+{
+	NMDeviceEthernetPrivate *priv = NM_DEVICE_ETHERNET_GET_PRIVATE (device);
+
+	if (priv->dcb_timeout_id) {
+		g_source_remove (priv->dcb_timeout_id);
+		priv->dcb_timeout_id = 0;
+	}
+}
+
+static void
+dcb_carrier_cleanup (NMDevice *device)
+{
+	NMDeviceEthernetPrivate *priv = NM_DEVICE_ETHERNET_GET_PRIVATE (device);
+
+	if (priv->dcb_carrier_id) {
+		g_signal_handler_disconnect (device, priv->dcb_carrier_id);
+		priv->dcb_carrier_id = 0;
+	}
+}
+
+static void dcb_state (NMDevice *device, gboolean timeout);
+
+static gboolean
+dcb_carrier_timeout (gpointer user_data)
+{
+	NMDevice *device = NM_DEVICE (user_data);
+	NMDeviceEthernetPrivate *priv = NM_DEVICE_ETHERNET_GET_PRIVATE (device);
+
+	g_return_val_if_fail (nm_device_get_state (device) == NM_DEVICE_STATE_CONFIG, G_SOURCE_REMOVE);
+
+	priv->dcb_timeout_id = 0;
+	if (priv->dcb_wait != DCB_WAIT_CARRIER_POSTCONFIG_DOWN) {
+		nm_log_warn (LOGD_DCB,
+		             "(%s): DCB: timed out waiting for carrier (step %d)",
+		             nm_device_get_iface (device),
+		             priv->dcb_wait);
+	}
+	dcb_state (device, TRUE);
+	return G_SOURCE_REMOVE;
+}
+
+static gboolean
+dcb_configure (NMDevice *device)
+{
+	NMDeviceEthernetPrivate *priv = NM_DEVICE_ETHERNET_GET_PRIVATE (device);
+	NMSettingDcb *s_dcb;
+	const char *iface = nm_device_get_iface (device);
+	GError *error = NULL;
+
+	dcb_timeout_cleanup (device);
+
+	s_dcb = (NMSettingDcb *) device_get_setting (device, NM_TYPE_SETTING_DCB);
+	g_assert (s_dcb);
+	if (!nm_dcb_setup (iface, s_dcb, &error)) {
+		nm_log_warn (LOGD_DCB,
+		             "Activation (%s/wired) failed to enable DCB/FCoE: %s",
+		             iface, error->message);
+		g_clear_error (&error);
+		return FALSE;
+	}
+
+	/* Pause again just in case the device takes the carrier down when
+	 * setting specific DCB attributes.
+	 */
+	nm_log_dbg (LOGD_DCB, "(%s): waiting for carrier (postconfig down)", iface);
+	priv->dcb_wait = DCB_WAIT_CARRIER_POSTCONFIG_DOWN;
+	priv->dcb_timeout_id = g_timeout_add_seconds (3, dcb_carrier_timeout, device);
+	return TRUE;
+}
+
+static gboolean
+dcb_enable (NMDevice *device)
+{
+	NMDeviceEthernetPrivate *priv = NM_DEVICE_ETHERNET_GET_PRIVATE (device);
+	const char *iface = nm_device_get_iface (device);
+	GError *error = NULL;
+
+	dcb_timeout_cleanup (device);
+	if (!nm_dcb_enable (iface, TRUE, &error)) {
+		nm_log_warn (LOGD_DCB,
+		             "Activation (%s/wired) failed to enable DCB/FCoE: %s",
+		             iface, error->message);
+		g_clear_error (&error);
+		return FALSE;
+	}
+
+	/* Pause for 3 seconds after enabling DCB to let the card reconfigure
+	 * itself.  Drivers will often re-initialize internal settings which
+	 * takes the carrier down for 2 or more seconds.  During this time,
+	 * lldpad will refuse to do anything else with the card since the carrier
+	 * is down.  But NM might get the carrier-down signal long after calling
+	 * "dcbtool dcb on", so we have to first wait for the carrier to go down.
+	 */
+	nm_log_dbg (LOGD_DCB, "(%s): waiting for carrier (preconfig down)", iface);
+	priv->dcb_wait = DCB_WAIT_CARRIER_PRECONFIG_DOWN;
+	priv->dcb_timeout_id = g_timeout_add_seconds (3, dcb_carrier_timeout, device);
+	return TRUE;
+}
+
+static void
+dcb_state (NMDevice *device, gboolean timeout)
+{
+	NMDeviceEthernetPrivate *priv = NM_DEVICE_ETHERNET_GET_PRIVATE (device);
+	const char *iface = nm_device_get_iface (device);
+	gboolean carrier;
+
+	g_return_if_fail (nm_device_get_state (device) == NM_DEVICE_STATE_CONFIG);
+
+
+	carrier = nm_platform_link_is_connected (nm_device_get_ifindex (device));
+	nm_log_dbg (LOGD_DCB, "(%s): dcb_state() wait %d carrier %d timeout %d", iface, priv->dcb_wait, carrier, timeout);
+
+	switch (priv->dcb_wait) {
+	case DCB_WAIT_CARRIER_PREENABLE_UP:
+		if (timeout || carrier) {
+			nm_log_dbg (LOGD_DCB, "(%s): dcb_state() enabling DCB", iface);
+			dcb_timeout_cleanup (device);
+			if (!dcb_enable (device)) {
+				dcb_carrier_cleanup (device);
+				nm_device_state_changed (device,
+				                         NM_ACT_STAGE_RETURN_FAILURE,
+				                         NM_DEVICE_STATE_REASON_DCB_FCOE_FAILED);
+			}
+		}
+		break;
+	case DCB_WAIT_CARRIER_PRECONFIG_DOWN:
+		dcb_timeout_cleanup (device);
+		priv->dcb_wait = DCB_WAIT_CARRIER_PRECONFIG_UP;
+
+		if (!carrier) {
+			/* Wait for the carrier to come back up */
+			nm_log_dbg (LOGD_DCB, "(%s): waiting for carrier (preconfig up)", iface);
+			priv->dcb_timeout_id = g_timeout_add_seconds (5, dcb_carrier_timeout, device);
+			break;
+		}
+		nm_log_dbg (LOGD_DCB, "(%s): dcb_state() preconfig down falling through", iface);
+		/* carrier never went down? fall through */
+	case DCB_WAIT_CARRIER_PRECONFIG_UP:
+		if (timeout || carrier) {
+			nm_log_dbg (LOGD_DCB, "(%s): dcb_state() preconfig up configuring DCB", iface);
+			dcb_timeout_cleanup (device);
+			if (!dcb_configure (device)) {
+				dcb_carrier_cleanup (device);
+				nm_device_state_changed (device,
+				                         NM_ACT_STAGE_RETURN_FAILURE,
+				                         NM_DEVICE_STATE_REASON_DCB_FCOE_FAILED);
+			}
+		}
+		break;
+	case DCB_WAIT_CARRIER_POSTCONFIG_DOWN:
+		dcb_timeout_cleanup (device);
+		priv->dcb_wait = DCB_WAIT_CARRIER_POSTCONFIG_UP;
+
+		if (!carrier) {
+			/* Wait for the carrier to come back up */
+			nm_log_dbg (LOGD_DCB, "(%s): waiting for carrier (postconfig up)", iface);
+			priv->dcb_timeout_id = g_timeout_add_seconds (5, dcb_carrier_timeout, device);
+			break;
+		}
+		nm_log_dbg (LOGD_DCB, "(%s): dcb_state() postconfig down falling through", iface);
+		/* carrier never went down? fall through */
+	case DCB_WAIT_CARRIER_POSTCONFIG_UP:
+		if (timeout || carrier) {
+			nm_log_dbg (LOGD_DCB, "(%s): dcb_state() postconfig up starting IP", iface);
+			dcb_timeout_cleanup (device);
+			dcb_carrier_cleanup (device);
+			priv->dcb_wait = DCB_WAIT_UNKNOWN;
+			nm_device_activate_schedule_stage3_ip_config_start (device);
+		}
+		break;
+	default:
+		g_assert_not_reached ();
+	}
+}
+
+static void
+dcb_carrier_changed (NMDevice *device, GParamSpec *pspec, gpointer unused)
+{
+	NMDeviceEthernetPrivate *priv = NM_DEVICE_ETHERNET_GET_PRIVATE (device);
+
+	g_return_if_fail (nm_device_get_state (device) == NM_DEVICE_STATE_CONFIG);
+
+	if (priv->dcb_timeout_id) {
+		nm_log_dbg (LOGD_DCB, "(%s): carrier_changed() calling dcb_state()", nm_device_get_iface (device));
+		dcb_state (device, FALSE);
+	}
+}
+
+/****************************************************************/
+
 static NMActStageReturn
 act_stage2_config (NMDevice *device, NMDeviceStateReason *reason)
 {
+	NMDeviceEthernetPrivate *priv = NM_DEVICE_ETHERNET_GET_PRIVATE (device);
 	NMSettingConnection *s_con;
 	const char *connection_type;
 	NMActStageReturn ret = NM_ACT_STAGE_RETURN_SUCCESS;
 	NMSettingDcb *s_dcb;
-	GError *error = NULL;
 
 	g_return_val_if_fail (reason != NULL, NM_ACT_STAGE_RETURN_FAILURE);
 
-	/* DCB and FCoE setup */
-	s_dcb = (NMSettingDcb *) device_get_setting (device, NM_TYPE_SETTING_DCB);
-	if (s_dcb) {
-		if (!nm_dcb_setup (nm_device_get_iface (device), s_dcb, &error)) {
-			nm_log_warn (LOGD_DEVICE | LOGD_HW,
-			             "Activation (%s/wired) failed to enable DCB/FCoE: %s",
-			             nm_device_get_iface (device), error->message);
-			g_clear_error (&error);
-			*reason = NM_DEVICE_STATE_REASON_DCB_FCOE_FAILED;
-			return NM_ACT_STAGE_RETURN_FAILURE;
-		}
-	}
-
 	s_con = NM_SETTING_CONNECTION (device_get_setting (device, NM_TYPE_SETTING_CONNECTION));
 	g_assert (s_con);
 
+	dcb_timeout_cleanup (device);
+	dcb_carrier_cleanup (device);
+
 	/* 802.1x has to run before any IP configuration since the 802.1x auth
 	 * process opens the port up for normal traffic.
 	 */
@@ -1129,8 +1332,36 @@ act_stage2_config (NMDevice *device, NMDeviceStateReason *reason)
 		NMSetting8021x *security;
 
 		security = (NMSetting8021x *) device_get_setting (device, NM_TYPE_SETTING_802_1X);
-		if (security)
-			ret = nm_8021x_stage2_config (NM_DEVICE_ETHERNET (device), reason);
+		if (security) {
+			/* FIXME: for now 802.1x is mutually exclusive with DCB */
+			return nm_8021x_stage2_config (NM_DEVICE_ETHERNET (device), reason);
+		}
+	}
+
+	/* DCB and FCoE setup */
+	s_dcb = (NMSettingDcb *) device_get_setting (device, NM_TYPE_SETTING_DCB);
+	if (s_dcb) {
+		/* lldpad really really wants the carrier to be up */
+		if (nm_platform_link_is_connected (nm_device_get_ifindex (device))) {
+			if (!dcb_enable (device)) {
+				*reason = NM_DEVICE_STATE_REASON_DCB_FCOE_FAILED;
+				return NM_ACT_STAGE_RETURN_FAILURE;
+			}
+		} else {
+			nm_log_dbg (LOGD_DCB, "(%s): waiting for carrier (preenable up)",
+			            nm_device_get_iface (device));
+			priv->dcb_wait = DCB_WAIT_CARRIER_PREENABLE_UP;
+			priv->dcb_timeout_id = g_timeout_add_seconds (4, dcb_carrier_timeout, device);
+		}
+
+		/* Watch carrier independently of NMDeviceClass::carrier_changed so
+		 * we get instant notifications of disconnection that aren't deferred.
+		 */
+		priv->dcb_carrier_id = g_signal_connect (device,
+		                                         "notify::" NM_DEVICE_CARRIER,
+		                                         G_CALLBACK (dcb_carrier_changed),
+		                                         NULL);
+		ret = NM_ACT_STAGE_RETURN_POSTPONE;
 	}
 
 	return ret;
@@ -1206,6 +1437,10 @@ deactivate (NMDevice *device)
 
 	supplicant_interface_release (self);
 
+	priv->dcb_wait = DCB_WAIT_UNKNOWN;
+	dcb_timeout_cleanup (device);
+	dcb_carrier_cleanup (device);
+
 	/* Tear down DCB/FCoE if it was enabled */
 	s_dcb = (NMSettingDcb *) device_get_setting (device, NM_TYPE_SETTING_DCB);
 	if (s_dcb) {
@@ -1409,6 +1644,9 @@ dispose (GObject *object)
 		priv->pppoe_wait_id = 0;
 	}
 
+	dcb_timeout_cleanup (NM_DEVICE (self));
+	dcb_carrier_cleanup (NM_DEVICE (self));
+
 	G_OBJECT_CLASS (nm_device_ethernet_parent_class)->dispose (object);
 }
 
diff --git a/src/nm-dcb.c b/src/nm-dcb.c
index 5e25cd84e6..428c61e67d 100644
--- a/src/nm-dcb.c
+++ b/src/nm-dcb.c
@@ -92,6 +92,19 @@ out:
 	return success;
 }
 
+gboolean
+_dcb_enable (const char *iface,
+             gboolean enable,
+             DcbFunc run_func,
+             gpointer user_data,
+             GError **error)
+{
+	if (enable)
+		return do_helper (iface, DCBTOOL, run_func, user_data, error, "dcb on");
+	else
+		return do_helper (iface, DCBTOOL, run_func, user_data, error, "dcb off");
+}
+
 #define SET_FLAGS(f, tag) \
 G_STMT_START { \
 	if (!do_helper (iface, DCBTOOL, run_func, user_data, error, tag " e:%c a:%c w:%c", \
@@ -124,9 +137,6 @@ _dcb_setup (const char *iface,
 
 	g_assert (s_dcb);
 
-	if (!do_helper (iface, DCBTOOL, run_func, user_data, error, "dcb on"))
-		return FALSE;
-
 	/* FCoE */
 	flags = nm_setting_dcb_get_app_fcoe_flags (s_dcb);
 	SET_APP (flags, s_dcb, fcoe);
@@ -229,7 +239,6 @@ _dcb_cleanup (const char *iface,
               GError **error)
 {
 	const char *cmds[] = {
-		"dcb off",
 		"app:fcoe e:0",
 		"app:iscsi e:0",
 		"app:fip e:0",
@@ -247,6 +256,9 @@ _dcb_cleanup (const char *iface,
 		iter++;
 	}
 
+	if (!_dcb_enable (iface, FALSE, run_func, user_data, success ? error : NULL))
+		success = FALSE;
+
 	return success;
 }
 
@@ -355,6 +367,12 @@ run_helper (char **argv, guint which, gpointer user_data, GError **error)
 }
 
 gboolean
+nm_dcb_enable (const char *iface, gboolean enable, GError **error)
+{
+	return _dcb_enable (iface, enable, run_helper, GUINT_TO_POINTER (DCBTOOL), error);
+}
+
+gboolean
 nm_dcb_setup (const char *iface, NMSettingDcb *s_dcb, GError **error)
 {
 	gboolean success;
@@ -369,14 +387,8 @@ nm_dcb_setup (const char *iface, NMSettingDcb *s_dcb, GError **error)
 gboolean
 nm_dcb_cleanup (const char *iface, GError **error)
 {
-	gboolean success;
-
-	success = _dcb_cleanup (iface, run_helper, GUINT_TO_POINTER (DCBTOOL), error);
-	if (success) {
-		/* Only report FCoE errors if DCB cleanup was successful */
-		success = _fcoe_cleanup (iface, run_helper, GUINT_TO_POINTER (FCOEADM), success ? error : NULL);
-	}
-
-	return success;
+	/* Ignore FCoE cleanup errors */
+	_fcoe_cleanup (iface, run_helper, GUINT_TO_POINTER (FCOEADM), NULL);
+	return _dcb_cleanup (iface, run_helper, GUINT_TO_POINTER (DCBTOOL), error);
 }
 
diff --git a/src/nm-dcb.h b/src/nm-dcb.h
index 3dbd5e6f4f..bfe5cedffb 100644
--- a/src/nm-dcb.h
+++ b/src/nm-dcb.h
@@ -48,6 +48,7 @@ GQuark nm_dcb_error_quark (void);
 GType  nm_dcb_error_get_type (void);
 
 
+gboolean nm_dcb_enable (const char *iface, gboolean enable, GError **error);
 gboolean nm_dcb_setup (const char *iface, NMSettingDcb *s_dcb, GError **error);
 gboolean nm_dcb_cleanup (const char *iface, GError **error);
 
@@ -68,6 +69,12 @@ gboolean do_helper (const char *iface,
                     const char *fmt,
                     ...) G_GNUC_PRINTF(6, 7);
 
+gboolean _dcb_enable (const char *iface,
+                      gboolean enable,
+                      DcbFunc run_func,
+                      gpointer user_data,
+                      GError **error);
+
 gboolean _dcb_setup (const char *iface,
                      NMSettingDcb *s_dcb,
                      DcbFunc run_func,
diff --git a/src/tests/test-dcb.c b/src/tests/test-dcb.c
index 74dcca5de1..aead8f186c 100644
--- a/src/tests/test-dcb.c
+++ b/src/tests/test-dcb.c
@@ -54,8 +54,7 @@ static void
 test_dcb_fcoe (void)
 {
 	static DcbExpected expected = { 0,
-		{ "dcbtool sc eth0 dcb on",
-		  "dcbtool sc eth0 app:fcoe e:1 a:1 w:1",
+		{ "dcbtool sc eth0 app:fcoe e:1 a:1 w:1",
 		  "dcbtool sc eth0 app:fcoe appcfg:40",
 		  "dcbtool sc eth0 app:iscsi e:0 a:0 w:0",
 		  "dcbtool sc eth0 app:fip e:0 a:0 w:0",
@@ -85,8 +84,7 @@ static void
 test_dcb_iscsi (void)
 {
 	static DcbExpected expected = { 0,
-		{ "dcbtool sc eth0 dcb on",
-		  "dcbtool sc eth0 app:fcoe e:0 a:0 w:0",
+		{ "dcbtool sc eth0 app:fcoe e:0 a:0 w:0",
 		  "dcbtool sc eth0 app:iscsi e:1 a:0 w:1",
 		  "dcbtool sc eth0 app:iscsi appcfg:08",
 		  "dcbtool sc eth0 app:fip e:0 a:0 w:0",
@@ -116,8 +114,7 @@ static void
 test_dcb_fip (void)
 {
 	static DcbExpected expected = { 0,
-		{ "dcbtool sc eth0 dcb on",
-		  "dcbtool sc eth0 app:fcoe e:0 a:0 w:0",
+		{ "dcbtool sc eth0 app:fcoe e:0 a:0 w:0",
 		  "dcbtool sc eth0 app:iscsi e:0 a:0 w:0",
 		  "dcbtool sc eth0 app:fip e:1 a:1 w:0",
 		  "dcbtool sc eth0 app:fip appcfg:01",
@@ -147,8 +144,7 @@ static void
 test_dcb_fip_default_prio (void)
 {
 	static DcbExpected expected = { 0,
-		{ "dcbtool sc eth0 dcb on",
-		  "dcbtool sc eth0 app:fcoe e:0 a:0 w:0",
+		{ "dcbtool sc eth0 app:fcoe e:0 a:0 w:0",
 		  "dcbtool sc eth0 app:iscsi e:0 a:0 w:0",
 		  "dcbtool sc eth0 app:fip e:1 a:1 w:0",
 		  "dcbtool sc eth0 pfc e:0 a:0 w:0",
@@ -177,8 +173,7 @@ static void
 test_dcb_pfc (void)
 {
 	static DcbExpected expected = { 0,
-		{ "dcbtool sc eth0 dcb on",
-		  "dcbtool sc eth0 app:fcoe e:0 a:0 w:0",
+		{ "dcbtool sc eth0 app:fcoe e:0 a:0 w:0",
 		  "dcbtool sc eth0 app:iscsi e:0 a:0 w:0",
 		  "dcbtool sc eth0 app:fip e:0 a:0 w:0",
 		  "dcbtool sc eth0 pfc e:1 a:1 w:1",
@@ -216,8 +211,7 @@ static void
 test_dcb_priority_groups (void)
 {
 	static DcbExpected expected = { 0,
-		{ "dcbtool sc eth0 dcb on",
-		  "dcbtool sc eth0 app:fcoe e:0 a:0 w:0",
+		{ "dcbtool sc eth0 app:fcoe e:0 a:0 w:0",
 		  "dcbtool sc eth0 app:iscsi e:0 a:0 w:0",
 		  "dcbtool sc eth0 app:fip e:0 a:0 w:0",
 		  "dcbtool sc eth0 pfc e:0 a:0 w:0",
@@ -268,20 +262,27 @@ static void
 test_dcb_cleanup (void)
 {
 	static DcbExpected expected = { 0,
-		{ "dcbtool sc eth0 dcb off",
+		{ "fcoeadm -d eth0",
 		  "dcbtool sc eth0 app:fcoe e:0",
 		  "dcbtool sc eth0 app:iscsi e:0",
 		  "dcbtool sc eth0 app:fip e:0",
 		  "dcbtool sc eth0 pfc e:0",
 		  "dcbtool sc eth0 pg e:0",
+		  "dcbtool sc eth0 dcb off",
 		  NULL },
 	};
 	GError *error = NULL;
 	gboolean success;
 
+	success = _fcoe_cleanup ("eth0", test_dcb_func, &expected, &error);
+	g_assert_no_error (error);
+	g_assert (success);
+
 	success = _dcb_cleanup ("eth0", test_dcb_func, &expected, &error);
 	g_assert_no_error (error);
 	g_assert (success);
+
+	g_assert_cmpstr (expected.cmds[expected.num], ==, NULL);
 }
 
 static void