lvmlockd: vgcreate/vgremove call init_vg/free_vgdev-dct-lvmlockd4-initfree

vgcreate calls lvmlockd_init_vg() to do any create/initialize
steps that are needed in lvmlockd for the given lock_type.

vgcreate calls lvmlockd_free_vg_before() to do any removal/freeing
steps that are needed in lvmlockd for the given lock_type
before the VG is removed on disk.

vgcreate calls lvmlockd_free_vg_final() to do any removal/freeing
steps that are needed in lvmlockd for the given lock_type
after the VG is removed on disk.

When the lock_type is sanlock, the init/free also include
lvm client side steps to create/remove an internal LV on
which sanlock will store the locks for the VG.

5 files changed, 655 insertions, 1 deletions

diff --git a/daemons/lvmlockd/lvmlockd-client.h b/daemons/lvmlockd/lvmlockd-client.h
index 97402cee5..bde0191ea 100644
--- a/daemons/lvmlockd/lvmlockd-client.h
+++ b/daemons/lvmlockd/lvmlockd-client.h
@@ -33,4 +33,20 @@ static inline void lvmlockd_close(daemon_handle h)
 	return daemon_close(h);
 }
 
+/*
+ * Also see lvmlockd-sanlock GL_LOCK_BEGIN, VG_LOCK_BEGIN, LV_LOCK_BEGIN.
+ * gl lock at sanlock lease area 65
+ * vg lock at sanlock lease area 66
+ * lv locks begin at sanlock lease area 67
+ *
+ * LV_LOCK_BEGIN + MAX_LVS_IN_VG = sanlock lease areas required
+ * with 512 byte sectors, each lease area is 1MB
+ * with 4k byte sectors, each lease area is 8MB (use this for sizing)
+ *
+ * 66+190 = 256 sanlock lease areas,
+ * so we need 256 * 8MB = 2GB lock lv size to hold 190 lv leases.
+ */
+#define LVMLOCKD_SANLOCK_MAX_LVS_IN_VG 190
+#define LVMLOCKD_SANLOCK_LV_SIZE       2147483648 /* 2GB */
+
 #endif
diff --git a/lib/locking/lvmlockd.c b/lib/locking/lvmlockd.c
index ab33b6e72..6f392d994 100644
--- a/lib/locking/lvmlockd.c
+++ b/lib/locking/lvmlockd.c
@@ -105,3 +105,605 @@ void lvmlockd_set_socket(const char *sock)
 	_lvmlockd_socket = sock;
 }
 
+static void _result_str_to_flags(const char *str, uint32_t *flags)
+{
+}
+
+/*
+ * evaluate the reply from lvmlockd, check for errors, extract
+ * the result and result_flags returned by lvmlockd.
+ * 0 failure (no result/result_flags set)
+ * 1 success (result/result_flags set)
+ */
+
+static int _lvmlockd_result(daemon_reply reply, int *result, uint32_t *result_flags)
+{
+	int reply_result;
+	const char *reply_flags;
+	const char *lock_type;
+
+	if (reply.error) {
+		log_error("lvmlockd_result reply error %d", reply.error);
+		return 0;
+	}
+
+	if (strcmp(daemon_reply_str(reply, "response", ""), "OK")) {
+		log_error("lvmlockd_result bad response");
+		return 0;
+	}
+
+	/* -1000 is a random number that we know is not returned. */
+
+	reply_result = daemon_reply_int(reply, "op_result", -1000);
+	if (reply_result == -1000) {
+		log_error("lvmlockd_result no op_result");
+		return 0;
+	}
+
+	/* The lock_type that lvmlockd used for locking. */
+	lock_type = daemon_reply_str(reply, "lock_type", "none");
+
+	*result = reply_result;
+
+	if (!result_flags)
+		goto out;
+
+	reply_flags = daemon_reply_str(reply, "result_flags", NULL);
+	if (reply_flags)
+		_result_str_to_flags(reply_flags, result_flags);
+
+ out:
+	log_debug("lvmlockd_result %d %s lm %s", reply_result, reply_flags, lock_type);
+	return 1;
+}
+
+static daemon_reply _lvmlockd_send(const char *req_name, ...)
+{
+	va_list ap;
+	daemon_reply repl;
+	daemon_request req;
+
+	req = daemon_request_make(req_name);
+
+	va_start(ap, req_name);
+	daemon_request_extend_v(req, ap);
+	va_end(ap);
+
+	repl = daemon_send(_lvmlockd, req);
+
+	daemon_request_destroy(req);
+
+	return repl;
+}
+
+/*
+ * result/result_flags are values returned from lvmlockd.
+ *
+ * return 0 (failure)
+ * return 1 (result/result_flags indicate success/failure)
+ *
+ * return 1 result 0   (success)
+ * return 1 result < 0 (failure)
+ *
+ * caller may ignore result < 0 failure depending on
+ * result_flags and the specific command/mode.
+ *
+ * When this function returns 0 (failure), no result/result_flags
+ * were obtained from lvmlockd.
+ *
+ * When this function returns 1 (success), result/result_flags may
+ * have been obtained from lvmlockd.  This lvmlockd result may
+ * indicate a locking failure.
+ */
+
+int lvmlockd_send(struct cmd_context *cmd,
+		  const char *cmd_name,
+		  const char *req_name,
+		  const char *vg_name,
+		  const char *vg_lock_type,
+		  const char *vg_lock_args,
+		  const char *lv_name,
+		  const char *lv_lock_args,
+		  const char *mode,
+		  const char *opts,
+		  int *result,
+		  uint32_t *result_flags)
+{
+	daemon_reply reply;
+	int pid = getpid();
+
+	*result = 0;
+	*result_flags = 0;
+
+	if (!strcmp(mode, "na"))
+		return 1;
+
+	if (!_lvmlockd_active)
+		return 1;
+	if (!lvmlockd_connected())
+		return 0;
+
+	/* cmd and pid are passed for informational and debugging purposes */
+
+	if (vg_name && lv_name) {
+		reply = _lvmlockd_send(req_name,
+					"cmd = %s", cmd_name,
+					"pid = %d", pid,
+					"mode = %s", mode,
+					"opts = %s", opts ?: "none",
+					"vg_name = %s", vg_name,
+					"lv_name = %s", lv_name,
+					"vg_lock_type = %s", vg_lock_type ?: "none",
+					"vg_lock_args = %s", vg_lock_args ?: "none",
+					"lv_lock_args = %s", lv_lock_args ?: "none",
+					NULL);
+
+		if (!_lvmlockd_result(reply, result, result_flags))
+			goto fail;
+
+		log_debug("lvmlockd %s %s vg %s lv %s result %d %x",
+			  req_name, mode, vg_name, lv_name, *result, *result_flags);
+
+	} else if (vg_name) {
+		reply = _lvmlockd_send(req_name,
+					"cmd = %s", cmd_name,
+					"pid = %d", pid,
+					"mode = %s", mode,
+					"opts = %s", opts ?: "none",
+					"vg_name = %s", vg_name,
+					"vg_lock_type = %s", vg_lock_type ?: "none",
+					"vg_lock_args = %s", vg_lock_args ?: "none",
+					NULL);
+
+		if (!_lvmlockd_result(reply, result, result_flags))
+			goto fail;
+
+		log_debug("lvmlockd %s %s vg %s result %d %x",
+			  req_name, mode, vg_name, *result, *result_flags);
+
+	} else {
+		reply = _lvmlockd_send(req_name,
+					"cmd = %s", cmd_name,
+					"pid = %d", pid,
+					"mode = %s", mode,
+					"opts = %s", opts ?: "none",
+					"vg_lock_type = %s", vg_lock_type ?: "none",
+					NULL);
+
+		if (!_lvmlockd_result(reply, result, result_flags))
+			goto fail;
+
+		log_debug("lvmlockd %s %s result %d %x",
+			  req_name, mode, *result, *result_flags);
+	}
+
+	daemon_reply_destroy(reply);
+
+	/* result/result_flags have lvmlockd result */
+	return 1;
+
+ fail:
+	/* no result was obtained from lvmlockd */
+
+	log_error("lvmlockd %s %s failed no result", req_name, mode);
+
+	daemon_reply_destroy(reply);
+	return 0;
+}
+
+/* The name of the internal lv created to hold sanlock locks. */
+#define LVMLOCKD_SANLOCK_LV_NAME "lvmlock"
+
+static struct logical_volume *_find_sanlock_lv(struct volume_group *vg,
+					       const char *lock_lv_name)
+{
+	struct lv_list *lvl;
+
+	dm_list_iterate_items(lvl, &vg->lvs) {
+		if (!strcmp(lvl->lv->name, lock_lv_name))
+			return lvl->lv;
+	}
+	return NULL;
+}
+
+/*
+ * Eventually add an option to specify which pv the lvmlock lv should be placed on.
+ */
+
+static int _create_sanlock_lv(struct cmd_context *cmd, struct volume_group *vg,
+			      const char *lock_lv_name)
+{
+	struct logical_volume *lv;
+	struct lvcreate_params lp = {
+		.activate = CHANGE_ALY,
+		.alloc = ALLOC_INHERIT,
+		.extents = LVMLOCKD_SANLOCK_LV_SIZE / (vg->extent_size * SECTOR_SIZE),
+		.major = -1,
+		.minor = -1,
+		.permission = LVM_READ | LVM_WRITE,
+		.pvh = &vg->pvs,
+		.read_ahead = DM_READ_AHEAD_NONE,
+		.stripes = 1,
+		.vg_name = vg->name,
+		.lv_name = dm_pool_strdup(cmd->mem, lock_lv_name),
+		.zero = 1,
+	};
+
+	dm_list_init(&lp.tags);
+
+	if (!(lp.segtype = get_segtype_from_string(vg->cmd, "striped")))
+		return_0;
+
+	lv = lv_create_single(vg, &lp);
+	if (!lv) {
+		log_error("Failed to create sanlock lv %s in vg %s", lock_lv_name, vg->name);
+		return 0;
+	}
+
+	lv_set_hidden(lv);
+	return 1;
+}
+
+static int _remove_sanlock_lv(struct cmd_context *cmd, struct volume_group *vg,
+			      const char *lock_lv_name)
+{
+	struct logical_volume *lv;
+
+	lv = _find_sanlock_lv(vg, lock_lv_name);
+	if (!lv) {
+		log_error("Failed to find sanlock LV %s in VG %s", lock_lv_name, vg->name);
+		return 0;
+	}
+
+	if (!lv_remove(lv)) {
+		log_error("Failed to remove sanlock LV %s/%s", vg->name, lock_lv_name);
+		return 0;
+	}
+
+	return 1;
+}
+
+static int _activate_sanlock_lv(struct cmd_context *cmd, struct volume_group *vg)
+{
+	struct logical_volume *lv;
+	const char *lock_lv_name = LVMLOCKD_SANLOCK_LV_NAME;
+
+	lv = _find_sanlock_lv(vg, lock_lv_name);
+	if (!lv) {
+		log_error("Failed to find sanlock lv %s in vg %s", lock_lv_name, vg->name);
+		return 0;
+	}
+
+	if (!activate_lv(cmd, lv)) {
+		log_error("Failed to activate sanlock lv %s/%s", vg->name, lock_lv_name);
+		return 0;
+	}
+
+	return 1;
+}
+
+static int _deactivate_sanlock_lv(struct cmd_context *cmd, struct volume_group *vg)
+{
+	struct logical_volume *lv;
+	const char *lock_lv_name = LVMLOCKD_SANLOCK_LV_NAME;
+
+	lv = _find_sanlock_lv(vg, lock_lv_name);
+	if (!lv) {
+		log_error("Failed to find sanlock lv %s in vg %s", lock_lv_name, vg->name);
+		return 0;
+	}
+
+	if (!deactivate_lv(cmd, lv)) {
+		log_error("Failed to deactivate sanlock lv %s/%s", vg->name, lock_lv_name);
+		return 0;
+	}
+
+	return 1;
+}
+
+static int _init_vg_dlm(struct cmd_context *cmd, struct volume_group *vg)
+{
+	daemon_reply reply;
+	const char *reply_str;
+	const char *vg_lock_args = NULL;
+	int result;
+	int ret;
+
+	log_debug("_init_vg_dlm %s", vg->name);
+
+	if (!_lvmlockd_active)
+		return 1;
+	if (!lvmlockd_connected())
+		return 0;
+
+	reply = _lvmlockd_send("init_vg",
+				"pid = %d", getpid(),
+				"vg_name = %s", vg->name,
+				"vg_lock_type = %s", "dlm",
+				NULL);
+
+	if (!_lvmlockd_result(reply, &result, NULL)) {
+		ret = 0;
+	} else {
+		ret = (result < 0) ? 0 : 1;
+	}
+
+	if (!ret) {
+		log_error("_init_vg_dlm lvmlockd result %d", result);
+		goto out;
+	}
+
+	reply_str = daemon_reply_str(reply, "vg_lock_args", NULL);
+	if (!reply_str) {
+		log_error("vg_lock_args not returned");
+		ret = 0;
+		goto out;
+	}
+
+	vg_lock_args = dm_pool_strdup(cmd->mem, reply_str);
+	if (!vg_lock_args) {
+		log_error("vg_lock_args allocation failed");
+		ret = 0;
+	}
+out:
+	daemon_reply_destroy(reply);
+
+	vg->lock_args = vg_lock_args;
+	return ret;
+}
+
+static int _init_vg_sanlock(struct cmd_context *cmd, struct volume_group *vg)
+{
+	daemon_reply reply;
+	const char *reply_str;
+	const char *vg_lock_args = NULL;
+	const char *lock_lv_name = LVMLOCKD_SANLOCK_LV_NAME;
+	const char *opts = NULL;
+	int result;
+	int ret;
+
+	log_debug("_init_vg_sanlock %s", vg->name);
+
+	if (!_lvmlockd_active)
+		return 1;
+	if (!lvmlockd_connected())
+		return 0;
+
+	if (!_create_sanlock_lv(cmd, vg, lock_lv_name)) {
+		log_error("Failed to create internal lv.");
+		return 0;
+	}
+
+	/*
+	 * N.B. this passes the lock_lv_name as vg_lock_args
+	 * even though it is only part of the final args string
+	 * which will be returned from lvmlockd.
+	 */
+
+	reply = _lvmlockd_send("init_vg",
+				"pid = %d", getpid(),
+				"vg_name = %s", vg->name,
+				"vg_lock_type = %s", "sanlock",
+				"vg_lock_args = %s", lock_lv_name,
+				"opts = %s", opts ?: "none",
+				NULL);
+
+	if (!_lvmlockd_result(reply, &result, NULL)) {
+		ret = 0;
+	} else {
+		ret = (result < 0) ? 0 : 1;
+	}
+
+	if (!ret) {
+		log_error("_init_vg_sanlock lvmlockd result %d", result);
+		_remove_sanlock_lv(cmd, vg, lock_lv_name);
+		goto out;
+	}
+
+	reply_str = daemon_reply_str(reply, "vg_lock_args", NULL);
+	if (!reply_str) {
+		log_error("vg_lock_args not returned");
+		ret = 0;
+		goto out;
+	}
+
+	vg_lock_args = dm_pool_strdup(cmd->mem, reply_str);
+	if (!vg_lock_args) {
+		log_error("vg_lock_args allocation failed");
+		ret = 0;
+	}
+out:
+	daemon_reply_destroy(reply);
+
+	vg->lock_args = vg_lock_args;
+	return ret;
+}
+
+/* called after vg_remove on disk */
+
+static int _free_vg_dlm(struct cmd_context *cmd, struct volume_group *vg)
+{
+	uint32_t result_flags;
+	int result;
+	int ret;
+
+	/*
+	 * Unlocking the vg lock here preempts the dlock_vg("un") in
+	 * toollib.c which happens too late since the lockspace is
+	 * left here.
+	 */
+
+	log_debug("_free_vg_dlm un for vgremove %s", vg->name);
+
+	/* Equivalent to dlock_vg(cmd, vg->name, "un", 0); */
+	ret = lvmlockd_send(cmd, "vgremove", "lock_vg", vg->name,
+			    NULL, NULL, NULL, NULL, "un", NULL,
+			    &result, &result_flags);
+
+	if (!ret || result < 0) {
+		log_error("_free_vg_dlm lvmlockd result %d", result);
+		return 0;
+	}
+
+	/*
+	 * Leave the dlm lockspace.
+	 * Joining and leaving the lockspaces to be added by later patch.
+	 * lvmlockd_stop_vg(cmd, vg);
+	 */
+
+	return 1;
+}
+
+/* called before vg_remove on disk */
+
+static int _free_vg_sanlock(struct cmd_context *cmd, struct volume_group *vg)
+{
+	daemon_reply reply;
+	const char *lock_lv_name = LVMLOCKD_SANLOCK_LV_NAME;
+	int result;
+	int ret;
+
+	log_debug("_free_vg_sanlock for vgremove %s", vg->name);
+
+	if (!_lvmlockd_active)
+		return 1;
+	if (!lvmlockd_connected())
+		return 0;
+
+	if (!vg->lock_args || !strlen(vg->lock_args)) {
+		/* Shouldn't happen in general, but maybe in some error cases? */
+		log_debug("_free_vg_sanlock %s no lock_args", vg->name);
+		return 1;
+	}
+
+	reply = _lvmlockd_send("free_vg",
+				"pid = %d", getpid(),
+				"vg_name = %s", vg->name,
+				"vg_lock_type = %s", vg->lock_type,
+				"vg_lock_args = %s", vg->lock_args,
+				NULL);
+
+	if (!_lvmlockd_result(reply, &result, NULL)) {
+		ret = 0;
+	} else {
+		ret = (result < 0) ? 0 : 1;
+	}
+
+	/*
+	 * Other hosts could still be joined to the lockspace, which means they
+	 * are using the internal sanlock LV, which means we cannot remove the
+	 * VG.  Once other hosts stop using the VG it can be removed.
+	 */
+	if (result == -EBUSY) {
+		log_error("Lockspace for \"%s\" not stopped on other hosts", vg->name);
+		goto out;
+	}
+
+	if (!ret) {
+		log_error("_free_vg_sanlock lvmlockd result %d", result);
+		goto out;
+	}
+
+	_deactivate_sanlock_lv(cmd, vg);
+
+	_remove_sanlock_lv(cmd, vg, lock_lv_name);
+ out:
+	daemon_reply_destroy(reply);
+
+	return ret;
+}
+
+/*
+ * Called to remove lvmlockd's record of the local vg which it caches as an
+ * optimization.
+ */
+
+static int _free_vg_local(struct cmd_context *cmd, struct volume_group *vg)
+{
+	daemon_reply reply;
+	char uuid[64] __attribute__((aligned(8)));
+	int result;
+	int ret;
+
+	memset(uuid, 0, sizeof(uuid));
+	id_write_format(&vg->id, uuid, sizeof(uuid));
+
+	log_debug("_free_vg_local for vgremove %s", vg->name);
+
+	reply = _lvmlockd_send("rem_local",
+				"pid = %d", getpid(),
+				"vg_name = %s", vg->name,
+				"vg_uuid = %s", uuid[0] ? uuid : "none",
+				"vg_lock_type = %s", "none",
+				NULL);
+
+	if (!_lvmlockd_result(reply, &result, NULL)) {
+		ret = 0;
+	} else {
+		ret = (result < 0) ? 0 : 1;
+	}
+	
+	if (!ret) {
+		log_error("_free_vg_local lvmlockd result %d", result);
+	}
+
+	daemon_reply_destroy(reply);
+
+	return ret;
+}
+
+/* vgcreate */
+
+int lvmlockd_init_vg(struct cmd_context *cmd, struct volume_group *vg)
+{       
+	switch (lock_type_to_num(vg->lock_type)) {
+	case LOCK_TYPE_NONE:
+	case LOCK_TYPE_CLVM:
+		return 1;
+	case LOCK_TYPE_DLM:
+		return _init_vg_dlm(cmd, vg);
+	case LOCK_TYPE_SANLOCK:
+		return _init_vg_sanlock(cmd, vg);
+	default:
+		log_error("Unknown lock_type.");
+		return 0;
+	}
+}
+
+/* vgremove before the vg is removed */
+
+int lvmlockd_free_vg_before(struct cmd_context *cmd, struct volume_group *vg)
+{
+	switch (lock_type_to_num(vg->lock_type)) {
+	case LOCK_TYPE_NONE:
+	case LOCK_TYPE_CLVM:
+	case LOCK_TYPE_DLM:
+		return 1;
+	case LOCK_TYPE_SANLOCK:
+		/* returning an error will prevent vg_remove() */
+		return _free_vg_sanlock(cmd, vg);
+	default:
+		log_error("Unknown lock_type.");
+		return 0;
+	}
+}
+
+/* vgremove after the vg is removed */
+
+void lvmlockd_free_vg_final(struct cmd_context *cmd, struct volume_group *vg)
+{
+	switch (lock_type_to_num(vg->lock_type)) {
+	case LOCK_TYPE_NONE:
+		_free_vg_local(cmd, vg);
+		break;
+	case LOCK_TYPE_CLVM:
+	case LOCK_TYPE_SANLOCK:
+		break;
+	case LOCK_TYPE_DLM:
+		_free_vg_dlm(cmd, vg);
+		break;
+	default:
+		log_error("Unknown lock_type.");
+	}
+}
+
diff --git a/lib/locking/lvmlockd.h b/lib/locking/lvmlockd.h
index 95b85256f..f2ab1a826 100644
--- a/lib/locking/lvmlockd.h
+++ b/lib/locking/lvmlockd.h
@@ -63,7 +63,7 @@ static inline int is_dlock_type(const char *lock_type)
 
 #ifdef LVMLOCKD_SUPPORT
 
-/* daemon management */
+/* lvmlockd connection and communication */
 
 void lvmlockd_init(struct cmd_context *);
 void lvmlockd_set_active(int);
@@ -72,6 +72,25 @@ void lvmlockd_disconnect(void);
 void lvmlockd_connect_or_warn(void);
 int lvmlockd_connected(void);
 
+int lvmlockd_send(struct cmd_context *cmd,
+                  const char *cmd_name,
+                  const char *req_name,
+                  const char *vg_name,
+                  const char *vg_lock_type,
+                  const char *vg_lock_args,
+                  const char *lv_name,
+                  const char *lv_lock_args,
+                  const char *mode,
+                  const char *opts,
+                  int *result,
+                  uint32_t *result_flags);
+
+/* vgcreate/vgremove use init/free */
+
+int lvmlockd_init_vg(struct cmd_context *cmd, struct volume_group *vg);
+int lvmlockd_free_vg_before(struct cmd_context *cmd, struct volume_group *vg);
+void lvmlockd_free_vg_final(struct cmd_context *cmd, struct volume_group *vg);
+
 #else /* LVMLOCKD_SUPPORT */
 
 #define lvmlockd_init(cmd)          do { } while (0)
@@ -81,6 +100,12 @@ int lvmlockd_connected(void);
 #define lvmlockd_connect_or_warn()  do { } while (0)
 #define lvmlockd_connected          (0)
 
+#define lvmlockd_send(cmd, cmd_name, req_name, vg_name, vg_lock_type, vg_lock_args, lv_name, lv_lock_args, mode, opts, result, result_flags) (1)
+
+#define lvmlockd_init_vg(cmd, vg)        (0)
+#define lvmlockd_free_vg_before(cmd, vg) (0)
+#define lvmlockd_free_vg_final(cmd, vg)  do { } while (0)
+
 #endif /* LVMLOCKD_SUPPORT */
 
 #endif
diff --git a/tools/vgcreate.c b/tools/vgcreate.c
index 7304064fa..7c6d9187e 100644
--- a/tools/vgcreate.c
+++ b/tools/vgcreate.c
@@ -108,6 +108,12 @@ int vgcreate(struct cmd_context *cmd, int argc, char **argv)
 		}
 	}
 
+	if (!lvmlockd_init_vg(cmd, vg)) {
+		log_error("Failed to initialize lock args for lock type %s",
+			  vp_new.lock_type);
+		goto_bad;
+	}
+
 	if (vg_is_clustered(vg))
 		clustered_message = "Clustered ";
 	else if (locking_is_clustered())
diff --git a/tools/vgremove.c b/tools/vgremove.c
index 1dce41f24..98336362d 100644
--- a/tools/vgremove.c
+++ b/tools/vgremove.c
@@ -54,6 +54,9 @@ static int vgremove_single(struct cmd_context *cmd, const char *vg_name,
 		}
 	}
 
+	if (!lvmlockd_free_vg_before(cmd, vg))
+		return_ECMD_FAILED;
+
 	if (!force && !vg_remove_check(vg))
 		return_ECMD_FAILED;
 
@@ -62,6 +65,8 @@ static int vgremove_single(struct cmd_context *cmd, const char *vg_name,
 	if (!vg_remove(vg))
 		return_ECMD_FAILED;
 
+	lvmlockd_free_vg_final(cmd, vg);
+
 	return ECMD_PROCESSED;
 }