lvmlockd: extend the sanlock lv when neededdev-dct-lvmlockd-AA

6 files changed, 165 insertions, 56 deletions

diff --git a/daemons/lvmlockd/lvmlockd-client.h b/daemons/lvmlockd/lvmlockd-client.h
index 7010078a2..0f0c3051e 100644
--- a/daemons/lvmlockd/lvmlockd-client.h
+++ b/daemons/lvmlockd/lvmlockd-client.h
@@ -43,20 +43,4 @@ static inline void lvmlockd_close(daemon_handle h)
 #define EOTHERVG  212 /* vg sysid specifies other host */
 #define ESTARTING 213 /* lockspace is starting */
 
-/*
- * Also see lvmlockd-sanlock GL_LOCK_BEGIN, VG_LOCK_BEGIN, LV_LOCK_BEGIN.
- * gl lock at sanlock lease area 65
- * vg lock at sanlock lease area 66
- * lv locks begin at sanlock lease area 67
- *
- * LV_LOCK_BEGIN + MAX_LVS_IN_VG = sanlock lease areas required
- * with 512 byte sectors, each lease area is 1MB
- * with 4k byte sectors, each lease area is 8MB (use this for sizing)
- *
- * 66+190 = 256 sanlock lease areas,
- * so we need 256 * 8MB = 2GB lock lv size to hold 190 lv leases.
- */
-#define LVMLOCKD_SANLOCK_MAX_LVS_IN_VG 190
-#define LVMLOCKD_SANLOCK_LV_SIZE       2147483648 /* 2GB */
-
 #endif
diff --git a/daemons/lvmlockd/lvmlockd-sanlock.c b/daemons/lvmlockd/lvmlockd-sanlock.c
index 3451a8646..5ac02c17d 100644
--- a/daemons/lvmlockd/lvmlockd-sanlock.c
+++ b/daemons/lvmlockd/lvmlockd-sanlock.c
@@ -383,10 +383,15 @@ int lm_init_vg_sanlock(char *ls_name, char *vg_name, uint32_t flags, char *vg_ar
 
 	log_debug("S %s init_vg_san clearing lv lease areas", ls_name);
 
-	for (i = 0; i < LVMLOCKD_SANLOCK_MAX_LVS_IN_VG; i++) {
+	for (i = 0; ; i++) {
 		rd.rs.disks[0].offset = offset;
 
 		rv = sanlock_write_resource(&rd.rs, 0, 0, 0);
+		if (rv == -EMSGSIZE) {
+			/* This indicates the end of the device is reached. */
+			break;
+		}
+
 		if (rv) {
 			log_error("clear lv resource area %llu error %d",
 				  (unsigned long long)offset, rv);
@@ -414,7 +419,6 @@ int lm_init_lv_sanlock(char *ls_name, char *vg_name, char *lv_name,
 	char lock_args_version[MAX_ARGS];
 	uint64_t offset;
 	int align_size;
-	int lv_count = 0;
 	int rv;
 
 	memset(&rd, 0, sizeof(rd));
@@ -456,6 +460,13 @@ int lm_init_lv_sanlock(char *ls_name, char *vg_name, char *lv_name,
 		memset(rd.rs.name, 0, SANLK_NAME_LEN);
 
 		rv = sanlock_read_resource(&rd.rs, 0);
+		if (rv == -EMSGSIZE) {
+			/* This indicates the end of the device is reached. */
+			log_debug("S %s init_lv_san read limit offset %llu",
+				  ls_name, (unsigned long long)offset);
+			return rv;
+		}
+
 		if (rv) {
 			log_error("S %s init_lv_san read error %d offset %llu",
 				  ls_name, rv, (unsigned long long)offset);
@@ -486,12 +497,6 @@ int lm_init_lv_sanlock(char *ls_name, char *vg_name, char *lv_name,
 		}
 
 		offset += align_size;
-
-		if (lv_count++ >= LVMLOCKD_SANLOCK_MAX_LVS_IN_VG) {
-			log_error("S %s init_lv_san too many lvs %d", ls_name, lv_count);
-			rv = -ENOENT;
-			break;
-		}
 	}
 
 	return rv;
@@ -625,13 +630,18 @@ int lm_rename_vg_sanlock(char *ls_name, char *vg_name, uint32_t flags, char *vg_
 
 	offset = align_size * LV_LOCK_BEGIN;
 
-	for (i = 0; i < LVMLOCKD_SANLOCK_MAX_LVS_IN_VG; i++) {
+	for (i = 0; ; i++) {
 		memset(&rd, 0, sizeof(rd));
 		memcpy(rd.rs.disks[0].path, disk.path, SANLK_PATH_LEN);
 		rd.rs.disks[0].offset = offset;
 		rd.rs.num_disks = 1;
 
 		rv = sanlock_read_resource(&rd.rs, 0);
+		if (rv == -EMSGSIZE) {
+			/* This indicates the end of the device is reached. */
+			break;
+		}
+
 		if (rv < 0) {
 			log_error("S %s rename_vg_san read_resource resource area %llu error %d",
 				  ls_name, (unsigned long long)offset, rv);
@@ -1091,6 +1101,11 @@ static int find_lv_offset(struct lockspace *ls, struct resource *r,
 		memset(rd.rs.name, 0, SANLK_NAME_LEN);
 
 		rv = sanlock_read_resource(&rd.rs, 0);
+		if (rv == -EMSGSIZE) {
+			/* This indicates the end of the device is reached. */
+			break;
+		}
+
 		if (rv) {
 			log_error("S %s find_lv_offset read error %d offset %llu",
 				  ls->name, rv, (unsigned long long)offset);
@@ -1105,12 +1120,6 @@ static int find_lv_offset(struct lockspace *ls, struct resource *r,
 		}
 
 		offset += align_size;
-
-		if (lv_count++ >= LVMLOCKD_SANLOCK_MAX_LVS_IN_VG) {
-			rv = -EBADSLT;
-			break;
-		}
-
 	}
 	return rv;
 }
@@ -1259,6 +1268,21 @@ int lm_lock_sanlock(struct lockspace *ls, struct resource *r, int ld_mode,
 		return -EAGAIN;
 	}
 
+	if ((rv == -EMSGSIZE) && (r->type == LD_RT_LV)) {
+		/*
+		 * sanlock tried to read beyond the end of the device,
+		 * so the offset of the lv lease is beyond the end of the
+		 * device, which means that the lease lv was extended, and
+		 * the lease for this lv was allocated in the new space.
+		 * The lvm command will see this error, refresh the lvmlock
+		 * lv, and try again.
+		 */
+		log_debug("S %s R %s lock_san acquire offset %llu rv EMSGSIZE",
+			  ls->name, r->name, (unsigned long long)rs->disks[0].offset);
+		*retry = 0;
+		return -EMSGSIZE;
+	}
+
 	if (adopt && (rv == -EUCLEAN)) {
 		/*
 		 * The orphan lock exists but in a different mode than we asked
diff --git a/lib/locking/lvmlockd.c b/lib/locking/lvmlockd.c
index 484fdede7..d11725575 100644
--- a/lib/locking/lvmlockd.c
+++ b/lib/locking/lvmlockd.c
@@ -313,9 +313,18 @@ static int _lockd_request(struct cmd_context *cmd,
 	return 0;
 }
 
-/* The name of the internal lv created to hold sanlock locks. */
+/*
+ * The name of the internal lv created to hold sanlock locks.
+ */
 #define LVMLOCKD_SANLOCK_LV_NAME "lvmlock"
 
+/*
+ * The internal sanlock lv starts at 1GB and is increased by 1GB whenever it
+ * runs out of space.
+ */
+
+#define LVMLOCKD_SANLOCK_LV_EXTEND 1073741824 /* 1GB */
+
 static struct logical_volume *_find_sanlock_lv(struct volume_group *vg,
 					       const char *lock_lv_name)
 {
@@ -339,7 +348,7 @@ static int _create_sanlock_lv(struct cmd_context *cmd, struct volume_group *vg,
 	struct lvcreate_params lp = {
 		.activate = CHANGE_ALY,
 		.alloc = ALLOC_INHERIT,
-		.extents = LVMLOCKD_SANLOCK_LV_SIZE / (vg->extent_size * SECTOR_SIZE),
+		.extents = LVMLOCKD_SANLOCK_LV_EXTEND / (vg->extent_size * SECTOR_SIZE),
 		.major = -1,
 		.minor = -1,
 		.permission = LVM_READ | LVM_WRITE,
@@ -371,8 +380,7 @@ static int _remove_sanlock_lv(struct cmd_context *cmd, struct volume_group *vg,
 {
 	struct logical_volume *lv;
 
-	lv = _find_sanlock_lv(vg, lock_lv_name);
-	if (!lv) {
+	if (!(lv = _find_sanlock_lv(vg, lock_lv_name))) {
 		log_error("Failed to find sanlock LV %s in VG %s", lock_lv_name, vg->name);
 		return 0;
 	}
@@ -385,13 +393,62 @@ static int _remove_sanlock_lv(struct cmd_context *cmd, struct volume_group *vg,
 	return 1;
 }
 
+static int _extend_sanlock_lv(struct cmd_context *cmd, struct volume_group *vg)
+{
+	const char *lock_lv_name = LVMLOCKD_SANLOCK_LV_NAME;
+	struct logical_volume *lv;
+	struct lvresize_params lp = {
+		.lv_name = lock_lv_name,
+		.sign = SIGN_NONE,
+		.percent = PERCENT_NONE,
+		.resize = LV_EXTEND,
+		.ac_force = 1,
+		.sizeargs = 1,
+	};
+
+	if (!(lv = _find_sanlock_lv(vg, lock_lv_name))) {
+		log_error("Extend failed to find sanlock LV %s in VG %s", lock_lv_name, vg->name);
+		return 0;
+	}
+
+	lp.size = lv->size + (LVMLOCKD_SANLOCK_LV_EXTEND / SECTOR_SIZE);
+
+	if (!lv_resize_prepare(cmd, lv, &lp, &vg->pvs) ||
+	    !lv_resize(cmd, lv, &lp, &vg->pvs)) {
+		log_error("Extend LV %s/%s to size %llu failed.",
+			  vg->name, lv->name, (unsigned long long)lp.size);
+		return 0;
+	}
+
+	return 1;
+}
+
+/* When one host does _extend_sanlock_lv, the others need to refresh the size. */
+
+static int _refresh_sanlock_lv(struct cmd_context *cmd, struct volume_group *vg)
+{
+	struct logical_volume *lv;
+	const char *lock_lv_name = LVMLOCKD_SANLOCK_LV_NAME;
+
+	if (!(lv = _find_sanlock_lv(vg, lock_lv_name))) {
+		log_error("Refresh failed to find sanlock lv %s in vg %s", lock_lv_name, vg->name);
+		return 0;
+	}
+
+	if (!lv_refresh_suspend_resume(cmd, lv)) {
+		log_error("Failed to refresh %s.", lv->name);
+		return 0;
+	}
+
+	return 1;
+}
+
 static int _activate_sanlock_lv(struct cmd_context *cmd, struct volume_group *vg)
 {
 	struct logical_volume *lv;
 	const char *lock_lv_name = LVMLOCKD_SANLOCK_LV_NAME;
 
-	lv = _find_sanlock_lv(vg, lock_lv_name);
-	if (!lv) {
+	if (!(lv = _find_sanlock_lv(vg, lock_lv_name))) {
 		log_error("Failed to find sanlock lv %s in vg %s", lock_lv_name, vg->name);
 		return 0;
 	}
@@ -409,8 +466,7 @@ static int _deactivate_sanlock_lv(struct cmd_context *cmd, struct volume_group *
 	struct logical_volume *lv;
 	const char *lock_lv_name = LVMLOCKD_SANLOCK_LV_NAME;
 
-	lv = _find_sanlock_lv(vg, lock_lv_name);
-	if (!lv) {
+	if (!(lv = _find_sanlock_lv(vg, lock_lv_name))) {
 		log_error("Failed to find sanlock lv %s in vg %s", lock_lv_name, vg->name);
 		return 0;
 	}
@@ -1537,6 +1593,7 @@ int lockd_lv_name(struct cmd_context *cmd, struct volume_group *vg,
 	const char *mode = NULL;
 	const char *opts = NULL;
 	uint32_t result_flags;
+	int refreshed = 0;
 	int result;
 
 	if (cmd->lockd_lv_disable)
@@ -1580,6 +1637,7 @@ int lockd_lv_name(struct cmd_context *cmd, struct volume_group *vg,
 	if (flags & LDLV_PERSISTENT)
 		opts = "persistent";
 
+ retry:
 	if (!_lockd_request(cmd, "lock_lv",
 			       vg->name, vg->lock_type, vg->lock_args,
 			       lv_name, lock_args, mode, opts,
@@ -1601,6 +1659,15 @@ int lockd_lv_name(struct cmd_context *cmd, struct volume_group *vg,
 		return 0;
 	}
 
+	if (result == -EMSGSIZE) {
+		/* Another host probably extended lvmlock. */
+		if (!refreshed++) {
+			log_debug("Refresh lvmlock");
+		       	_refresh_sanlock_lv(cmd, vg);
+			goto retry;
+		}
+	}
+
 	if (result < 0) {
 		log_error("LV lock %s error %d: %s/%s", mode, result, vg->name, lv_name);
 		return 0;
@@ -1697,6 +1764,8 @@ static int _init_lv_sanlock(struct cmd_context *cmd, struct volume_group *vg,
 	daemon_reply reply;
 	const char *reply_str;
 	const char *lv_lock_args = NULL;
+	int refreshed = 0;
+	int extended = 0;
 	int result;
 	int ret;
 
@@ -1705,6 +1774,7 @@ static int _init_lv_sanlock(struct cmd_context *cmd, struct volume_group *vg,
 	if (!lvmlockd_connected())
 		return 0;
 
+ retry:
 	reply = _lockd_send("init_lv",
 				"pid = %d", getpid(),
 				"vg_name = %s", vg->name,
@@ -1724,6 +1794,25 @@ static int _init_lv_sanlock(struct cmd_context *cmd, struct volume_group *vg,
 		goto out;
 	}
 
+	if (result == -EMSGSIZE) {
+		/*
+		 * No space on the lvmlock lv for a new lease.
+		 * Check if another host has extended lvmlock,
+		 * and extend lvmlock if needed.
+		 */
+		if (!refreshed++) {
+			log_debug("Refresh lvmlock");
+			_refresh_sanlock_lv(cmd, vg);
+			goto retry;
+		}
+		if (!extended++) {
+			log_debug("Extend lvmlock");
+			_extend_sanlock_lv(cmd, vg);
+			goto retry;
+		}
+		goto out;
+	}
+
 	if (!ret) {
 		log_error("_init_lv_sanlock lvmlockd result %d", result);
 		goto out;
diff --git a/lib/metadata/lv_manip.c b/lib/metadata/lv_manip.c
index 6854bfad0..485c55d92 100644
--- a/lib/metadata/lv_manip.c
+++ b/lib/metadata/lv_manip.c
@@ -1398,6 +1398,28 @@ int replace_lv_with_error_segment(struct logical_volume *lv)
 	return 1;
 }
 
+int lv_refresh_suspend_resume(struct cmd_context *cmd, struct logical_volume *lv)
+{
+	if (!cmd->partial_activation && (lv->status & PARTIAL_LV)) {
+		log_error("Refusing refresh of partial LV %s."
+			  " Use '--activationmode partial' to override.",
+			  lv->name);
+		return 0;
+	}
+
+	if (!suspend_lv(cmd, lv)) {
+		log_error("Failed to suspend %s.", lv->name);
+		return 0;
+	}
+
+	if (!resume_lv(cmd, lv)) {
+		log_error("Failed to reactivate %s.", lv->name);
+		return 0;
+	}
+
+	return 1;
+}
+
 /*
  * Remove given number of extents from LV.
  */
@@ -4420,7 +4442,9 @@ static int _lvresize_check_lv(struct cmd_context *cmd, struct logical_volume *lv
 		return 0;
 	}
 
-	if (!lv_is_visible(lv) && !lv_is_thin_pool_metadata(lv)) {
+	/* FIXME: use a status flag instead of the name "lvmlock". */
+
+	if (!lv_is_visible(lv) && !lv_is_thin_pool_metadata(lv) && strcmp(lv->name, "lvmlock")) {
 		log_error("Can't resize internal logical volume %s", lv->name);
 		return 0;
 	}
diff --git a/lib/metadata/metadata-exported.h b/lib/metadata/metadata-exported.h
index 4355d1404..6bec90297 100644
--- a/lib/metadata/metadata-exported.h
+++ b/lib/metadata/metadata-exported.h
@@ -719,6 +719,8 @@ int lv_empty(struct logical_volume *lv);
 /* Empty an LV and add error segment */
 int replace_lv_with_error_segment(struct logical_volume *lv);
 
+int lv_refresh_suspend_resume(struct cmd_context *cmd, struct logical_volume *lv);
+
 /* Entry point for all LV extent allocations */
 int lv_extend(struct logical_volume *lv,
 	      const struct segment_type *segtype,
diff --git a/tools/toollib.c b/tools/toollib.c
index b31c8325f..7ca60c25b 100644
--- a/tools/toollib.c
+++ b/tools/toollib.c
@@ -1030,22 +1030,8 @@ int lv_change_activate(struct cmd_context *cmd, struct logical_volume *lv,
 
 int lv_refresh(struct cmd_context *cmd, struct logical_volume *lv)
 {
-	if (!cmd->partial_activation && (lv->status & PARTIAL_LV)) {
-		log_error("Refusing refresh of partial LV %s."
-			  " Use '--activationmode partial' to override.",
-			  lv->name);
-		return 0;
-	}
-
-	if (!suspend_lv(cmd, lv)) {
-		log_error("Failed to suspend %s.", lv->name);
-		return 0;
-	}
-
-	if (!resume_lv(cmd, lv)) {
-		log_error("Failed to reactivate %s.", lv->name);
-		return 0;
-	}
+	if (!lv_refresh_suspend_resume(cmd, lv))
+		return_0;
 
 	/*
 	 * check if snapshot merge should be polled