lib lvmlockd: retry busy locks

By default retry lock operations that return EAGAIN
because the lock is held by another host.

These retries occur at the client/command level, and
can be adjusted using global/lock_retries.

lvmlockd already has some automatic retries to handle
most incidental lock conflicts.

3 files changed, 36 insertions, 0 deletions

diff --git a/lib/config/config_settings.h b/lib/config/config_settings.h
index 2c97b8fc8..adb803843 100644
--- a/lib/config/config_settings.h
+++ b/lib/config/config_settings.h
@@ -833,6 +833,9 @@ cfg(global_use_lvmetad_CFG, "use_lvmetad", global_CFG_SECTION, 0, CFG_TYPE_BOOL,
 cfg(global_use_lvmlockd_CFG, "use_lvmlockd", global_CFG_SECTION, 0, CFG_TYPE_BOOL, 0, vsn(2, 2, 120), NULL, 0, NULL,
 	"Use lvmlockd for locking among hosts using LVM on shared storage.\n")
 
+cfg(global_lock_retries_CFG, "lock_retries", global_CFG_SECTION, CFG_DEFAULT_COMMENTED, CFG_TYPE_INT, DEFAULT_LOCK_RETRIES, vsn(2, 2, 12), NULL, 0, NULL,
+	"Retry lvmlockd lock requests this many times.\n")
+
 cfg(global_allow_override_lock_modes_CFG, "allow_override_lock_modes", global_CFG_SECTION, 0, CFG_TYPE_BOOL, 0, vsn(2, 2, 120), NULL, 0, NULL,
 	"Allow command options to override normal locking.\n")
 
diff --git a/lib/config/defaults.h b/lib/config/defaults.h
index efe9544dd..b5b14331c 100644
--- a/lib/config/defaults.h
+++ b/lib/config/defaults.h
@@ -51,6 +51,7 @@
 #define DEFAULT_FALLBACK_TO_LOCAL_LOCKING 1
 #define DEFAULT_FALLBACK_TO_CLUSTERED_LOCKING 1
 #define DEFAULT_WAIT_FOR_LOCKS 1
+#define DEFAULT_LOCK_RETRIES 3
 #define DEFAULT_PRIORITISE_WRITE_LOCKS 1
 #define DEFAULT_USE_MLOCKALL 0
 #define DEFAULT_METADATA_READ_ONLY 0
diff --git a/lib/locking/lvmlockd.c b/lib/locking/lvmlockd.c
index 49ab40c38..b5cfa5f40 100644
--- a/lib/locking/lvmlockd.c
+++ b/lib/locking/lvmlockd.c
@@ -1084,6 +1084,7 @@ int lockd_gl_create(struct cmd_context *cmd, const char *def_mode, const char *v
 {
 	const char *mode = NULL;
 	uint32_t lockd_flags;
+	int retries = 0;
 	int result;
 
 	/* A specific lock mode was given on the command line. */
@@ -1111,6 +1112,7 @@ int lockd_gl_create(struct cmd_context *cmd, const char *def_mode, const char *v
 		return 0;
 	}
 
+ req:
 	if (!_lockd_request(cmd, "lock_gl",
 			      NULL, vg_lock_type, NULL, NULL, NULL, mode, NULL,
 			      &result, &lockd_flags)) {
@@ -1119,6 +1121,15 @@ int lockd_gl_create(struct cmd_context *cmd, const char *def_mode, const char *v
 		return 0;
 	}
 
+	if (result == -EAGAIN) {
+		if (retries < find_config_tree_int(cmd, global_lock_retries_CFG, NULL)) {
+			log_warn("Retrying %s global lock", mode);
+			sleep(1);
+			retries++;
+			goto req;
+		}
+	}
+
 	/*
 	 * result and lockd_flags were returned from lvmlockd.
 	 *
@@ -1267,6 +1278,7 @@ int lockd_gl(struct cmd_context *cmd, const char *def_mode, uint32_t flags)
 	const char *mode = NULL;
 	const char *opts = NULL;
 	uint32_t lockd_flags;
+	int retries = 0;
 	int result;
 
 	/* A specific lock mode was given on the command line. */
@@ -1294,6 +1306,7 @@ int lockd_gl(struct cmd_context *cmd, const char *def_mode, uint32_t flags)
 		return 0;
 	}
 
+ req:
 	if (!_lockd_request(cmd, "lock_gl",
 			    NULL, NULL, NULL, NULL, NULL, mode, opts,
 			    &result, &lockd_flags)) {
@@ -1321,6 +1334,15 @@ int lockd_gl(struct cmd_context *cmd, const char *def_mode, uint32_t flags)
 		return 0;
 	}
 
+	if (result == -EAGAIN) {
+		if (retries < find_config_tree_int(cmd, global_lock_retries_CFG, NULL)) {
+			log_warn("Retrying %s global lock", mode);
+			sleep(1);
+			retries++;
+			goto req;
+		}
+	}
+
 	/*
 	 * result and lockd_flags were returned from lvmlockd.
 	 *
@@ -1436,6 +1458,7 @@ int lockd_vg(struct cmd_context *cmd, const char *vg_name, const char *def_mode,
 {
 	const char *mode = NULL;
 	uint32_t lockd_flags;
+	int retries = 0;
 	int result;
 	int ret;
 
@@ -1528,6 +1551,15 @@ int lockd_vg(struct cmd_context *cmd, const char *vg_name, const char *def_mode,
 		return 1;
 	}
 
+	if (result == -EAGAIN) {
+		if (retries < find_config_tree_int(cmd, global_lock_retries_CFG, NULL)) {
+			log_warn("Retrying %s lock on VG %s", mode, vg_name);
+			sleep(1);
+			retries++;
+			goto req;
+		}
+	}
+
 	switch (result) {
 	case 0:
 		/* success */