From 250737991a85d661c9889a7d6283f1401dc574c9 Mon Sep 17 00:00:00 2001 From: David Teigland Date: Wed, 12 Nov 2014 16:24:53 -0600 Subject: lvmlockd: vgcreate/vgremove call init_vg/free_vg vgcreate calls lvmlockd_init_vg() to do any create/initialize steps that are needed in lvmlockd for the given lock_type. vgcreate calls lvmlockd_free_vg_before() to do any removal/freeing steps that are needed in lvmlockd for the given lock_type before the VG is removed on disk. vgcreate calls lvmlockd_free_vg_final() to do any removal/freeing steps that are needed in lvmlockd for the given lock_type after the VG is removed on disk. When the lock_type is sanlock, the init/free also include lvm client side steps to create/remove an internal LV on which sanlock will store the locks for the VG. --- daemons/lvmlockd/lvmlockd-client.h | 16 + lib/locking/lvmlockd.c | 602 +++++++++++++++++++++++++++++++++++++ lib/locking/lvmlockd.h | 27 +- tools/vgcreate.c | 6 + tools/vgremove.c | 5 + 5 files changed, 655 insertions(+), 1 deletion(-) diff --git a/daemons/lvmlockd/lvmlockd-client.h b/daemons/lvmlockd/lvmlockd-client.h index 97402cee5..bde0191ea 100644 --- a/daemons/lvmlockd/lvmlockd-client.h +++ b/daemons/lvmlockd/lvmlockd-client.h @@ -33,4 +33,20 @@ static inline void lvmlockd_close(daemon_handle h) return daemon_close(h); } +/* + * Also see lvmlockd-sanlock GL_LOCK_BEGIN, VG_LOCK_BEGIN, LV_LOCK_BEGIN. + * gl lock at sanlock lease area 65 + * vg lock at sanlock lease area 66 + * lv locks begin at sanlock lease area 67 + * + * LV_LOCK_BEGIN + MAX_LVS_IN_VG = sanlock lease areas required + * with 512 byte sectors, each lease area is 1MB + * with 4k byte sectors, each lease area is 8MB (use this for sizing) + * + * 66+190 = 256 sanlock lease areas, + * so we need 256 * 8MB = 2GB lock lv size to hold 190 lv leases. + */ +#define LVMLOCKD_SANLOCK_MAX_LVS_IN_VG 190 +#define LVMLOCKD_SANLOCK_LV_SIZE 2147483648 /* 2GB */ + #endif diff --git a/lib/locking/lvmlockd.c b/lib/locking/lvmlockd.c index ab33b6e72..6f392d994 100644 --- a/lib/locking/lvmlockd.c +++ b/lib/locking/lvmlockd.c @@ -105,3 +105,605 @@ void lvmlockd_set_socket(const char *sock) _lvmlockd_socket = sock; } +static void _result_str_to_flags(const char *str, uint32_t *flags) +{ +} + +/* + * evaluate the reply from lvmlockd, check for errors, extract + * the result and result_flags returned by lvmlockd. + * 0 failure (no result/result_flags set) + * 1 success (result/result_flags set) + */ + +static int _lvmlockd_result(daemon_reply reply, int *result, uint32_t *result_flags) +{ + int reply_result; + const char *reply_flags; + const char *lock_type; + + if (reply.error) { + log_error("lvmlockd_result reply error %d", reply.error); + return 0; + } + + if (strcmp(daemon_reply_str(reply, "response", ""), "OK")) { + log_error("lvmlockd_result bad response"); + return 0; + } + + /* -1000 is a random number that we know is not returned. */ + + reply_result = daemon_reply_int(reply, "op_result", -1000); + if (reply_result == -1000) { + log_error("lvmlockd_result no op_result"); + return 0; + } + + /* The lock_type that lvmlockd used for locking. */ + lock_type = daemon_reply_str(reply, "lock_type", "none"); + + *result = reply_result; + + if (!result_flags) + goto out; + + reply_flags = daemon_reply_str(reply, "result_flags", NULL); + if (reply_flags) + _result_str_to_flags(reply_flags, result_flags); + + out: + log_debug("lvmlockd_result %d %s lm %s", reply_result, reply_flags, lock_type); + return 1; +} + +static daemon_reply _lvmlockd_send(const char *req_name, ...) +{ + va_list ap; + daemon_reply repl; + daemon_request req; + + req = daemon_request_make(req_name); + + va_start(ap, req_name); + daemon_request_extend_v(req, ap); + va_end(ap); + + repl = daemon_send(_lvmlockd, req); + + daemon_request_destroy(req); + + return repl; +} + +/* + * result/result_flags are values returned from lvmlockd. + * + * return 0 (failure) + * return 1 (result/result_flags indicate success/failure) + * + * return 1 result 0 (success) + * return 1 result < 0 (failure) + * + * caller may ignore result < 0 failure depending on + * result_flags and the specific command/mode. + * + * When this function returns 0 (failure), no result/result_flags + * were obtained from lvmlockd. + * + * When this function returns 1 (success), result/result_flags may + * have been obtained from lvmlockd. This lvmlockd result may + * indicate a locking failure. + */ + +int lvmlockd_send(struct cmd_context *cmd, + const char *cmd_name, + const char *req_name, + const char *vg_name, + const char *vg_lock_type, + const char *vg_lock_args, + const char *lv_name, + const char *lv_lock_args, + const char *mode, + const char *opts, + int *result, + uint32_t *result_flags) +{ + daemon_reply reply; + int pid = getpid(); + + *result = 0; + *result_flags = 0; + + if (!strcmp(mode, "na")) + return 1; + + if (!_lvmlockd_active) + return 1; + if (!lvmlockd_connected()) + return 0; + + /* cmd and pid are passed for informational and debugging purposes */ + + if (vg_name && lv_name) { + reply = _lvmlockd_send(req_name, + "cmd = %s", cmd_name, + "pid = %d", pid, + "mode = %s", mode, + "opts = %s", opts ?: "none", + "vg_name = %s", vg_name, + "lv_name = %s", lv_name, + "vg_lock_type = %s", vg_lock_type ?: "none", + "vg_lock_args = %s", vg_lock_args ?: "none", + "lv_lock_args = %s", lv_lock_args ?: "none", + NULL); + + if (!_lvmlockd_result(reply, result, result_flags)) + goto fail; + + log_debug("lvmlockd %s %s vg %s lv %s result %d %x", + req_name, mode, vg_name, lv_name, *result, *result_flags); + + } else if (vg_name) { + reply = _lvmlockd_send(req_name, + "cmd = %s", cmd_name, + "pid = %d", pid, + "mode = %s", mode, + "opts = %s", opts ?: "none", + "vg_name = %s", vg_name, + "vg_lock_type = %s", vg_lock_type ?: "none", + "vg_lock_args = %s", vg_lock_args ?: "none", + NULL); + + if (!_lvmlockd_result(reply, result, result_flags)) + goto fail; + + log_debug("lvmlockd %s %s vg %s result %d %x", + req_name, mode, vg_name, *result, *result_flags); + + } else { + reply = _lvmlockd_send(req_name, + "cmd = %s", cmd_name, + "pid = %d", pid, + "mode = %s", mode, + "opts = %s", opts ?: "none", + "vg_lock_type = %s", vg_lock_type ?: "none", + NULL); + + if (!_lvmlockd_result(reply, result, result_flags)) + goto fail; + + log_debug("lvmlockd %s %s result %d %x", + req_name, mode, *result, *result_flags); + } + + daemon_reply_destroy(reply); + + /* result/result_flags have lvmlockd result */ + return 1; + + fail: + /* no result was obtained from lvmlockd */ + + log_error("lvmlockd %s %s failed no result", req_name, mode); + + daemon_reply_destroy(reply); + return 0; +} + +/* The name of the internal lv created to hold sanlock locks. */ +#define LVMLOCKD_SANLOCK_LV_NAME "lvmlock" + +static struct logical_volume *_find_sanlock_lv(struct volume_group *vg, + const char *lock_lv_name) +{ + struct lv_list *lvl; + + dm_list_iterate_items(lvl, &vg->lvs) { + if (!strcmp(lvl->lv->name, lock_lv_name)) + return lvl->lv; + } + return NULL; +} + +/* + * Eventually add an option to specify which pv the lvmlock lv should be placed on. + */ + +static int _create_sanlock_lv(struct cmd_context *cmd, struct volume_group *vg, + const char *lock_lv_name) +{ + struct logical_volume *lv; + struct lvcreate_params lp = { + .activate = CHANGE_ALY, + .alloc = ALLOC_INHERIT, + .extents = LVMLOCKD_SANLOCK_LV_SIZE / (vg->extent_size * SECTOR_SIZE), + .major = -1, + .minor = -1, + .permission = LVM_READ | LVM_WRITE, + .pvh = &vg->pvs, + .read_ahead = DM_READ_AHEAD_NONE, + .stripes = 1, + .vg_name = vg->name, + .lv_name = dm_pool_strdup(cmd->mem, lock_lv_name), + .zero = 1, + }; + + dm_list_init(&lp.tags); + + if (!(lp.segtype = get_segtype_from_string(vg->cmd, "striped"))) + return_0; + + lv = lv_create_single(vg, &lp); + if (!lv) { + log_error("Failed to create sanlock lv %s in vg %s", lock_lv_name, vg->name); + return 0; + } + + lv_set_hidden(lv); + return 1; +} + +static int _remove_sanlock_lv(struct cmd_context *cmd, struct volume_group *vg, + const char *lock_lv_name) +{ + struct logical_volume *lv; + + lv = _find_sanlock_lv(vg, lock_lv_name); + if (!lv) { + log_error("Failed to find sanlock LV %s in VG %s", lock_lv_name, vg->name); + return 0; + } + + if (!lv_remove(lv)) { + log_error("Failed to remove sanlock LV %s/%s", vg->name, lock_lv_name); + return 0; + } + + return 1; +} + +static int _activate_sanlock_lv(struct cmd_context *cmd, struct volume_group *vg) +{ + struct logical_volume *lv; + const char *lock_lv_name = LVMLOCKD_SANLOCK_LV_NAME; + + lv = _find_sanlock_lv(vg, lock_lv_name); + if (!lv) { + log_error("Failed to find sanlock lv %s in vg %s", lock_lv_name, vg->name); + return 0; + } + + if (!activate_lv(cmd, lv)) { + log_error("Failed to activate sanlock lv %s/%s", vg->name, lock_lv_name); + return 0; + } + + return 1; +} + +static int _deactivate_sanlock_lv(struct cmd_context *cmd, struct volume_group *vg) +{ + struct logical_volume *lv; + const char *lock_lv_name = LVMLOCKD_SANLOCK_LV_NAME; + + lv = _find_sanlock_lv(vg, lock_lv_name); + if (!lv) { + log_error("Failed to find sanlock lv %s in vg %s", lock_lv_name, vg->name); + return 0; + } + + if (!deactivate_lv(cmd, lv)) { + log_error("Failed to deactivate sanlock lv %s/%s", vg->name, lock_lv_name); + return 0; + } + + return 1; +} + +static int _init_vg_dlm(struct cmd_context *cmd, struct volume_group *vg) +{ + daemon_reply reply; + const char *reply_str; + const char *vg_lock_args = NULL; + int result; + int ret; + + log_debug("_init_vg_dlm %s", vg->name); + + if (!_lvmlockd_active) + return 1; + if (!lvmlockd_connected()) + return 0; + + reply = _lvmlockd_send("init_vg", + "pid = %d", getpid(), + "vg_name = %s", vg->name, + "vg_lock_type = %s", "dlm", + NULL); + + if (!_lvmlockd_result(reply, &result, NULL)) { + ret = 0; + } else { + ret = (result < 0) ? 0 : 1; + } + + if (!ret) { + log_error("_init_vg_dlm lvmlockd result %d", result); + goto out; + } + + reply_str = daemon_reply_str(reply, "vg_lock_args", NULL); + if (!reply_str) { + log_error("vg_lock_args not returned"); + ret = 0; + goto out; + } + + vg_lock_args = dm_pool_strdup(cmd->mem, reply_str); + if (!vg_lock_args) { + log_error("vg_lock_args allocation failed"); + ret = 0; + } +out: + daemon_reply_destroy(reply); + + vg->lock_args = vg_lock_args; + return ret; +} + +static int _init_vg_sanlock(struct cmd_context *cmd, struct volume_group *vg) +{ + daemon_reply reply; + const char *reply_str; + const char *vg_lock_args = NULL; + const char *lock_lv_name = LVMLOCKD_SANLOCK_LV_NAME; + const char *opts = NULL; + int result; + int ret; + + log_debug("_init_vg_sanlock %s", vg->name); + + if (!_lvmlockd_active) + return 1; + if (!lvmlockd_connected()) + return 0; + + if (!_create_sanlock_lv(cmd, vg, lock_lv_name)) { + log_error("Failed to create internal lv."); + return 0; + } + + /* + * N.B. this passes the lock_lv_name as vg_lock_args + * even though it is only part of the final args string + * which will be returned from lvmlockd. + */ + + reply = _lvmlockd_send("init_vg", + "pid = %d", getpid(), + "vg_name = %s", vg->name, + "vg_lock_type = %s", "sanlock", + "vg_lock_args = %s", lock_lv_name, + "opts = %s", opts ?: "none", + NULL); + + if (!_lvmlockd_result(reply, &result, NULL)) { + ret = 0; + } else { + ret = (result < 0) ? 0 : 1; + } + + if (!ret) { + log_error("_init_vg_sanlock lvmlockd result %d", result); + _remove_sanlock_lv(cmd, vg, lock_lv_name); + goto out; + } + + reply_str = daemon_reply_str(reply, "vg_lock_args", NULL); + if (!reply_str) { + log_error("vg_lock_args not returned"); + ret = 0; + goto out; + } + + vg_lock_args = dm_pool_strdup(cmd->mem, reply_str); + if (!vg_lock_args) { + log_error("vg_lock_args allocation failed"); + ret = 0; + } +out: + daemon_reply_destroy(reply); + + vg->lock_args = vg_lock_args; + return ret; +} + +/* called after vg_remove on disk */ + +static int _free_vg_dlm(struct cmd_context *cmd, struct volume_group *vg) +{ + uint32_t result_flags; + int result; + int ret; + + /* + * Unlocking the vg lock here preempts the dlock_vg("un") in + * toollib.c which happens too late since the lockspace is + * left here. + */ + + log_debug("_free_vg_dlm un for vgremove %s", vg->name); + + /* Equivalent to dlock_vg(cmd, vg->name, "un", 0); */ + ret = lvmlockd_send(cmd, "vgremove", "lock_vg", vg->name, + NULL, NULL, NULL, NULL, "un", NULL, + &result, &result_flags); + + if (!ret || result < 0) { + log_error("_free_vg_dlm lvmlockd result %d", result); + return 0; + } + + /* + * Leave the dlm lockspace. + * Joining and leaving the lockspaces to be added by later patch. + * lvmlockd_stop_vg(cmd, vg); + */ + + return 1; +} + +/* called before vg_remove on disk */ + +static int _free_vg_sanlock(struct cmd_context *cmd, struct volume_group *vg) +{ + daemon_reply reply; + const char *lock_lv_name = LVMLOCKD_SANLOCK_LV_NAME; + int result; + int ret; + + log_debug("_free_vg_sanlock for vgremove %s", vg->name); + + if (!_lvmlockd_active) + return 1; + if (!lvmlockd_connected()) + return 0; + + if (!vg->lock_args || !strlen(vg->lock_args)) { + /* Shouldn't happen in general, but maybe in some error cases? */ + log_debug("_free_vg_sanlock %s no lock_args", vg->name); + return 1; + } + + reply = _lvmlockd_send("free_vg", + "pid = %d", getpid(), + "vg_name = %s", vg->name, + "vg_lock_type = %s", vg->lock_type, + "vg_lock_args = %s", vg->lock_args, + NULL); + + if (!_lvmlockd_result(reply, &result, NULL)) { + ret = 0; + } else { + ret = (result < 0) ? 0 : 1; + } + + /* + * Other hosts could still be joined to the lockspace, which means they + * are using the internal sanlock LV, which means we cannot remove the + * VG. Once other hosts stop using the VG it can be removed. + */ + if (result == -EBUSY) { + log_error("Lockspace for \"%s\" not stopped on other hosts", vg->name); + goto out; + } + + if (!ret) { + log_error("_free_vg_sanlock lvmlockd result %d", result); + goto out; + } + + _deactivate_sanlock_lv(cmd, vg); + + _remove_sanlock_lv(cmd, vg, lock_lv_name); + out: + daemon_reply_destroy(reply); + + return ret; +} + +/* + * Called to remove lvmlockd's record of the local vg which it caches as an + * optimization. + */ + +static int _free_vg_local(struct cmd_context *cmd, struct volume_group *vg) +{ + daemon_reply reply; + char uuid[64] __attribute__((aligned(8))); + int result; + int ret; + + memset(uuid, 0, sizeof(uuid)); + id_write_format(&vg->id, uuid, sizeof(uuid)); + + log_debug("_free_vg_local for vgremove %s", vg->name); + + reply = _lvmlockd_send("rem_local", + "pid = %d", getpid(), + "vg_name = %s", vg->name, + "vg_uuid = %s", uuid[0] ? uuid : "none", + "vg_lock_type = %s", "none", + NULL); + + if (!_lvmlockd_result(reply, &result, NULL)) { + ret = 0; + } else { + ret = (result < 0) ? 0 : 1; + } + + if (!ret) { + log_error("_free_vg_local lvmlockd result %d", result); + } + + daemon_reply_destroy(reply); + + return ret; +} + +/* vgcreate */ + +int lvmlockd_init_vg(struct cmd_context *cmd, struct volume_group *vg) +{ + switch (lock_type_to_num(vg->lock_type)) { + case LOCK_TYPE_NONE: + case LOCK_TYPE_CLVM: + return 1; + case LOCK_TYPE_DLM: + return _init_vg_dlm(cmd, vg); + case LOCK_TYPE_SANLOCK: + return _init_vg_sanlock(cmd, vg); + default: + log_error("Unknown lock_type."); + return 0; + } +} + +/* vgremove before the vg is removed */ + +int lvmlockd_free_vg_before(struct cmd_context *cmd, struct volume_group *vg) +{ + switch (lock_type_to_num(vg->lock_type)) { + case LOCK_TYPE_NONE: + case LOCK_TYPE_CLVM: + case LOCK_TYPE_DLM: + return 1; + case LOCK_TYPE_SANLOCK: + /* returning an error will prevent vg_remove() */ + return _free_vg_sanlock(cmd, vg); + default: + log_error("Unknown lock_type."); + return 0; + } +} + +/* vgremove after the vg is removed */ + +void lvmlockd_free_vg_final(struct cmd_context *cmd, struct volume_group *vg) +{ + switch (lock_type_to_num(vg->lock_type)) { + case LOCK_TYPE_NONE: + _free_vg_local(cmd, vg); + break; + case LOCK_TYPE_CLVM: + case LOCK_TYPE_SANLOCK: + break; + case LOCK_TYPE_DLM: + _free_vg_dlm(cmd, vg); + break; + default: + log_error("Unknown lock_type."); + } +} + diff --git a/lib/locking/lvmlockd.h b/lib/locking/lvmlockd.h index 95b85256f..f2ab1a826 100644 --- a/lib/locking/lvmlockd.h +++ b/lib/locking/lvmlockd.h @@ -63,7 +63,7 @@ static inline int is_dlock_type(const char *lock_type) #ifdef LVMLOCKD_SUPPORT -/* daemon management */ +/* lvmlockd connection and communication */ void lvmlockd_init(struct cmd_context *); void lvmlockd_set_active(int); @@ -72,6 +72,25 @@ void lvmlockd_disconnect(void); void lvmlockd_connect_or_warn(void); int lvmlockd_connected(void); +int lvmlockd_send(struct cmd_context *cmd, + const char *cmd_name, + const char *req_name, + const char *vg_name, + const char *vg_lock_type, + const char *vg_lock_args, + const char *lv_name, + const char *lv_lock_args, + const char *mode, + const char *opts, + int *result, + uint32_t *result_flags); + +/* vgcreate/vgremove use init/free */ + +int lvmlockd_init_vg(struct cmd_context *cmd, struct volume_group *vg); +int lvmlockd_free_vg_before(struct cmd_context *cmd, struct volume_group *vg); +void lvmlockd_free_vg_final(struct cmd_context *cmd, struct volume_group *vg); + #else /* LVMLOCKD_SUPPORT */ #define lvmlockd_init(cmd) do { } while (0) @@ -81,6 +100,12 @@ int lvmlockd_connected(void); #define lvmlockd_connect_or_warn() do { } while (0) #define lvmlockd_connected (0) +#define lvmlockd_send(cmd, cmd_name, req_name, vg_name, vg_lock_type, vg_lock_args, lv_name, lv_lock_args, mode, opts, result, result_flags) (1) + +#define lvmlockd_init_vg(cmd, vg) (0) +#define lvmlockd_free_vg_before(cmd, vg) (0) +#define lvmlockd_free_vg_final(cmd, vg) do { } while (0) + #endif /* LVMLOCKD_SUPPORT */ #endif diff --git a/tools/vgcreate.c b/tools/vgcreate.c index 7304064fa..7c6d9187e 100644 --- a/tools/vgcreate.c +++ b/tools/vgcreate.c @@ -108,6 +108,12 @@ int vgcreate(struct cmd_context *cmd, int argc, char **argv) } } + if (!lvmlockd_init_vg(cmd, vg)) { + log_error("Failed to initialize lock args for lock type %s", + vp_new.lock_type); + goto_bad; + } + if (vg_is_clustered(vg)) clustered_message = "Clustered "; else if (locking_is_clustered()) diff --git a/tools/vgremove.c b/tools/vgremove.c index 1dce41f24..98336362d 100644 --- a/tools/vgremove.c +++ b/tools/vgremove.c @@ -54,6 +54,9 @@ static int vgremove_single(struct cmd_context *cmd, const char *vg_name, } } + if (!lvmlockd_free_vg_before(cmd, vg)) + return_ECMD_FAILED; + if (!force && !vg_remove_check(vg)) return_ECMD_FAILED; @@ -62,6 +65,8 @@ static int vgremove_single(struct cmd_context *cmd, const char *vg_name, if (!vg_remove(vg)) return_ECMD_FAILED; + lvmlockd_free_vg_final(cmd, vg); + return ECMD_PROCESSED; } -- cgit v1.2.1