lvmcache: reread a VG if the lvmetad copy is stale

and update the lvmetad copy after it is reread from disk.

To test this:

- Create VG foo that is visible to two hosts and usable by both.
- On both run 'lvmeta vg_lookup_name foo' to see the cached copy of
  foo and its seqno.  Say the seqno is 8.
- On host1 run 'lvcreate -n lv1 -L1G foo'.
- On host1 run 'lvmeta vg_lookup_name foo' to see the new version
  of foo in lvmetad.  It should have seqno 9.
- On host2 run 'lvmeta vg_lookup_name foo' to see the old cached
  version of foo.  It should have seqno 8.
- On host2 run 'lvmeta set_vg_version <uuid of foo> 9'.
- On host2 run 'lvmeta vg_lookup_name foo' to see that the vg_invalid
  config node is reported along with the old cached version of foo.
- On host2 run 'lvs foo'.  It should reread foo from disk and display lv1.
- On host2 run 'lvmeta vg_lookup_name foo' to see that the cached version
  of foo is now updated to seqno 9, and the vg_invalid node is not reported.

4 files changed, 164 insertions, 57 deletions

diff --git a/daemons/lvmetad/lvmetad-core.c b/daemons/lvmetad/lvmetad-core.c
index ffa1c2867..e7b80c113 100644
--- a/daemons/lvmetad/lvmetad-core.c
+++ b/daemons/lvmetad/lvmetad-core.c
@@ -26,9 +26,6 @@
 #include <stdint.h>
 #include <unistd.h>
 
-#include <math.h>  /* fabs() */
-#include <float.h> /* DBL_EPSILON */
-
 #define LVMETAD_SOCKET DEFAULT_RUN_DIR "/lvmetad.socket"
 
 struct vg_info {
@@ -552,60 +549,6 @@ bad:
 	return reply_fail("out of memory");
 }
 
-/* Test if the doubles are close enough to be considered equal */
-static int close_enough(double d1, double d2)
-{
-	return fabs(d1 - d2) < DBL_EPSILON;
-}
-
-static int compare_value(struct dm_config_value *a, struct dm_config_value *b)
-{
-	int r = 0;
-
-	if (a->type > b->type)
-		return 1;
-	if (a->type < b->type)
-		return -1;
-
-	switch (a->type) {
-	case DM_CFG_STRING: r = strcmp(a->v.str, b->v.str); break;
-	case DM_CFG_FLOAT: r = close_enough(a->v.f, b->v.f) ? 0 : (a->v.f > b->v.f) ? 1 : -1; break;
-	case DM_CFG_INT: r = (a->v.i == b->v.i) ? 0 : (a->v.i > b->v.i) ? 1 : -1; break;
-	case DM_CFG_EMPTY_ARRAY: return 0;
-	}
-
-	if (r == 0 && a->next && b->next)
-		r = compare_value(a->next, b->next);
-	return r;
-}
-
-static int compare_config(struct dm_config_node *a, struct dm_config_node *b)
-{
-	int result = 0;
-	if (a->v && b->v)
-		result = compare_value(a->v, b->v);
-	if (a->v && !b->v)
-		result = 1;
-	if (!a->v && b->v)
-		result = -1;
-	if (a->child && b->child)
-		result = compare_config(a->child, b->child);
-
-	if (result) {
-		// DEBUGLOG("config inequality at %s / %s", a->key, b->key);
-		return result;
-	}
-
-	if (a->sib && b->sib)
-		result = compare_config(a->sib, b->sib);
-	if (a->sib && !b->sib)
-		result = 1;
-	if (!a->sib && b->sib)
-		result = -1;
-
-	return result;
-}
-
 static int vg_remove_if_missing(lvmetad_state *s, const char *vgid, int update_pvids);
 
 /* You need to be holding the pvid_to_vgid lock already to call this. */
diff --git a/lib/cache/lvmetad.c b/lib/cache/lvmetad.c
index ccdf63a35..f0429c4d1 100644
--- a/lib/cache/lvmetad.c
+++ b/lib/cache/lvmetad.c
@@ -34,6 +34,8 @@ static char *_lvmetad_token = NULL;
 static const char *_lvmetad_socket = NULL;
 static struct cmd_context *_lvmetad_cmd = NULL;
 
+static struct volume_group *lvmetad_pvscan_vg(struct cmd_context *cmd, struct volume_group *vg);
+
 void lvmetad_disconnect(void)
 {
 	if (_lvmetad_connected)
@@ -362,6 +364,7 @@ static struct lvmcache_info *_pv_populate_lvmcache(struct cmd_context *cmd,
 struct volume_group *lvmetad_vg_lookup(struct cmd_context *cmd, const char *vgname, const char *vgid)
 {
 	struct volume_group *vg = NULL;
+	struct volume_group *vg2 = NULL;
 	daemon_reply reply;
 	int found;
 	char uuid[64];
@@ -442,6 +445,17 @@ struct volume_group *lvmetad_vg_lookup(struct cmd_context *cmd, const char *vgna
 				pvl->pv->status |= MISSING_PV; /* probably missing */
 		}
 
+		/*
+		 * locking may have detected a newer vg version and
+		 * invalidated the cached vg.
+		 */
+		if (dm_config_find_node(reply.cft->root, "vg_invalid")) {
+			log_debug_lvmetad("Update invalid lvmetad cache for VG %s", vgname);
+			vg2 = lvmetad_pvscan_vg(cmd, vg);
+			release_vg(vg);
+			vg = vg2;
+		}
+
 		lvmcache_update_vg(vg, 0);
 		vg_mark_partial_lvs(vg, 1);
 	}
@@ -902,6 +916,97 @@ static int _lvmetad_pvscan_single(struct metadata_area *mda, void *baton)
 	return 1;
 }
 
+/*
+ * The lock manager may detect that the vg cached in lvmetad is out of date,
+ * due to something like an lvcreate from another host.
+ * This is limited to changes that only affect the vg (not global state like
+ * orphan PVs), so we only need to reread mdas on the vg's existing pvs.
+ */
+
+static struct volume_group *lvmetad_pvscan_vg(struct cmd_context *cmd, struct volume_group *vg)
+{
+	struct volume_group *vg_ret = NULL;
+	struct dm_config_tree *vgmeta_ret = NULL;
+	struct dm_config_tree *vgmeta;
+	struct pv_list *pvl;
+	struct lvmcache_info *info;
+	struct format_instance *fid;
+	struct format_instance_ctx fic = { .type = 0 };
+	struct _lvmetad_pvscan_baton baton;
+
+	dm_list_iterate_items(pvl, &vg->pvs) {
+		/* missing pv */
+		if (!pvl->pv->dev)
+			continue;
+
+		info = lvmcache_info_from_pvid((const char *)&pvl->pv->id, 0);
+
+		baton.vg = NULL;
+		baton.fid = lvmcache_fmt(info)->ops->create_instance(lvmcache_fmt(info), &fic);
+
+		if (!baton.fid)
+			return NULL;
+
+		if (baton.fid->fmt->features & FMT_OBSOLETE) {
+			log_error("WARNING: Ignoring obsolete format of metadata (%s) on device %s when using lvmetad",
+			  	baton.fid->fmt->name, dev_name(pvl->pv->dev));
+			lvmcache_fmt(info)->ops->destroy_instance(baton.fid);
+			return NULL;
+		}
+
+		lvmcache_foreach_mda(info, _lvmetad_pvscan_single, &baton);
+
+		if (!baton.vg) {
+			lvmcache_fmt(info)->ops->destroy_instance(baton.fid);
+			return NULL;
+		}
+
+		if (!(vgmeta = export_vg_to_config_tree(baton.vg))) {
+			log_error("VG export to config tree failed");
+			release_vg(baton.vg);
+			return NULL;
+		}
+
+		if (!vgmeta_ret) {
+			vgmeta_ret = vgmeta;
+		} else {
+			if (!compare_config(vgmeta_ret->root, vgmeta->root)) {
+				log_error("VG metadata comparison failed");
+				dm_config_destroy(vgmeta);
+				dm_config_destroy(vgmeta_ret);
+				release_vg(baton.vg);
+				return NULL;
+			}
+			dm_config_destroy(vgmeta);
+		}
+
+		release_vg(baton.vg);
+	}
+
+	if (vgmeta_ret) {
+		fid = lvmcache_fmt(info)->ops->create_instance(lvmcache_fmt(info), &fic);
+		if (!(vg_ret = import_vg_from_config_tree(vgmeta_ret, fid))) {
+			log_error("VG import from config tree failed");
+			lvmcache_fmt(info)->ops->destroy_instance(fid);
+			goto out;
+		}
+
+		/*
+		 * Update lvmetad with the newly read version of the VG.
+		 * The "precommitted" name is a misnomer in this case,
+		 * but that is the field which lvmetad_vg_update() uses
+		 * to send the metadata cft to lvmetad.
+		 */
+		vg_ret->cft_precommitted = vgmeta_ret;
+		if (!lvmetad_vg_update(vg_ret))
+			log_error("Failed to update lvmetad with new VG meta");
+		vg_ret->cft_precommitted = NULL;
+		dm_config_destroy(vgmeta_ret);
+	}
+out:
+	return vg_ret;
+}
+
 int lvmetad_pvscan_single(struct cmd_context *cmd, struct device *dev,
 			  activation_handler handler)
 {
diff --git a/libdaemon/client/config-util.c b/libdaemon/client/config-util.c
index e5f4e2205..d6e23d08d 100644
--- a/libdaemon/client/config-util.c
+++ b/libdaemon/client/config-util.c
@@ -20,6 +20,9 @@
 #include <string.h>
 #include <unistd.h>
 
+#include <math.h>  /* fabs() */
+#include <float.h> /* DBL_EPSILON */
+
 int buffer_append_vf(struct buffer *buf, va_list ap)
 {
 	char *append;
@@ -277,6 +280,60 @@ struct dm_config_node *config_make_nodes(struct dm_config_tree *cft,
 	return res;
 }
 
+/* Test if the doubles are close enough to be considered equal */
+static int close_enough(double d1, double d2)
+{
+	return fabs(d1 - d2) < DBL_EPSILON;
+}
+
+static int compare_value(struct dm_config_value *a, struct dm_config_value *b)
+{
+	int r = 0;
+
+	if (a->type > b->type)
+		return 1;
+	if (a->type < b->type)
+		return -1;
+
+	switch (a->type) {
+	case DM_CFG_STRING: r = strcmp(a->v.str, b->v.str); break;
+	case DM_CFG_FLOAT: r = close_enough(a->v.f, b->v.f) ? 0 : (a->v.f > b->v.f) ? 1 : -1; break;
+	case DM_CFG_INT: r = (a->v.i == b->v.i) ? 0 : (a->v.i > b->v.i) ? 1 : -1; break;
+	case DM_CFG_EMPTY_ARRAY: return 0;
+	}
+
+	if (r == 0 && a->next && b->next)
+		r = compare_value(a->next, b->next);
+	return r;
+}
+
+int compare_config(struct dm_config_node *a, struct dm_config_node *b)
+{
+	int result = 0;
+	if (a->v && b->v)
+		result = compare_value(a->v, b->v);
+	if (a->v && !b->v)
+		result = 1;
+	if (!a->v && b->v)
+		result = -1;
+	if (a->child && b->child)
+		result = compare_config(a->child, b->child);
+
+	if (result) {
+		// DEBUGLOG("config inequality at %s / %s", a->key, b->key);
+		return result;
+	}
+
+	if (a->sib && b->sib)
+		result = compare_config(a->sib, b->sib);
+	if (a->sib && !b->sib)
+		result = 1;
+	if (!a->sib && b->sib)
+		result = -1;
+
+	return result;
+}
+
 int buffer_realloc(struct buffer *buf, int needed)
 {
 	char *new;
diff --git a/libdaemon/client/config-util.h b/libdaemon/client/config-util.h
index 47fab6bcb..d1ee3b345 100644
--- a/libdaemon/client/config-util.h
+++ b/libdaemon/client/config-util.h
@@ -42,6 +42,8 @@ struct dm_config_node *make_config_node(struct dm_config_tree *cft,
 					struct dm_config_node *parent,
 					struct dm_config_node *pre_sib);
 
+int compare_config(struct dm_config_node *a, struct dm_config_node *b);
+
 struct dm_config_node *make_text_node(struct dm_config_tree *cft,
 				      const char *key,
 				      const char *value,