diff options
author | David Teigland <teigland@redhat.com> | 2017-08-09 16:09:48 -0500 |
---|---|---|
committer | David Teigland <teigland@redhat.com> | 2017-10-23 15:17:32 -0500 |
commit | 83a57110ddf61a8422d7fbde2d848efca98d2505 (patch) | |
tree | 831ab0ff63e1f414fff19f86b061393e63528e96 | |
parent | 543027b3eff23feeec1838e6833bd9a6c5908b53 (diff) | |
download | lvm2-83a57110ddf61a8422d7fbde2d848efca98d2505.tar.gz |
scanning: rewrite lvmetad_pvscan_vg to use new label reading
This is used to refresh the lvmetad content for a VG after
lvmlockd has invalidated the cached copy of the metadata in
lvmetad.
-rw-r--r-- | lib/cache/lvmcache.c | 21 | ||||
-rw-r--r-- | lib/cache/lvmcache.h | 7 | ||||
-rw-r--r-- | lib/cache/lvmetad.c | 456 | ||||
-rw-r--r-- | lib/label/label.c | 209 | ||||
-rw-r--r-- | lib/label/label.h | 13 |
5 files changed, 511 insertions, 195 deletions
diff --git a/lib/cache/lvmcache.c b/lib/cache/lvmcache.c index 66ef30441..9ab762395 100644 --- a/lib/cache/lvmcache.c +++ b/lib/cache/lvmcache.c @@ -1140,8 +1140,8 @@ int lvmcache_label_rescan_vg(struct cmd_context *cmd, const char *vgname, const dm_list_add(&devs, &devl->list); } - if (!cmd->use_aio || !label_rescan_async(cmd, &devs)) - label_rescan_sync(cmd, &devs); + if (!cmd->use_aio || !label_scan_devs_async(cmd, &devs)) + label_scan_devs_sync(cmd, &devs); /* * TODO: grab vginfo again, and compare vginfo->infos @@ -1500,6 +1500,23 @@ struct dm_list *lvmcache_get_pvids(struct cmd_context *cmd, const char *vgname, return pvids; } +int lvmcache_get_vg_devs(struct cmd_context *cmd, + struct lvmcache_vginfo *vginfo, + struct dm_list *devs) +{ + struct lvmcache_info *info; + struct device_list *devl; + + dm_list_iterate_items(info, &vginfo->infos) { + if (!(devl = dm_pool_zalloc(cmd->mem, sizeof(*devl)))) + return_0; + + devl->dev = info->dev; + dm_list_add(devs, &devl->list); + } + return 1; +} + static struct device *_device_from_pvid(const struct id *pvid, uint64_t *label_sector) { struct lvmcache_info *info; diff --git a/lib/cache/lvmcache.h b/lib/cache/lvmcache.h index 526efb0b0..51ce5933b 100644 --- a/lib/cache/lvmcache.h +++ b/lib/cache/lvmcache.h @@ -214,4 +214,11 @@ void lvmcache_remove_unchosen_duplicate(struct device *dev); int lvmcache_pvid_in_unchosen_duplicates(const char *pvid); +void lvmcache_save_suspended_vg(struct volume_group *vg, int precommitted); +struct volume_group *lvmcache_get_suspended_vg(const char *vgid); +void lvmcache_drop_suspended_vg(struct volume_group *vg); + +int lvmcache_get_vg_devs(struct cmd_context *cmd, + struct lvmcache_vginfo *vginfo, + struct dm_list *devs); #endif diff --git a/lib/cache/lvmetad.c b/lib/cache/lvmetad.c index 73243a6ba..9f88441f4 100644 --- a/lib/cache/lvmetad.c +++ b/lib/cache/lvmetad.c @@ -39,7 +39,7 @@ static int64_t _lvmetad_update_timeout; static int _found_lvm1_metadata = 0; -static struct volume_group *_lvmetad_pvscan_vg(struct cmd_context *cmd, struct volume_group *vg); +static struct volume_group *_lvmetad_pvscan_vg(struct cmd_context *cmd, struct volume_group *vg, const char *vgid, struct format_type *fmt); static uint64_t _monotonic_seconds(void) { @@ -1090,7 +1090,7 @@ struct volume_group *lvmetad_vg_lookup(struct cmd_context *cmd, const char *vgna * invalidated the cached vg. */ if (rescan) { - if (!(vg2 = _lvmetad_pvscan_vg(cmd, vg))) { + if (!(vg2 = _lvmetad_pvscan_vg(cmd, vg, vgid, fmt))) { log_debug_lvmetad("VG %s from lvmetad not found during rescan.", vgname); fid = NULL; release_vg(vg); @@ -1098,6 +1098,7 @@ struct volume_group *lvmetad_vg_lookup(struct cmd_context *cmd, const char *vgna goto out; } release_vg(vg); + fmt->ops->destroy_instance(fid); vg = vg2; fid = vg2->fid; } @@ -1105,14 +1106,14 @@ struct volume_group *lvmetad_vg_lookup(struct cmd_context *cmd, const char *vgna dm_list_iterate_items(pvl, &vg->pvs) { if (!_pv_update_struct_pv(pvl->pv, fid)) { vg = NULL; - goto_out; /* FIXME error path */ + goto_out; /* FIXME: use an error path that disables lvmetad */ } } dm_list_iterate_items(pvl, &vg->pvs_outdated) { if (!_pv_update_struct_pv(pvl->pv, fid)) { vg = NULL; - goto_out; /* FIXME error path */ + goto_out; /* FIXME: use an error path that disables lvmetad */ } } @@ -1784,6 +1785,37 @@ static int _lvmetad_pvscan_single(struct metadata_area *mda, void *baton) } /* + * FIXME: handle errors and do proper comparison of metadata from each area + * like vg_read and fall back to real vg_read from disk if there's any problem. + */ + +static int _lvmetad_pvscan_vg_single(struct metadata_area *mda, void *baton) +{ + struct _lvmetad_pvscan_baton *b = baton; + struct device *mda_dev = mda_get_device(mda); + struct label_read_data *ld; + struct volume_group *vg = NULL; + + if (mda_is_ignored(mda)) + return 1; + + ld = get_label_read_data(b->cmd, mda_dev); + + if (!(vg = mda->ops->vg_read(b->fid, "", mda, ld, NULL, NULL))) + return 1; + + if (!b->vg) + b->vg = vg; + else if (vg->seqno > b->vg->seqno) { + release_vg(b->vg); + b->vg = vg; + } else + release_vg(vg); + + return 1; +} + +/* * The lock manager may detect that the vg cached in lvmetad is out of date, * due to something like an lvcreate from another host. * This is limited to changes that only affect the vg (not global state like @@ -1792,41 +1824,41 @@ static int _lvmetad_pvscan_single(struct metadata_area *mda, void *baton) * the VG, and that PV may have been reused for another VG. */ -static struct volume_group *_lvmetad_pvscan_vg(struct cmd_context *cmd, struct volume_group *vg) +static struct volume_group *_lvmetad_pvscan_vg(struct cmd_context *cmd, struct volume_group *vg, + const char *vgid, struct format_type *fmt) { char pvid_s[ID_LEN + 1] __attribute__((aligned(8))); char uuid[64] __attribute__((aligned(8))); - struct label *label; - struct volume_group *vg_ret = NULL; - struct dm_config_tree *vgmeta_ret = NULL; struct dm_config_tree *vgmeta; struct pv_list *pvl, *pvl_new; - struct device_list *devl, *devl_new, *devlsafe; + struct device_list *devl, *devlsafe; struct dm_list pvs_scan; struct dm_list pvs_drop; - struct dm_list pvs_new; + struct lvmcache_vginfo *vginfo = NULL; struct lvmcache_info *info = NULL; struct format_instance *fid; struct format_instance_ctx fic = { .type = 0 }; struct _lvmetad_pvscan_baton baton; + struct volume_group *save_vg; + struct dm_config_tree *save_meta; struct device *save_dev = NULL; uint32_t save_seqno = 0; - int missing_devs = 0; - int check_new_pvs = 0; + int found_new_pvs = 0; + int retried_reads = 0; int found; + save_vg = NULL; + save_meta = NULL; + save_dev = NULL; + save_seqno = 0; + dm_list_init(&pvs_scan); dm_list_init(&pvs_drop); - dm_list_init(&pvs_new); - log_debug_lvmetad("Rescanning VG %s (seqno %u).", vg->name, vg->seqno); + log_debug_lvmetad("Rescan VG %s to update lvmetad (seqno %u).", vg->name, vg->seqno); /* - * Another host may have added a PV to the VG, and some - * commands do not always populate their lvmcache with - * all devs from lvmetad, so they would fail to find - * the new PV when scanning the VG. So make sure this - * command knows about all PVs from lvmetad. + * Make sure this command knows about all PVs from lvmetad. */ lvmcache_seed_infos_from_lvmetad(cmd); @@ -1841,55 +1873,112 @@ static struct volume_group *_lvmetad_pvscan_vg(struct cmd_context *cmd, struct v dm_list_add(&pvs_scan, &devl->list); } -scan_more: + /* + * Rescan labels/metadata only from devs that we previously + * saw in the VG. If we find below that there are new PVs + * in the VG, we'll have to rescan all devices to find which + * device(s) are now being used. + */ + log_debug_lvmetad("Rescan VG %s scanning data from devs in previous metadata.", vg->name); + + if (!cmd->use_aio || !label_scan_devs_async(cmd, &pvs_scan)) + label_scan_devs_sync(cmd, &pvs_scan); /* - * Run the equivalent of lvmetad_pvscan_single on each dev in the VG. + * Check if any pvs_scan entries are no longer PVs. + * In that case, label_read/_find_label_header will have + * found no label_header, and would have dropped the + * info struct for the device from lvmcache. So, if + * we look up the info struct here and don't find it, + * we can infer it's no longer a PV. + * + * FIXME: we should record specific results from the + * label_read and then check specifically for whatever + * result means "no label was found", rather than going + * about this indirectly via the lvmcache side effects. */ dm_list_iterate_items_safe(devl, devlsafe, &pvs_scan) { - if (!devl->dev) - continue; + if (!(info = lvmcache_info_from_pvid(devl->dev->pvid, devl->dev, 0))) { + /* Another host removed this PV from the VG. */ + log_debug_lvmetad("Rescan VG %s from %s dropping dev (no label).", + vg->name, dev_name(devl->dev)); + dm_list_move(&pvs_drop, &devl->list); + } + } - log_debug_lvmetad("Rescan VG %s scanning %s.", vg->name, dev_name(devl->dev)); + fic.type = FMT_INSTANCE_MDAS | FMT_INSTANCE_AUX_MDAS; + fic.context.vg_ref.vg_name = vg->name; + fic.context.vg_ref.vg_id = vgid; - if (!label_read(devl->dev, &label, 0)) { - /* Another host removed this PV from the VG. */ - log_debug_lvmetad("Rescan VG %s found %s was removed.", vg->name, dev_name(devl->dev)); + retry_reads: - if ((info = lvmcache_info_from_pvid(devl->dev->pvid, NULL, 0))) - lvmcache_del(info); + if (!(fid = fmt->ops->create_instance(fmt, &fic))) { + /* FIXME: are there only internal reasons for failures here? */ + log_error("Reading VG %s failed to create format instance.", vg->name); + return NULL; + } - dm_list_move(&pvs_drop, &devl->list); - continue; - } + /* FIXME: not sure if this is necessary */ + fid->ref_count++; - info = (struct lvmcache_info *) label->info; + baton.fid = fid; + baton.cmd = cmd; - baton.cmd = cmd; - baton.vg = NULL; - baton.fid = lvmcache_fmt(info)->ops->create_instance(lvmcache_fmt(info), &fic); - if (!baton.fid) - return_NULL; + /* + * FIXME: this vg_read path does not have the ability to repair + * any problems with the VG, e.g. VG on one dev has an older + * seqno. When vg_read() is reworked, we need to fall back + * to using that from here (and vg_read's from lvmetad) when + * there is a problem. Perhaps by disabling lvmetad when a + * VG problem is detected, causing commands to fully fall + * back to disk, which will repair the VG. Then lvmetad can + * be repopulated and re-enabled (possibly automatically.) + */ - if (baton.fid->fmt->features & FMT_OBSOLETE) { - log_debug_lvmetad("Ignoring obsolete format on PV %s in VG %s.", dev_name(devl->dev), vg->name); - lvmcache_fmt(info)->ops->destroy_instance(baton.fid); + /* + * Do a low level vg_read on each dev, verify the vg returned + * from metadata on each device is for the VG being read + * (the PV may have been removed from the VG being read and + * added to a different one), and return this vg to the caller + * as the current vg to use. + * + * The label scan above will have saved in lvmcache which + * vg each device is used in, so we could figure that part + * out without doing the vg_read. + */ + dm_list_iterate_items_safe(devl, devlsafe, &pvs_scan) { + if (!devl->dev) + continue; + + log_debug_lvmetad("Rescan VG %s getting metadata from %s.", + vg->name, dev_name(devl->dev)); + + /* + * The info struct for this dev knows what and where + * the mdas are for this dev (the label scan saved + * the mda locations for this dev on the lvmcache info struct). + */ + if (!(info = lvmcache_info_from_pvid(devl->dev->pvid, devl->dev, 0))) { + log_debug_lvmetad("Rescan VG %s from %s dropping dev (no info).", + vg->name, dev_name(devl->dev)); dm_list_move(&pvs_drop, &devl->list); continue; } + baton.vg = NULL; + /* * Read VG metadata from this dev's mdas. */ - lvmcache_foreach_mda(info, _lvmetad_pvscan_single, &baton); + lvmcache_foreach_mda(info, _lvmetad_pvscan_vg_single, &baton); /* * The PV may have been removed from the VG by another host * since we last read the VG. */ if (!baton.vg) { - log_debug_lvmetad("Rescan VG %s did not find %s.", vg->name, dev_name(devl->dev)); - lvmcache_fmt(info)->ops->destroy_instance(baton.fid); + log_debug_lvmetad("Rescan VG %s from %s dropping dev (no metadata).", + vg->name, dev_name(devl->dev)); dm_list_move(&pvs_drop, &devl->list); continue; } @@ -1899,10 +1988,15 @@ scan_more: * different VG since we last read the VG. */ if (strcmp(baton.vg->name, vg->name)) { - log_debug_lvmetad("Rescan VG %s found different VG %s on PV %s.", - vg->name, baton.vg->name, dev_name(devl->dev)); + log_debug_lvmetad("Rescan VG %s from %s dropping dev (other VG %s).", + vg->name, dev_name(devl->dev), baton.vg->name); + release_vg(baton.vg); + continue; + } + + if (!(vgmeta = export_vg_to_config_tree(baton.vg))) { + log_error("VG export to config tree failed"); release_vg(baton.vg); - dm_list_move(&pvs_drop, &devl->list); continue; } @@ -1912,20 +2006,35 @@ scan_more: * read from each other dev. */ - if (!save_seqno) - save_seqno = baton.vg->seqno; - - if (!(vgmeta = export_vg_to_config_tree(baton.vg))) { - log_error("VG export to config tree failed"); - release_vg(baton.vg); - return NULL; + if (save_vg && (save_seqno != baton.vg->seqno)) { + /* FIXME: fall back to vg_read to correct this. */ + log_warn("WARNING: inconsistent metadata for VG %s on devices %s seqno %u and %s seqno %u.", + vg->name, dev_name(save_dev), save_seqno, + dev_name(devl->dev), baton.vg->seqno); + log_warn("WARNING: temporarily disable lvmetad to repair metadata."); + + /* Use the most recent */ + if (save_seqno < baton.vg->seqno) { + release_vg(save_vg); + dm_config_destroy(save_meta); + save_vg = baton.vg; + save_meta = vgmeta; + save_seqno = baton.vg->seqno; + save_dev = devl->dev; + } else { + release_vg(baton.vg); + dm_config_destroy(vgmeta); + } + continue; } - if (!vgmeta_ret) { - vgmeta_ret = vgmeta; + if (!save_vg) { + save_vg = baton.vg; + save_meta = vgmeta; + save_seqno = baton.vg->seqno; save_dev = devl->dev; } else { - struct dm_config_node *meta1 = vgmeta_ret->root; + struct dm_config_node *meta1 = save_meta->root; struct dm_config_node *meta2 = vgmeta->root; struct dm_config_node *sib1 = meta1->sib; struct dm_config_node *sib2 = meta2->sib; @@ -1950,73 +2059,129 @@ scan_more: meta2->sib = NULL; if (compare_config(meta1, meta2)) { + /* FIXME: fall back to vg_read to correct this. */ + log_warn("WARNING: inconsistent metadata for VG %s on devices %s seqno %u and %s seqno %u.", + vg->name, dev_name(save_dev), save_seqno, + dev_name(devl->dev), baton.vg->seqno); + log_warn("WARNING: temporarily disable lvmetad to repair metadata."); log_error("VG %s metadata comparison failed for device %s vs %s", vg->name, dev_name(devl->dev), save_dev ? dev_name(save_dev) : "none"); - _log_debug_inequality(vg->name, vgmeta_ret->root, vgmeta->root); + _log_debug_inequality(vg->name, save_meta->root, vgmeta->root); meta1->sib = sib1; meta2->sib = sib2; - dm_config_destroy(vgmeta); - dm_config_destroy(vgmeta_ret); + + /* no right choice, just use the previous copy */ release_vg(baton.vg); - return NULL; + dm_config_destroy(vgmeta); } meta1->sib = sib1; meta2->sib = sib2; + release_vg(baton.vg); dm_config_destroy(vgmeta); } + } - /* - * Look for any new PVs in the VG metadata that were not in our - * previous version of the VG. Add them to pvs_new to be - * scanned in this loop just like the old PVs. - */ - if (!check_new_pvs) { - check_new_pvs = 1; - dm_list_iterate_items(pvl_new, &baton.vg->pvs) { - found = 0; - dm_list_iterate_items(pvl, &vg->pvs) { - if (pvl_new->pv->dev != pvl->pv->dev) - continue; - found = 1; - break; - } - if (found) - continue; - if (!pvl_new->pv->dev) { - strncpy(pvid_s, (char *) &pvl_new->pv->id, sizeof(pvid_s) - 1); - if (!id_write_format((const struct id *)&pvid_s, uuid, sizeof(uuid))) - stack; - log_error("Device not found for PV %s in VG %s", uuid, vg->name); - missing_devs++; + /* FIXME: see above */ + fid->ref_count--; + + /* + * Look for any new PVs in the VG metadata that were not in our + * previous version of the VG. + * + * (Don't look for new PVs after a rescan and retry.) + */ + found_new_pvs = 0; + + if (save_vg && !retried_reads) { + dm_list_iterate_items(pvl_new, &save_vg->pvs) { + found = 0; + dm_list_iterate_items(pvl, &vg->pvs) { + if (pvl_new->pv->dev != pvl->pv->dev) continue; - } - if (!(devl_new = dm_pool_zalloc(cmd->mem, sizeof(*devl_new)))) - return_NULL; - devl_new->dev = pvl_new->pv->dev; - dm_list_add(&pvs_new, &devl_new->list); - log_debug_lvmetad("Rescan VG %s found %s was added.", vg->name, dev_name(devl_new->dev)); + found = 1; + break; + } + + /* + * PV in new VG metadata not found in old VG metadata. + * There's a good chance we don't know about this new + * PV or what device it's on; a label scan is needed + * of all devices so we know which device the VG is + * now using. + */ + if (!found) { + found_new_pvs++; + strncpy(pvid_s, (char *) &pvl_new->pv->id, sizeof(pvid_s) - 1); + if (!id_write_format((const struct id *)&pvid_s, uuid, sizeof(uuid))) + stack; + log_debug_lvmetad("Rescan VG %s found new PV %s.", vg->name, uuid); } } + } - release_vg(baton.vg); + if (!save_vg && retried_reads) { + log_error("VG %s not found after rescanning devices.", vg->name); + goto out; } /* - * Do the same scanning above for any new PVs. + * Do a full rescan of devices, then look up which devices the + * scan found for this VG name, and select those devices to + * read metadata from in the loop above (rather than the list + * of devices we created from our last copy of the vg metadata.) + * + * Case 1: VG we knew is no longer on any of the devices we knew it + * to be on (save_vg is NULL, which means the metadata wasn't found + * when reading mdas on each of the initial pvs_scan devices). + * Rescan all devs and then retry reading metadata from the devs that + * the scan finds associated with this VG. + * + * Case 2: VG has new PVs but we don't know what devices they are + * so rescan all devs and then retry reading metadata from the devs + * that the scan finds associated with this VG. + * + * (N.B. after a retry, we don't check for found_new_pvs.) */ - if (!dm_list_empty(&pvs_new)) { + if (!save_vg || found_new_pvs) { + if (!save_vg) + log_debug_lvmetad("Rescan VG %s did not find VG on previous devs.", vg->name); + if (found_new_pvs) + log_debug_lvmetad("Rescan VG %s scanning all devs to find new PVs.", vg->name); + + if (!cmd->use_aio || !label_scan_async_force(cmd)) + label_scan_sync_force(cmd); + + if (!(vginfo = lvmcache_vginfo_from_vgname(vg->name, NULL))) { + log_error("VG %s vg info not found after rescanning devices.", vg->name); + goto out; + } + + /* + * Set pvs_scan to devs that the label scan found + * in the VG and retry the metadata reading loop. + */ dm_list_init(&pvs_scan); - dm_list_splice(&pvs_scan, &pvs_new); - dm_list_init(&pvs_new); - log_debug_lvmetad("Rescan VG %s found new PVs to scan.", vg->name); - goto scan_more; - } - if (missing_devs) { - if (vgmeta_ret) - dm_config_destroy(vgmeta_ret); - return_NULL; + if (!lvmcache_get_vg_devs(cmd, vginfo, &pvs_scan)) { + log_error("VG %s info devs not found after rescanning devices.", vg->name); + goto out; + } + + log_debug_lvmetad("Rescan VG %s has %d PVs after label scan.", + vg->name, dm_list_size(&pvs_scan)); + + if (save_vg) + release_vg(save_vg); + if (save_meta) + dm_config_destroy(save_meta); + save_vg = NULL; + save_meta = NULL; + save_dev = NULL; + save_seqno = 0; + found_new_pvs = 0; + retried_reads = 1; + goto retry_reads; } /* @@ -2025,52 +2190,50 @@ scan_more: dm_list_iterate_items(devl, &pvs_drop) { if (!devl->dev) continue; - log_debug_lvmetad("Rescan VG %s dropping %s.", vg->name, dev_name(devl->dev)); - if (!lvmetad_pv_gone_by_dev(devl->dev)) - return_NULL; + log_debug_lvmetad("Rescan VG %s removing %s from lvmetad.", vg->name, dev_name(devl->dev)); + if (!lvmetad_pv_gone_by_dev(devl->dev)) { + /* FIXME: use an error path that disables lvmetad */ + log_error("Failed to remove %s from lvmetad.", dev_name(devl->dev)); + } } /* - * Update the VG in lvmetad. + * Update lvmetad with the newly read version of the VG. + * When the seqno is unchanged the cached VG can be left. */ - if (vgmeta_ret) { - fid = lvmcache_fmt(info)->ops->create_instance(lvmcache_fmt(info), &fic); - if (!(vg_ret = import_vg_from_config_tree(vgmeta_ret, fid))) { - log_error("VG import from config tree failed"); - lvmcache_fmt(info)->ops->destroy_instance(fid); - goto out; + if (save_vg && (save_seqno != vg->seqno)) { + dm_list_iterate_items(devl, &pvs_scan) { + if (!devl->dev) + continue; + log_debug_lvmetad("Rescan VG %s removing %s from lvmetad to replace.", + vg->name, dev_name(devl->dev)); + if (!lvmetad_pv_gone_by_dev(devl->dev)) { + /* FIXME: use an error path that disables lvmetad */ + log_error("Failed to remove %s from lvmetad.", dev_name(devl->dev)); + } } + log_debug_lvmetad("Rescan VG %s updating lvmetad from seqno %u to seqno %u.", + vg->name, vg->seqno, save_seqno); + /* - * Update lvmetad with the newly read version of the VG. - * When the seqno is unchanged the cached VG can be left. + * If this vg_update fails the cached metadata in + * lvmetad will remain invalid. */ - if (save_seqno != vg->seqno) { - dm_list_iterate_items(devl, &pvs_scan) { - if (!devl->dev) - continue; - log_debug_lvmetad("Rescan VG %s dropping to replace %s.", vg->name, dev_name(devl->dev)); - if (!lvmetad_pv_gone_by_dev(devl->dev)) - return_NULL; - } - - log_debug_lvmetad("Rescan VG %s updating lvmetad from seqno %u to seqno %u.", - vg->name, vg->seqno, save_seqno); - - /* - * If this vg_update fails the cached metadata in - * lvmetad will remain invalid. - */ - vg_ret->lvmetad_update_pending = 1; - if (!lvmetad_vg_update_finish(vg_ret)) - log_error("Failed to update lvmetad with new VG meta"); + save_vg->lvmetad_update_pending = 1; + if (!lvmetad_vg_update_finish(save_vg)) { + /* FIXME: use an error path that disables lvmetad */ + log_error("Failed to update lvmetad with new VG meta"); } - dm_config_destroy(vgmeta_ret); } out: - if (vg_ret) - log_debug_lvmetad("Rescan VG %s done (seqno %u).", vg_ret->name, vg_ret->seqno); - return vg_ret; + if (!save_vg && fid) + fmt->ops->destroy_instance(fid); + if (save_meta) + dm_config_destroy(save_meta); + if (save_vg) + log_debug_lvmetad("Rescan VG %s done (new seqno %u).", save_vg->name, save_vg->seqno); + return save_vg; } int lvmetad_pvscan_single(struct cmd_context *cmd, struct device *dev, @@ -2080,9 +2243,12 @@ int lvmetad_pvscan_single(struct cmd_context *cmd, struct device *dev, struct label *label; struct lvmcache_info *info; struct _lvmetad_pvscan_baton baton; + const struct format_type *fmt; /* Create a dummy instance. */ struct format_instance_ctx fic = { .type = 0 }; + log_debug_lvmetad("Scan metadata from dev %s", dev_name(dev)); + if (!lvmetad_used()) { log_error("Cannot proceed since lvmetad is not active."); return 0; @@ -2107,15 +2273,17 @@ int lvmetad_pvscan_single(struct cmd_context *cmd, struct device *dev, return 1; } + fmt = lvmcache_fmt(info); + baton.cmd = cmd; baton.vg = NULL; - baton.fid = lvmcache_fmt(info)->ops->create_instance(lvmcache_fmt(info), &fic); + baton.fid = fmt->ops->create_instance(fmt, &fic); if (!baton.fid) goto_bad; - if (baton.fid->fmt->features & FMT_OBSOLETE) { - lvmcache_fmt(info)->ops->destroy_instance(baton.fid); + if (fmt->features & FMT_OBSOLETE) { + fmt->ops->destroy_instance(baton.fid); log_warn("WARNING: Disabling lvmetad cache which does not support obsolete (lvm1) metadata."); lvmetad_set_disabled(cmd, LVMETAD_DISABLE_REASON_LVM1); _found_lvm1_metadata = 1; @@ -2129,9 +2297,9 @@ int lvmetad_pvscan_single(struct cmd_context *cmd, struct device *dev, lvmcache_foreach_mda(info, _lvmetad_pvscan_single, &baton); if (!baton.vg) - lvmcache_fmt(info)->ops->destroy_instance(baton.fid); + fmt->ops->destroy_instance(baton.fid); - if (!lvmetad_pv_found(cmd, (const struct id *) &dev->pvid, dev, lvmcache_fmt(info), + if (!lvmetad_pv_found(cmd, (const struct id *) &dev->pvid, dev, fmt, label->sector, baton.vg, found_vgnames, changed_vgnames)) { release_vg(baton.vg); goto_bad; @@ -2570,6 +2738,8 @@ void lvmetad_validate_global_cache(struct cmd_context *cmd, int force) */ _lvmetad_get_pv_cache_list(cmd, &pvc_before); + log_debug_lvmetad("Rescan all devices to validate global cache."); + /* * Update the local lvmetad cache so it correctly reflects any * changes made on remote hosts. (It's possible that this command @@ -2638,7 +2808,7 @@ void lvmetad_validate_global_cache(struct cmd_context *cmd, int force) _update_changed_pvs_in_udev(cmd, &pvc_before, &pvc_after); } - log_debug_lvmetad("Validating global lvmetad cache finished"); + log_debug_lvmetad("Rescanned all devices"); } int lvmetad_vg_is_foreign(struct cmd_context *cmd, const char *vgname, const char *vgid) diff --git a/lib/label/label.c b/lib/label/label.c index 0fc93f0b9..45ec7a5fc 100644 --- a/lib/label/label.c +++ b/lib/label/label.c @@ -462,7 +462,7 @@ static void _free_label_read_list(int do_close) dm_list_iterate_items_safe(ld, ld2, &label_read_list) { dm_list_del(&ld->list); - free(ld); + dm_free(ld); } } @@ -552,6 +552,31 @@ static int _label_read_async_process(struct cmd_context *cmd, struct label_read_ } /* + * ld->buf is the buffer into which the first ASYNC_SCAN_SIZE bytes + * of each device are read. The memory for buf needs to be aligned. + * + * This data is meant to big large enough to cover all the + * headers and metadata that need to be read from the device + * during the label scan for most common cases. + * + * 1. one of the first four sectors holds: + * label_header, pv_header, pv_header_extention + * + * 2. the mda_header whose location is found from 1. + * + * 3. the metadata whose location is from found 2. + * + * If during processing, metadata needs to be read in a region + * beyond this buffer, then the code will revert do doing a + * synchronous read of the data it needs. + */ +static int _get_async_scan_size(struct cmd_context *cmd) +{ + /* FIXME: make this a config setting, default to a multiple of optimal_io_size? */ + return ASYNC_SCAN_SIZE; +} + +/* * label_scan run at the start of a command iterates over all visible devices, * looking for any that belong to lvm, and fills lvmcache with basic info about * them. It's main job is to prepare for subsequent vg_reads. vg_read(vgname) @@ -564,9 +589,16 @@ static int _label_read_async_process(struct cmd_context *cmd, struct label_read_ /* * Scan labels/metadata for all devices (async) + * + * Reads and looks at label_header, pv_header, pv_header_extension, + * mda_header, raw_locns, vg metadata from each device. + * + * Effect is populating lvmcache with latest info/vginfo (PV/VG) data + * from the devs. If a scanned device does not have a label_header, + * its info is removed from lvmcache. */ -int label_scan_async(struct cmd_context *cmd) +static int _label_scan_async(struct cmd_context *cmd, int skip_cached) { struct label_read_data *ld, *ld2; struct dev_iter *iter; @@ -580,26 +612,7 @@ int label_scan_async(struct cmd_context *cmd) _free_label_read_list(0); - /* - * "buf" is the buffer into which the first ASYNC_SCAN_SIZE bytes - * of each device are read. The memory for buf needs to be aligned. - * - * This data is meant to big large enough to cover all the - * headers and metadata that need to be read from the device - * during the label scan for most common cases. - * - * 1. one of the first four sectors holds: - * label_header, pv_header, pv_header_extention - * - * 2. the mda_header whose location is found from 1. - * - * 3. the metadata whose location is from found 2. - * - * If during processing, metadata needs to be read in a region - * beyond this buffer, then the code will revert do doing a - * synchronous read of the data it needs. - */ - buf_len = ASYNC_SCAN_SIZE; + buf_len = _get_async_scan_size(cmd); /* * if aio setup fails, caller will revert to sync scan @@ -621,10 +634,10 @@ int label_scan_async(struct cmd_context *cmd) dev_cache_full_scan(cmd->full_filter); - log_debug_devs("Scanning all labels async"); + log_debug_devs("Scanning data from all devs async"); if (!(iter = dev_iter_create(cmd->full_filter, 0))) { - log_error("Scanning labels failed to get devices."); + log_error("Scanning data failed to get devices."); return 0; } @@ -639,7 +652,7 @@ int label_scan_async(struct cmd_context *cmd) * FIXME: fix code so it's not scanning labels when it's not needed, * then stuff like this can be removed. */ - if ((info = lvmcache_info_from_pvid(dev->pvid, dev, 1))) { + if (skip_cached && (info = lvmcache_info_from_pvid(dev->pvid, dev, 1))) { log_debug_devs("Reading label skipped in cache %s", dev_name(dev)); continue; } @@ -749,19 +762,20 @@ int label_scan_async(struct cmd_context *cmd) dm_list_iterate_items(ld, &label_read_list) dev_close(ld->dev); - log_debug_devs("Scanned %d labels async", dev_count); + log_debug_devs("Scanned data from all %d devs async", dev_count); return 1; bad: /* caller will try sync scan */ - log_error("async label scan failed, reverting to sync scan."); + log_error("async data scan failed, reverting to sync scan."); dev_iter_destroy(iter); dev_async_context_destroy(ac); dm_list_iterate_items(ld, &label_read_list) { dev_close(ld->dev); - dev_async_io_destroy(ld->aio); + if (ld->aio) + dev_async_io_destroy(ld->aio); } dm_list_iterate_items_safe(ld, ld2, &label_read_list) { @@ -772,22 +786,41 @@ bad: return 0; } +int label_scan_async(struct cmd_context *cmd) +{ + return _label_scan_async(cmd, 1); +} + +int label_scan_async_force(struct cmd_context *cmd) +{ + return _label_scan_async(cmd, 0); +} + /* - * Rescan labels/metadata for select devices (async) - * (devs from a specific VG) + * Read or reread label/metadata from selected devs (async). + * + * Reads and looks at label_header, pv_header, pv_header_extension, + * mda_header, raw_locns, vg metadata from each device. + * + * Effect is populating lvmcache with latest info/vginfo (PV/VG) data + * from the devs. If a scanned device does not have a label_header, + * its info is removed from lvmcache. */ -int label_rescan_async(struct cmd_context *cmd, struct dm_list *devs) +int label_scan_devs_async(struct cmd_context *cmd, struct dm_list *devs) { struct dm_list tmp_label_read_list; struct label_read_data *ld, *ld2; struct device_list *devl; struct device *dev; struct dev_async_context *ac; + int buf_len; int need_wait_count; int need_process_count; int dev_count = 0; + buf_len = _get_async_scan_size(cmd); + dm_list_init(&tmp_label_read_list); if (!(ac = dev_async_context_setup(0))) { @@ -795,18 +828,32 @@ int label_rescan_async(struct cmd_context *cmd, struct dm_list *devs) return_0; } - log_debug_devs("Scanning labels for VG devs async"); + log_debug_devs("Scanning data from devs async"); dm_list_iterate_items(devl, devs) { dev = devl->dev; if (!(ld = get_label_read_data(cmd, dev))) { - log_warn("WARNING: label rescan cannot find dev %s", dev_name(dev)); - continue; + /* New device hasn't been scanned before. */ + + if (!(ld = dm_malloc(sizeof(*ld)))) + goto_bad; + + memset(ld, 0, sizeof(*ld)); + + if (!(ld->aio = dev_async_io_alloc(buf_len))) { + dm_free(ld); + goto_bad; + } + + ld->buf = ld->aio->buf; + ld->buf_len = buf_len; + ld->dev = dev; + } else { + /* Temporarily move structs being reread onto local list. */ + dm_list_del(&ld->list); } - /* Temporarily move structs being reread onto local list. */ - dm_list_del(&ld->list); dm_list_add(&tmp_label_read_list, &ld->list); if (!dev_open_readonly(ld->dev)) { @@ -860,7 +907,8 @@ int label_rescan_async(struct cmd_context *cmd, struct dm_list *devs) /* * Process devices that have finished reading label sectors. - * Processing includes sync i/o to read mda locations and vg metadata. + * Processing can include sync i/o to read metadata areas + * beyond the ASYNC_SCAN_SIZE. * * FIXME: we shouldn't need to fully reprocess everything when rescanning. * lvmcache is already populated from the previous scan, and if nothing @@ -902,12 +950,54 @@ int label_rescan_async(struct cmd_context *cmd, struct dm_list *devs) dm_list_add(&label_read_list, &ld->list); } - log_debug_devs("Scanned %d labels for VG devs async", dev_count); + log_debug_devs("Scanned data from %d devs async", dev_count); return 1; + +bad: + /* caller will try sync scan */ + log_error("async data scan failed, reverting to sync scan."); + + dev_async_context_destroy(ac); + + dm_list_iterate_items(ld, &label_read_list) { + dev_close(ld->dev); + if (ld->aio) + dev_async_io_destroy(ld->aio); + } + + dm_list_iterate_items(ld, &tmp_label_read_list) { + dev_close(ld->dev); + if (ld->aio) + dev_async_io_destroy(ld->aio); + } + + dm_list_iterate_items_safe(ld, ld2, &label_read_list) { + dm_list_del(&ld->list); + dm_free(ld); + } + + dm_list_iterate_items_safe(ld, ld2, &tmp_label_read_list) { + dm_list_del(&ld->list); + dm_free(ld); + } + + return 0; } /* * Read label/metadata for one dev (sync) + * + * Reads and looks at label_header, pv_header, pv_header_extension, + * mda_header, raw_locns, vg metadata from each device. + * + * Effect is populating lvmcache with latest info/vginfo (PV/VG) data + * from the devs. If a scanned device does not have a label_header, + * its info is removed from lvmcache. + * + * FIXME: allocate label_read_data structs for devs and read data + * into there rather than into stack buffers. Then the data can + * be found and reused throughout the scan path, rather than doing + * a separate disk read for everything. */ static int _label_read_sync(struct cmd_context *cmd, struct device *dev) @@ -959,9 +1049,16 @@ static int _label_read_sync(struct cmd_context *cmd, struct device *dev) /* * Scan labels/metadata for all devices (sync) + * + * Reads and looks at label_header, pv_header, pv_header_extension, + * mda_header, raw_locns, vg metadata from each device. + * + * Effect is populating lvmcache with latest info/vginfo (PV/VG) data + * from the devs. If a scanned device does not have a label_header, + * its info is removed from lvmcache. */ -int label_scan_sync(struct cmd_context *cmd) +static int _label_scan_sync(struct cmd_context *cmd, int skip_cached) { struct dev_iter *iter; struct device *dev; @@ -974,15 +1071,15 @@ int label_scan_sync(struct cmd_context *cmd) dev_cache_full_scan(cmd->full_filter); - log_very_verbose("Scanning all labels sync"); + log_very_verbose("Scanning data from all devs sync"); if (!(iter = dev_iter_create(cmd->full_filter, 0))) { - log_error("Scanning labels failed to get devices."); + log_error("Scanning data failed to get devices."); return 0; } while ((dev = dev_iter_get(iter))) { - if ((info = lvmcache_info_from_pvid(dev->pvid, dev, 1))) { + if (skip_cached && (info = lvmcache_info_from_pvid(dev->pvid, dev, 1))) { log_debug_devs("Reading label skipped in cache %s", dev_name(dev)); continue; } @@ -1001,21 +1098,37 @@ int label_scan_sync(struct cmd_context *cmd) } dev_iter_destroy(iter); - log_very_verbose("Scanned %d labels sync", dev_count); + log_very_verbose("Scanned data from all %d devs sync", dev_count); return 1; } +int label_scan_sync(struct cmd_context *cmd) +{ + return _label_scan_sync(cmd, 1); +} + +int label_scan_sync_force(struct cmd_context *cmd) +{ + return _label_scan_sync(cmd, 0); +} + /* - * Rescan labels/metadata for select devices (sync) - * (devs from a specific VG) + * Read or reread label/metadata from selected devs (sync). + * + * Reads and looks at label_header, pv_header, pv_header_extension, + * mda_header, raw_locns, vg metadata from each device. + * + * Effect is populating lvmcache with latest info/vginfo (PV/VG) data + * from the devs. If a scanned device does not have a label_header, + * its info is removed from lvmcache. */ -int label_rescan_sync(struct cmd_context *cmd, struct dm_list *devs) +int label_scan_devs_sync(struct cmd_context *cmd, struct dm_list *devs) { struct device_list *devl; int dev_count = 0; - log_debug_devs("Scanning labels for VG devs sync"); + log_debug_devs("Scanning data from devs sync"); dm_list_iterate_items(devl, devs) { if (!dev_open_readonly(devl->dev)) { @@ -1031,7 +1144,7 @@ int label_rescan_sync(struct cmd_context *cmd, struct dm_list *devs) dev_count++; } - log_very_verbose("Scanned %d labels for VG devs sync", dev_count); + log_very_verbose("Scanned data from %d devs sync", dev_count); return 1; } diff --git a/lib/label/label.h b/lib/label/label.h index e3e4520f3..e9cfcda92 100644 --- a/lib/label/label.h +++ b/lib/label/label.h @@ -127,8 +127,17 @@ void label_destroy(struct label *label); int label_scan_async(struct cmd_context *cmd); int label_scan_sync(struct cmd_context *cmd); -int label_rescan_async(struct cmd_context *cmd, struct dm_list *devs); -int label_rescan_sync(struct cmd_context *cmd, struct dm_list *devs); +int label_scan_devs_async(struct cmd_context *cmd, struct dm_list *devs); +int label_scan_devs_sync(struct cmd_context *cmd, struct dm_list *devs); struct label_read_data *get_label_read_data(struct cmd_context *cmd, struct device *dev); +/* + * FIXME: get rid of these force variations by making label_scan + * never skip scanning when info is cached. + * _force versions don't skip scanning label when info exists + * in lvmcache. + */ +int label_scan_async_force(struct cmd_context *cmd); +int label_scan_sync_force(struct cmd_context *cmd); + #endif |