From 97b933cfd9e92757563c6d033fb937c3d8e44a28 Mon Sep 17 00:00:00 2001 From: David Teigland Date: Wed, 4 Dec 2019 16:41:00 -0600 Subject: dm-integrity with raid1 support Create a raid1 LV where the raid images use dm-integrity. Only external metadata is currently enabled. lvcreate --type raid1 --mirrors Num --integrity y ... --- device_mapper/all.h | 2 + device_mapper/libdm-deptree.c | 6 + lib/metadata/integrity_manip.c | 280 ++++++++++++++++++++++++++++++++++++++- lib/metadata/lv_manip.c | 13 +- lib/metadata/metadata-exported.h | 8 +- lib/raid/raid.c | 30 +++++ tools/command-lines.in | 3 +- tools/lvcreate.c | 12 +- 8 files changed, 338 insertions(+), 16 deletions(-) diff --git a/device_mapper/all.h b/device_mapper/all.h index 780cdb9da..1bf71a582 100644 --- a/device_mapper/all.h +++ b/device_mapper/all.h @@ -1001,6 +1001,8 @@ struct integrity_settings { unsigned block_size_set:1; unsigned bitmap_flush_interval_set:1; unsigned sectors_per_bit_set:1; + + int recalculate; /* not persistent */ }; int dm_tree_node_add_integrity_target(struct dm_tree_node *node, diff --git a/device_mapper/libdm-deptree.c b/device_mapper/libdm-deptree.c index d20f2f7a8..d037b8f01 100644 --- a/device_mapper/libdm-deptree.c +++ b/device_mapper/libdm-deptree.c @@ -2750,6 +2750,9 @@ static int _integrity_emit_segment_line(struct dm_task *dmt, if (set->sectors_per_bit_set) count++; + if (set->recalculate) + count++; + EMIT_PARAMS(pos, "%s 0 %u %s %d internal_hash:%s", origin_dev, set->tag_size, @@ -2784,6 +2787,9 @@ static int _integrity_emit_segment_line(struct dm_task *dmt, if (set->sectors_per_bit_set) EMIT_PARAMS(pos, " sectors_per_bit:%llu", (unsigned long long)set->sectors_per_bit); + if (set->recalculate) + EMIT_PARAMS(pos, " recalculate"); + return 1; } diff --git a/lib/metadata/integrity_manip.c b/lib/metadata/integrity_manip.c index 77ea18b46..eb0af222f 100644 --- a/lib/metadata/integrity_manip.c +++ b/lib/metadata/integrity_manip.c @@ -203,10 +203,277 @@ int lv_remove_integrity(struct logical_volume *lv) return 1; } +/* + * Add integrity to each raid image. + * + * for each rimage_N: + * . create and allocate a new linear LV rimage_N_imeta + * . move the segments from rimage_N to a new rimage_N_iorig + * . add an integrity segment to rimage_N with + * origin=rimage_N_iorig, meta_dev=rimage_N_imeta + * + * Before: + * rimage_0 + * segment1: striped: pv0:A + * rimage_1 + * segment1: striped: pv1:B + * + * After: + * rimage_0 + * segment1: integrity: rimage_0_iorig, rimage_0_imeta + * rimage_1 + * segment1: integrity: rimage_1_iorig, rimage_1_imeta + * rimage_0_iorig + * segment1: striped: pv0:A + * rimage_1_iorig + * segment1: striped: pv1:B + * rimage_0_imeta + * segment1: striped: pv2:A + * rimage_1_imeta + * segment1: striped: pv2:B + * + */ + +static int _lv_add_integrity_to_raid(struct logical_volume *lv, const char *arg, + struct integrity_settings *settings, + struct dm_list *pvh, + uint64_t *zero_data_sectors) +{ + struct lvcreate_params lp; + struct logical_volume *imeta_lvs[DEFAULT_RAID_MAX_IMAGES]; + struct cmd_context *cmd = lv->vg->cmd; + struct volume_group *vg = lv->vg; + struct logical_volume *lv_image, *lv_imeta, *lv_iorig; + struct lv_segment *seg_top, *seg_image; + const struct segment_type *segtype; + struct integrity_settings *set; + uint64_t lv_image_size; /* sectors */ + uint64_t status_data_sectors; + uint32_t area_count, s; + int internal_metadata; + int ret = 1; + + memset(imeta_lvs, 0, sizeof(imeta_lvs)); + + if (dm_list_size(&lv->segments) != 1) + return_0; + + seg_top = first_seg(lv); + area_count = seg_top->area_count; + + if ((internal_metadata = !strcmp(arg, "internal"))) { + /* + * FIXME: raid on internal integrity might not be used widely + * enough to enable, given the additional complexity/support. + * i.e. nearly everyone may just use external metadata. + * + * FIXME: _info_run() needs code to adjust the length, like + * is done for if (lv_is_integrity()) length = ... + */ + /* goto skip_imeta; */ + log_error("Internal integrity metadata is not yet supported with raid."); + return 0; + } + + /* + * For each rimage, create an _imeta LV for integrity metadata. + * Each needs to be zeroed. + */ + for (s = 0; s < area_count; s++) { + struct logical_volume *meta_lv; + struct wipe_params wipe; + + if (s >= DEFAULT_RAID_MAX_IMAGES) + return_0; + + lv_image = seg_lv(seg_top, s); + + if (!seg_is_striped(first_seg(lv_image))) { + log_error("raid1 image must be linear to add integrity"); + return_0; + } + + /* + * allocate a new linear LV NAME_rimage_N_imeta + * lv_create_integrity_metadata() returns its result in lp + */ + memset(&lp, 0, sizeof(lp)); + lp.lv_name = lv_image->name; + lp.pvh = pvh; + lp.extents = lv_image->size / vg->extent_size; + + if (!lv_create_integrity_metadata(cmd, vg, &lp)) { + return_0; + } + meta_lv = lp.integrity_meta_lv; + + /* + * dm-integrity requires the metadata LV header to be zeroed. + */ + + if (!activate_lv(cmd, meta_lv)) { + log_error("Failed to activate LV %s to zero", display_lvname(meta_lv)); + return 0; + } + + memset(&wipe, 0, sizeof(wipe)); + wipe.do_zero = 1; + wipe.zero_sectors = 8; + + if (!wipe_lv(meta_lv, wipe)) { + log_error("Failed to zero LV for integrity metadata %s", display_lvname(meta_lv)); + return 0; + } + + if (!deactivate_lv(cmd, meta_lv)) { + log_error("Failed to deactivate LV %s after zero", display_lvname(meta_lv)); + return 0; + } + + /* Used below to set up the new integrity segment. */ + imeta_lvs[s] = meta_lv; + } + +/* skip_imeta: */ + + /* + * For each rimage, move its segments to a new rimage_iorig and give + * the rimage a new integrity segment. + */ + for (s = 0; s < area_count; s++) { + lv_image = seg_lv(seg_top, s); + + lv_image_size = lv_image->size; /* sectors */ + + if (!(segtype = get_segtype_from_string(cmd, SEG_TYPE_NAME_INTEGRITY))) + return_0; + + log_debug("Adding integrity to raid image %s", lv_image->name); + + /* + * "lv_iorig" is a new LV with new id, but with the segments + * from "lv_image". "lv_image" keeps the existing name and id, + * but gets a new integrity segment, in place of the segments + * that were moved to lv_iorig. + */ + if (!(lv_iorig = insert_layer_for_lv(cmd, lv_image, INTEGRITY, "_iorig"))) + return_0; + + lv_image->status |= INTEGRITY; + + /* + * Set up the new first segment of lv_image as integrity. + */ + seg_image = first_seg(lv_image); + seg_image->segtype = segtype; + + if (internal_metadata) { + /* dm-integrity wants temp/fake size of 1 to report usable size */ + lv_image->size = 1; + seg_image->integrity_data_sectors = 1; + } else { + lv_imeta = imeta_lvs[s]; /* external metadata lv created above */ + lv_imeta->status |= INTEGRITY_METADATA; + lv_set_hidden(lv_imeta); + seg_image->integrity_data_sectors = lv_image_size; + seg_image->integrity_meta_dev = lv_imeta; + } + + memcpy(&seg_image->integrity_settings, settings, sizeof(struct integrity_settings)); + set = &seg_image->integrity_settings; + + if (!set->mode[0]) + set->mode[0] = DEFAULT_MODE; + + if (!set->tag_size) + set->tag_size = DEFAULT_TAG_SIZE; + + if (!set->internal_hash) + set->internal_hash = DEFAULT_INTERNAL_HASH; + + set->recalculate = 1; /* enable integrity initialization in the kernel */ + } + + if (internal_metadata) { + /* Get the size from the first image, the others will be the same. */ + lv_image = seg_lv(seg_top, 0); + + lv_image->status |= LV_TEMPORARY; + lv_image->status |= LV_NOSCAN; + lv_image->status |= LV_UNCOMMITTED; + + log_debug("Activating temporary integrity LV to get data sectors."); + + if (!activate_lv(cmd, lv_image)) { + log_error("Failed to activate temporary integrity."); + ret = 0; + goto out; + } + + if (!_get_provided_data_sectors(lv_image, &status_data_sectors)) { + log_error("Failed to get data sectors from dm-integrity"); + ret = 0; + } else { + log_print("Found integrity provided_data_sectors %llu", (unsigned long long)status_data_sectors); + ret = 1; + } + + if (!status_data_sectors) { + log_error("LV size too small to include metadata."); + ret = 0; + } + + lv_image->status |= LV_UNCOMMITTED; + + if (!deactivate_lv(cmd, lv_image)) { + log_error("Failed to deactivate temporary integrity."); + ret = 0; + } + + if (!ret) + goto_out; + + lv_image->status &= ~LV_UNCOMMITTED; + lv_image->status &= ~LV_NOSCAN; + lv_image->status &= ~LV_TEMPORARY; + + for (s = 0; s < area_count; s++) { + lv_image = seg_lv(seg_top, s); + lv_image->size = lv_image_size; /* was 1 temporarily */ + seg_image = first_seg(lv_image); + seg_image->integrity_data_sectors = status_data_sectors; + } + } + + /* + * recalculate above enables automatic integrity metadata intialization + * in the kernel when first activated, instead of activating the raid + * LV and writing zeroes to the entire thing from the lvcreate command + * to initialize the metadata in the integrity images. + * + * Zeroing the raid LV to intialize integrity metadata on the images + * does not work because dm-raid immediately reads the integrity + * images when activated, and gets errors back because of no init. + * + * When activated with recalculate, dm-integrity will not return errors + * to dm-raid when it reads uninitialized blocks. + */ + *zero_data_sectors = 0; /* disables intialization by lvcreate */ + + log_debug("Write VG with integrity added to LV"); + + if (!vg_write(vg) || !vg_commit(vg)) + ret = 0; +out: + return ret; +} + int lv_add_integrity(struct logical_volume *lv, const char *arg, struct logical_volume *meta_lv_created, const char *meta_name, - struct integrity_settings *settings) + struct integrity_settings *settings, + struct dm_list *pvh, + uint64_t *zero_data_sectors) { struct cmd_context *cmd = lv->vg->cmd; struct volume_group *vg = lv->vg; @@ -217,7 +484,10 @@ int lv_add_integrity(struct logical_volume *lv, const char *arg, struct lv_segment *seg; uint64_t meta_bytes, meta_sectors; uint64_t lv_size_sectors; - int ret; + int ret = 1; + + if (lv_is_raid(lv)) + return _lv_add_integrity_to_raid(lv, arg, settings, pvh, zero_data_sectors); lv_size_sectors = lv->size; @@ -335,7 +605,6 @@ int lv_add_integrity(struct logical_volume *lv, const char *arg, seg->integrity_meta_dev = meta_lv; lv_set_hidden(meta_lv); /* TODO: give meta_lv a suffix? e.g. _imeta */ - ret = 1; } else { /* dm-integrity wants temp/fake size of 1 to report usable size */ seg->integrity_data_sectors = 1; @@ -377,6 +646,11 @@ int lv_add_integrity(struct logical_volume *lv, const char *arg, lv->status &= ~LV_TEMPORARY; } + /* + * Tells lvcreate to zero the final LV at the end of commands. + */ + *zero_data_sectors = seg->integrity_data_sectors; + log_debug("Write VG with integrity added to LV"); if (!vg_write(vg) || !vg_commit(vg)) diff --git a/lib/metadata/lv_manip.c b/lib/metadata/lv_manip.c index 7ad5b1ff6..d15f4774e 100644 --- a/lib/metadata/lv_manip.c +++ b/lib/metadata/lv_manip.c @@ -7473,11 +7473,12 @@ int insert_layer_for_segments_on_pv(struct cmd_context *cmd, return 1; } -int zero_lv_name(struct cmd_context *cmd, const char *vg_name, const char *lv_name, uint64_t lv_size_bytes) +int zero_lv_name(struct cmd_context *cmd, const char *vg_name, const char *lv_name, uint64_t lv_size_sectors) { char name[PATH_MAX]; struct device *dev; uint64_t off = 0, i = 0, j = 0; + uint64_t lv_size_bytes = lv_size_sectors * 512; uint64_t zero_bytes; uint32_t extra_bytes; @@ -7831,7 +7832,7 @@ static int _should_wipe_lv(struct lvcreate_params *lp, * TODO: print a warning or error if the user specifically * asks for no wiping or zeroing? */ - if (seg_is_integrity(lp)) + if (seg_is_integrity(lp) || (seg_is_raid1(lp) && lp->integrity_arg)) return 1; /* Cannot zero read-only volume */ @@ -8438,7 +8439,7 @@ static struct logical_volume *_lv_create_an_lv(struct volume_group *vg, } lv->status &= ~LV_TEMPORARY; - } else if (seg_is_integrity(lp)) { + } else if (seg_is_integrity(lp) || (seg_is_raid1(lp) && lp->integrity_arg)) { /* * Activate the new origin LV so it can be zeroed/wiped * below before adding integrity. @@ -8469,7 +8470,7 @@ static struct logical_volume *_lv_create_an_lv(struct volume_group *vg, } } - if (seg_is_integrity(lp)) { + if (seg_is_integrity(lp) || (seg_is_raid1(lp) && lp->integrity_arg)) { log_verbose("Adding integrity to new LV"); /* Origin is active from zeroing, deactivate to add integrity. */ @@ -8480,7 +8481,8 @@ static struct logical_volume *_lv_create_an_lv(struct volume_group *vg, } if (!lv_add_integrity(lv, lp->integrity_arg, lp->integrity_meta_lv, - lp->integrity_meta_name, &lp->integrity_settings)) + lp->integrity_meta_name, &lp->integrity_settings, lp->pvh, + &lp->zero_data_sectors)) goto revert_new_lv; backup(vg); @@ -8512,7 +8514,6 @@ static struct logical_volume *_lv_create_an_lv(struct volume_group *vg, * problems (if the user doesn't want to wait, or wants * to do the zeroing themselves.) */ - lp->integrity_bytes_to_zero = first_seg(lv)->integrity_data_sectors * 512; goto out; } diff --git a/lib/metadata/metadata-exported.h b/lib/metadata/metadata-exported.h index 7d488e705..117163f29 100644 --- a/lib/metadata/metadata-exported.h +++ b/lib/metadata/metadata-exported.h @@ -1007,7 +1007,7 @@ struct lvcreate_params { const char *integrity_meta_name; /* external LV is user-specified */ struct logical_volume *integrity_meta_lv; /* external LV we create */ struct integrity_settings integrity_settings; - uint64_t integrity_bytes_to_zero; /* zeros the final LV after it's created */ + uint64_t zero_data_sectors; /* the resulting size that should be zeroed at the end */ struct dm_list tags; /* all */ @@ -1406,12 +1406,14 @@ void vg_write_commit_bad_mdas(struct cmd_context *cmd, struct volume_group *vg); int lv_add_integrity(struct logical_volume *lv, const char *arg, struct logical_volume *meta_lv_created, - const char *meta_name, struct integrity_settings *settings); + const char *meta_name, struct integrity_settings *settings, + struct dm_list *pvh, + uint64_t *zero_data_sectors); int lv_create_integrity_metadata(struct cmd_context *cmd, struct volume_group *vg, struct lvcreate_params *lp); int lv_remove_integrity(struct logical_volume *lv); -int zero_lv_name(struct cmd_context *cmd, const char *vg_name, const char *lv_name, uint64_t zero_bytes); +int zero_lv_name(struct cmd_context *cmd, const char *vg_name, const char *lv_name, uint64_t lv_size_sectors); #endif diff --git a/lib/raid/raid.c b/lib/raid/raid.c index e88a15408..c315a2cee 100644 --- a/lib/raid/raid.c +++ b/lib/raid/raid.c @@ -240,6 +240,26 @@ static int _raid_text_export(const struct lv_segment *seg, struct formatter *f) return _raid_text_export_raid(seg, f); } +static int _image_integrity_data_sectors(struct lv_segment *seg, uint64_t *data_sectors) +{ + struct logical_volume *lv_image; + struct lv_segment *seg_image; + int s; + + *data_sectors = 0; + + for (s = 0; s < seg->area_count; s++) { + lv_image = seg_lv(seg, s); + + if (lv_is_integrity(lv_image)) { + seg_image = first_seg(lv_image); + *data_sectors = seg_image->integrity_data_sectors; + return 1; + } + } + return 1; +} + static int _raid_add_target_line(struct dev_manager *dm __attribute__((unused)), struct dm_pool *mem __attribute__((unused)), struct cmd_context *cmd __attribute__((unused)), @@ -256,6 +276,7 @@ static int _raid_add_target_line(struct dev_manager *dm __attribute__((unused)), uint64_t writemostly[RAID_BITMAP_SIZE] = { 0 }; struct dm_tree_node_raid_params_v2 params = { 0 }; unsigned attrs; + uint64_t integrity_data_sectors = 0; if (seg_is_raid4(seg)) { if (!_raid_target_present(cmd, NULL, &attrs) || @@ -351,6 +372,15 @@ static int _raid_add_target_line(struct dev_manager *dm __attribute__((unused)), params.stripe_size = seg->stripe_size; params.flags = flags; + if (!_image_integrity_data_sectors(seg, &integrity_data_sectors)) + return_0; + + if (integrity_data_sectors) { + log_debug("Reducing raid size from %llu to integrity_data_sectors %llu", + (unsigned long long)len, (unsigned long long)integrity_data_sectors); + len = integrity_data_sectors; + } + if (!dm_tree_node_add_raid_target_with_params_v2(node, len, ¶ms)) return_0; diff --git a/tools/command-lines.in b/tools/command-lines.in index 8ed4b5e78..032621029 100644 --- a/tools/command-lines.in +++ b/tools/command-lines.in @@ -878,7 +878,8 @@ DESC: Create a raid1 or mirror LV (infers --type raid1|mirror). # R9,R10,R11,R12 (--type raid with any use of --stripes/--mirrors) lvcreate --type raid --size SizeMB VG OO: --mirrors PNumber, --stripes Number, --stripesize SizeKB, ---regionsize RegionSize, --minrecoveryrate SizeKB, --maxrecoveryrate SizeKB, OO_LVCREATE +--regionsize RegionSize, --minrecoveryrate SizeKB, --maxrecoveryrate SizeKB, +--integrity String, --integritysettings String, OO_LVCREATE OP: PV ... ID: lvcreate_raid_any DESC: Create a raid LV (a specific raid level must be used, e.g. raid1). diff --git a/tools/lvcreate.c b/tools/lvcreate.c index d19047e8c..e52d751f6 100644 --- a/tools/lvcreate.c +++ b/tools/lvcreate.c @@ -848,6 +848,8 @@ static int _lvcreate_params(struct cmd_context *cmd, poolmetadataspare_ARG #define RAID_ARGS \ + integrity_ARG,\ + integritysettings_ARG,\ maxrecoveryrate_ARG,\ minrecoveryrate_ARG,\ raidmaxrecoveryrate_ARG,\ @@ -1220,7 +1222,7 @@ static int _lvcreate_params(struct cmd_context *cmd, } } - if (seg_is_integrity(lp)) { + if (seg_is_integrity(lp) || seg_is_raid(lp)) { if (!get_integrity_options(cmd, &lp->integrity_arg, &lp->integrity_meta_name, &lp->integrity_settings)) return 0; } @@ -1808,11 +1810,15 @@ int lvcreate(struct cmd_context *cmd, int argc, char **argv) _destroy_lvcreate_params(&lp); destroy_processing_handle(cmd, handle); - if (lp.integrity_bytes_to_zero) { + /* + * When set, the final LV should be zeroed (set by add_integrity + * to intialize integrity metadata/checksums). + */ + if (lp.zero_data_sectors) { if (!lp.zero) log_warn("WARNING: not zeroing integrity LV, read errors are possible."); else - zero_lv_name(cmd, lp.vg_name, lp.lv_name, lp.integrity_bytes_to_zero); + zero_lv_name(cmd, lp.vg_name, lp.lv_name, lp.zero_data_sectors); } return ret; -- cgit v1.2.1