dm-integrity with raid1 supportdev-dct-integrity9

Create a raid1 LV where the raid images use dm-integrity.
Only external metadata is currently enabled.

lvcreate --type raid1 --mirrors Num --integrity y ...

8 files changed, 338 insertions, 16 deletions

diff --git a/device_mapper/all.h b/device_mapper/all.h
index 780cdb9da..1bf71a582 100644
--- a/device_mapper/all.h
+++ b/device_mapper/all.h
@@ -1001,6 +1001,8 @@ struct integrity_settings {
 	unsigned block_size_set:1;
 	unsigned bitmap_flush_interval_set:1;
 	unsigned sectors_per_bit_set:1;
+
+	int recalculate; /* not persistent */
 };
 
 int dm_tree_node_add_integrity_target(struct dm_tree_node *node,
diff --git a/device_mapper/libdm-deptree.c b/device_mapper/libdm-deptree.c
index d20f2f7a8..d037b8f01 100644
--- a/device_mapper/libdm-deptree.c
+++ b/device_mapper/libdm-deptree.c
@@ -2750,6 +2750,9 @@ static int _integrity_emit_segment_line(struct dm_task *dmt,
 	if (set->sectors_per_bit_set)
 		count++;
 
+	if (set->recalculate)
+		count++;
+
 	EMIT_PARAMS(pos, "%s 0 %u %s %d internal_hash:%s",
 		    origin_dev,
 		    set->tag_size,
@@ -2784,6 +2787,9 @@ static int _integrity_emit_segment_line(struct dm_task *dmt,
 	if (set->sectors_per_bit_set)
 		EMIT_PARAMS(pos, " sectors_per_bit:%llu", (unsigned long long)set->sectors_per_bit);
 
+	if (set->recalculate)
+		EMIT_PARAMS(pos, " recalculate");
+
 	return 1;
 }
 
diff --git a/lib/metadata/integrity_manip.c b/lib/metadata/integrity_manip.c
index 77ea18b46..eb0af222f 100644
--- a/lib/metadata/integrity_manip.c
+++ b/lib/metadata/integrity_manip.c
@@ -203,10 +203,277 @@ int lv_remove_integrity(struct logical_volume *lv)
 	return 1;
 }
 
+/*
+ * Add integrity to each raid image.
+ *
+ * for each rimage_N:
+ * . create and allocate a new linear LV rimage_N_imeta
+ * . move the segments from rimage_N to a new rimage_N_iorig
+ * . add an integrity segment to rimage_N with
+ *   origin=rimage_N_iorig, meta_dev=rimage_N_imeta
+ *
+ * Before:
+ * rimage_0
+ *   segment1: striped: pv0:A
+ * rimage_1
+ *   segment1: striped: pv1:B
+ *
+ * After:
+ * rimage_0
+ *   segment1: integrity: rimage_0_iorig, rimage_0_imeta
+ * rimage_1
+ *   segment1: integrity: rimage_1_iorig, rimage_1_imeta
+ * rimage_0_iorig
+ *   segment1: striped: pv0:A
+ * rimage_1_iorig
+ *   segment1: striped: pv1:B
+ * rimage_0_imeta
+ *   segment1: striped: pv2:A
+ * rimage_1_imeta
+ *   segment1: striped: pv2:B
+ *    
+ */
+
+static int _lv_add_integrity_to_raid(struct logical_volume *lv, const char *arg,
+				     struct integrity_settings *settings,
+				     struct dm_list *pvh,
+				     uint64_t *zero_data_sectors)
+{
+	struct lvcreate_params lp;
+	struct logical_volume *imeta_lvs[DEFAULT_RAID_MAX_IMAGES];
+	struct cmd_context *cmd = lv->vg->cmd;
+	struct volume_group *vg = lv->vg;
+	struct logical_volume *lv_image, *lv_imeta, *lv_iorig;
+	struct lv_segment *seg_top, *seg_image;
+	const struct segment_type *segtype;
+	struct integrity_settings *set;
+	uint64_t lv_image_size; /* sectors */
+	uint64_t status_data_sectors;
+	uint32_t area_count, s;
+	int internal_metadata;
+	int ret = 1;
+
+	memset(imeta_lvs, 0, sizeof(imeta_lvs));
+
+	if (dm_list_size(&lv->segments) != 1)
+		return_0;
+
+	seg_top = first_seg(lv);
+	area_count = seg_top->area_count;
+
+	if ((internal_metadata = !strcmp(arg, "internal"))) {
+		/*
+		 * FIXME: raid on internal integrity might not be used widely
+		 * enough to enable, given the additional complexity/support.
+		 * i.e. nearly everyone may just use external metadata.
+		 *
+		 * FIXME: _info_run() needs code to adjust the length, like
+		 * is done for if (lv_is_integrity()) length = ...
+		 */
+		/* goto skip_imeta; */
+		log_error("Internal integrity metadata is not yet supported with raid.");
+		return 0;
+	}
+
+	/*
+	 * For each rimage, create an _imeta LV for integrity metadata.
+	 * Each needs to be zeroed.
+	 */
+	for (s = 0; s < area_count; s++) {
+		struct logical_volume *meta_lv;
+		struct wipe_params wipe;
+
+		if (s >= DEFAULT_RAID_MAX_IMAGES)
+			return_0;
+
+		lv_image = seg_lv(seg_top, s);
+
+		if (!seg_is_striped(first_seg(lv_image))) {
+			log_error("raid1 image must be linear to add integrity");
+			return_0;
+		}
+
+		/*
+		 * allocate a new linear LV NAME_rimage_N_imeta
+		 * lv_create_integrity_metadata() returns its result in lp
+		 */
+		memset(&lp, 0, sizeof(lp));
+		lp.lv_name = lv_image->name;
+		lp.pvh = pvh;
+		lp.extents = lv_image->size / vg->extent_size;
+
+		if (!lv_create_integrity_metadata(cmd, vg, &lp)) {
+			return_0;
+		}
+		meta_lv = lp.integrity_meta_lv;
+
+		/*
+		 * dm-integrity requires the metadata LV header to be zeroed.
+		 */
+
+		if (!activate_lv(cmd, meta_lv)) {
+			log_error("Failed to activate LV %s to zero", display_lvname(meta_lv));
+			return 0;
+		}
+
+		memset(&wipe, 0, sizeof(wipe));
+		wipe.do_zero = 1;
+		wipe.zero_sectors = 8;
+
+		if (!wipe_lv(meta_lv, wipe)) {
+			log_error("Failed to zero LV for integrity metadata %s", display_lvname(meta_lv));
+			return 0;
+		}
+
+		if (!deactivate_lv(cmd, meta_lv)) {
+			log_error("Failed to deactivate LV %s after zero", display_lvname(meta_lv));
+			return 0;
+		}
+
+		/* Used below to set up the new integrity segment. */
+		imeta_lvs[s] = meta_lv;
+	}
+
+/* skip_imeta: */
+
+	/*
+	 * For each rimage, move its segments to a new rimage_iorig and give
+	 * the rimage a new integrity segment.
+	 */
+	for (s = 0; s < area_count; s++) {
+		lv_image = seg_lv(seg_top, s);
+
+		lv_image_size = lv_image->size; /* sectors */
+
+		if (!(segtype = get_segtype_from_string(cmd, SEG_TYPE_NAME_INTEGRITY)))
+			return_0;
+
+		log_debug("Adding integrity to raid image %s", lv_image->name);
+
+		/*
+		 * "lv_iorig" is a new LV with new id, but with the segments
+		 * from "lv_image". "lv_image" keeps the existing name and id,
+		 * but gets a new integrity segment, in place of the segments
+		 * that were moved to lv_iorig.
+		 */
+		if (!(lv_iorig = insert_layer_for_lv(cmd, lv_image, INTEGRITY, "_iorig")))
+			return_0;
+
+		lv_image->status |= INTEGRITY;
+
+		/*
+		 * Set up the new first segment of lv_image as integrity.
+		 */
+		seg_image = first_seg(lv_image);
+		seg_image->segtype = segtype;
+
+		if (internal_metadata) {
+			/* dm-integrity wants temp/fake size of 1 to report usable size */
+			lv_image->size = 1;
+			seg_image->integrity_data_sectors = 1;
+		} else {
+			lv_imeta = imeta_lvs[s]; /* external metadata lv created above */
+			lv_imeta->status |= INTEGRITY_METADATA;
+			lv_set_hidden(lv_imeta);
+			seg_image->integrity_data_sectors = lv_image_size;
+			seg_image->integrity_meta_dev = lv_imeta;
+		}
+
+		memcpy(&seg_image->integrity_settings, settings, sizeof(struct integrity_settings));
+		set = &seg_image->integrity_settings;
+
+		if (!set->mode[0])
+			set->mode[0] = DEFAULT_MODE;
+
+		if (!set->tag_size)
+			set->tag_size = DEFAULT_TAG_SIZE;
+
+		if (!set->internal_hash)
+	       		set->internal_hash = DEFAULT_INTERNAL_HASH;
+
+		set->recalculate = 1; /* enable integrity initialization in the kernel */
+	}
+
+	if (internal_metadata) {
+		/* Get the size from the first image, the others will be the same. */
+		lv_image = seg_lv(seg_top, 0);
+
+		lv_image->status |= LV_TEMPORARY;
+		lv_image->status |= LV_NOSCAN;
+		lv_image->status |= LV_UNCOMMITTED;
+
+		log_debug("Activating temporary integrity LV to get data sectors.");
+
+		if (!activate_lv(cmd, lv_image)) {
+			log_error("Failed to activate temporary integrity.");
+			ret = 0;
+			goto out;
+		}
+
+		if (!_get_provided_data_sectors(lv_image, &status_data_sectors)) {
+			log_error("Failed to get data sectors from dm-integrity");
+			ret = 0;
+		} else {
+			log_print("Found integrity provided_data_sectors %llu", (unsigned long long)status_data_sectors);
+			ret = 1;
+		}
+
+		if (!status_data_sectors) {
+			log_error("LV size too small to include metadata.");
+			ret = 0;
+		}
+
+		lv_image->status |= LV_UNCOMMITTED;
+
+		if (!deactivate_lv(cmd, lv_image)) {
+			log_error("Failed to deactivate temporary integrity.");
+			ret = 0;
+		}
+
+		if (!ret)
+			goto_out;
+
+		lv_image->status &= ~LV_UNCOMMITTED;
+		lv_image->status &= ~LV_NOSCAN;
+		lv_image->status &= ~LV_TEMPORARY;
+
+		for (s = 0; s < area_count; s++) {
+			lv_image = seg_lv(seg_top, s);
+			lv_image->size = lv_image_size; /* was 1 temporarily */
+			seg_image = first_seg(lv_image);
+			seg_image->integrity_data_sectors = status_data_sectors;
+		}
+	}
+
+	/*
+	 * recalculate above enables automatic integrity metadata intialization
+	 * in the kernel when first activated, instead of activating the raid
+	 * LV and writing zeroes to the entire thing from the lvcreate command
+	 * to initialize the metadata in the integrity images.
+	 *
+	 * Zeroing the raid LV to intialize integrity metadata on the images
+	 * does not work because dm-raid immediately reads the integrity
+	 * images when activated, and gets errors back because of no init.
+	 *
+	 * When activated with recalculate, dm-integrity will not return errors
+	 * to dm-raid when it reads uninitialized blocks.
+	 */
+	*zero_data_sectors = 0; /* disables intialization by lvcreate */
+
+	log_debug("Write VG with integrity added to LV");
+
+	if (!vg_write(vg) || !vg_commit(vg))
+		ret = 0;
+out:
+	return ret;
+}
+
 int lv_add_integrity(struct logical_volume *lv, const char *arg,
 		     struct logical_volume *meta_lv_created,
 		     const char *meta_name,
-		     struct integrity_settings *settings)
+		     struct integrity_settings *settings,
+		     struct dm_list *pvh,
+		     uint64_t *zero_data_sectors)
 {
 	struct cmd_context *cmd = lv->vg->cmd;
 	struct volume_group *vg = lv->vg;
@@ -217,7 +484,10 @@ int lv_add_integrity(struct logical_volume *lv, const char *arg,
 	struct lv_segment *seg;
 	uint64_t meta_bytes, meta_sectors;
 	uint64_t lv_size_sectors;
-	int ret;
+	int ret = 1;
+
+	if (lv_is_raid(lv))
+		return _lv_add_integrity_to_raid(lv, arg, settings, pvh, zero_data_sectors);
 
 	lv_size_sectors = lv->size;
 
@@ -335,7 +605,6 @@ int lv_add_integrity(struct logical_volume *lv, const char *arg,
 		seg->integrity_meta_dev = meta_lv;
 		lv_set_hidden(meta_lv);
 		/* TODO: give meta_lv a suffix? e.g. _imeta */
-		ret = 1;
 	} else {
 		/* dm-integrity wants temp/fake size of 1 to report usable size */
 		seg->integrity_data_sectors = 1;
@@ -377,6 +646,11 @@ int lv_add_integrity(struct logical_volume *lv, const char *arg,
 		lv->status &= ~LV_TEMPORARY;
 	}
 
+	/*
+	 * Tells lvcreate to zero the final LV at the end of commands.
+	 */
+	*zero_data_sectors = seg->integrity_data_sectors;
+
 	log_debug("Write VG with integrity added to LV");
 
 	if (!vg_write(vg) || !vg_commit(vg))
diff --git a/lib/metadata/lv_manip.c b/lib/metadata/lv_manip.c
index 7ad5b1ff6..d15f4774e 100644
--- a/lib/metadata/lv_manip.c
+++ b/lib/metadata/lv_manip.c
@@ -7473,11 +7473,12 @@ int insert_layer_for_segments_on_pv(struct cmd_context *cmd,
 	return 1;
 }
 
-int zero_lv_name(struct cmd_context *cmd, const char *vg_name, const char *lv_name, uint64_t lv_size_bytes)
+int zero_lv_name(struct cmd_context *cmd, const char *vg_name, const char *lv_name, uint64_t lv_size_sectors)
 {
 	char name[PATH_MAX];
 	struct device *dev;
 	uint64_t off = 0, i = 0, j = 0;
+	uint64_t lv_size_bytes = lv_size_sectors * 512;
 	uint64_t zero_bytes;
 	uint32_t extra_bytes;
 
@@ -7831,7 +7832,7 @@ static int _should_wipe_lv(struct lvcreate_params *lp,
 	 * TODO: print a warning or error if the user specifically
 	 * asks for no wiping or zeroing?
 	 */
-	if (seg_is_integrity(lp))
+	if (seg_is_integrity(lp) || (seg_is_raid1(lp) && lp->integrity_arg))
 		return 1;
 
 	/* Cannot zero read-only volume */
@@ -8438,7 +8439,7 @@ static struct logical_volume *_lv_create_an_lv(struct volume_group *vg,
 		}
 		lv->status &= ~LV_TEMPORARY;
 
-	} else if (seg_is_integrity(lp)) {
+	} else if (seg_is_integrity(lp) || (seg_is_raid1(lp) && lp->integrity_arg)) {
 		/*
 		 * Activate the new origin LV so it can be zeroed/wiped
 		 * below before adding integrity.
@@ -8469,7 +8470,7 @@ static struct logical_volume *_lv_create_an_lv(struct volume_group *vg,
 		}
 	}
 
-	if (seg_is_integrity(lp)) {
+	if (seg_is_integrity(lp) || (seg_is_raid1(lp) && lp->integrity_arg)) {
 		log_verbose("Adding integrity to new LV");
 
 		/* Origin is active from zeroing, deactivate to add integrity. */
@@ -8480,7 +8481,8 @@ static struct logical_volume *_lv_create_an_lv(struct volume_group *vg,
 		}
 
 		if (!lv_add_integrity(lv, lp->integrity_arg, lp->integrity_meta_lv,
-				      lp->integrity_meta_name, &lp->integrity_settings))
+				      lp->integrity_meta_name, &lp->integrity_settings, lp->pvh,
+				      &lp->zero_data_sectors))
 			goto revert_new_lv;
 
 		backup(vg);
@@ -8512,7 +8514,6 @@ static struct logical_volume *_lv_create_an_lv(struct volume_group *vg,
 		 * problems (if the user doesn't want to wait, or wants
 		 * to do the zeroing themselves.)
 		 */
-		lp->integrity_bytes_to_zero = first_seg(lv)->integrity_data_sectors * 512;
 		goto out;
 	}
 
diff --git a/lib/metadata/metadata-exported.h b/lib/metadata/metadata-exported.h
index 7d488e705..117163f29 100644
--- a/lib/metadata/metadata-exported.h
+++ b/lib/metadata/metadata-exported.h
@@ -1007,7 +1007,7 @@ struct lvcreate_params {
 	const char *integrity_meta_name; /* external LV is user-specified */
 	struct logical_volume *integrity_meta_lv; /* external LV we create */
 	struct integrity_settings integrity_settings;
-	uint64_t integrity_bytes_to_zero; /* zeros the final LV after it's created */
+	uint64_t zero_data_sectors; /* the resulting size that should be zeroed at the end */
 
 	struct dm_list tags;	/* all */
 
@@ -1406,12 +1406,14 @@ void vg_write_commit_bad_mdas(struct cmd_context *cmd, struct volume_group *vg);
 
 int lv_add_integrity(struct logical_volume *lv, const char *arg,
 		     struct logical_volume *meta_lv_created,
-		     const char *meta_name, struct integrity_settings *settings);
+		     const char *meta_name, struct integrity_settings *settings,
+		     struct dm_list *pvh,
+		     uint64_t *zero_data_sectors);
 int lv_create_integrity_metadata(struct cmd_context *cmd,
 			struct volume_group *vg,
 			struct lvcreate_params *lp);
 int lv_remove_integrity(struct logical_volume *lv);
 
-int zero_lv_name(struct cmd_context *cmd, const char *vg_name, const char *lv_name, uint64_t zero_bytes);
+int zero_lv_name(struct cmd_context *cmd, const char *vg_name, const char *lv_name, uint64_t lv_size_sectors);
 
 #endif
diff --git a/lib/raid/raid.c b/lib/raid/raid.c
index e88a15408..c315a2cee 100644
--- a/lib/raid/raid.c
+++ b/lib/raid/raid.c
@@ -240,6 +240,26 @@ static int _raid_text_export(const struct lv_segment *seg, struct formatter *f)
 	return _raid_text_export_raid(seg, f);
 }
 
+static int _image_integrity_data_sectors(struct lv_segment *seg, uint64_t *data_sectors)
+{
+	struct logical_volume *lv_image;
+	struct lv_segment *seg_image;
+	int s;
+
+	*data_sectors = 0;
+
+	for (s = 0; s < seg->area_count; s++) {
+		 lv_image = seg_lv(seg, s);
+
+		 if (lv_is_integrity(lv_image)) {
+			 seg_image = first_seg(lv_image);
+			 *data_sectors = seg_image->integrity_data_sectors;
+			 return 1;
+		 }
+	}
+	return 1;
+}
+
 static int _raid_add_target_line(struct dev_manager *dm __attribute__((unused)),
 				 struct dm_pool *mem __attribute__((unused)),
 				 struct cmd_context *cmd __attribute__((unused)),
@@ -256,6 +276,7 @@ static int _raid_add_target_line(struct dev_manager *dm __attribute__((unused)),
 	uint64_t writemostly[RAID_BITMAP_SIZE] = { 0 };
 	struct dm_tree_node_raid_params_v2 params = { 0 };
 	unsigned attrs;
+	uint64_t integrity_data_sectors = 0;
 
 	if (seg_is_raid4(seg)) {
 		if (!_raid_target_present(cmd, NULL, &attrs) ||
@@ -351,6 +372,15 @@ static int _raid_add_target_line(struct dev_manager *dm __attribute__((unused)),
 	params.stripe_size = seg->stripe_size;
 	params.flags = flags;
 
+	if (!_image_integrity_data_sectors(seg, &integrity_data_sectors))
+		return_0;
+
+	if (integrity_data_sectors) {
+		log_debug("Reducing raid size from %llu to integrity_data_sectors %llu",
+			  (unsigned long long)len, (unsigned long long)integrity_data_sectors);
+		len = integrity_data_sectors;
+	}
+
 	if (!dm_tree_node_add_raid_target_with_params_v2(node, len, &params))
 		return_0;
 
diff --git a/tools/command-lines.in b/tools/command-lines.in
index 8ed4b5e78..032621029 100644
--- a/tools/command-lines.in
+++ b/tools/command-lines.in
@@ -878,7 +878,8 @@ DESC: Create a raid1 or mirror LV (infers --type raid1|mirror).
 # R9,R10,R11,R12 (--type raid with any use of --stripes/--mirrors)
 lvcreate --type raid --size SizeMB VG
 OO: --mirrors PNumber, --stripes Number, --stripesize SizeKB,
---regionsize RegionSize, --minrecoveryrate SizeKB, --maxrecoveryrate SizeKB, OO_LVCREATE
+--regionsize RegionSize, --minrecoveryrate SizeKB, --maxrecoveryrate SizeKB,
+--integrity String, --integritysettings String, OO_LVCREATE
 OP: PV ...
 ID: lvcreate_raid_any
 DESC: Create a raid LV (a specific raid level must be used, e.g. raid1).
diff --git a/tools/lvcreate.c b/tools/lvcreate.c
index d19047e8c..e52d751f6 100644
--- a/tools/lvcreate.c
+++ b/tools/lvcreate.c
@@ -848,6 +848,8 @@ static int _lvcreate_params(struct cmd_context *cmd,
 	poolmetadataspare_ARG
 
 #define RAID_ARGS \
+	integrity_ARG,\
+	integritysettings_ARG,\
 	maxrecoveryrate_ARG,\
 	minrecoveryrate_ARG,\
 	raidmaxrecoveryrate_ARG,\
@@ -1220,7 +1222,7 @@ static int _lvcreate_params(struct cmd_context *cmd,
 		}
 	}
 
-	if (seg_is_integrity(lp)) {
+	if (seg_is_integrity(lp) || seg_is_raid(lp)) {
 		if (!get_integrity_options(cmd, &lp->integrity_arg, &lp->integrity_meta_name, &lp->integrity_settings))
 			return 0;
 	}
@@ -1808,11 +1810,15 @@ int lvcreate(struct cmd_context *cmd, int argc, char **argv)
 	_destroy_lvcreate_params(&lp);
 	destroy_processing_handle(cmd, handle);
 
-	if (lp.integrity_bytes_to_zero) {
+	/*
+	 * When set, the final LV should be zeroed (set by add_integrity
+	 * to intialize integrity metadata/checksums).
+	 */
+	if (lp.zero_data_sectors) {
 		if (!lp.zero)
 			log_warn("WARNING: not zeroing integrity LV, read errors are possible.");
 		else
-			zero_lv_name(cmd, lp.vg_name, lp.lv_name, lp.integrity_bytes_to_zero);
+			zero_lv_name(cmd, lp.vg_name, lp.lv_name, lp.zero_data_sectors);
 	}
 
 	return ret;