From d2edf6646d670597b247ec3cf57b824a8469fa00 Mon Sep 17 00:00:00 2001 From: Alistair Coles Date: Mon, 8 Aug 2022 14:54:29 +0100 Subject: swift-manage-shard-ranges: add 'merge' subcommand Adds a subcommand to swift-manage-shard-ranges that allows arbitrary shard ranges to be read from a file and merged into a container DB. The file should be a JSON serialized list of dicts. The merge subcommand is only recommended for emergency shard range manipulation by expert users. Change-Id: Ic9ffcc042399f3834027a7935b64292d1fffe8d5 --- swift/cli/manage_shard_ranges.py | 74 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 73 insertions(+), 1 deletion(-) (limited to 'swift/cli') diff --git a/swift/cli/manage_shard_ranges.py b/swift/cli/manage_shard_ranges.py index 9bfd89e54..33e870d45 100644 --- a/swift/cli/manage_shard_ranges.py +++ b/swift/cli/manage_shard_ranges.py @@ -168,7 +168,8 @@ from six.moves import input from swift.common.utils import Timestamp, get_logger, ShardRange, readconf, \ ShardRangeList, non_negative_int, config_positive_int_value -from swift.container.backend import ContainerBroker, UNSHARDED +from swift.container.backend import ContainerBroker, UNSHARDED, \ + sift_shard_ranges from swift.container.sharder import make_shard_ranges, sharding_enabled, \ CleavingContext, process_compactible_shard_sequences, \ find_compactible_shard_sequences, find_overlapping_ranges, \ @@ -427,6 +428,61 @@ def delete_shard_ranges(broker, args): return EXIT_SUCCESS +def combine_shard_ranges(new_shard_ranges, existing_shard_ranges): + """ + Combines new and existing shard ranges based on most recent state. + + :param new_shard_ranges: a list of ShardRange instances. + :param existing_shard_ranges: a list of ShardRange instances. + :return: a list of ShardRange instances. + """ + new_shard_ranges = [dict(sr) for sr in new_shard_ranges] + existing_shard_ranges = [dict(sr) for sr in existing_shard_ranges] + to_add, to_delete = sift_shard_ranges( + new_shard_ranges, + dict((sr['name'], sr) for sr in existing_shard_ranges)) + result = [ShardRange.from_dict(existing) + for existing in existing_shard_ranges + if existing['name'] not in to_delete] + result.extend([ShardRange.from_dict(sr) for sr in to_add]) + return sorted([sr for sr in result if not sr.deleted], + key=ShardRange.sort_key) + + +def merge_shard_ranges(broker, args): + _check_own_shard_range(broker, args) + shard_data = _load_and_validate_shard_data(args, require_index=False) + new_shard_ranges = ShardRangeList([ShardRange.from_dict(sr) + for sr in shard_data]) + new_shard_ranges.sort(key=ShardRange.sort_key) + + # do some checks before merging... + existing_shard_ranges = ShardRangeList( + broker.get_shard_ranges(include_deleted=True)) + outcome = combine_shard_ranges(new_shard_ranges, existing_shard_ranges) + if args.verbose: + print('This change will result in the following shard ranges in the ' + 'affected namespace:') + print(json.dumps([dict(sr) for sr in outcome], indent=2)) + overlaps = find_overlapping_ranges(outcome) + if overlaps: + print('WARNING: this change will result in shard ranges overlaps!') + paths_with_gaps = find_paths_with_gaps(outcome) + gaps = [gap for start_path, gap, end_path in paths_with_gaps + if existing_shard_ranges.includes(gap)] + if gaps: + print('WARNING: this change will result in shard ranges gaps!') + + if not _proceed(args): + return EXIT_USER_QUIT + + with broker.updated_timeout(args.replace_timeout): + broker.merge_shard_ranges(new_shard_ranges) + print('Injected %d shard ranges.' % len(new_shard_ranges)) + print('Run container-replicator to replicate them to other nodes.') + return EXIT_SUCCESS + + def _replace_shard_ranges(broker, args, shard_data, timeout=0): own_shard_range = _check_own_shard_range(broker, args) shard_ranges = make_shard_ranges( @@ -957,6 +1013,22 @@ def _make_parser(): 'info', help='Print container db info') info_parser.set_defaults(func=db_info) + # merge + merge_parser = subparsers.add_parser( + 'merge', + help='Merge shard range(s) from file with existing shard ranges. This ' + 'subcommand should only be used if you are confident that you ' + 'know what you are doing. Shard ranges should not typically be ' + 'modified in this way.') + merge_parser.add_argument('input', metavar='input_file', + type=str, help='Name of file') + merge_parser.add_argument( + '--replace-timeout', type=int, default=600, + help='Minimum DB timeout to use when merging shard ranges.') + _add_account_prefix_arg(merge_parser) + _add_prompt_args(merge_parser) + merge_parser.set_defaults(func=merge_shard_ranges) + # replace replace_parser = subparsers.add_parser( 'replace', -- cgit v1.2.1