diff options
author | Jenkins <jenkins@review.openstack.org> | 2014-09-22 15:44:14 +0000 |
---|---|---|
committer | Gerrit Code Review <review@openstack.org> | 2014-09-22 15:44:14 +0000 |
commit | 607e2ed2e30ca02d114d04484763d92afb9158ab (patch) | |
tree | 2cca10b249dddba3a6480d8a09730c284742ee9f | |
parent | 0d0c16d4521b6be16a9fed8f5567c4a3c766d5a7 (diff) | |
parent | 4faf1702706b289521329243e5802cf86e54c7f7 (diff) | |
download | swift-607e2ed2e30ca02d114d04484763d92afb9158ab.tar.gz |
Merge "Add "--no-overlap" option to swift-dispersion populate"
-rwxr-xr-x | bin/swift-dispersion-populate | 57 | ||||
-rw-r--r-- | doc/manpages/swift-dispersion-populate.1 | 2 |
2 files changed, 53 insertions, 6 deletions
diff --git a/bin/swift-dispersion-populate b/bin/swift-dispersion-populate index a475e40e4..93ca6ee2c 100755 --- a/bin/swift-dispersion-populate +++ b/bin/swift-dispersion-populate @@ -101,6 +101,10 @@ Usage: %%prog [options] [conf_file] parser.add_option('--insecure', action='store_true', default=False, help='Allow accessing insecure keystone server. ' 'The keystone\'s certificate will not be verified.') + parser.add_option('--no-overlap', action='store_true', default=False, + help='No overlap of partitions if running populate \ + more than once. Will increase coverage by amount shown \ + in dispersion.conf file') options, args = parser.parse_args() if args: @@ -144,6 +148,19 @@ Usage: %%prog [options] [conf_file] container_ring = Ring(swift_dir, ring_name='container') parts_left = dict((x, x) for x in xrange(container_ring.partition_count)) + + if options.no_overlap: + with connpool.item() as conn: + containers = [cont['name'] for cont in conn.get_account( + prefix='dispersion_', full_listing=True)[1]] + containers_listed = len(containers) + if containers_listed > 0: + for container in containers: + partition, _junk = container_ring.get_nodes(account, + container) + if partition in parts_left: + del parts_left[partition] + item_type = 'containers' created = 0 retries_done = 0 @@ -152,9 +169,9 @@ Usage: %%prog [options] [conf_file] begun = next_report = time() next_report += 2 suffix = 0 - while need_to_queue >= 1: + while need_to_queue >= 1 and parts_left: container = 'dispersion_%d' % suffix - part, _junk = container_ring.get_nodes(account, container) + part = container_ring.get_part(account, container) if part in parts_left: if suffix >= options.container_suffix_start: coropool.spawn(put_container, connpool, container, report) @@ -168,7 +185,13 @@ Usage: %%prog [options] [conf_file] elapsed, elapsed_unit = get_time_units(time() - begun) print '\r\x1B[KCreated %d containers for dispersion reporting, ' \ '%d%s, %d retries' % \ - (need_to_create, round(elapsed), elapsed_unit, retries_done) + ((need_to_create - need_to_queue), round(elapsed), elapsed_unit, + retries_done) + if options.no_overlap: + con_coverage = container_ring.partition_count - len(parts_left) + print '\r\x1B[KTotal container coverage is now %.2f%%.' % \ + ((float(con_coverage) / container_ring.partition_count + * 100)) stdout.flush() if object_populate: @@ -176,6 +199,23 @@ Usage: %%prog [options] [conf_file] put_container(connpool, container, None) object_ring = Ring(swift_dir, ring_name='object') parts_left = dict((x, x) for x in xrange(object_ring.partition_count)) + + if options.no_overlap: + with connpool.item() as conn: + obj_container = [cont_b['name'] for cont_b in conn.get_account( + prefix=container, full_listing=True)[1]] + if obj_container: + with connpool.item() as conn: + objects = [o['name'] for o in + conn.get_container(container, + prefix='dispersion_', + full_listing=True)[1]] + for my_object in objects: + partition = object_ring.get_part(account, container, + my_object) + if partition in parts_left: + del parts_left[partition] + item_type = 'objects' created = 0 retries_done = 0 @@ -184,9 +224,9 @@ Usage: %%prog [options] [conf_file] begun = next_report = time() next_report += 2 suffix = 0 - while need_to_queue >= 1: + while need_to_queue >= 1 and parts_left: obj = 'dispersion_%d' % suffix - part, _junk = object_ring.get_nodes(account, container, obj) + part = object_ring.get_part(account, container, obj) if part in parts_left: if suffix >= options.object_suffix_start: coropool.spawn( @@ -201,5 +241,10 @@ Usage: %%prog [options] [conf_file] elapsed, elapsed_unit = get_time_units(time() - begun) print '\r\x1B[KCreated %d objects for dispersion reporting, ' \ '%d%s, %d retries' % \ - (need_to_create, round(elapsed), elapsed_unit, retries_done) + ((need_to_create - need_to_queue), round(elapsed), elapsed_unit, + retries_done) + if options.no_overlap: + obj_coverage = object_ring.partition_count - len(parts_left) + print '\r\x1B[KTotal object coverage is now %.2f%%.' % \ + ((float(obj_coverage) / object_ring.partition_count * 100)) stdout.flush() diff --git a/doc/manpages/swift-dispersion-populate.1 b/doc/manpages/swift-dispersion-populate.1 index 01c5fe329..dd45e4e8e 100644 --- a/doc/manpages/swift-dispersion-populate.1 +++ b/doc/manpages/swift-dispersion-populate.1 @@ -72,6 +72,8 @@ Only run object population Only run container population .IP "\fB--object-only\fR" Only run object population +.IP "\fB--no-overlap\fR" +Increase coverage by amount in dispersion_coverage option with no overlap of existing partitions (if run more than once) .SH CONFIGURATION .PD 0 |