Add improvements to MD5 validation.

With MD5Sum checking being added, a concern was brought up that It was
a change with no possibility of reverting to the old behaviour.
This change adds the flag '--ignore-checksum' to the upload subcommand
allowing the checks to be turned off.

Changed occurrences of the magic string for a null md5 to use a descriptive
constant instead.

Updated Error messages generated when validation fails. They should now be more descriptive
and not output a literal newline sequence.

Change-Id: Id1756cbb6700bb7e38f0ee0e75bc535e37f777ed

3 files changed, 34 insertions, 23 deletions

diff --git a/swiftclient/service.py b/swiftclient/service.py
index f24d430..232e9c3 100644
--- a/swiftclient/service.py
+++ b/swiftclient/service.py
@@ -40,7 +40,7 @@ from swiftclient.command_helpers import (
     stat_account, stat_container, stat_object
 )
 from swiftclient.utils import (
-    config_true_value, ReadableToIterable, LengthWrapper
+    config_true_value, ReadableToIterable, LengthWrapper, EMPTY_ETAG
 )
 from swiftclient.exceptions import ClientException
 from swiftclient.multithreading import MultiThreadingManager
@@ -174,7 +174,8 @@ _default_local_options = {
     'delimiter': None,
     'fail_fast': False,
     'human': False,
-    'dir_marker': False
+    'dir_marker': False,
+    'checksum': True
 }
 
 POLICY = 'X-Storage-Policy'
@@ -1410,16 +1411,16 @@ class SwiftService(object):
         res['headers'] = put_headers
         if options['changed']:
             try:
-                _empty_string_etag = 'd41d8cd98f00b204e9800998ecf8427e'
                 headers = conn.head_object(container, obj)
                 ct = headers.get('content-type')
                 cl = int(headers.get('content-length'))
                 et = headers.get('etag')
                 mt = headers.get('x-object-meta-mtime')
-                if ct.split(';', 1)[0] == 'text/directory' and \
-                        cl == 0 and \
-                        et == _empty_string_etag and \
-                        mt == put_headers['x-object-meta-mtime']:
+
+                if (ct.split(';', 1)[0] == 'text/directory' and
+                        cl == 0 and
+                        et == EMPTY_ETAG and
+                        mt == put_headers['x-object-meta-mtime']):
                     res['success'] = True
                     return res
             except ClientException as err:
@@ -1467,17 +1468,19 @@ class SwiftService(object):
             fp = open(path, 'rb')
             fp.seek(segment_start)
 
-            contents = LengthWrapper(fp, segment_size, md5=True)
+            contents = LengthWrapper(fp, segment_size, md5=options['checksum'])
             etag = conn.put_object(segment_container,
                                    segment_name, contents,
                                    content_length=segment_size,
                                    response_dict=results_dict)
 
-            if etag and etag != contents.get_md5sum():
-                raise SwiftError('Segment upload failed: remote and local '
-                                 'object md5 did not match, {0} != {1}\n'
-                                 'remote segment has not been removed.'
-                                 .format(etag, contents.get_md5sum()))
+            if options['checksum'] and etag and etag != contents.get_md5sum():
+                raise SwiftError('Segment {0}: upload verification failed: '
+                                 'md5 mismatch, local {1} != remote {2} '
+                                 '(remote segment has not been removed)'
+                                 .format(segment_index,
+                                         contents.get_md5sum(),
+                                         etag))
 
             res.update({
                 'success': True,
@@ -1707,11 +1710,13 @@ class SwiftService(object):
                 obr = {}
                 if path is not None:
                     content_length = getsize(path)
-                    contents = LengthWrapper(open(path, 'rb'), content_length,
-                                             md5=True)
+                    contents = LengthWrapper(open(path, 'rb'),
+                                             content_length,
+                                             md5=options['checksum'])
                 else:
                     content_length = None
-                    contents = ReadableToIterable(stream, md5=True)
+                    contents = ReadableToIterable(stream,
+                                                  md5=options['checksum'])
 
                 etag = conn.put_object(
                     container, obj, contents,
@@ -1720,11 +1725,12 @@ class SwiftService(object):
                 )
                 res['response_dict'] = obr
 
-                if etag and etag != contents.get_md5sum():
-                    raise SwiftError('Object upload failed: remote and local '
-                                     'object md5 did not match, {0} != {1}\n'
-                                     'remote object has not been removed.'
-                                     .format(etag, contents.get_md5sum()))
+                if (options['checksum'] and
+                        etag and etag != contents.get_md5sum()):
+                    raise SwiftError('Object upload verification failed: '
+                                     'md5 mismatch, local {0} != remote {1} '
+                                     '(remote object has not been removed)'
+                                     .format(contents.get_md5sum(), etag))
 
             if old_manifest or old_slo_manifest_paths:
                 drs = []
diff --git a/swiftclient/shell.py b/swiftclient/shell.py
index 439a92f..3d3ef51 100755
--- a/swiftclient/shell.py
+++ b/swiftclient/shell.py
@@ -637,7 +637,7 @@ def st_post(parser, args, output_manager):
 st_upload_options = '''[--changed] [--skip-identical] [--segment-size <size>]
                     [--segment-container <container>] [--leave-segments]
                     [--object-threads <thread>] [--segment-threads <threads>]
-                    [--header <header>] [--use-slo]
+                    [--header <header>] [--use-slo] [--ignore-checksum]
                     [--object-name <object-name>]
                     <container> <file_or_directory>
 '''
@@ -681,6 +681,7 @@ Optional arguments:
                         Upload file and name object to <object-name> or upload
                         dir and use <object-name> as object prefix instead of
                         folder name.
+  --ignore-checksum     Turn off checksum validation for uploads.
 '''.strip('\n')
 
 
@@ -733,6 +734,9 @@ def st_upload(parser, args, output_manager):
         '', '--object-name', dest='object_name',
         help='Upload file and name object to <object-name> or upload dir and '
         'use <object-name> as object prefix instead of folder name.')
+    parser.add_option(
+        '', '--ignore-checksum', dest='checksum', default=True,
+        action='store_false', help='Turn off checksum validation for uploads.')
     (options, args) = parse_args(parser, args)
     args = args[1:]
     if len(args) < 2:
@@ -848,7 +852,7 @@ def st_upload(parser, args, output_manager):
                         output_manager.error("%s" % error)
 
         except SwiftError as e:
-            output_manager.error("%s" % e)
+            output_manager.error(e.value)
 
 
 st_capabilities_options = "[<proxy_url>]"
diff --git a/swiftclient/utils.py b/swiftclient/utils.py
index f0fcc01..2f4ec3f 100644
--- a/swiftclient/utils.py
+++ b/swiftclient/utils.py
@@ -21,6 +21,7 @@ import time
 import six
 
 TRUE_VALUES = set(('true', '1', 'yes', 'on', 't', 'y'))
+EMPTY_ETAG = 'd41d8cd98f00b204e9800998ecf8427e'
 
 
 def config_true_value(value):