summaryrefslogtreecommitdiff
path: root/swiftclient
diff options
context:
space:
mode:
Diffstat (limited to 'swiftclient')
-rw-r--r--swiftclient/client.py6
-rw-r--r--swiftclient/service.py43
-rwxr-xr-xswiftclient/shell.py68
-rw-r--r--swiftclient/utils.py73
4 files changed, 141 insertions, 49 deletions
diff --git a/swiftclient/client.py b/swiftclient/client.py
index 80b6eda..80bc4a3 100644
--- a/swiftclient/client.py
+++ b/swiftclient/client.py
@@ -1286,8 +1286,10 @@ def put_object(url, token=None, container=None, name=None, contents=None,
if content_type is not None:
headers['Content-Type'] = content_type
elif 'Content-Type' not in headers:
- # python-requests sets application/x-www-form-urlencoded otherwise
- headers['Content-Type'] = ''
+ if StrictVersion(requests.__version__) < StrictVersion('2.4.0'):
+ # python-requests sets application/x-www-form-urlencoded otherwise
+ # if using python3.
+ headers['Content-Type'] = ''
if not contents:
headers['Content-Length'] = '0'
diff --git a/swiftclient/service.py b/swiftclient/service.py
index c842d23..65e2436 100644
--- a/swiftclient/service.py
+++ b/swiftclient/service.py
@@ -43,7 +43,8 @@ from swiftclient.command_helpers import (
)
from swiftclient.utils import (
config_true_value, ReadableToIterable, LengthWrapper, EMPTY_ETAG,
- parse_api_response, report_traceback, n_groups, split_request_headers
+ parse_api_response, report_traceback, n_groups, split_request_headers,
+ n_at_a_time
)
from swiftclient.exceptions import ClientException
from swiftclient.multithreading import MultiThreadingManager
@@ -1996,7 +1997,6 @@ class SwiftService(object):
} for d in segment_results
])
- put_headers['x-static-large-object'] = 'true'
mr = {}
conn.put_object(
container, obj, manifest_data,
@@ -2164,11 +2164,15 @@ class SwiftService(object):
rq = Queue()
obj_dels = {}
- if self._should_bulk_delete(objects):
- for obj_slice in n_groups(
- objects, self._options['object_dd_threads']):
- self._bulk_delete(container, obj_slice, options,
- obj_dels)
+ bulk_page_size = self._bulk_delete_page_size(objects)
+ if bulk_page_size > 1:
+ page_at_a_time = n_at_a_time(objects, bulk_page_size)
+ for page_slice in page_at_a_time:
+ for obj_slice in n_groups(
+ page_slice,
+ self._options['object_dd_threads']):
+ self._bulk_delete(container, obj_slice, options,
+ obj_dels)
else:
self._per_item_delete(container, objects, options,
obj_dels, rq)
@@ -2221,23 +2225,36 @@ class SwiftService(object):
and not res['success']):
cancelled = True
- def _should_bulk_delete(self, objects):
- if len(objects) < 2 * self._options['object_dd_threads']:
+ def _bulk_delete_page_size(self, objects):
+ '''
+ Given the iterable 'objects', will return how many items should be
+ deleted at a time.
+
+ :param objects: An iterable that supports 'len()'
+ :returns: The bulk delete page size (i.e. the max number of
+ objects that can be bulk deleted at once, as reported by
+ the cluster). If bulk delete is disabled, return 1
+ '''
+ if len(objects) <= 2 * self._options['object_dd_threads']:
# Not many objects; may as well delete one-by-one
- return False
+ return 1
try:
cap_result = self.capabilities()
if not cap_result['success']:
# This shouldn't actually happen, but just in case we start
# being more nuanced about our capabilities result...
- return False
+ return 1
except ClientException:
# Old swift, presumably; assume no bulk middleware
- return False
+ return 1
swift_info = cap_result['capabilities']
- return 'bulk_delete' in swift_info
+ if 'bulk_delete' in swift_info:
+ return swift_info['bulk_delete'].get(
+ 'max_deletes_per_request', 10000)
+ else:
+ return 1
def _per_item_delete(self, container, objects, options, rdict, rq):
for obj in objects:
diff --git a/swiftclient/shell.py b/swiftclient/shell.py
index 4a8a356..841ed6e 100755
--- a/swiftclient/shell.py
+++ b/swiftclient/shell.py
@@ -1224,8 +1224,8 @@ def st_auth(parser, args, thread_manager):
print('export OS_AUTH_TOKEN=%s' % sh_quote(token))
-st_tempurl_options = '''[--absolute] [--prefix-based]
- <method> <seconds> <path> <key>'''
+st_tempurl_options = '''[--absolute] [--prefix-based] [--iso8601]
+ <method> <time> <path> <key>'''
st_tempurl_help = '''
@@ -1234,9 +1234,35 @@ Generates a temporary URL for a Swift object.
Positional arguments:
<method> An HTTP method to allow for this temporary URL.
Usually 'GET' or 'PUT'.
- <seconds> The amount of time in seconds the temporary URL will be
- valid for; or, if --absolute is passed, the Unix
- timestamp when the temporary URL will expire.
+ <time> The amount of time the temporary URL will be
+ valid. The time can be specified in two ways:
+ an integer representing the time in seconds or an
+ ISO 8601 timestamp in a specific format.
+ If --absolute is passed and time
+ is an integer, the seconds are intepreted as the Unix
+ timestamp when the temporary URL will expire. The ISO
+ 8601 timestamp can be specified in one of following
+ formats:
+
+ i) Complete date: YYYY-MM-DD (eg 1997-07-16)
+
+ ii) Complete date plus hours, minutes and seconds:
+
+ YYYY-MM-DDThh:mm:ss
+
+ (eg 1997-07-16T19:20:30)
+
+ iii) Complete date plus hours, minutes and seconds with
+ UTC designator:
+
+ YYYY-MM-DDThh:mm:ssZ
+
+ (eg 1997-07-16T19:20:30Z)
+
+ Please be aware that if you don't provide the UTC
+ designator (i.e., Z) the timestamp is generated using
+ your local timezone. If only a date is specified,
+ the time part used will equal to 00:00:00.
<path> The full path or storage URL to the Swift object.
Example: /v1/AUTH_account/c/o
or: http://saio:8080/v1/AUTH_account/c/o
@@ -1245,10 +1271,14 @@ Positional arguments:
"Temp-URL-Key:b3968d0207b54ece87cccc06515a89d4"\'
Optional arguments:
- --absolute Interpret the <seconds> positional argument as a Unix
+ --absolute Interpret the <time> positional argument as a Unix
timestamp rather than a number of seconds in the
- future.
- --prefix-based If present, a prefix-based tempURL will be generated.
+ future. If an ISO 8601 timestamp is passed for <time>,
+ this argument is ignored.
+ --prefix-based If present, a prefix-based temporary URL will be
+ generated.
+ --iso8601 If present, the generated temporary URL will contain an
+ ISO 8601 UTC timestamp instead of a Unix timestamp.
'''.strip('\n')
@@ -1256,14 +1286,21 @@ def st_tempurl(parser, args, thread_manager):
parser.add_argument(
'--absolute', action='store_true',
dest='absolute_expiry', default=False,
- help=("If present, seconds argument will be interpreted as a Unix "
- "timestamp representing when the tempURL should expire, rather "
- "than an offset from the current time"),
+ help=("If present, and time argument is an integer, "
+ "time argument will be interpreted as a Unix "
+ "timestamp representing when the temporary URL should expire, "
+ "rather than an offset from the current time."),
)
parser.add_argument(
'--prefix-based', action='store_true',
default=False,
- help=("If present, a prefix-based tempURL will be generated."),
+ help=("If present, a prefix-based temporary URL will be generated."),
+ )
+ parser.add_argument(
+ '--iso8601', action='store_true',
+ default=False,
+ help=("If present, the temporary URL will contain an ISO 8601 UTC "
+ "timestamp instead of a Unix timestamp."),
)
(options, args) = parse_args(parser, args)
@@ -1272,7 +1309,7 @@ def st_tempurl(parser, args, thread_manager):
thread_manager.error('Usage: %s tempurl %s\n%s', BASENAME,
st_tempurl_options, st_tempurl_help)
return
- method, seconds, path, key = args[:4]
+ method, timestamp, path, key = args[:4]
parsed = urlparse(path)
@@ -1281,9 +1318,10 @@ def st_tempurl(parser, args, thread_manager):
'tempurl specified, possibly an error' %
method.upper())
try:
- path = generate_temp_url(parsed.path, seconds, key, method,
+ path = generate_temp_url(parsed.path, timestamp, key, method,
absolute=options['absolute_expiry'],
- prefix=options['prefix_based'],)
+ iso8601=options['iso8601'],
+ prefix=options['prefix_based'])
except ValueError as err:
thread_manager.error(err)
return
diff --git a/swiftclient/utils.py b/swiftclient/utils.py
index 08cd7b2..8afcde9 100644
--- a/swiftclient/utils.py
+++ b/swiftclient/utils.py
@@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Miscellaneous utility functions for use with Swift."""
+from calendar import timegm
import collections
import gzip
import hashlib
@@ -25,6 +26,10 @@ import traceback
TRUE_VALUES = set(('true', '1', 'yes', 'on', 't', 'y'))
EMPTY_ETAG = 'd41d8cd98f00b204e9800998ecf8427e'
+EXPIRES_ISO8601_FORMAT = '%Y-%m-%dT%H:%M:%SZ'
+SHORT_EXPIRES_ISO8601_FORMAT = '%Y-%m-%d'
+TIME_ERRMSG = ('time must either be a whole number or in specific '
+ 'ISO 8601 format.')
def config_true_value(value):
@@ -64,39 +69,65 @@ def prt_bytes(num_bytes, human_flag):
def generate_temp_url(path, seconds, key, method, absolute=False,
- prefix=False):
+ prefix=False, iso8601=False):
"""Generates a temporary URL that gives unauthenticated access to the
Swift object.
:param path: The full path to the Swift object or prefix if
a prefix-based temporary URL should be generated. Example:
/v1/AUTH_account/c/o or /v1/AUTH_account/c/prefix.
- :param seconds: If absolute is False then this specifies the amount of time
- in seconds for which the temporary URL will be valid. If absolute is
- True then this specifies an absolute time at which the temporary URL
- will expire.
+ :param seconds: time in seconds or ISO 8601 timestamp.
+ If absolute is False and this is the string representation of an
+ integer, then this specifies the amount of time in seconds for which
+ the temporary URL will be valid.
+ If absolute is True then this specifies an absolute time at which the
+ temporary URL will expire.
:param key: The secret temporary URL key set on the Swift
cluster. To set a key, run 'swift post -m
"Temp-URL-Key: <substitute tempurl key here>"'
:param method: A HTTP method, typically either GET or PUT, to allow
for this temporary URL.
- :param absolute: if True then the seconds parameter is interpreted as an
- absolute Unix time, otherwise seconds is interpreted as a relative time
- offset from current time.
+ :param absolute: if True then the seconds parameter is interpreted as a
+ Unix timestamp, if seconds represents an integer.
:param prefix: if True then a prefix-based temporary URL will be generated.
- :raises ValueError: if seconds is not a whole number or path is not to
- an object.
+ :param iso8601: if True, a URL containing an ISO 8601 UTC timestamp
+ instead of a UNIX timestamp will be created.
+ :raises ValueError: if timestamp or path is not in valid format.
:return: the path portion of a temporary URL
"""
try:
- seconds = float(seconds)
- if not seconds.is_integer():
- raise ValueError()
- seconds = int(seconds)
- if seconds < 0:
- raise ValueError()
+ try:
+ timestamp = float(seconds)
+ except ValueError:
+ formats = (
+ EXPIRES_ISO8601_FORMAT,
+ EXPIRES_ISO8601_FORMAT[:-1],
+ SHORT_EXPIRES_ISO8601_FORMAT)
+ for f in formats:
+ try:
+ t = time.strptime(seconds, f)
+ except ValueError:
+ t = None
+ else:
+ if f == EXPIRES_ISO8601_FORMAT:
+ timestamp = timegm(t)
+ else:
+ # Use local time if UTC designator is missing.
+ timestamp = int(time.mktime(t))
+
+ absolute = True
+ break
+
+ if t is None:
+ raise ValueError()
+ else:
+ if not timestamp.is_integer():
+ raise ValueError()
+ timestamp = int(timestamp)
+ if timestamp < 0:
+ raise ValueError()
except ValueError:
- raise ValueError('seconds must be a whole number')
+ raise ValueError(TIME_ERRMSG)
if isinstance(path, six.binary_type):
try:
@@ -121,9 +152,9 @@ def generate_temp_url(path, seconds, key, method, absolute=False,
'possibly an error', method.upper())
if not absolute:
- expiration = int(time.time() + seconds)
+ expiration = int(time.time() + timestamp)
else:
- expiration = seconds
+ expiration = timestamp
hmac_body = u'\n'.join([method.upper(), str(expiration),
('prefix:' if prefix else '') + path_for_body])
@@ -132,6 +163,10 @@ def generate_temp_url(path, seconds, key, method, absolute=False,
key = key.encode('utf-8')
sig = hmac.new(key, hmac_body.encode('utf-8'), hashlib.sha1).hexdigest()
+ if iso8601:
+ expiration = time.strftime(
+ EXPIRES_ISO8601_FORMAT, time.gmtime(expiration))
+
temp_url = u'{path}?temp_url_sig={sig}&temp_url_expires={exp}'.format(
path=path_for_body, sig=sig, exp=expiration)
if prefix:
View Lorry Controller Status