summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorTim Burke <tim.burke@gmail.com>2023-04-05 20:11:38 -0700
committerTim Burke <tim.burke@gmail.com>2023-04-12 13:17:10 -0700
commitc78a5962b5f6c9e75f154cac924a226815236e98 (patch)
tree8dcc6df39642dfe7542e74022575ffadd906a314
parent984cca9263c1589e1f4858ee1b0ba5d2ac1bf702 (diff)
downloadswift-c78a5962b5f6c9e75f154cac924a226815236e98.tar.gz
Pull libc-related functions out to a separate module
Partial-Bug: #2015274 Change-Id: I3e26f8d4e5de0835212ebc2314cac713950c85d7
-rw-r--r--swift/common/utils/__init__.py474
-rw-r--r--swift/common/utils/libc.py487
-rw-r--r--test/unit/common/test_utils.py560
-rw-r--r--test/unit/common/utils/test_libc.py599
-rw-r--r--test/unit/obj/test_diskfile.py3
5 files changed, 1099 insertions, 1024 deletions
diff --git a/swift/common/utils/__init__.py b/swift/common/utils/__init__.py
index 16e37b972..16dc58807 100644
--- a/swift/common/utils/__init__.py
+++ b/swift/common/utils/__init__.py
@@ -36,12 +36,9 @@ import sys
import time
import uuid
import functools
-import platform
import email.parser
from random import random, shuffle
from contextlib import contextmanager, closing
-import ctypes
-import ctypes.util
from optparse import OptionParser
import traceback
import warnings
@@ -96,6 +93,17 @@ from swift.common.linkat import linkat
# For backwards compatability with 3rd party middlewares
from swift.common.registry import register_swift_info, get_swift_info # noqa
+from swift.common.utils.libc import ( # noqa
+ F_SETPIPE_SZ,
+ load_libc_function,
+ config_fallocate_value,
+ disable_fallocate,
+ fallocate,
+ punch_hole,
+ drop_buffer_cache,
+ get_md5_socket,
+ modify_priority,
+)
from swift.common.utils.timestamp import ( # noqa
NORMAL_FORMAT,
INTERNAL_FORMAT,
@@ -122,79 +130,6 @@ NOTICE = 25
logging.addLevelName(NOTICE, 'NOTICE')
SysLogHandler.priority_map['NOTICE'] = 'notice'
-# These are lazily pulled from libc elsewhere
-_sys_fallocate = None
-_posix_fadvise = None
-_libc_socket = None
-_libc_bind = None
-_libc_accept = None
-# see man -s 2 setpriority
-_libc_setpriority = None
-# see man -s 2 syscall
-_posix_syscall = None
-
-# If set to non-zero, fallocate routines will fail based on free space
-# available being at or below this amount, in bytes.
-FALLOCATE_RESERVE = 0
-# Indicates if FALLOCATE_RESERVE is the percentage of free space (True) or
-# the number of bytes (False).
-FALLOCATE_IS_PERCENT = False
-
-# from /usr/include/linux/falloc.h
-FALLOC_FL_KEEP_SIZE = 1
-FALLOC_FL_PUNCH_HOLE = 2
-
-# from /usr/src/linux-headers-*/include/uapi/linux/resource.h
-PRIO_PROCESS = 0
-
-
-# /usr/include/x86_64-linux-gnu/asm/unistd_64.h defines syscalls there
-# are many like it, but this one is mine, see man -s 2 ioprio_set
-def NR_ioprio_set():
- """Give __NR_ioprio_set value for your system."""
- architecture = os.uname()[4]
- arch_bits = platform.architecture()[0]
- # check if supported system, now support x86_64 and AArch64
- if architecture == 'x86_64' and arch_bits == '64bit':
- return 251
- elif architecture == 'aarch64' and arch_bits == '64bit':
- return 30
- raise OSError("Swift doesn't support ionice priority for %s %s" %
- (architecture, arch_bits))
-
-
-# this syscall integer probably only works on x86_64 linux systems, you
-# can check if it's correct on yours with something like this:
-"""
-#include <stdio.h>
-#include <sys/syscall.h>
-
-int main(int argc, const char* argv[]) {
- printf("%d\n", __NR_ioprio_set);
- return 0;
-}
-"""
-
-# this is the value for "which" that says our who value will be a pid
-# pulled out of /usr/src/linux-headers-*/include/linux/ioprio.h
-IOPRIO_WHO_PROCESS = 1
-
-
-IO_CLASS_ENUM = {
- 'IOPRIO_CLASS_RT': 1,
- 'IOPRIO_CLASS_BE': 2,
- 'IOPRIO_CLASS_IDLE': 3,
-}
-
-# the IOPRIO_PRIO_VALUE "macro" is also pulled from
-# /usr/src/linux-headers-*/include/linux/ioprio.h
-IOPRIO_CLASS_SHIFT = 13
-
-
-def IOPRIO_PRIO_VALUE(class_, data):
- return (((class_) << IOPRIO_CLASS_SHIFT) | data)
-
-
# Used by hash_path to offer a bit more security when generating hashes for
# paths. It simply appends this value to all paths; guessing the hash a path
# will end up with would also require knowing this suffix.
@@ -203,12 +138,6 @@ HASH_PATH_PREFIX = b''
SWIFT_CONF_FILE = '/etc/swift/swift.conf'
-# These constants are Linux-specific, and Python doesn't seem to know
-# about them. We ask anyway just in case that ever gets fixed.
-#
-# The values were copied from the Linux 3.x kernel headers.
-AF_ALG = getattr(socket, 'AF_ALG', 38)
-F_SETPIPE_SZ = getattr(fcntl, 'F_SETPIPE_SZ', 1031)
O_TMPFILE = getattr(os, 'O_TMPFILE', 0o20000000 | os.O_DIRECTORY)
# Used by the parse_socket_string() function to validate IPv6 addresses
@@ -528,10 +457,6 @@ def eventlet_monkey_patch():
logging.logThreads = 0
-def noop_libc_function(*args):
- return 0
-
-
def validate_configuration():
try:
validate_hash_conf()
@@ -539,39 +464,6 @@ def validate_configuration():
sys.exit("Error: %s" % e)
-def load_libc_function(func_name, log_error=True,
- fail_if_missing=False, errcheck=False):
- """
- Attempt to find the function in libc, otherwise return a no-op func.
-
- :param func_name: name of the function to pull from libc.
- :param log_error: log an error when a function can't be found
- :param fail_if_missing: raise an exception when a function can't be found.
- Default behavior is to return a no-op function.
- :param errcheck: boolean, if true install a wrapper on the function
- to check for a return values of -1 and call
- ctype.get_errno and raise an OSError
- """
- try:
- libc = ctypes.CDLL(ctypes.util.find_library('c'), use_errno=True)
- func = getattr(libc, func_name)
- except AttributeError:
- if fail_if_missing:
- raise
- if log_error:
- logging.warning(_("Unable to locate %s in libc. Leaving as a "
- "no-op."), func_name)
- return noop_libc_function
- if errcheck:
- def _errcheck(result, f, args):
- if result == -1:
- errcode = ctypes.get_errno()
- raise OSError(errcode, os.strerror(errcode))
- return result
- func.errcheck = _errcheck
- return func
-
-
def generate_trans_id(trans_id_suffix):
return 'tx%s-%010x%s' % (
uuid.uuid4().hex[:21], int(time.time()), quote(trans_id_suffix))
@@ -768,25 +660,6 @@ def get_trans_id_time(trans_id):
return None
-def config_fallocate_value(reserve_value):
- """
- Returns fallocate reserve_value as an int or float.
- Returns is_percent as a boolean.
- Returns a ValueError on invalid fallocate value.
- """
- try:
- if str(reserve_value[-1:]) == '%':
- reserve_value = float(reserve_value[:-1])
- is_percent = True
- else:
- reserve_value = int(reserve_value)
- is_percent = False
- except ValueError:
- raise ValueError('Error: %s is an invalid value for fallocate'
- '_reserve.' % reserve_value)
- return reserve_value, is_percent
-
-
class FileLikeIter(object):
def __init__(self, iterable):
@@ -937,164 +810,6 @@ def fs_has_free_space(fs_path, space_needed, is_percent):
return free_bytes >= space_needed
-class _LibcWrapper(object):
- """
- A callable object that forwards its calls to a C function from libc.
-
- These objects are lazy. libc will not be checked until someone tries to
- either call the function or check its availability.
-
- _LibcWrapper objects have an "available" property; if true, then libc
- has the function of that name. If false, then calls will fail with a
- NotImplementedError.
- """
-
- def __init__(self, func_name):
- self._func_name = func_name
- self._func_handle = None
- self._loaded = False
-
- def _ensure_loaded(self):
- if not self._loaded:
- func_name = self._func_name
- try:
- # Keep everything in this try-block in local variables so
- # that a typo in self.some_attribute_name doesn't raise a
- # spurious AttributeError.
- func_handle = load_libc_function(
- func_name, fail_if_missing=True)
- self._func_handle = func_handle
- except AttributeError:
- # We pass fail_if_missing=True to load_libc_function and
- # then ignore the error. It's weird, but otherwise we have
- # to check if self._func_handle is noop_libc_function, and
- # that's even weirder.
- pass
- self._loaded = True
-
- @property
- def available(self):
- self._ensure_loaded()
- return bool(self._func_handle)
-
- def __call__(self, *args):
- if self.available:
- return self._func_handle(*args)
- else:
- raise NotImplementedError(
- "No function %r found in libc" % self._func_name)
-
-
-_fallocate_enabled = True
-_fallocate_warned_about_missing = False
-_sys_fallocate = _LibcWrapper('fallocate')
-_sys_posix_fallocate = _LibcWrapper('posix_fallocate')
-
-
-def disable_fallocate():
- global _fallocate_enabled
- _fallocate_enabled = False
-
-
-def fallocate(fd, size, offset=0):
- """
- Pre-allocate disk space for a file.
-
- This function can be disabled by calling disable_fallocate(). If no
- suitable C function is available in libc, this function is a no-op.
-
- :param fd: file descriptor
- :param size: size to allocate (in bytes)
- """
- global _fallocate_enabled
- if not _fallocate_enabled:
- return
-
- if size < 0:
- size = 0 # Done historically; not really sure why
- if size >= (1 << 63):
- raise ValueError('size must be less than 2 ** 63')
- if offset < 0:
- raise ValueError('offset must be non-negative')
- if offset >= (1 << 63):
- raise ValueError('offset must be less than 2 ** 63')
-
- # Make sure there's some (configurable) amount of free space in
- # addition to the number of bytes we're allocating.
- if FALLOCATE_RESERVE:
- st = os.fstatvfs(fd)
- free = st.f_frsize * st.f_bavail - size
- if FALLOCATE_IS_PERCENT:
- free = (float(free) / float(st.f_frsize * st.f_blocks)) * 100
- if float(free) <= float(FALLOCATE_RESERVE):
- raise OSError(
- errno.ENOSPC,
- 'FALLOCATE_RESERVE fail %g <= %g' %
- (free, FALLOCATE_RESERVE))
-
- if _sys_fallocate.available:
- # Parameters are (fd, mode, offset, length).
- #
- # mode=FALLOC_FL_KEEP_SIZE pre-allocates invisibly (without
- # affecting the reported file size).
- ret = _sys_fallocate(
- fd, FALLOC_FL_KEEP_SIZE, ctypes.c_uint64(offset),
- ctypes.c_uint64(size))
- err = ctypes.get_errno()
- elif _sys_posix_fallocate.available:
- # Parameters are (fd, offset, length).
- ret = _sys_posix_fallocate(fd, ctypes.c_uint64(offset),
- ctypes.c_uint64(size))
- err = ctypes.get_errno()
- else:
- # No suitable fallocate-like function is in our libc. Warn about it,
- # but just once per process, and then do nothing.
- global _fallocate_warned_about_missing
- if not _fallocate_warned_about_missing:
- logging.warning(_("Unable to locate fallocate, posix_fallocate in "
- "libc. Leaving as a no-op."))
- _fallocate_warned_about_missing = True
- return
-
- if ret and err not in (0, errno.ENOSYS, errno.EOPNOTSUPP,
- errno.EINVAL):
- raise OSError(err, 'Unable to fallocate(%s)' % size)
-
-
-def punch_hole(fd, offset, length):
- """
- De-allocate disk space in the middle of a file.
-
- :param fd: file descriptor
- :param offset: index of first byte to de-allocate
- :param length: number of bytes to de-allocate
- """
- if offset < 0:
- raise ValueError('offset must be non-negative')
- if offset >= (1 << 63):
- raise ValueError('offset must be less than 2 ** 63')
- if length <= 0:
- raise ValueError('length must be positive')
- if length >= (1 << 63):
- raise ValueError('length must be less than 2 ** 63')
-
- if _sys_fallocate.available:
- # Parameters are (fd, mode, offset, length).
- ret = _sys_fallocate(
- fd,
- FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE,
- ctypes.c_uint64(offset),
- ctypes.c_uint64(length))
- err = ctypes.get_errno()
- if ret and err:
- mode_str = "FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE"
- raise OSError(err, "Unable to fallocate(%d, %s, %d, %d)" % (
- fd, mode_str, offset, length))
- else:
- raise OSError(errno.ENOTSUP,
- 'No suitable C function found for hole punching')
-
-
def fsync(fd):
"""
Sync modified file data and metadata to disk.
@@ -1144,26 +859,6 @@ def fsync_dir(dirpath):
os.close(dirfd)
-def drop_buffer_cache(fd, offset, length):
- """
- Drop 'buffer' cache for the given range of the given file.
-
- :param fd: file descriptor
- :param offset: start offset
- :param length: length
- """
- global _posix_fadvise
- if _posix_fadvise is None:
- _posix_fadvise = load_libc_function('posix_fadvise64')
- # 4 means "POSIX_FADV_DONTNEED"
- ret = _posix_fadvise(fd, ctypes.c_uint64(offset),
- ctypes.c_uint64(length), 4)
- if ret != 0:
- logging.warning("posix_fadvise64(%(fd)s, %(offset)s, %(length)s, 4) "
- "-> %(ret)s", {'fd': fd, 'offset': offset,
- 'length': length, 'ret': ret})
-
-
def mkdirs(path):
"""
Ensures the path is a directory or makes it if not. Errors if the path
@@ -4749,87 +4444,6 @@ def parse_content_disposition(header):
return header, attributes
-class sockaddr_alg(ctypes.Structure):
- _fields_ = [("salg_family", ctypes.c_ushort),
- ("salg_type", ctypes.c_ubyte * 14),
- ("salg_feat", ctypes.c_uint),
- ("salg_mask", ctypes.c_uint),
- ("salg_name", ctypes.c_ubyte * 64)]
-
-
-_bound_md5_sockfd = None
-
-
-def get_md5_socket():
- """
- Get an MD5 socket file descriptor. One can MD5 data with it by writing it
- to the socket with os.write, then os.read the 16 bytes of the checksum out
- later.
-
- NOTE: It is the caller's responsibility to ensure that os.close() is
- called on the returned file descriptor. This is a bare file descriptor,
- not a Python object. It doesn't close itself.
- """
-
- # Linux's AF_ALG sockets work like this:
- #
- # First, initialize a socket with socket() and bind(). This tells the
- # socket what algorithm to use, as well as setting up any necessary bits
- # like crypto keys. Of course, MD5 doesn't need any keys, so it's just the
- # algorithm name.
- #
- # Second, to hash some data, get a second socket by calling accept() on
- # the first socket. Write data to the socket, then when finished, read the
- # checksum from the socket and close it. This lets you checksum multiple
- # things without repeating all the setup code each time.
- #
- # Since we only need to bind() one socket, we do that here and save it for
- # future re-use. That way, we only use one file descriptor to get an MD5
- # socket instead of two, and we also get to save some syscalls.
-
- global _bound_md5_sockfd
- global _libc_socket
- global _libc_bind
- global _libc_accept
-
- if _libc_accept is None:
- _libc_accept = load_libc_function('accept', fail_if_missing=True)
- if _libc_socket is None:
- _libc_socket = load_libc_function('socket', fail_if_missing=True)
- if _libc_bind is None:
- _libc_bind = load_libc_function('bind', fail_if_missing=True)
-
- # Do this at first call rather than at import time so that we don't use a
- # file descriptor on systems that aren't using any MD5 sockets.
- if _bound_md5_sockfd is None:
- sockaddr_setup = sockaddr_alg(
- AF_ALG,
- (ord('h'), ord('a'), ord('s'), ord('h'), 0),
- 0, 0,
- (ord('m'), ord('d'), ord('5'), 0))
- hash_sockfd = _libc_socket(ctypes.c_int(AF_ALG),
- ctypes.c_int(socket.SOCK_SEQPACKET),
- ctypes.c_int(0))
- if hash_sockfd < 0:
- raise IOError(ctypes.get_errno(),
- "Failed to initialize MD5 socket")
-
- bind_result = _libc_bind(ctypes.c_int(hash_sockfd),
- ctypes.pointer(sockaddr_setup),
- ctypes.c_int(ctypes.sizeof(sockaddr_alg)))
- if bind_result < 0:
- os.close(hash_sockfd)
- raise IOError(ctypes.get_errno(), "Failed to bind MD5 socket")
-
- _bound_md5_sockfd = hash_sockfd
-
- md5_sockfd = _libc_accept(ctypes.c_int(_bound_md5_sockfd), None, 0)
- if md5_sockfd < 0:
- raise IOError(ctypes.get_errno(), "Failed to accept MD5 socket")
-
- return md5_sockfd
-
-
try:
_test_md5 = hashlib.md5(usedforsecurity=False) # nosec
@@ -5994,72 +5608,6 @@ def filter_shard_ranges(shard_ranges, includes, marker, end_marker):
return shard_ranges
-def modify_priority(conf, logger):
- """
- Modify priority by nice and ionice.
- """
-
- global _libc_setpriority
- if _libc_setpriority is None:
- _libc_setpriority = load_libc_function('setpriority',
- errcheck=True)
-
- def _setpriority(nice_priority):
- """
- setpriority for this pid
-
- :param nice_priority: valid values are -19 to 20
- """
- try:
- _libc_setpriority(PRIO_PROCESS, os.getpid(),
- int(nice_priority))
- except (ValueError, OSError):
- print(_("WARNING: Unable to modify scheduling priority of process."
- " Keeping unchanged! Check logs for more info. "))
- logger.exception('Unable to modify nice priority')
- else:
- logger.debug('set nice priority to %s' % nice_priority)
-
- nice_priority = conf.get('nice_priority')
- if nice_priority is not None:
- _setpriority(nice_priority)
-
- global _posix_syscall
- if _posix_syscall is None:
- _posix_syscall = load_libc_function('syscall', errcheck=True)
-
- def _ioprio_set(io_class, io_priority):
- """
- ioprio_set for this process
-
- :param io_class: the I/O class component, can be
- IOPRIO_CLASS_RT, IOPRIO_CLASS_BE,
- or IOPRIO_CLASS_IDLE
- :param io_priority: priority value in the I/O class
- """
- try:
- io_class = IO_CLASS_ENUM[io_class]
- io_priority = int(io_priority)
- _posix_syscall(NR_ioprio_set(),
- IOPRIO_WHO_PROCESS,
- os.getpid(),
- IOPRIO_PRIO_VALUE(io_class, io_priority))
- except (KeyError, ValueError, OSError):
- print(_("WARNING: Unable to modify I/O scheduling class "
- "and priority of process. Keeping unchanged! "
- "Check logs for more info."))
- logger.exception("Unable to modify ionice priority")
- else:
- logger.debug('set ionice class %s priority %s',
- io_class, io_priority)
-
- io_class = conf.get("ionice_class")
- if io_class is None:
- return
- io_priority = conf.get("ionice_priority", 0)
- _ioprio_set(io_class, io_priority)
-
-
def o_tmpfile_in_path_supported(dirpath):
fd = None
try:
diff --git a/swift/common/utils/libc.py b/swift/common/utils/libc.py
new file mode 100644
index 000000000..df2179020
--- /dev/null
+++ b/swift/common/utils/libc.py
@@ -0,0 +1,487 @@
+# Copyright (c) 2010-2023 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Functions Swift uses to interact with libc and other low-level APIs."""
+
+import ctypes
+import ctypes.util
+import errno
+import fcntl
+import logging
+import os
+import platform
+import socket
+
+
+# These are lazily pulled from libc elsewhere
+_sys_fallocate = None
+_posix_fadvise = None
+_libc_socket = None
+_libc_bind = None
+_libc_accept = None
+# see man -s 2 setpriority
+_libc_setpriority = None
+# see man -s 2 syscall
+_posix_syscall = None
+
+# If set to non-zero, fallocate routines will fail based on free space
+# available being at or below this amount, in bytes.
+FALLOCATE_RESERVE = 0
+# Indicates if FALLOCATE_RESERVE is the percentage of free space (True) or
+# the number of bytes (False).
+FALLOCATE_IS_PERCENT = False
+
+# from /usr/include/linux/falloc.h
+FALLOC_FL_KEEP_SIZE = 1
+FALLOC_FL_PUNCH_HOLE = 2
+
+# from /usr/src/linux-headers-*/include/uapi/linux/resource.h
+PRIO_PROCESS = 0
+
+
+# /usr/include/x86_64-linux-gnu/asm/unistd_64.h defines syscalls there
+# are many like it, but this one is mine, see man -s 2 ioprio_set
+def NR_ioprio_set():
+ """Give __NR_ioprio_set value for your system."""
+ architecture = os.uname()[4]
+ arch_bits = platform.architecture()[0]
+ # check if supported system, now support x86_64 and AArch64
+ if architecture == 'x86_64' and arch_bits == '64bit':
+ return 251
+ elif architecture == 'aarch64' and arch_bits == '64bit':
+ return 30
+ raise OSError("Swift doesn't support ionice priority for %s %s" %
+ (architecture, arch_bits))
+
+
+# this syscall integer probably only works on x86_64 linux systems, you
+# can check if it's correct on yours with something like this:
+"""
+#include <stdio.h>
+#include <sys/syscall.h>
+
+int main(int argc, const char* argv[]) {
+ printf("%d\n", __NR_ioprio_set);
+ return 0;
+}
+"""
+
+# this is the value for "which" that says our who value will be a pid
+# pulled out of /usr/src/linux-headers-*/include/linux/ioprio.h
+IOPRIO_WHO_PROCESS = 1
+
+
+IO_CLASS_ENUM = {
+ 'IOPRIO_CLASS_RT': 1,
+ 'IOPRIO_CLASS_BE': 2,
+ 'IOPRIO_CLASS_IDLE': 3,
+}
+
+# the IOPRIO_PRIO_VALUE "macro" is also pulled from
+# /usr/src/linux-headers-*/include/linux/ioprio.h
+IOPRIO_CLASS_SHIFT = 13
+
+
+def IOPRIO_PRIO_VALUE(class_, data):
+ return (((class_) << IOPRIO_CLASS_SHIFT) | data)
+
+
+# These constants are Linux-specific, and Python doesn't seem to know
+# about them. We ask anyway just in case that ever gets fixed.
+#
+# The values were copied from the Linux 3.x kernel headers.
+AF_ALG = getattr(socket, 'AF_ALG', 38)
+F_SETPIPE_SZ = getattr(fcntl, 'F_SETPIPE_SZ', 1031)
+
+
+def noop_libc_function(*args):
+ return 0
+
+
+def load_libc_function(func_name, log_error=True,
+ fail_if_missing=False, errcheck=False):
+ """
+ Attempt to find the function in libc, otherwise return a no-op func.
+
+ :param func_name: name of the function to pull from libc.
+ :param log_error: log an error when a function can't be found
+ :param fail_if_missing: raise an exception when a function can't be found.
+ Default behavior is to return a no-op function.
+ :param errcheck: boolean, if true install a wrapper on the function
+ to check for a return values of -1 and call
+ ctype.get_errno and raise an OSError
+ """
+ try:
+ libc = ctypes.CDLL(ctypes.util.find_library('c'), use_errno=True)
+ func = getattr(libc, func_name)
+ except AttributeError:
+ if fail_if_missing:
+ raise
+ if log_error:
+ logging.warning("Unable to locate %s in libc. Leaving as a "
+ "no-op.", func_name)
+ return noop_libc_function
+ if errcheck:
+ def _errcheck(result, f, args):
+ if result == -1:
+ errcode = ctypes.get_errno()
+ raise OSError(errcode, os.strerror(errcode))
+ return result
+ func.errcheck = _errcheck
+ return func
+
+
+class _LibcWrapper(object):
+ """
+ A callable object that forwards its calls to a C function from libc.
+
+ These objects are lazy. libc will not be checked until someone tries to
+ either call the function or check its availability.
+
+ _LibcWrapper objects have an "available" property; if true, then libc
+ has the function of that name. If false, then calls will fail with a
+ NotImplementedError.
+ """
+
+ def __init__(self, func_name):
+ self._func_name = func_name
+ self._func_handle = None
+ self._loaded = False
+
+ def _ensure_loaded(self):
+ if not self._loaded:
+ func_name = self._func_name
+ try:
+ # Keep everything in this try-block in local variables so
+ # that a typo in self.some_attribute_name doesn't raise a
+ # spurious AttributeError.
+ func_handle = load_libc_function(
+ func_name, fail_if_missing=True)
+ self._func_handle = func_handle
+ except AttributeError:
+ # We pass fail_if_missing=True to load_libc_function and
+ # then ignore the error. It's weird, but otherwise we have
+ # to check if self._func_handle is noop_libc_function, and
+ # that's even weirder.
+ pass
+ self._loaded = True
+
+ @property
+ def available(self):
+ self._ensure_loaded()
+ return bool(self._func_handle)
+
+ def __call__(self, *args):
+ if self.available:
+ return self._func_handle(*args)
+ else:
+ raise NotImplementedError(
+ "No function %r found in libc" % self._func_name)
+
+
+def config_fallocate_value(reserve_value):
+ """
+ Returns fallocate reserve_value as an int or float.
+ Returns is_percent as a boolean.
+ Returns a ValueError on invalid fallocate value.
+ """
+ try:
+ if str(reserve_value[-1:]) == '%':
+ reserve_value = float(reserve_value[:-1])
+ is_percent = True
+ else:
+ reserve_value = int(reserve_value)
+ is_percent = False
+ except ValueError:
+ raise ValueError('Error: %s is an invalid value for fallocate'
+ '_reserve.' % reserve_value)
+ return reserve_value, is_percent
+
+
+_fallocate_enabled = True
+_fallocate_warned_about_missing = False
+_sys_fallocate = _LibcWrapper('fallocate')
+_sys_posix_fallocate = _LibcWrapper('posix_fallocate')
+
+
+def disable_fallocate():
+ global _fallocate_enabled
+ _fallocate_enabled = False
+
+
+def fallocate(fd, size, offset=0):
+ """
+ Pre-allocate disk space for a file.
+
+ This function can be disabled by calling disable_fallocate(). If no
+ suitable C function is available in libc, this function is a no-op.
+
+ :param fd: file descriptor
+ :param size: size to allocate (in bytes)
+ """
+ global _fallocate_enabled
+ if not _fallocate_enabled:
+ return
+
+ if size < 0:
+ size = 0 # Done historically; not really sure why
+ if size >= (1 << 63):
+ raise ValueError('size must be less than 2 ** 63')
+ if offset < 0:
+ raise ValueError('offset must be non-negative')
+ if offset >= (1 << 63):
+ raise ValueError('offset must be less than 2 ** 63')
+
+ # Make sure there's some (configurable) amount of free space in
+ # addition to the number of bytes we're allocating.
+ if FALLOCATE_RESERVE:
+ st = os.fstatvfs(fd)
+ free = st.f_frsize * st.f_bavail - size
+ if FALLOCATE_IS_PERCENT:
+ free = (float(free) / float(st.f_frsize * st.f_blocks)) * 100
+ if float(free) <= float(FALLOCATE_RESERVE):
+ raise OSError(
+ errno.ENOSPC,
+ 'FALLOCATE_RESERVE fail %g <= %g' %
+ (free, FALLOCATE_RESERVE))
+
+ if _sys_fallocate.available:
+ # Parameters are (fd, mode, offset, length).
+ #
+ # mode=FALLOC_FL_KEEP_SIZE pre-allocates invisibly (without
+ # affecting the reported file size).
+ ret = _sys_fallocate(
+ fd, FALLOC_FL_KEEP_SIZE, ctypes.c_uint64(offset),
+ ctypes.c_uint64(size))
+ err = ctypes.get_errno()
+ elif _sys_posix_fallocate.available:
+ # Parameters are (fd, offset, length).
+ ret = _sys_posix_fallocate(fd, ctypes.c_uint64(offset),
+ ctypes.c_uint64(size))
+ err = ctypes.get_errno()
+ else:
+ # No suitable fallocate-like function is in our libc. Warn about it,
+ # but just once per process, and then do nothing.
+ global _fallocate_warned_about_missing
+ if not _fallocate_warned_about_missing:
+ logging.warning("Unable to locate fallocate, posix_fallocate in "
+ "libc. Leaving as a no-op.")
+ _fallocate_warned_about_missing = True
+ return
+
+ if ret and err not in (0, errno.ENOSYS, errno.EOPNOTSUPP,
+ errno.EINVAL):
+ raise OSError(err, 'Unable to fallocate(%s)' % size)
+
+
+def punch_hole(fd, offset, length):
+ """
+ De-allocate disk space in the middle of a file.
+
+ :param fd: file descriptor
+ :param offset: index of first byte to de-allocate
+ :param length: number of bytes to de-allocate
+ """
+ if offset < 0:
+ raise ValueError('offset must be non-negative')
+ if offset >= (1 << 63):
+ raise ValueError('offset must be less than 2 ** 63')
+ if length <= 0:
+ raise ValueError('length must be positive')
+ if length >= (1 << 63):
+ raise ValueError('length must be less than 2 ** 63')
+
+ if _sys_fallocate.available:
+ # Parameters are (fd, mode, offset, length).
+ ret = _sys_fallocate(
+ fd,
+ FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE,
+ ctypes.c_uint64(offset),
+ ctypes.c_uint64(length))
+ err = ctypes.get_errno()
+ if ret and err:
+ mode_str = "FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE"
+ raise OSError(err, "Unable to fallocate(%d, %s, %d, %d)" % (
+ fd, mode_str, offset, length))
+ else:
+ raise OSError(errno.ENOTSUP,
+ 'No suitable C function found for hole punching')
+
+
+def drop_buffer_cache(fd, offset, length):
+ """
+ Drop 'buffer' cache for the given range of the given file.
+
+ :param fd: file descriptor
+ :param offset: start offset
+ :param length: length
+ """
+ global _posix_fadvise
+ if _posix_fadvise is None:
+ _posix_fadvise = load_libc_function('posix_fadvise64')
+ # 4 means "POSIX_FADV_DONTNEED"
+ ret = _posix_fadvise(fd, ctypes.c_uint64(offset),
+ ctypes.c_uint64(length), 4)
+ if ret != 0:
+ logging.warning("posix_fadvise64(%(fd)s, %(offset)s, %(length)s, 4) "
+ "-> %(ret)s", {'fd': fd, 'offset': offset,
+ 'length': length, 'ret': ret})
+
+
+class sockaddr_alg(ctypes.Structure):
+ _fields_ = [("salg_family", ctypes.c_ushort),
+ ("salg_type", ctypes.c_ubyte * 14),
+ ("salg_feat", ctypes.c_uint),
+ ("salg_mask", ctypes.c_uint),
+ ("salg_name", ctypes.c_ubyte * 64)]
+
+
+_bound_md5_sockfd = None
+
+
+def get_md5_socket():
+ """
+ Get an MD5 socket file descriptor. One can MD5 data with it by writing it
+ to the socket with os.write, then os.read the 16 bytes of the checksum out
+ later.
+
+ NOTE: It is the caller's responsibility to ensure that os.close() is
+ called on the returned file descriptor. This is a bare file descriptor,
+ not a Python object. It doesn't close itself.
+ """
+
+ # Linux's AF_ALG sockets work like this:
+ #
+ # First, initialize a socket with socket() and bind(). This tells the
+ # socket what algorithm to use, as well as setting up any necessary bits
+ # like crypto keys. Of course, MD5 doesn't need any keys, so it's just the
+ # algorithm name.
+ #
+ # Second, to hash some data, get a second socket by calling accept() on
+ # the first socket. Write data to the socket, then when finished, read the
+ # checksum from the socket and close it. This lets you checksum multiple
+ # things without repeating all the setup code each time.
+ #
+ # Since we only need to bind() one socket, we do that here and save it for
+ # future re-use. That way, we only use one file descriptor to get an MD5
+ # socket instead of two, and we also get to save some syscalls.
+
+ global _bound_md5_sockfd
+ global _libc_socket
+ global _libc_bind
+ global _libc_accept
+
+ if _libc_accept is None:
+ _libc_accept = load_libc_function('accept', fail_if_missing=True)
+ if _libc_socket is None:
+ _libc_socket = load_libc_function('socket', fail_if_missing=True)
+ if _libc_bind is None:
+ _libc_bind = load_libc_function('bind', fail_if_missing=True)
+
+ # Do this at first call rather than at import time so that we don't use a
+ # file descriptor on systems that aren't using any MD5 sockets.
+ if _bound_md5_sockfd is None:
+ sockaddr_setup = sockaddr_alg(
+ AF_ALG,
+ (ord('h'), ord('a'), ord('s'), ord('h'), 0),
+ 0, 0,
+ (ord('m'), ord('d'), ord('5'), 0))
+ hash_sockfd = _libc_socket(ctypes.c_int(AF_ALG),
+ ctypes.c_int(socket.SOCK_SEQPACKET),
+ ctypes.c_int(0))
+ if hash_sockfd < 0:
+ raise IOError(ctypes.get_errno(),
+ "Failed to initialize MD5 socket")
+
+ bind_result = _libc_bind(ctypes.c_int(hash_sockfd),
+ ctypes.pointer(sockaddr_setup),
+ ctypes.c_int(ctypes.sizeof(sockaddr_alg)))
+ if bind_result < 0:
+ os.close(hash_sockfd)
+ raise IOError(ctypes.get_errno(), "Failed to bind MD5 socket")
+
+ _bound_md5_sockfd = hash_sockfd
+
+ md5_sockfd = _libc_accept(ctypes.c_int(_bound_md5_sockfd), None, 0)
+ if md5_sockfd < 0:
+ raise IOError(ctypes.get_errno(), "Failed to accept MD5 socket")
+
+ return md5_sockfd
+
+
+def modify_priority(conf, logger):
+ """
+ Modify priority by nice and ionice.
+ """
+
+ global _libc_setpriority
+ if _libc_setpriority is None:
+ _libc_setpriority = load_libc_function('setpriority',
+ errcheck=True)
+
+ def _setpriority(nice_priority):
+ """
+ setpriority for this pid
+
+ :param nice_priority: valid values are -19 to 20
+ """
+ try:
+ _libc_setpriority(PRIO_PROCESS, os.getpid(),
+ int(nice_priority))
+ except (ValueError, OSError):
+ print("WARNING: Unable to modify scheduling priority of process."
+ " Keeping unchanged! Check logs for more info. ")
+ logger.exception('Unable to modify nice priority')
+ else:
+ logger.debug('set nice priority to %s' % nice_priority)
+
+ nice_priority = conf.get('nice_priority')
+ if nice_priority is not None:
+ _setpriority(nice_priority)
+
+ global _posix_syscall
+ if _posix_syscall is None:
+ _posix_syscall = load_libc_function('syscall', errcheck=True)
+
+ def _ioprio_set(io_class, io_priority):
+ """
+ ioprio_set for this process
+
+ :param io_class: the I/O class component, can be
+ IOPRIO_CLASS_RT, IOPRIO_CLASS_BE,
+ or IOPRIO_CLASS_IDLE
+ :param io_priority: priority value in the I/O class
+ """
+ try:
+ io_class = IO_CLASS_ENUM[io_class]
+ io_priority = int(io_priority)
+ _posix_syscall(NR_ioprio_set(),
+ IOPRIO_WHO_PROCESS,
+ os.getpid(),
+ IOPRIO_PRIO_VALUE(io_class, io_priority))
+ except (KeyError, ValueError, OSError):
+ print("WARNING: Unable to modify I/O scheduling class "
+ "and priority of process. Keeping unchanged! "
+ "Check logs for more info.")
+ logger.exception("Unable to modify ionice priority")
+ else:
+ logger.debug('set ionice class %s priority %s',
+ io_class, io_priority)
+
+ io_class = conf.get("ionice_class")
+ if io_class is None:
+ return
+ io_priority = conf.get("ionice_priority", 0)
+ _ioprio_set(io_class, io_priority)
diff --git a/test/unit/common/test_utils.py b/test/unit/common/test_utils.py
index 10fe383da..e477ee85d 100644
--- a/test/unit/common/test_utils.py
+++ b/test/unit/common/test_utils.py
@@ -23,7 +23,6 @@ from test.debug_logger import debug_logger
from test.unit import temptree, make_timestamp_iter, with_tempdir, \
mock_timestamp_now, FakeIterable
-import ctypes
import contextlib
import errno
import eventlet
@@ -33,7 +32,6 @@ import eventlet.patcher
import functools
import grp
import logging
-import platform
import os
import mock
import posix
@@ -2679,44 +2677,6 @@ cluster_dfw1 = http://dfw1.host/v1/
ts = utils.get_trans_id_time('tx1df4ff4f55ea45f7b2ec2-almostright')
self.assertIsNone(ts)
- def test_config_fallocate_value(self):
- fallocate_value, is_percent = utils.config_fallocate_value('10%')
- self.assertEqual(fallocate_value, 10)
- self.assertTrue(is_percent)
- fallocate_value, is_percent = utils.config_fallocate_value('10')
- self.assertEqual(fallocate_value, 10)
- self.assertFalse(is_percent)
- try:
- fallocate_value, is_percent = utils.config_fallocate_value('ab%')
- except ValueError as err:
- exc = err
- self.assertEqual(str(exc), 'Error: ab% is an invalid value for '
- 'fallocate_reserve.')
- try:
- fallocate_value, is_percent = utils.config_fallocate_value('ab')
- except ValueError as err:
- exc = err
- self.assertEqual(str(exc), 'Error: ab is an invalid value for '
- 'fallocate_reserve.')
- try:
- fallocate_value, is_percent = utils.config_fallocate_value('1%%')
- except ValueError as err:
- exc = err
- self.assertEqual(str(exc), 'Error: 1%% is an invalid value for '
- 'fallocate_reserve.')
- try:
- fallocate_value, is_percent = utils.config_fallocate_value('10.0')
- except ValueError as err:
- exc = err
- self.assertEqual(str(exc), 'Error: 10.0 is an invalid value for '
- 'fallocate_reserve.')
- fallocate_value, is_percent = utils.config_fallocate_value('10.5%')
- self.assertEqual(fallocate_value, 10.5)
- self.assertTrue(is_percent)
- fallocate_value, is_percent = utils.config_fallocate_value('10.000%')
- self.assertEqual(fallocate_value, 10.000)
- self.assertTrue(is_percent)
-
def test_lock_file(self):
flags = os.O_CREAT | os.O_RDWR
with NamedTemporaryFile(delete=False) as nt:
@@ -3544,110 +3504,6 @@ cluster_dfw1 = http://dfw1.host/v1/
self.assertRaises(ValueError, utils.make_db_file_path,
'/path/to/hash.db', 'bad epoch')
- def test_modify_priority(self):
- pid = os.getpid()
- logger = debug_logger()
- called = {}
-
- def _fake_setpriority(*args):
- called['setpriority'] = args
-
- def _fake_syscall(*args):
- called['syscall'] = args
-
- # Test if current architecture supports changing of priority
- try:
- utils.NR_ioprio_set()
- except OSError as e:
- raise unittest.SkipTest(e)
-
- with patch('swift.common.utils._libc_setpriority',
- _fake_setpriority), \
- patch('swift.common.utils._posix_syscall', _fake_syscall):
- called = {}
- # not set / default
- utils.modify_priority({}, logger)
- self.assertEqual(called, {})
- called = {}
- # just nice
- utils.modify_priority({'nice_priority': '1'}, logger)
- self.assertEqual(called, {'setpriority': (0, pid, 1)})
- called = {}
- # just ionice class uses default priority 0
- utils.modify_priority({'ionice_class': 'IOPRIO_CLASS_RT'}, logger)
- architecture = os.uname()[4]
- arch_bits = platform.architecture()[0]
- if architecture == 'x86_64' and arch_bits == '64bit':
- self.assertEqual(called, {'syscall': (251, 1, pid, 1 << 13)})
- elif architecture == 'aarch64' and arch_bits == '64bit':
- self.assertEqual(called, {'syscall': (30, 1, pid, 1 << 13)})
- else:
- self.fail("Unexpected call: %r" % called)
- called = {}
- # just ionice priority is ignored
- utils.modify_priority({'ionice_priority': '4'}, logger)
- self.assertEqual(called, {})
- called = {}
- # bad ionice class
- utils.modify_priority({'ionice_class': 'class_foo'}, logger)
- self.assertEqual(called, {})
- called = {}
- # ionice class & priority
- utils.modify_priority({
- 'ionice_class': 'IOPRIO_CLASS_BE',
- 'ionice_priority': '4',
- }, logger)
- if architecture == 'x86_64' and arch_bits == '64bit':
- self.assertEqual(called, {
- 'syscall': (251, 1, pid, 2 << 13 | 4)
- })
- elif architecture == 'aarch64' and arch_bits == '64bit':
- self.assertEqual(called, {
- 'syscall': (30, 1, pid, 2 << 13 | 4)
- })
- else:
- self.fail("Unexpected call: %r" % called)
- called = {}
- # all
- utils.modify_priority({
- 'nice_priority': '-15',
- 'ionice_class': 'IOPRIO_CLASS_IDLE',
- 'ionice_priority': '6',
- }, logger)
- if architecture == 'x86_64' and arch_bits == '64bit':
- self.assertEqual(called, {
- 'setpriority': (0, pid, -15),
- 'syscall': (251, 1, pid, 3 << 13 | 6),
- })
- elif architecture == 'aarch64' and arch_bits == '64bit':
- self.assertEqual(called, {
- 'setpriority': (0, pid, -15),
- 'syscall': (30, 1, pid, 3 << 13 | 6),
- })
- else:
- self.fail("Unexpected call: %r" % called)
-
- def test__NR_ioprio_set(self):
- with patch('os.uname', return_value=('', '', '', '', 'x86_64')), \
- patch('platform.architecture', return_value=('64bit', '')):
- self.assertEqual(251, utils.NR_ioprio_set())
-
- with patch('os.uname', return_value=('', '', '', '', 'x86_64')), \
- patch('platform.architecture', return_value=('32bit', '')):
- self.assertRaises(OSError, utils.NR_ioprio_set)
-
- with patch('os.uname', return_value=('', '', '', '', 'aarch64')), \
- patch('platform.architecture', return_value=('64bit', '')):
- self.assertEqual(30, utils.NR_ioprio_set())
-
- with patch('os.uname', return_value=('', '', '', '', 'aarch64')), \
- patch('platform.architecture', return_value=('32bit', '')):
- self.assertRaises(OSError, utils.NR_ioprio_set)
-
- with patch('os.uname', return_value=('', '', '', '', 'alpha')), \
- patch('platform.architecture', return_value=('64bit', '')):
- self.assertRaises(OSError, utils.NR_ioprio_set)
-
@requires_o_tmpfile_support_in_tmp
def test_link_fd_to_path_linkat_success(self):
tempdir = mkdtemp()
@@ -8882,422 +8738,6 @@ class TestShardRangeList(unittest.TestCase):
do_test([utils.ShardRange.ACTIVE]))
-@patch('ctypes.get_errno')
-@patch.object(utils, '_sys_posix_fallocate')
-@patch.object(utils, '_sys_fallocate')
-@patch.object(utils, 'FALLOCATE_RESERVE', 0)
-class TestFallocate(unittest.TestCase):
- def test_fallocate(self, sys_fallocate_mock,
- sys_posix_fallocate_mock, get_errno_mock):
- sys_fallocate_mock.available = True
- sys_fallocate_mock.return_value = 0
-
- utils.fallocate(1234, 5000 * 2 ** 20)
-
- # We can't use sys_fallocate_mock.assert_called_once_with because no
- # two ctypes.c_uint64 objects are equal even if their values are
- # equal. Yes, ctypes.c_uint64(123) != ctypes.c_uint64(123).
- calls = sys_fallocate_mock.mock_calls
- self.assertEqual(len(calls), 1)
- args = calls[0][1]
- self.assertEqual(len(args), 4)
- self.assertEqual(args[0], 1234)
- self.assertEqual(args[1], utils.FALLOC_FL_KEEP_SIZE)
- self.assertEqual(args[2].value, 0)
- self.assertEqual(args[3].value, 5000 * 2 ** 20)
-
- sys_posix_fallocate_mock.assert_not_called()
-
- def test_fallocate_offset(self, sys_fallocate_mock,
- sys_posix_fallocate_mock, get_errno_mock):
- sys_fallocate_mock.available = True
- sys_fallocate_mock.return_value = 0
-
- utils.fallocate(1234, 5000 * 2 ** 20, offset=3 * 2 ** 30)
- calls = sys_fallocate_mock.mock_calls
- self.assertEqual(len(calls), 1)
- args = calls[0][1]
- self.assertEqual(len(args), 4)
- self.assertEqual(args[0], 1234)
- self.assertEqual(args[1], utils.FALLOC_FL_KEEP_SIZE)
- self.assertEqual(args[2].value, 3 * 2 ** 30)
- self.assertEqual(args[3].value, 5000 * 2 ** 20)
-
- sys_posix_fallocate_mock.assert_not_called()
-
- def test_fallocate_fatal_error(self, sys_fallocate_mock,
- sys_posix_fallocate_mock, get_errno_mock):
- sys_fallocate_mock.available = True
- sys_fallocate_mock.return_value = -1
- get_errno_mock.return_value = errno.EIO
-
- with self.assertRaises(OSError) as cm:
- utils.fallocate(1234, 5000 * 2 ** 20)
- self.assertEqual(cm.exception.errno, errno.EIO)
-
- def test_fallocate_silent_errors(self, sys_fallocate_mock,
- sys_posix_fallocate_mock, get_errno_mock):
- sys_fallocate_mock.available = True
- sys_fallocate_mock.return_value = -1
-
- for silent_error in (0, errno.ENOSYS, errno.EOPNOTSUPP, errno.EINVAL):
- get_errno_mock.return_value = silent_error
- try:
- utils.fallocate(1234, 5678)
- except OSError:
- self.fail("fallocate() raised an error on %d", silent_error)
-
- def test_posix_fallocate_fallback(self, sys_fallocate_mock,
- sys_posix_fallocate_mock,
- get_errno_mock):
- sys_fallocate_mock.available = False
- sys_fallocate_mock.side_effect = NotImplementedError
-
- sys_posix_fallocate_mock.available = True
- sys_posix_fallocate_mock.return_value = 0
-
- utils.fallocate(1234, 567890)
- sys_fallocate_mock.assert_not_called()
-
- calls = sys_posix_fallocate_mock.mock_calls
- self.assertEqual(len(calls), 1)
- args = calls[0][1]
- self.assertEqual(len(args), 3)
- self.assertEqual(args[0], 1234)
- self.assertEqual(args[1].value, 0)
- self.assertEqual(args[2].value, 567890)
-
- def test_posix_fallocate_offset(self, sys_fallocate_mock,
- sys_posix_fallocate_mock, get_errno_mock):
- sys_fallocate_mock.available = False
- sys_fallocate_mock.side_effect = NotImplementedError
-
- sys_posix_fallocate_mock.available = True
- sys_posix_fallocate_mock.return_value = 0
-
- utils.fallocate(1234, 5000 * 2 ** 20, offset=3 * 2 ** 30)
- calls = sys_posix_fallocate_mock.mock_calls
- self.assertEqual(len(calls), 1)
- args = calls[0][1]
- self.assertEqual(len(args), 3)
- self.assertEqual(args[0], 1234)
- self.assertEqual(args[1].value, 3 * 2 ** 30)
- self.assertEqual(args[2].value, 5000 * 2 ** 20)
-
- sys_fallocate_mock.assert_not_called()
-
- def test_no_fallocates_available(self, sys_fallocate_mock,
- sys_posix_fallocate_mock, get_errno_mock):
- sys_fallocate_mock.available = False
- sys_posix_fallocate_mock.available = False
-
- with mock.patch("logging.warning") as warning_mock, \
- mock.patch.object(utils, "_fallocate_warned_about_missing",
- False):
- utils.fallocate(321, 654)
- utils.fallocate(321, 654)
-
- sys_fallocate_mock.assert_not_called()
- sys_posix_fallocate_mock.assert_not_called()
- get_errno_mock.assert_not_called()
-
- self.assertEqual(len(warning_mock.mock_calls), 1)
-
- def test_arg_bounds(self, sys_fallocate_mock,
- sys_posix_fallocate_mock, get_errno_mock):
- sys_fallocate_mock.available = True
- sys_fallocate_mock.return_value = 0
- with self.assertRaises(ValueError):
- utils.fallocate(0, 1 << 64, 0)
- with self.assertRaises(ValueError):
- utils.fallocate(0, 0, -1)
- with self.assertRaises(ValueError):
- utils.fallocate(0, 0, 1 << 64)
- self.assertEqual([], sys_fallocate_mock.mock_calls)
- # sanity check
- utils.fallocate(0, 0, 0)
- self.assertEqual(
- [mock.call(0, utils.FALLOC_FL_KEEP_SIZE, mock.ANY, mock.ANY)],
- sys_fallocate_mock.mock_calls)
- # Go confirm the ctypes values separately; apparently == doesn't
- # work the way you'd expect with ctypes :-/
- self.assertEqual(sys_fallocate_mock.mock_calls[0][1][2].value, 0)
- self.assertEqual(sys_fallocate_mock.mock_calls[0][1][3].value, 0)
- sys_fallocate_mock.reset_mock()
-
- # negative size will be adjusted as 0
- utils.fallocate(0, -1, 0)
- self.assertEqual(
- [mock.call(0, utils.FALLOC_FL_KEEP_SIZE, mock.ANY, mock.ANY)],
- sys_fallocate_mock.mock_calls)
- self.assertEqual(sys_fallocate_mock.mock_calls[0][1][2].value, 0)
- self.assertEqual(sys_fallocate_mock.mock_calls[0][1][3].value, 0)
-
-
-@patch.object(os, 'fstatvfs')
-@patch.object(utils, '_sys_fallocate', available=True, return_value=0)
-@patch.object(utils, 'FALLOCATE_RESERVE', 0)
-@patch.object(utils, 'FALLOCATE_IS_PERCENT', False)
-@patch.object(utils, '_fallocate_enabled', True)
-class TestFallocateReserve(unittest.TestCase):
- def _statvfs_result(self, f_frsize, f_bavail):
- # Only 3 values are relevant to us, so use zeros for the rest
- f_blocks = 100
- return posix.statvfs_result((0, f_frsize, f_blocks, 0, f_bavail,
- 0, 0, 0, 0, 0))
-
- def test_disabled(self, sys_fallocate_mock, fstatvfs_mock):
- utils.disable_fallocate()
- utils.fallocate(123, 456)
-
- sys_fallocate_mock.assert_not_called()
- fstatvfs_mock.assert_not_called()
-
- def test_zero_reserve(self, sys_fallocate_mock, fstatvfs_mock):
- utils.fallocate(123, 456)
-
- fstatvfs_mock.assert_not_called()
- self.assertEqual(len(sys_fallocate_mock.mock_calls), 1)
-
- def test_enough_space(self, sys_fallocate_mock, fstatvfs_mock):
- # Want 1024 bytes in reserve plus 1023 allocated, and have 2 blocks
- # of size 1024 free, so succeed
- utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \
- utils.config_fallocate_value('1024')
-
- fstatvfs_mock.return_value = self._statvfs_result(1024, 2)
- utils.fallocate(88, 1023)
-
- def test_not_enough_space(self, sys_fallocate_mock, fstatvfs_mock):
- # Want 1024 bytes in reserve plus 1024 allocated, and have 2 blocks
- # of size 1024 free, so fail
- utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \
- utils.config_fallocate_value('1024')
-
- fstatvfs_mock.return_value = self._statvfs_result(1024, 2)
- with self.assertRaises(OSError) as catcher:
- utils.fallocate(88, 1024)
- self.assertEqual(
- str(catcher.exception),
- '[Errno %d] FALLOCATE_RESERVE fail 1024 <= 1024'
- % errno.ENOSPC)
- sys_fallocate_mock.assert_not_called()
-
- def test_not_enough_space_large(self, sys_fallocate_mock, fstatvfs_mock):
- # Want 1024 bytes in reserve plus 1GB allocated, and have 2 blocks
- # of size 1024 free, so fail
- utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \
- utils.config_fallocate_value('1024')
-
- fstatvfs_mock.return_value = self._statvfs_result(1024, 2)
- with self.assertRaises(OSError) as catcher:
- utils.fallocate(88, 1 << 30)
- self.assertEqual(
- str(catcher.exception),
- '[Errno %d] FALLOCATE_RESERVE fail %g <= 1024'
- % (errno.ENOSPC, ((2 * 1024) - (1 << 30))))
- sys_fallocate_mock.assert_not_called()
-
- def test_enough_space_small_blocks(self, sys_fallocate_mock,
- fstatvfs_mock):
- # Want 1024 bytes in reserve plus 1023 allocated, and have 4 blocks
- # of size 512 free, so succeed
- utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \
- utils.config_fallocate_value('1024')
-
- fstatvfs_mock.return_value = self._statvfs_result(512, 4)
- utils.fallocate(88, 1023)
-
- def test_not_enough_space_small_blocks(self, sys_fallocate_mock,
- fstatvfs_mock):
- # Want 1024 bytes in reserve plus 1024 allocated, and have 4 blocks
- # of size 512 free, so fail
- utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \
- utils.config_fallocate_value('1024')
-
- fstatvfs_mock.return_value = self._statvfs_result(512, 4)
- with self.assertRaises(OSError) as catcher:
- utils.fallocate(88, 1024)
- self.assertEqual(
- str(catcher.exception),
- '[Errno %d] FALLOCATE_RESERVE fail 1024 <= 1024'
- % errno.ENOSPC)
- sys_fallocate_mock.assert_not_called()
-
- def test_free_space_under_reserve(self, sys_fallocate_mock, fstatvfs_mock):
- # Want 2048 bytes in reserve but have only 3 blocks of size 512, so
- # allocating even 0 bytes fails
- utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \
- utils.config_fallocate_value('2048')
-
- fstatvfs_mock.return_value = self._statvfs_result(512, 3)
- with self.assertRaises(OSError) as catcher:
- utils.fallocate(88, 0)
- self.assertEqual(
- str(catcher.exception),
- '[Errno %d] FALLOCATE_RESERVE fail 1536 <= 2048'
- % errno.ENOSPC)
- sys_fallocate_mock.assert_not_called()
-
- def test_all_reserved(self, sys_fallocate_mock, fstatvfs_mock):
- # Filesystem is empty, but our reserve is bigger than the
- # filesystem, so any allocation will fail
- utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \
- utils.config_fallocate_value('9999999999999')
-
- fstatvfs_mock.return_value = self._statvfs_result(1024, 100)
- self.assertRaises(OSError, utils.fallocate, 88, 0)
- sys_fallocate_mock.assert_not_called()
-
- def test_enough_space_pct(self, sys_fallocate_mock, fstatvfs_mock):
- # Want 1% reserved, filesystem has 3/100 blocks of size 1024 free
- # and file size is 2047, so succeed
- utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \
- utils.config_fallocate_value('1%')
-
- fstatvfs_mock.return_value = self._statvfs_result(1024, 3)
- utils.fallocate(88, 2047)
-
- def test_not_enough_space_pct(self, sys_fallocate_mock, fstatvfs_mock):
- # Want 1% reserved, filesystem has 3/100 blocks of size 1024 free
- # and file size is 2048, so fail
- utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \
- utils.config_fallocate_value('1%')
-
- fstatvfs_mock.return_value = self._statvfs_result(1024, 3)
- with self.assertRaises(OSError) as catcher:
- utils.fallocate(88, 2048)
- self.assertEqual(
- str(catcher.exception),
- '[Errno %d] FALLOCATE_RESERVE fail 1 <= 1'
- % errno.ENOSPC)
- sys_fallocate_mock.assert_not_called()
-
- def test_all_space_reserved_pct(self, sys_fallocate_mock, fstatvfs_mock):
- # Filesystem is empty, but our reserve is the whole filesystem, so
- # any allocation will fail
- utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \
- utils.config_fallocate_value('100%')
-
- fstatvfs_mock.return_value = self._statvfs_result(1024, 100)
- with self.assertRaises(OSError) as catcher:
- utils.fallocate(88, 0)
- self.assertEqual(
- str(catcher.exception),
- '[Errno %d] FALLOCATE_RESERVE fail 100 <= 100'
- % errno.ENOSPC)
- sys_fallocate_mock.assert_not_called()
-
-
-@patch('ctypes.get_errno')
-@patch.object(utils, '_sys_fallocate')
-class TestPunchHole(unittest.TestCase):
- def test_punch_hole(self, sys_fallocate_mock, get_errno_mock):
- sys_fallocate_mock.available = True
- sys_fallocate_mock.return_value = 0
-
- utils.punch_hole(123, 456, 789)
-
- calls = sys_fallocate_mock.mock_calls
- self.assertEqual(len(calls), 1)
- args = calls[0][1]
- self.assertEqual(len(args), 4)
- self.assertEqual(args[0], 123)
- self.assertEqual(
- args[1], utils.FALLOC_FL_PUNCH_HOLE | utils.FALLOC_FL_KEEP_SIZE)
- self.assertEqual(args[2].value, 456)
- self.assertEqual(args[3].value, 789)
-
- def test_error(self, sys_fallocate_mock, get_errno_mock):
- sys_fallocate_mock.available = True
- sys_fallocate_mock.return_value = -1
- get_errno_mock.return_value = errno.EISDIR
-
- with self.assertRaises(OSError) as cm:
- utils.punch_hole(123, 456, 789)
- self.assertEqual(cm.exception.errno, errno.EISDIR)
-
- def test_arg_bounds(self, sys_fallocate_mock, get_errno_mock):
- sys_fallocate_mock.available = True
- sys_fallocate_mock.return_value = 0
-
- with self.assertRaises(ValueError):
- utils.punch_hole(0, 1, -1)
- with self.assertRaises(ValueError):
- utils.punch_hole(0, 1 << 64, 1)
- with self.assertRaises(ValueError):
- utils.punch_hole(0, -1, 1)
- with self.assertRaises(ValueError):
- utils.punch_hole(0, 1, 0)
- with self.assertRaises(ValueError):
- utils.punch_hole(0, 1, 1 << 64)
- self.assertEqual([], sys_fallocate_mock.mock_calls)
-
- # sanity check
- utils.punch_hole(0, 0, 1)
- self.assertEqual(
- [mock.call(
- 0, utils.FALLOC_FL_PUNCH_HOLE | utils.FALLOC_FL_KEEP_SIZE,
- mock.ANY, mock.ANY)],
- sys_fallocate_mock.mock_calls)
- # Go confirm the ctypes values separately; apparently == doesn't
- # work the way you'd expect with ctypes :-/
- self.assertEqual(sys_fallocate_mock.mock_calls[0][1][2].value, 0)
- self.assertEqual(sys_fallocate_mock.mock_calls[0][1][3].value, 1)
-
- def test_no_fallocate(self, sys_fallocate_mock, get_errno_mock):
- sys_fallocate_mock.available = False
-
- with self.assertRaises(OSError) as cm:
- utils.punch_hole(123, 456, 789)
- self.assertEqual(cm.exception.errno, errno.ENOTSUP)
-
-
-class TestPunchHoleReally(unittest.TestCase):
- def setUp(self):
- if not utils._sys_fallocate.available:
- raise unittest.SkipTest("utils._sys_fallocate not available")
-
- def test_punch_a_hole(self):
- with TemporaryFile() as tf:
- tf.write(b"x" * 64 + b"y" * 64 + b"z" * 64)
- tf.flush()
-
- # knock out the first half of the "y"s
- utils.punch_hole(tf.fileno(), 64, 32)
-
- tf.seek(0)
- contents = tf.read(4096)
- self.assertEqual(
- contents,
- b"x" * 64 + b"\0" * 32 + b"y" * 32 + b"z" * 64)
-
-
-class Test_LibcWrapper(unittest.TestCase):
- def test_available_function(self):
- # This should pretty much always exist
- getpid_wrapper = utils._LibcWrapper('getpid')
- self.assertTrue(getpid_wrapper.available)
- self.assertEqual(getpid_wrapper(), os.getpid())
-
- def test_unavailable_function(self):
- # This won't exist
- no_func_wrapper = utils._LibcWrapper('diffractively_protectorship')
- self.assertFalse(no_func_wrapper.available)
- self.assertRaises(NotImplementedError, no_func_wrapper)
-
- def test_argument_plumbing(self):
- lseek_wrapper = utils._LibcWrapper('lseek')
- with TemporaryFile() as tf:
- tf.write(b"abcdefgh")
- tf.flush()
- lseek_wrapper(tf.fileno(),
- ctypes.c_uint64(3),
- # 0 is SEEK_SET
- 0)
- self.assertEqual(tf.read(100), b"defgh")
-
-
class TestWatchdog(unittest.TestCase):
def test_start_stop(self):
w = utils.Watchdog()
diff --git a/test/unit/common/utils/test_libc.py b/test/unit/common/utils/test_libc.py
new file mode 100644
index 000000000..5357ce34d
--- /dev/null
+++ b/test/unit/common/utils/test_libc.py
@@ -0,0 +1,599 @@
+# Copyright (c) 2010-2023 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Tests for swift.common.utils.libc"""
+
+import ctypes
+import errno
+import os
+import platform
+import posix
+import tempfile
+import unittest
+
+import mock
+
+from swift.common.utils import libc
+
+from test.debug_logger import debug_logger
+
+
+@mock.patch('ctypes.get_errno')
+@mock.patch.object(libc, '_sys_posix_fallocate')
+@mock.patch.object(libc, '_sys_fallocate')
+@mock.patch.object(libc, 'FALLOCATE_RESERVE', 0)
+class TestFallocate(unittest.TestCase):
+ def test_config_fallocate_value(self, sys_fallocate_mock,
+ sys_posix_fallocate_mock, get_errno_mock):
+ fallocate_value, is_percent = libc.config_fallocate_value('10%')
+ self.assertEqual(fallocate_value, 10)
+ self.assertTrue(is_percent)
+ fallocate_value, is_percent = libc.config_fallocate_value('10')
+ self.assertEqual(fallocate_value, 10)
+ self.assertFalse(is_percent)
+ try:
+ fallocate_value, is_percent = libc.config_fallocate_value('ab%')
+ except ValueError as err:
+ exc = err
+ self.assertEqual(str(exc), 'Error: ab% is an invalid value for '
+ 'fallocate_reserve.')
+ try:
+ fallocate_value, is_percent = libc.config_fallocate_value('ab')
+ except ValueError as err:
+ exc = err
+ self.assertEqual(str(exc), 'Error: ab is an invalid value for '
+ 'fallocate_reserve.')
+ try:
+ fallocate_value, is_percent = libc.config_fallocate_value('1%%')
+ except ValueError as err:
+ exc = err
+ self.assertEqual(str(exc), 'Error: 1%% is an invalid value for '
+ 'fallocate_reserve.')
+ try:
+ fallocate_value, is_percent = libc.config_fallocate_value('10.0')
+ except ValueError as err:
+ exc = err
+ self.assertEqual(str(exc), 'Error: 10.0 is an invalid value for '
+ 'fallocate_reserve.')
+ fallocate_value, is_percent = libc.config_fallocate_value('10.5%')
+ self.assertEqual(fallocate_value, 10.5)
+ self.assertTrue(is_percent)
+ fallocate_value, is_percent = libc.config_fallocate_value('10.000%')
+ self.assertEqual(fallocate_value, 10.000)
+ self.assertTrue(is_percent)
+
+ def test_fallocate(self, sys_fallocate_mock,
+ sys_posix_fallocate_mock, get_errno_mock):
+ sys_fallocate_mock.available = True
+ sys_fallocate_mock.return_value = 0
+
+ libc.fallocate(1234, 5000 * 2 ** 20)
+
+ # We can't use sys_fallocate_mock.assert_called_once_with because no
+ # two ctypes.c_uint64 objects are equal even if their values are
+ # equal. Yes, ctypes.c_uint64(123) != ctypes.c_uint64(123).
+ calls = sys_fallocate_mock.mock_calls
+ self.assertEqual(len(calls), 1)
+ args = calls[0][1]
+ self.assertEqual(len(args), 4)
+ self.assertEqual(args[0], 1234)
+ self.assertEqual(args[1], libc.FALLOC_FL_KEEP_SIZE)
+ self.assertEqual(args[2].value, 0)
+ self.assertEqual(args[3].value, 5000 * 2 ** 20)
+
+ sys_posix_fallocate_mock.assert_not_called()
+
+ def test_fallocate_offset(self, sys_fallocate_mock,
+ sys_posix_fallocate_mock, get_errno_mock):
+ sys_fallocate_mock.available = True
+ sys_fallocate_mock.return_value = 0
+
+ libc.fallocate(1234, 5000 * 2 ** 20, offset=3 * 2 ** 30)
+ calls = sys_fallocate_mock.mock_calls
+ self.assertEqual(len(calls), 1)
+ args = calls[0][1]
+ self.assertEqual(len(args), 4)
+ self.assertEqual(args[0], 1234)
+ self.assertEqual(args[1], libc.FALLOC_FL_KEEP_SIZE)
+ self.assertEqual(args[2].value, 3 * 2 ** 30)
+ self.assertEqual(args[3].value, 5000 * 2 ** 20)
+
+ sys_posix_fallocate_mock.assert_not_called()
+
+ def test_fallocate_fatal_error(self, sys_fallocate_mock,
+ sys_posix_fallocate_mock, get_errno_mock):
+ sys_fallocate_mock.available = True
+ sys_fallocate_mock.return_value = -1
+ get_errno_mock.return_value = errno.EIO
+
+ with self.assertRaises(OSError) as cm:
+ libc.fallocate(1234, 5000 * 2 ** 20)
+ self.assertEqual(cm.exception.errno, errno.EIO)
+
+ def test_fallocate_silent_errors(self, sys_fallocate_mock,
+ sys_posix_fallocate_mock, get_errno_mock):
+ sys_fallocate_mock.available = True
+ sys_fallocate_mock.return_value = -1
+
+ for silent_error in (0, errno.ENOSYS, errno.EOPNOTSUPP, errno.EINVAL):
+ get_errno_mock.return_value = silent_error
+ try:
+ libc.fallocate(1234, 5678)
+ except OSError:
+ self.fail("fallocate() raised an error on %d", silent_error)
+
+ def test_posix_fallocate_fallback(self, sys_fallocate_mock,
+ sys_posix_fallocate_mock,
+ get_errno_mock):
+ sys_fallocate_mock.available = False
+ sys_fallocate_mock.side_effect = NotImplementedError
+
+ sys_posix_fallocate_mock.available = True
+ sys_posix_fallocate_mock.return_value = 0
+
+ libc.fallocate(1234, 567890)
+ sys_fallocate_mock.assert_not_called()
+
+ calls = sys_posix_fallocate_mock.mock_calls
+ self.assertEqual(len(calls), 1)
+ args = calls[0][1]
+ self.assertEqual(len(args), 3)
+ self.assertEqual(args[0], 1234)
+ self.assertEqual(args[1].value, 0)
+ self.assertEqual(args[2].value, 567890)
+
+ def test_posix_fallocate_offset(self, sys_fallocate_mock,
+ sys_posix_fallocate_mock, get_errno_mock):
+ sys_fallocate_mock.available = False
+ sys_fallocate_mock.side_effect = NotImplementedError
+
+ sys_posix_fallocate_mock.available = True
+ sys_posix_fallocate_mock.return_value = 0
+
+ libc.fallocate(1234, 5000 * 2 ** 20, offset=3 * 2 ** 30)
+ calls = sys_posix_fallocate_mock.mock_calls
+ self.assertEqual(len(calls), 1)
+ args = calls[0][1]
+ self.assertEqual(len(args), 3)
+ self.assertEqual(args[0], 1234)
+ self.assertEqual(args[1].value, 3 * 2 ** 30)
+ self.assertEqual(args[2].value, 5000 * 2 ** 20)
+
+ sys_fallocate_mock.assert_not_called()
+
+ def test_no_fallocates_available(self, sys_fallocate_mock,
+ sys_posix_fallocate_mock, get_errno_mock):
+ sys_fallocate_mock.available = False
+ sys_posix_fallocate_mock.available = False
+
+ with mock.patch("logging.warning") as warning_mock, \
+ mock.patch.object(libc, "_fallocate_warned_about_missing",
+ False):
+ libc.fallocate(321, 654)
+ libc.fallocate(321, 654)
+
+ sys_fallocate_mock.assert_not_called()
+ sys_posix_fallocate_mock.assert_not_called()
+ get_errno_mock.assert_not_called()
+
+ self.assertEqual(len(warning_mock.mock_calls), 1)
+
+ def test_arg_bounds(self, sys_fallocate_mock,
+ sys_posix_fallocate_mock, get_errno_mock):
+ sys_fallocate_mock.available = True
+ sys_fallocate_mock.return_value = 0
+ with self.assertRaises(ValueError):
+ libc.fallocate(0, 1 << 64, 0)
+ with self.assertRaises(ValueError):
+ libc.fallocate(0, 0, -1)
+ with self.assertRaises(ValueError):
+ libc.fallocate(0, 0, 1 << 64)
+ self.assertEqual([], sys_fallocate_mock.mock_calls)
+ # sanity check
+ libc.fallocate(0, 0, 0)
+ self.assertEqual(
+ [mock.call(0, libc.FALLOC_FL_KEEP_SIZE, mock.ANY, mock.ANY)],
+ sys_fallocate_mock.mock_calls)
+ # Go confirm the ctypes values separately; apparently == doesn't
+ # work the way you'd expect with ctypes :-/
+ self.assertEqual(sys_fallocate_mock.mock_calls[0][1][2].value, 0)
+ self.assertEqual(sys_fallocate_mock.mock_calls[0][1][3].value, 0)
+ sys_fallocate_mock.reset_mock()
+
+ # negative size will be adjusted as 0
+ libc.fallocate(0, -1, 0)
+ self.assertEqual(
+ [mock.call(0, libc.FALLOC_FL_KEEP_SIZE, mock.ANY, mock.ANY)],
+ sys_fallocate_mock.mock_calls)
+ self.assertEqual(sys_fallocate_mock.mock_calls[0][1][2].value, 0)
+ self.assertEqual(sys_fallocate_mock.mock_calls[0][1][3].value, 0)
+
+
+@mock.patch.object(os, 'fstatvfs')
+@mock.patch.object(libc, '_sys_fallocate', available=True, return_value=0)
+@mock.patch.object(libc, 'FALLOCATE_RESERVE', 0)
+@mock.patch.object(libc, 'FALLOCATE_IS_PERCENT', False)
+@mock.patch.object(libc, '_fallocate_enabled', True)
+class TestFallocateReserve(unittest.TestCase):
+ def _statvfs_result(self, f_frsize, f_bavail):
+ # Only 3 values are relevant to us, so use zeros for the rest
+ f_blocks = 100
+ return posix.statvfs_result((0, f_frsize, f_blocks, 0, f_bavail,
+ 0, 0, 0, 0, 0))
+
+ def test_disabled(self, sys_fallocate_mock, fstatvfs_mock):
+ libc.disable_fallocate()
+ libc.fallocate(123, 456)
+
+ sys_fallocate_mock.assert_not_called()
+ fstatvfs_mock.assert_not_called()
+
+ def test_zero_reserve(self, sys_fallocate_mock, fstatvfs_mock):
+ libc.fallocate(123, 456)
+
+ fstatvfs_mock.assert_not_called()
+ self.assertEqual(len(sys_fallocate_mock.mock_calls), 1)
+
+ def test_enough_space(self, sys_fallocate_mock, fstatvfs_mock):
+ # Want 1024 bytes in reserve plus 1023 allocated, and have 2 blocks
+ # of size 1024 free, so succeed
+ libc.FALLOCATE_RESERVE, libc.FALLOCATE_IS_PERCENT = \
+ libc.config_fallocate_value('1024')
+
+ fstatvfs_mock.return_value = self._statvfs_result(1024, 2)
+ libc.fallocate(88, 1023)
+
+ def test_not_enough_space(self, sys_fallocate_mock, fstatvfs_mock):
+ # Want 1024 bytes in reserve plus 1024 allocated, and have 2 blocks
+ # of size 1024 free, so fail
+ libc.FALLOCATE_RESERVE, libc.FALLOCATE_IS_PERCENT = \
+ libc.config_fallocate_value('1024')
+
+ fstatvfs_mock.return_value = self._statvfs_result(1024, 2)
+ with self.assertRaises(OSError) as catcher:
+ libc.fallocate(88, 1024)
+ self.assertEqual(
+ str(catcher.exception),
+ '[Errno %d] FALLOCATE_RESERVE fail 1024 <= 1024'
+ % errno.ENOSPC)
+ sys_fallocate_mock.assert_not_called()
+
+ def test_not_enough_space_large(self, sys_fallocate_mock, fstatvfs_mock):
+ # Want 1024 bytes in reserve plus 1GB allocated, and have 2 blocks
+ # of size 1024 free, so fail
+ libc.FALLOCATE_RESERVE, libc.FALLOCATE_IS_PERCENT = \
+ libc.config_fallocate_value('1024')
+
+ fstatvfs_mock.return_value = self._statvfs_result(1024, 2)
+ with self.assertRaises(OSError) as catcher:
+ libc.fallocate(88, 1 << 30)
+ self.assertEqual(
+ str(catcher.exception),
+ '[Errno %d] FALLOCATE_RESERVE fail %g <= 1024'
+ % (errno.ENOSPC, ((2 * 1024) - (1 << 30))))
+ sys_fallocate_mock.assert_not_called()
+
+ def test_enough_space_small_blocks(self, sys_fallocate_mock,
+ fstatvfs_mock):
+ # Want 1024 bytes in reserve plus 1023 allocated, and have 4 blocks
+ # of size 512 free, so succeed
+ libc.FALLOCATE_RESERVE, libc.FALLOCATE_IS_PERCENT = \
+ libc.config_fallocate_value('1024')
+
+ fstatvfs_mock.return_value = self._statvfs_result(512, 4)
+ libc.fallocate(88, 1023)
+
+ def test_not_enough_space_small_blocks(self, sys_fallocate_mock,
+ fstatvfs_mock):
+ # Want 1024 bytes in reserve plus 1024 allocated, and have 4 blocks
+ # of size 512 free, so fail
+ libc.FALLOCATE_RESERVE, libc.FALLOCATE_IS_PERCENT = \
+ libc.config_fallocate_value('1024')
+
+ fstatvfs_mock.return_value = self._statvfs_result(512, 4)
+ with self.assertRaises(OSError) as catcher:
+ libc.fallocate(88, 1024)
+ self.assertEqual(
+ str(catcher.exception),
+ '[Errno %d] FALLOCATE_RESERVE fail 1024 <= 1024'
+ % errno.ENOSPC)
+ sys_fallocate_mock.assert_not_called()
+
+ def test_free_space_under_reserve(self, sys_fallocate_mock, fstatvfs_mock):
+ # Want 2048 bytes in reserve but have only 3 blocks of size 512, so
+ # allocating even 0 bytes fails
+ libc.FALLOCATE_RESERVE, libc.FALLOCATE_IS_PERCENT = \
+ libc.config_fallocate_value('2048')
+
+ fstatvfs_mock.return_value = self._statvfs_result(512, 3)
+ with self.assertRaises(OSError) as catcher:
+ libc.fallocate(88, 0)
+ self.assertEqual(
+ str(catcher.exception),
+ '[Errno %d] FALLOCATE_RESERVE fail 1536 <= 2048'
+ % errno.ENOSPC)
+ sys_fallocate_mock.assert_not_called()
+
+ def test_all_reserved(self, sys_fallocate_mock, fstatvfs_mock):
+ # Filesystem is empty, but our reserve is bigger than the
+ # filesystem, so any allocation will fail
+ libc.FALLOCATE_RESERVE, libc.FALLOCATE_IS_PERCENT = \
+ libc.config_fallocate_value('9999999999999')
+
+ fstatvfs_mock.return_value = self._statvfs_result(1024, 100)
+ self.assertRaises(OSError, libc.fallocate, 88, 0)
+ sys_fallocate_mock.assert_not_called()
+
+ def test_enough_space_pct(self, sys_fallocate_mock, fstatvfs_mock):
+ # Want 1% reserved, filesystem has 3/100 blocks of size 1024 free
+ # and file size is 2047, so succeed
+ libc.FALLOCATE_RESERVE, libc.FALLOCATE_IS_PERCENT = \
+ libc.config_fallocate_value('1%')
+
+ fstatvfs_mock.return_value = self._statvfs_result(1024, 3)
+ libc.fallocate(88, 2047)
+
+ def test_not_enough_space_pct(self, sys_fallocate_mock, fstatvfs_mock):
+ # Want 1% reserved, filesystem has 3/100 blocks of size 1024 free
+ # and file size is 2048, so fail
+ libc.FALLOCATE_RESERVE, libc.FALLOCATE_IS_PERCENT = \
+ libc.config_fallocate_value('1%')
+
+ fstatvfs_mock.return_value = self._statvfs_result(1024, 3)
+ with self.assertRaises(OSError) as catcher:
+ libc.fallocate(88, 2048)
+ self.assertEqual(
+ str(catcher.exception),
+ '[Errno %d] FALLOCATE_RESERVE fail 1 <= 1'
+ % errno.ENOSPC)
+ sys_fallocate_mock.assert_not_called()
+
+ def test_all_space_reserved_pct(self, sys_fallocate_mock, fstatvfs_mock):
+ # Filesystem is empty, but our reserve is the whole filesystem, so
+ # any allocation will fail
+ libc.FALLOCATE_RESERVE, libc.FALLOCATE_IS_PERCENT = \
+ libc.config_fallocate_value('100%')
+
+ fstatvfs_mock.return_value = self._statvfs_result(1024, 100)
+ with self.assertRaises(OSError) as catcher:
+ libc.fallocate(88, 0)
+ self.assertEqual(
+ str(catcher.exception),
+ '[Errno %d] FALLOCATE_RESERVE fail 100 <= 100'
+ % errno.ENOSPC)
+ sys_fallocate_mock.assert_not_called()
+
+
+@mock.patch('ctypes.get_errno')
+@mock.patch.object(libc, '_sys_fallocate')
+class TestPunchHole(unittest.TestCase):
+ def test_punch_hole(self, sys_fallocate_mock, get_errno_mock):
+ sys_fallocate_mock.available = True
+ sys_fallocate_mock.return_value = 0
+
+ libc.punch_hole(123, 456, 789)
+
+ calls = sys_fallocate_mock.mock_calls
+ self.assertEqual(len(calls), 1)
+ args = calls[0][1]
+ self.assertEqual(len(args), 4)
+ self.assertEqual(args[0], 123)
+ self.assertEqual(
+ args[1], libc.FALLOC_FL_PUNCH_HOLE | libc.FALLOC_FL_KEEP_SIZE)
+ self.assertEqual(args[2].value, 456)
+ self.assertEqual(args[3].value, 789)
+
+ def test_error(self, sys_fallocate_mock, get_errno_mock):
+ sys_fallocate_mock.available = True
+ sys_fallocate_mock.return_value = -1
+ get_errno_mock.return_value = errno.EISDIR
+
+ with self.assertRaises(OSError) as cm:
+ libc.punch_hole(123, 456, 789)
+ self.assertEqual(cm.exception.errno, errno.EISDIR)
+
+ def test_arg_bounds(self, sys_fallocate_mock, get_errno_mock):
+ sys_fallocate_mock.available = True
+ sys_fallocate_mock.return_value = 0
+
+ with self.assertRaises(ValueError):
+ libc.punch_hole(0, 1, -1)
+ with self.assertRaises(ValueError):
+ libc.punch_hole(0, 1 << 64, 1)
+ with self.assertRaises(ValueError):
+ libc.punch_hole(0, -1, 1)
+ with self.assertRaises(ValueError):
+ libc.punch_hole(0, 1, 0)
+ with self.assertRaises(ValueError):
+ libc.punch_hole(0, 1, 1 << 64)
+ self.assertEqual([], sys_fallocate_mock.mock_calls)
+
+ # sanity check
+ libc.punch_hole(0, 0, 1)
+ self.assertEqual(
+ [mock.call(
+ 0, libc.FALLOC_FL_PUNCH_HOLE | libc.FALLOC_FL_KEEP_SIZE,
+ mock.ANY, mock.ANY)],
+ sys_fallocate_mock.mock_calls)
+ # Go confirm the ctypes values separately; apparently == doesn't
+ # work the way you'd expect with ctypes :-/
+ self.assertEqual(sys_fallocate_mock.mock_calls[0][1][2].value, 0)
+ self.assertEqual(sys_fallocate_mock.mock_calls[0][1][3].value, 1)
+
+ def test_no_fallocate(self, sys_fallocate_mock, get_errno_mock):
+ sys_fallocate_mock.available = False
+
+ with self.assertRaises(OSError) as cm:
+ libc.punch_hole(123, 456, 789)
+ self.assertEqual(cm.exception.errno, errno.ENOTSUP)
+
+
+class TestPunchHoleReally(unittest.TestCase):
+ def setUp(self):
+ if not libc._sys_fallocate.available:
+ raise unittest.SkipTest("libc._sys_fallocate not available")
+
+ def test_punch_a_hole(self):
+ with tempfile.TemporaryFile() as tf:
+ tf.write(b"x" * 64 + b"y" * 64 + b"z" * 64)
+ tf.flush()
+
+ # knock out the first half of the "y"s
+ libc.punch_hole(tf.fileno(), 64, 32)
+
+ tf.seek(0)
+ contents = tf.read(4096)
+ self.assertEqual(
+ contents,
+ b"x" * 64 + b"\0" * 32 + b"y" * 32 + b"z" * 64)
+
+
+class Test_LibcWrapper(unittest.TestCase):
+ def test_available_function(self):
+ # This should pretty much always exist
+ getpid_wrapper = libc._LibcWrapper('getpid')
+ self.assertTrue(getpid_wrapper.available)
+ self.assertEqual(getpid_wrapper(), os.getpid())
+
+ def test_unavailable_function(self):
+ # This won't exist
+ no_func_wrapper = libc._LibcWrapper('diffractively_protectorship')
+ self.assertFalse(no_func_wrapper.available)
+ self.assertRaises(NotImplementedError, no_func_wrapper)
+
+ def test_argument_plumbing(self):
+ lseek_wrapper = libc._LibcWrapper('lseek')
+ with tempfile.TemporaryFile() as tf:
+ tf.write(b"abcdefgh")
+ tf.flush()
+ lseek_wrapper(tf.fileno(),
+ ctypes.c_uint64(3),
+ # 0 is SEEK_SET
+ 0)
+ self.assertEqual(tf.read(100), b"defgh")
+
+
+class TestModifyPriority(unittest.TestCase):
+ def test_modify_priority(self):
+ pid = os.getpid()
+ logger = debug_logger()
+ called = {}
+
+ def _fake_setpriority(*args):
+ called['setpriority'] = args
+
+ def _fake_syscall(*args):
+ called['syscall'] = args
+
+ # Test if current architecture supports changing of priority
+ try:
+ libc.NR_ioprio_set()
+ except OSError as e:
+ raise unittest.SkipTest(e)
+
+ with mock.patch('swift.common.utils.libc._libc_setpriority',
+ _fake_setpriority), \
+ mock.patch('swift.common.utils.libc._posix_syscall',
+ _fake_syscall):
+ called = {}
+ # not set / default
+ libc.modify_priority({}, logger)
+ self.assertEqual(called, {})
+ called = {}
+ # just nice
+ libc.modify_priority({'nice_priority': '1'}, logger)
+ self.assertEqual(called, {'setpriority': (0, pid, 1)})
+ called = {}
+ # just ionice class uses default priority 0
+ libc.modify_priority({'ionice_class': 'IOPRIO_CLASS_RT'}, logger)
+ architecture = os.uname()[4]
+ arch_bits = platform.architecture()[0]
+ if architecture == 'x86_64' and arch_bits == '64bit':
+ self.assertEqual(called, {'syscall': (251, 1, pid, 1 << 13)})
+ elif architecture == 'aarch64' and arch_bits == '64bit':
+ self.assertEqual(called, {'syscall': (30, 1, pid, 1 << 13)})
+ else:
+ self.fail("Unexpected call: %r" % called)
+ called = {}
+ # just ionice priority is ignored
+ libc.modify_priority({'ionice_priority': '4'}, logger)
+ self.assertEqual(called, {})
+ called = {}
+ # bad ionice class
+ libc.modify_priority({'ionice_class': 'class_foo'}, logger)
+ self.assertEqual(called, {})
+ called = {}
+ # ionice class & priority
+ libc.modify_priority({
+ 'ionice_class': 'IOPRIO_CLASS_BE',
+ 'ionice_priority': '4',
+ }, logger)
+ if architecture == 'x86_64' and arch_bits == '64bit':
+ self.assertEqual(called, {
+ 'syscall': (251, 1, pid, 2 << 13 | 4)
+ })
+ elif architecture == 'aarch64' and arch_bits == '64bit':
+ self.assertEqual(called, {
+ 'syscall': (30, 1, pid, 2 << 13 | 4)
+ })
+ else:
+ self.fail("Unexpected call: %r" % called)
+ called = {}
+ # all
+ libc.modify_priority({
+ 'nice_priority': '-15',
+ 'ionice_class': 'IOPRIO_CLASS_IDLE',
+ 'ionice_priority': '6',
+ }, logger)
+ if architecture == 'x86_64' and arch_bits == '64bit':
+ self.assertEqual(called, {
+ 'setpriority': (0, pid, -15),
+ 'syscall': (251, 1, pid, 3 << 13 | 6),
+ })
+ elif architecture == 'aarch64' and arch_bits == '64bit':
+ self.assertEqual(called, {
+ 'setpriority': (0, pid, -15),
+ 'syscall': (30, 1, pid, 3 << 13 | 6),
+ })
+ else:
+ self.fail("Unexpected call: %r" % called)
+
+ def test__NR_ioprio_set(self):
+ with mock.patch('os.uname', return_value=('', '', '', '', 'x86_64')), \
+ mock.patch('platform.architecture',
+ return_value=('64bit', '')):
+ self.assertEqual(251, libc.NR_ioprio_set())
+
+ with mock.patch('os.uname', return_value=('', '', '', '', 'x86_64')), \
+ mock.patch('platform.architecture',
+ return_value=('32bit', '')):
+ self.assertRaises(OSError, libc.NR_ioprio_set)
+
+ with mock.patch('os.uname',
+ return_value=('', '', '', '', 'aarch64')), \
+ mock.patch('platform.architecture',
+ return_value=('64bit', '')):
+ self.assertEqual(30, libc.NR_ioprio_set())
+
+ with mock.patch('os.uname',
+ return_value=('', '', '', '', 'aarch64')), \
+ mock.patch('platform.architecture',
+ return_value=('32bit', '')):
+ self.assertRaises(OSError, libc.NR_ioprio_set)
+
+ with mock.patch('os.uname', return_value=('', '', '', '', 'alpha')), \
+ mock.patch('platform.architecture',
+ return_value=('64bit', '')):
+ self.assertRaises(OSError, libc.NR_ioprio_set)
diff --git a/test/unit/obj/test_diskfile.py b/test/unit/obj/test_diskfile.py
index 59baa38ab..8d3a484b7 100644
--- a/test/unit/obj/test_diskfile.py
+++ b/test/unit/obj/test_diskfile.py
@@ -47,6 +47,7 @@ from test.unit import (mock as unit_mock, temptree, mock_check_drive,
encode_frag_archive_bodies, skip_if_no_xattrs)
from swift.obj import diskfile
from swift.common import utils
+from swift.common.utils import libc
from swift.common.utils import hash_path, mkdirs, Timestamp, \
encode_timestamps, O_TMPFILE, md5 as _md5
from swift.common import ring
@@ -4748,7 +4749,7 @@ class DiskFileMixin(BaseDiskFileTestMixin):
# This is a horrible hack so you can run this test in isolation.
# Some of the ctypes machinery calls os.close(), and that runs afoul
# of our mock.
- with mock.patch.object(utils, '_sys_fallocate', None):
+ with mock.patch.object(libc, '_sys_fallocate', None):
utils.disable_fallocate()
df = self.df_mgr.get_diskfile(self.existing_device, '0', 'abc',