diff options
50 files changed, 667 insertions, 53 deletions
diff --git a/nova/cmd/conductor.py b/nova/cmd/conductor.py index 9697d9acdf..776e368917 100644 --- a/nova/cmd/conductor.py +++ b/nova/cmd/conductor.py @@ -16,12 +16,12 @@ import sys -from oslo.concurrency import processutils from oslo.config import cfg from nova import config from nova import objects from nova.openstack.common import log as logging +from nova.openstack.common import processutils from nova.openstack.common.report import guru_meditation_report as gmr from nova import service from nova import utils diff --git a/nova/console/xvp.py b/nova/console/xvp.py index 5a407a137d..282f008200 100644 --- a/nova/console/xvp.py +++ b/nova/console/xvp.py @@ -19,7 +19,6 @@ import os import signal import jinja2 -from oslo.concurrency import processutils from oslo.config import cfg from oslo.utils import excutils @@ -27,6 +26,7 @@ from nova import context from nova import db from nova.i18n import _, _LE from nova.openstack.common import log as logging +from nova.openstack.common import processutils from nova import paths from nova import utils diff --git a/nova/crypto.py b/nova/crypto.py index 36bf43858f..c547bcb6e1 100644 --- a/nova/crypto.py +++ b/nova/crypto.py @@ -28,7 +28,6 @@ import re import string import struct -from oslo.concurrency import processutils from oslo.config import cfg from oslo.utils import excutils from oslo.utils import timeutils @@ -41,6 +40,7 @@ from nova import exception from nova.i18n import _ from nova.openstack.common import fileutils from nova.openstack.common import log as logging +from nova.openstack.common import processutils from nova import paths from nova import utils diff --git a/nova/image/s3.py b/nova/image/s3.py index 9ea6f77008..01629b7083 100644 --- a/nova/image/s3.py +++ b/nova/image/s3.py @@ -26,7 +26,6 @@ import tempfile import boto.s3.connection import eventlet from lxml import etree -from oslo.concurrency import processutils from oslo.config import cfg from nova.api.ec2 import ec2utils @@ -36,6 +35,7 @@ from nova import exception from nova.i18n import _, _LE from nova.image import glance from nova.openstack.common import log as logging +from nova.openstack.common import processutils from nova import utils diff --git a/nova/network/base_api.py b/nova/network/base_api.py index 2d784ae104..d8ff36bc46 100644 --- a/nova/network/base_api.py +++ b/nova/network/base_api.py @@ -16,7 +16,6 @@ import functools import inspect -from oslo.concurrency import lockutils from oslo.utils import excutils from nova.db import base @@ -24,6 +23,7 @@ from nova import hooks from nova.i18n import _ from nova.network import model as network_model from nova import objects +from nova.openstack.common import lockutils from nova.openstack.common import log as logging diff --git a/nova/network/floating_ips.py b/nova/network/floating_ips.py index f92814d67c..5b61935531 100644 --- a/nova/network/floating_ips.py +++ b/nova/network/floating_ips.py @@ -15,7 +15,6 @@ # License for the specific language governing permissions and limitations # under the License. -from oslo.concurrency import processutils from oslo.config import cfg from oslo import messaging from oslo.utils import excutils @@ -29,6 +28,7 @@ from nova.i18n import _, _LE from nova.network import rpcapi as network_rpcapi from nova import objects from nova.openstack.common import log as logging +from nova.openstack.common import processutils from nova.openstack.common import uuidutils from nova import quota from nova import rpc diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index 89175b0287..da52d154c0 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -23,7 +23,6 @@ import os import re import netaddr -from oslo.concurrency import processutils from oslo.config import cfg from oslo.serialization import jsonutils from oslo.utils import excutils @@ -36,6 +35,7 @@ from nova.i18n import _, _LE from nova import objects from nova.openstack.common import fileutils from nova.openstack.common import log as logging +from nova.openstack.common import processutils from nova import paths from nova import utils diff --git a/nova/network/neutronv2/__init__.py b/nova/network/neutronv2/__init__.py index 67b82c832a..e442ae8e44 100644 --- a/nova/network/neutronv2/__init__.py +++ b/nova/network/neutronv2/__init__.py @@ -15,9 +15,9 @@ from neutronclient.common import exceptions from neutronclient.v2_0 import client as clientv20 -from oslo.concurrency import lockutils from oslo.config import cfg +from nova.openstack.common import lockutils from nova.openstack.common import log as logging CONF = cfg.CONF diff --git a/nova/network/neutronv2/api.py b/nova/network/neutronv2/api.py index 0597316f63..4b9dcb2275 100644 --- a/nova/network/neutronv2/api.py +++ b/nova/network/neutronv2/api.py @@ -19,7 +19,6 @@ import time import uuid from neutronclient.common import exceptions as neutron_client_exc -from oslo.concurrency import lockutils from oslo.config import cfg from oslo.utils import excutils import six @@ -34,6 +33,7 @@ from nova.network import model as network_model from nova.network import neutronv2 from nova.network.neutronv2 import constants from nova import objects +from nova.openstack.common import lockutils from nova.openstack.common import log as logging from nova.openstack.common import uuidutils from nova.pci import manager as pci_manager diff --git a/nova/openstack/common/lockutils.py b/nova/openstack/common/lockutils.py new file mode 100644 index 0000000000..6d5258762d --- /dev/null +++ b/nova/openstack/common/lockutils.py @@ -0,0 +1,326 @@ +# Copyright 2011 OpenStack Foundation. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import contextlib +import errno +import functools +import logging +import os +import shutil +import subprocess +import sys +import tempfile +import threading +import time +import weakref + +from oslo.config import cfg + +from nova.openstack.common import fileutils +from nova.openstack.common._i18n import _, _LE, _LI + + +LOG = logging.getLogger(__name__) + + +util_opts = [ + cfg.BoolOpt('disable_process_locking', default=False, + help='Enables or disables inter-process locks.'), + cfg.StrOpt('lock_path', + default=os.environ.get("NOVA_LOCK_PATH"), + help='Directory to use for lock files.') +] + + +CONF = cfg.CONF +CONF.register_opts(util_opts) + + +def set_defaults(lock_path): + cfg.set_defaults(util_opts, lock_path=lock_path) + + +class _FileLock(object): + """Lock implementation which allows multiple locks, working around + issues like bugs.debian.org/cgi-bin/bugreport.cgi?bug=632857 and does + not require any cleanup. Since the lock is always held on a file + descriptor rather than outside of the process, the lock gets dropped + automatically if the process crashes, even if __exit__ is not executed. + + There are no guarantees regarding usage by multiple green threads in a + single process here. This lock works only between processes. Exclusive + access between local threads should be achieved using the semaphores + in the @synchronized decorator. + + Note these locks are released when the descriptor is closed, so it's not + safe to close the file descriptor while another green thread holds the + lock. Just opening and closing the lock file can break synchronisation, + so lock files must be accessed only using this abstraction. + """ + + def __init__(self, name): + self.lockfile = None + self.fname = name + + def acquire(self): + basedir = os.path.dirname(self.fname) + + if not os.path.exists(basedir): + fileutils.ensure_tree(basedir) + LOG.info(_LI('Created lock path: %s'), basedir) + + self.lockfile = open(self.fname, 'w') + + while True: + try: + # Using non-blocking locks since green threads are not + # patched to deal with blocking locking calls. + # Also upon reading the MSDN docs for locking(), it seems + # to have a laughable 10 attempts "blocking" mechanism. + self.trylock() + LOG.debug('Got file lock "%s"', self.fname) + return True + except IOError as e: + if e.errno in (errno.EACCES, errno.EAGAIN): + # external locks synchronise things like iptables + # updates - give it some time to prevent busy spinning + time.sleep(0.01) + else: + raise threading.ThreadError(_("Unable to acquire lock on" + " `%(filename)s` due to" + " %(exception)s") % + {'filename': self.fname, + 'exception': e}) + + def __enter__(self): + self.acquire() + return self + + def release(self): + try: + self.unlock() + self.lockfile.close() + LOG.debug('Released file lock "%s"', self.fname) + except IOError: + LOG.exception(_LE("Could not release the acquired lock `%s`"), + self.fname) + + def __exit__(self, exc_type, exc_val, exc_tb): + self.release() + + def exists(self): + return os.path.exists(self.fname) + + def trylock(self): + raise NotImplementedError() + + def unlock(self): + raise NotImplementedError() + + +class _WindowsLock(_FileLock): + def trylock(self): + msvcrt.locking(self.lockfile.fileno(), msvcrt.LK_NBLCK, 1) + + def unlock(self): + msvcrt.locking(self.lockfile.fileno(), msvcrt.LK_UNLCK, 1) + + +class _FcntlLock(_FileLock): + def trylock(self): + fcntl.lockf(self.lockfile, fcntl.LOCK_EX | fcntl.LOCK_NB) + + def unlock(self): + fcntl.lockf(self.lockfile, fcntl.LOCK_UN) + + +if os.name == 'nt': + import msvcrt + InterProcessLock = _WindowsLock +else: + import fcntl + InterProcessLock = _FcntlLock + +_semaphores = weakref.WeakValueDictionary() +_semaphores_lock = threading.Lock() + + +def _get_lock_path(name, lock_file_prefix, lock_path=None): + # NOTE(mikal): the lock name cannot contain directory + # separators + name = name.replace(os.sep, '_') + if lock_file_prefix: + sep = '' if lock_file_prefix.endswith('-') else '-' + name = '%s%s%s' % (lock_file_prefix, sep, name) + + local_lock_path = lock_path or CONF.lock_path + + if not local_lock_path: + raise cfg.RequiredOptError('lock_path') + + return os.path.join(local_lock_path, name) + + +def external_lock(name, lock_file_prefix=None, lock_path=None): + LOG.debug('Attempting to grab external lock "%(lock)s"', + {'lock': name}) + + lock_file_path = _get_lock_path(name, lock_file_prefix, lock_path) + + return InterProcessLock(lock_file_path) + + +def remove_external_lock_file(name, lock_file_prefix=None): + """Remove an external lock file when it's not used anymore + This will be helpful when we have a lot of lock files + """ + with internal_lock(name): + lock_file_path = _get_lock_path(name, lock_file_prefix) + try: + os.remove(lock_file_path) + except OSError: + LOG.info(_LI('Failed to remove file %(file)s'), + {'file': lock_file_path}) + + +def internal_lock(name): + with _semaphores_lock: + try: + sem = _semaphores[name] + LOG.debug('Using existing semaphore "%s"', name) + except KeyError: + sem = threading.Semaphore() + _semaphores[name] = sem + LOG.debug('Created new semaphore "%s"', name) + + return sem + + +@contextlib.contextmanager +def lock(name, lock_file_prefix=None, external=False, lock_path=None): + """Context based lock + + This function yields a `threading.Semaphore` instance (if we don't use + eventlet.monkey_patch(), else `semaphore.Semaphore`) unless external is + True, in which case, it'll yield an InterProcessLock instance. + + :param lock_file_prefix: The lock_file_prefix argument is used to provide + lock files on disk with a meaningful prefix. + + :param external: The external keyword argument denotes whether this lock + should work across multiple processes. This means that if two different + workers both run a method decorated with @synchronized('mylock', + external=True), only one of them will execute at a time. + """ + int_lock = internal_lock(name) + with int_lock: + LOG.debug('Acquired semaphore "%(lock)s"', {'lock': name}) + try: + if external and not CONF.disable_process_locking: + ext_lock = external_lock(name, lock_file_prefix, lock_path) + with ext_lock: + yield ext_lock + else: + yield int_lock + finally: + LOG.debug('Releasing semaphore "%(lock)s"', {'lock': name}) + + +def synchronized(name, lock_file_prefix=None, external=False, lock_path=None): + """Synchronization decorator. + + Decorating a method like so:: + + @synchronized('mylock') + def foo(self, *args): + ... + + ensures that only one thread will execute the foo method at a time. + + Different methods can share the same lock:: + + @synchronized('mylock') + def foo(self, *args): + ... + + @synchronized('mylock') + def bar(self, *args): + ... + + This way only one of either foo or bar can be executing at a time. + """ + + def wrap(f): + @functools.wraps(f) + def inner(*args, **kwargs): + try: + with lock(name, lock_file_prefix, external, lock_path): + LOG.debug('Got semaphore / lock "%(function)s"', + {'function': f.__name__}) + return f(*args, **kwargs) + finally: + LOG.debug('Semaphore / lock released "%(function)s"', + {'function': f.__name__}) + return inner + return wrap + + +def synchronized_with_prefix(lock_file_prefix): + """Partial object generator for the synchronization decorator. + + Redefine @synchronized in each project like so:: + + (in oslo.utils.py) + from nova.openstack.common import lockutils + + synchronized = lockutils.synchronized_with_prefix('nova-') + + + (in nova/foo.py) + from nova import utils + + @utils.synchronized('mylock') + def bar(self, *args): + ... + + The lock_file_prefix argument is used to provide lock files on disk with a + meaningful prefix. + """ + + return functools.partial(synchronized, lock_file_prefix=lock_file_prefix) + + +def main(argv): + """Create a dir for locks and pass it to command from arguments + + If you run this: + python -m openstack.common.lockutils python setup.py testr <etc> + + a temporary directory will be created for all your locks and passed to all + your tests in an environment variable. The temporary dir will be deleted + afterwards and the return value will be preserved. + """ + + lock_dir = tempfile.mkdtemp() + os.environ["NOVA_LOCK_PATH"] = lock_dir + try: + ret_val = subprocess.call(argv[1:]) + finally: + shutil.rmtree(lock_dir, ignore_errors=True) + return ret_val + + +if __name__ == '__main__': + sys.exit(main(sys.argv)) diff --git a/nova/openstack/common/processutils.py b/nova/openstack/common/processutils.py new file mode 100644 index 0000000000..9de8e90db2 --- /dev/null +++ b/nova/openstack/common/processutils.py @@ -0,0 +1,289 @@ +# Copyright 2011 OpenStack Foundation. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +System-level utilities and helper functions. +""" + +import errno +import logging +import multiprocessing +import os +import random +import shlex +import signal + +from eventlet.green import subprocess +from eventlet import greenthread +from oslo.utils import strutils +import six + +from nova.openstack.common._i18n import _ + + +LOG = logging.getLogger(__name__) + + +class InvalidArgumentError(Exception): + def __init__(self, message=None): + super(InvalidArgumentError, self).__init__(message) + + +class UnknownArgumentError(Exception): + def __init__(self, message=None): + super(UnknownArgumentError, self).__init__(message) + + +class ProcessExecutionError(Exception): + def __init__(self, stdout=None, stderr=None, exit_code=None, cmd=None, + description=None): + self.exit_code = exit_code + self.stderr = stderr + self.stdout = stdout + self.cmd = cmd + self.description = description + + if description is None: + description = _("Unexpected error while running command.") + if exit_code is None: + exit_code = '-' + message = _('%(description)s\n' + 'Command: %(cmd)s\n' + 'Exit code: %(exit_code)s\n' + 'Stdout: %(stdout)r\n' + 'Stderr: %(stderr)r') % {'description': description, + 'cmd': cmd, + 'exit_code': exit_code, + 'stdout': stdout, + 'stderr': stderr} + super(ProcessExecutionError, self).__init__(message) + + +class NoRootWrapSpecified(Exception): + def __init__(self, message=None): + super(NoRootWrapSpecified, self).__init__(message) + + +def _subprocess_setup(): + # Python installs a SIGPIPE handler by default. This is usually not what + # non-Python subprocesses expect. + signal.signal(signal.SIGPIPE, signal.SIG_DFL) + + +def execute(*cmd, **kwargs): + """Helper method to shell out and execute a command through subprocess. + + Allows optional retry. + + :param cmd: Passed to subprocess.Popen. + :type cmd: string + :param process_input: Send to opened process. + :type process_input: string + :param env_variables: Environment variables and their values that + will be set for the process. + :type env_variables: dict + :param check_exit_code: Single bool, int, or list of allowed exit + codes. Defaults to [0]. Raise + :class:`ProcessExecutionError` unless + program exits with one of these code. + :type check_exit_code: boolean, int, or [int] + :param delay_on_retry: True | False. Defaults to True. If set to True, + wait a short amount of time before retrying. + :type delay_on_retry: boolean + :param attempts: How many times to retry cmd. + :type attempts: int + :param run_as_root: True | False. Defaults to False. If set to True, + the command is prefixed by the command specified + in the root_helper kwarg. + :type run_as_root: boolean + :param root_helper: command to prefix to commands called with + run_as_root=True + :type root_helper: string + :param shell: whether or not there should be a shell used to + execute this command. Defaults to false. + :type shell: boolean + :param loglevel: log level for execute commands. + :type loglevel: int. (Should be logging.DEBUG or logging.INFO) + :returns: (stdout, stderr) from process execution + :raises: :class:`UnknownArgumentError` on + receiving unknown arguments + :raises: :class:`ProcessExecutionError` + """ + + process_input = kwargs.pop('process_input', None) + env_variables = kwargs.pop('env_variables', None) + check_exit_code = kwargs.pop('check_exit_code', [0]) + ignore_exit_code = False + delay_on_retry = kwargs.pop('delay_on_retry', True) + attempts = kwargs.pop('attempts', 1) + run_as_root = kwargs.pop('run_as_root', False) + root_helper = kwargs.pop('root_helper', '') + shell = kwargs.pop('shell', False) + loglevel = kwargs.pop('loglevel', logging.DEBUG) + + if isinstance(check_exit_code, bool): + ignore_exit_code = not check_exit_code + check_exit_code = [0] + elif isinstance(check_exit_code, int): + check_exit_code = [check_exit_code] + + if kwargs: + raise UnknownArgumentError(_('Got unknown keyword args: %r') % kwargs) + + if run_as_root and hasattr(os, 'geteuid') and os.geteuid() != 0: + if not root_helper: + raise NoRootWrapSpecified( + message=_('Command requested root, but did not ' + 'specify a root helper.')) + cmd = shlex.split(root_helper) + list(cmd) + + cmd = map(str, cmd) + sanitized_cmd = strutils.mask_password(' '.join(cmd)) + + while attempts > 0: + attempts -= 1 + try: + LOG.log(loglevel, _('Running cmd (subprocess): %s'), sanitized_cmd) + _PIPE = subprocess.PIPE # pylint: disable=E1101 + + if os.name == 'nt': + preexec_fn = None + close_fds = False + else: + preexec_fn = _subprocess_setup + close_fds = True + + obj = subprocess.Popen(cmd, + stdin=_PIPE, + stdout=_PIPE, + stderr=_PIPE, + close_fds=close_fds, + preexec_fn=preexec_fn, + shell=shell, + env=env_variables) + result = None + for _i in six.moves.range(20): + # NOTE(russellb) 20 is an arbitrary number of retries to + # prevent any chance of looping forever here. + try: + if process_input is not None: + result = obj.communicate(process_input) + else: + result = obj.communicate() + except OSError as e: + if e.errno in (errno.EAGAIN, errno.EINTR): + continue + raise + break + obj.stdin.close() # pylint: disable=E1101 + _returncode = obj.returncode # pylint: disable=E1101 + LOG.log(loglevel, 'Result was %s' % _returncode) + if not ignore_exit_code and _returncode not in check_exit_code: + (stdout, stderr) = result + sanitized_stdout = strutils.mask_password(stdout) + sanitized_stderr = strutils.mask_password(stderr) + raise ProcessExecutionError(exit_code=_returncode, + stdout=sanitized_stdout, + stderr=sanitized_stderr, + cmd=sanitized_cmd) + return result + except ProcessExecutionError: + if not attempts: + raise + else: + LOG.log(loglevel, _('%r failed. Retrying.'), sanitized_cmd) + if delay_on_retry: + greenthread.sleep(random.randint(20, 200) / 100.0) + finally: + # NOTE(termie): this appears to be necessary to let the subprocess + # call clean something up in between calls, without + # it two execute calls in a row hangs the second one + greenthread.sleep(0) + + +def trycmd(*args, **kwargs): + """A wrapper around execute() to more easily handle warnings and errors. + + Returns an (out, err) tuple of strings containing the output of + the command's stdout and stderr. If 'err' is not empty then the + command can be considered to have failed. + + :discard_warnings True | False. Defaults to False. If set to True, + then for succeeding commands, stderr is cleared + + """ + discard_warnings = kwargs.pop('discard_warnings', False) + + try: + out, err = execute(*args, **kwargs) + failed = False + except ProcessExecutionError as exn: + out, err = '', six.text_type(exn) + failed = True + + if not failed and discard_warnings and err: + # Handle commands that output to stderr but otherwise succeed + err = '' + + return out, err + + +def ssh_execute(ssh, cmd, process_input=None, + addl_env=None, check_exit_code=True): + sanitized_cmd = strutils.mask_password(cmd) + LOG.debug('Running cmd (SSH): %s', sanitized_cmd) + if addl_env: + raise InvalidArgumentError(_('Environment not supported over SSH')) + + if process_input: + # This is (probably) fixable if we need it... + raise InvalidArgumentError(_('process_input not supported over SSH')) + + stdin_stream, stdout_stream, stderr_stream = ssh.exec_command(cmd) + channel = stdout_stream.channel + + # NOTE(justinsb): This seems suspicious... + # ...other SSH clients have buffering issues with this approach + stdout = stdout_stream.read() + sanitized_stdout = strutils.mask_password(stdout) + stderr = stderr_stream.read() + sanitized_stderr = strutils.mask_password(stderr) + + stdin_stream.close() + + exit_status = channel.recv_exit_status() + + # exit_status == -1 if no exit code was returned + if exit_status != -1: + LOG.debug('Result was %s' % exit_status) + if check_exit_code and exit_status != 0: + raise ProcessExecutionError(exit_code=exit_status, + stdout=sanitized_stdout, + stderr=sanitized_stderr, + cmd=sanitized_cmd) + + return (sanitized_stdout, sanitized_stderr) + + +def get_worker_count(): + """Utility to get the default worker count. + + @return: The number of CPUs if that can be determined, else a default + worker count of 1 is returned. + """ + try: + return multiprocessing.cpu_count() + except NotImplementedError: + return 1 diff --git a/nova/service.py b/nova/service.py index 6f27a3808b..2815f1fe28 100644 --- a/nova/service.py +++ b/nova/service.py @@ -21,7 +21,6 @@ import os import random import sys -from oslo.concurrency import processutils from oslo.config import cfg from oslo import messaging from oslo.utils import importutils @@ -34,6 +33,7 @@ from nova import exception from nova.i18n import _, _LE from nova.objects import base as objects_base from nova.openstack.common import log as logging +from nova.openstack.common import processutils from nova.openstack.common import service from nova import rpc from nova import servicegroup diff --git a/nova/storage/linuxscsi.py b/nova/storage/linuxscsi.py index 713a082845..08577b0f6d 100644 --- a/nova/storage/linuxscsi.py +++ b/nova/storage/linuxscsi.py @@ -14,11 +14,10 @@ """Generic linux scsi subsystem utilities.""" -from oslo.concurrency import processutils - from nova.i18n import _LW from nova.openstack.common import log as logging from nova.openstack.common import loopingcall +from nova.openstack.common import processutils from nova import utils LOG = logging.getLogger(__name__) diff --git a/nova/tests/db/test_migrations.py b/nova/tests/db/test_migrations.py index 1ebe8ad470..666e825a5a 100644 --- a/nova/tests/db/test_migrations.py +++ b/nova/tests/db/test_migrations.py @@ -44,7 +44,6 @@ import glob import os from migrate.versioning import repository -from oslo.concurrency import processutils from oslo.db.sqlalchemy import session from oslo.db.sqlalchemy import utils as oslodbutils import six.moves.urllib.parse as urlparse @@ -55,6 +54,7 @@ import nova.db.sqlalchemy.migrate_repo from nova.db.sqlalchemy import utils as db_utils from nova.i18n import _ from nova.openstack.common import log as logging +from nova.openstack.common import processutils from nova import test from nova import utils diff --git a/nova/tests/fake_processutils.py b/nova/tests/fake_processutils.py index 111540d1d1..7d7b4c23b4 100644 --- a/nova/tests/fake_processutils.py +++ b/nova/tests/fake_processutils.py @@ -12,15 +12,15 @@ # License for the specific language governing permissions and limitations # under the License. -"""This modules stubs out functions in oslo.concurrency.processutils.""" +"""This modules stubs out functions in openstack.common.processutils.""" import re from eventlet import greenthread -from oslo.concurrency import processutils import six from nova.openstack.common import log as logging +from nova.openstack.common import processutils LOG = logging.getLogger(__name__) diff --git a/nova/tests/network/test_linux_net.py b/nova/tests/network/test_linux_net.py index ef4c567af8..15a20eb7d5 100644 --- a/nova/tests/network/test_linux_net.py +++ b/nova/tests/network/test_linux_net.py @@ -20,7 +20,6 @@ import os import mock import mox -from oslo.concurrency import processutils from oslo.config import cfg from oslo.serialization import jsonutils from oslo.utils import timeutils @@ -33,6 +32,7 @@ from nova.network import linux_net from nova import objects from nova.openstack.common import fileutils from nova.openstack.common import log as logging +from nova.openstack.common import processutils from nova import test from nova import utils diff --git a/nova/tests/network/test_manager.py b/nova/tests/network/test_manager.py index 8c41164555..0dfa0598cd 100644 --- a/nova/tests/network/test_manager.py +++ b/nova/tests/network/test_manager.py @@ -21,7 +21,6 @@ import fixtures import mock import mox import netaddr -from oslo.concurrency import processutils from oslo.config import cfg from oslo.db import exception as db_exc from oslo import messaging @@ -41,6 +40,7 @@ from nova import objects from nova.objects import quotas as quotas_obj from nova.objects import virtual_interface as vif_obj from nova.openstack.common import log as logging +from nova.openstack.common import processutils from nova import quota from nova import test from nova.tests import fake_instance diff --git a/nova/tests/network/test_neutronv2.py b/nova/tests/network/test_neutronv2.py index f462fa8fa8..52389b46e4 100644 --- a/nova/tests/network/test_neutronv2.py +++ b/nova/tests/network/test_neutronv2.py @@ -2671,7 +2671,7 @@ class TestNeutronv2WithMock(test.TestCase): 'fake-user', 'fake-project', auth_token='bff4a5a6b9eb4ea2a6efec6eefb77936') - @mock.patch('oslo.concurrency.lockutils.lock') + @mock.patch('nova.openstack.common.lockutils.lock') def test_get_instance_nw_info_locks_per_instance(self, mock_lock): instance = objects.Instance(uuid=uuid.uuid4()) api = neutronapi.API() diff --git a/nova/tests/test_crypto.py b/nova/tests/test_crypto.py index 49634626a3..392937f864 100644 --- a/nova/tests/test_crypto.py +++ b/nova/tests/test_crypto.py @@ -20,11 +20,11 @@ import os import mock import mox -from oslo.concurrency import processutils from nova import crypto from nova import db from nova import exception +from nova.openstack.common import processutils from nova import test from nova import utils diff --git a/nova/tests/test_service.py b/nova/tests/test_service.py index 4bc7104945..4a00aacf7a 100644 --- a/nova/tests/test_service.py +++ b/nova/tests/test_service.py @@ -22,7 +22,6 @@ import sys import mock import mox -from oslo.concurrency import processutils from oslo.config import cfg import testtools @@ -30,6 +29,7 @@ from nova import context from nova import db from nova import exception from nova import manager +from nova.openstack.common import processutils from nova.openstack.common import service as _service from nova import rpc from nova import service diff --git a/nova/tests/test_utils.py b/nova/tests/test_utils.py index 548e973b6a..0c1da9535e 100644 --- a/nova/tests/test_utils.py +++ b/nova/tests/test_utils.py @@ -24,12 +24,12 @@ import tempfile import mox import netaddr -from oslo.concurrency import processutils from oslo.config import cfg from oslo.utils import timeutils import nova from nova import exception +from nova.openstack.common import processutils from nova import test from nova import utils diff --git a/nova/tests/virt/disk/test_api.py b/nova/tests/virt/disk/test_api.py index 1f62c33b51..c86e8bc0b4 100644 --- a/nova/tests/virt/disk/test_api.py +++ b/nova/tests/virt/disk/test_api.py @@ -16,8 +16,8 @@ import tempfile import fixtures -from oslo.concurrency import processutils +from nova.openstack.common import processutils from nova import test from nova import utils from nova.virt.disk import api diff --git a/nova/tests/virt/disk/vfs/test_localfs.py b/nova/tests/virt/disk/vfs/test_localfs.py index 16935498ac..7c88708b94 100644 --- a/nova/tests/virt/disk/vfs/test_localfs.py +++ b/nova/tests/virt/disk/vfs/test_localfs.py @@ -12,10 +12,10 @@ # License for the specific language governing permissions and limitations # under the License. -from oslo.concurrency import processutils from oslo.config import cfg from nova import exception +from nova.openstack.common import processutils from nova import test from nova.tests import utils as tests_utils import nova.utils diff --git a/nova/tests/virt/libvirt/test_driver.py b/nova/tests/virt/libvirt/test_driver.py index 3c346f03a4..d658ee5f3e 100644 --- a/nova/tests/virt/libvirt/test_driver.py +++ b/nova/tests/virt/libvirt/test_driver.py @@ -32,8 +32,6 @@ import fixtures from lxml import etree import mock import mox -from oslo.concurrency import lockutils -from oslo.concurrency import processutils from oslo.config import cfg from oslo.serialization import jsonutils from oslo.utils import importutils @@ -55,7 +53,9 @@ from nova import exception from nova.network import model as network_model from nova import objects from nova.openstack.common import fileutils +from nova.openstack.common import lockutils from nova.openstack.common import loopingcall +from nova.openstack.common import processutils from nova.openstack.common import uuidutils from nova.pci import manager as pci_manager from nova import test diff --git a/nova/tests/virt/libvirt/test_firewall.py b/nova/tests/virt/libvirt/test_firewall.py index f928825fba..3210760c4d 100644 --- a/nova/tests/virt/libvirt/test_firewall.py +++ b/nova/tests/virt/libvirt/test_firewall.py @@ -21,12 +21,12 @@ from xml.dom import minidom from lxml import etree import mock import mox -from oslo.concurrency import lockutils from nova.compute import utils as compute_utils from nova import exception from nova.network import linux_net from nova import objects +from nova.openstack.common import lockutils from nova import test from nova.tests import fake_network from nova.tests.virt.libvirt import fakelibvirt diff --git a/nova/tests/virt/libvirt/test_imagecache.py b/nova/tests/virt/libvirt/test_imagecache.py index a2eb4a1c67..37540d4449 100644 --- a/nova/tests/virt/libvirt/test_imagecache.py +++ b/nova/tests/virt/libvirt/test_imagecache.py @@ -20,7 +20,6 @@ import hashlib import os import time -from oslo.concurrency import processutils from oslo.config import cfg from oslo.serialization import jsonutils from oslo.utils import importutils @@ -28,6 +27,7 @@ from oslo.utils import importutils from nova import conductor from nova import db from nova.openstack.common import log as logging +from nova.openstack.common import processutils from nova import test from nova.tests import fake_instance from nova import utils diff --git a/nova/tests/virt/libvirt/test_lvm.py b/nova/tests/virt/libvirt/test_lvm.py index fdb3e4b9f6..e51ea9aeef 100644 --- a/nova/tests/virt/libvirt/test_lvm.py +++ b/nova/tests/virt/libvirt/test_lvm.py @@ -16,10 +16,10 @@ import contextlib import mock -from oslo.concurrency import processutils from oslo.config import cfg from nova import exception +from nova.openstack.common import processutils from nova import test from nova import utils from nova.virt.libvirt import lvm diff --git a/nova/tests/virt/libvirt/test_utils.py b/nova/tests/virt/libvirt/test_utils.py index 4114c03516..bf5663a575 100644 --- a/nova/tests/virt/libvirt/test_utils.py +++ b/nova/tests/virt/libvirt/test_utils.py @@ -18,11 +18,11 @@ import os import tempfile import mock -from oslo.concurrency import processutils from oslo.config import cfg from nova import exception from nova.openstack.common import fileutils +from nova.openstack.common import processutils from nova import test from nova import utils from nova.virt.disk import api as disk diff --git a/nova/tests/virt/libvirt/test_vif.py b/nova/tests/virt/libvirt/test_vif.py index 256ca58046..0e444dd849 100644 --- a/nova/tests/virt/libvirt/test_vif.py +++ b/nova/tests/virt/libvirt/test_vif.py @@ -16,12 +16,12 @@ import contextlib from lxml import etree import mock -from oslo.concurrency import processutils from oslo.config import cfg from nova import exception from nova.network import linux_net from nova.network import model as network_model +from nova.openstack.common import processutils from nova import test from nova.tests.virt.libvirt import fakelibvirt from nova import utils diff --git a/nova/tests/virt/libvirt/test_volume.py b/nova/tests/virt/libvirt/test_volume.py index 48437d1c6d..8688bb35e7 100644 --- a/nova/tests/virt/libvirt/test_volume.py +++ b/nova/tests/virt/libvirt/test_volume.py @@ -19,10 +19,10 @@ import time import fixtures import mock -from oslo.concurrency import processutils from oslo.config import cfg from nova import exception +from nova.openstack.common import processutils from nova.storage import linuxscsi from nova import test from nova.tests.virt.libvirt import fake_libvirt_utils diff --git a/nova/tests/virt/test_images.py b/nova/tests/virt/test_images.py index be5ea73ef1..f7057aba52 100644 --- a/nova/tests/virt/test_images.py +++ b/nova/tests/virt/test_images.py @@ -15,9 +15,9 @@ import os import mock -from oslo.concurrency import processutils from nova import exception +from nova.openstack.common import processutils from nova import test from nova import utils from nova.virt import images diff --git a/nova/tests/virt/xenapi/test_vm_utils.py b/nova/tests/virt/xenapi/test_vm_utils.py index 990c14d744..f65891fbec 100644 --- a/nova/tests/virt/xenapi/test_vm_utils.py +++ b/nova/tests/virt/xenapi/test_vm_utils.py @@ -20,7 +20,6 @@ from eventlet import greenthread import fixtures import mock import mox -from oslo.concurrency import processutils from oslo.config import cfg from oslo.utils import timeutils from oslo.utils import units @@ -32,6 +31,7 @@ from nova.compute import vm_mode from nova import context from nova import exception from nova.i18n import _ +from nova.openstack.common import processutils from nova import test from nova.tests.virt.xenapi import stubs from nova.tests.virt.xenapi import test_xenapi diff --git a/nova/utils.py b/nova/utils.py index 90524c2b96..60a3668094 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -36,8 +36,6 @@ from xml.sax import saxutils import eventlet import netaddr -from oslo.concurrency import lockutils -from oslo.concurrency import processutils from oslo.config import cfg from oslo import messaging from oslo.utils import excutils @@ -47,7 +45,9 @@ import six from nova import exception from nova.i18n import _, _LE +from nova.openstack.common import lockutils from nova.openstack.common import log as logging +from nova.openstack.common import processutils notify_decorator = 'nova.notifications.notify_decorator' diff --git a/nova/virt/disk/api.py b/nova/virt/disk/api.py index b808ab9eaf..b87f2b7d1d 100644 --- a/nova/virt/disk/api.py +++ b/nova/virt/disk/api.py @@ -30,7 +30,6 @@ import tempfile if os.name != 'nt': import crypt -from oslo.concurrency import processutils from oslo.config import cfg from oslo.serialization import jsonutils @@ -39,6 +38,7 @@ from nova.i18n import _ from nova.i18n import _LE from nova.i18n import _LW from nova.openstack.common import log as logging +from nova.openstack.common import processutils from nova import utils from nova.virt.disk.mount import api as mount from nova.virt.disk.vfs import api as vfs diff --git a/nova/virt/hyperv/vmops.py b/nova/virt/hyperv/vmops.py index 4f4ac2cce3..b1b8e01a6f 100644 --- a/nova/virt/hyperv/vmops.py +++ b/nova/virt/hyperv/vmops.py @@ -22,7 +22,6 @@ import os import time from eventlet import timeout as etimeout -from oslo.concurrency import processutils from oslo.config import cfg from oslo.utils import excutils from oslo.utils import importutils @@ -34,6 +33,7 @@ from nova.i18n import _, _LI, _LE, _LW from nova.openstack.common import fileutils from nova.openstack.common import log as logging from nova.openstack.common import loopingcall +from nova.openstack.common import processutils from nova.openstack.common import uuidutils from nova import utils from nova.virt import configdrive diff --git a/nova/virt/libvirt/driver.py b/nova/virt/libvirt/driver.py index 467d4fc853..e9a8a53a87 100644 --- a/nova/virt/libvirt/driver.py +++ b/nova/virt/libvirt/driver.py @@ -48,7 +48,6 @@ from eventlet import patcher from eventlet import tpool from eventlet import util as eventlet_util from lxml import etree -from oslo.concurrency import processutils from oslo.config import cfg from oslo.serialization import jsonutils from oslo.utils import excutils @@ -81,6 +80,7 @@ from nova import objects from nova.openstack.common import fileutils from nova.openstack.common import log as logging from nova.openstack.common import loopingcall +from nova.openstack.common import processutils from nova.pci import manager as pci_manager from nova.pci import utils as pci_utils from nova.pci import whitelist as pci_whitelist diff --git a/nova/virt/libvirt/imagecache.py b/nova/virt/libvirt/imagecache.py index d264482271..5c09b79ce6 100644 --- a/nova/virt/libvirt/imagecache.py +++ b/nova/virt/libvirt/imagecache.py @@ -25,7 +25,6 @@ import os import re import time -from oslo.concurrency import processutils from oslo.config import cfg from oslo.serialization import jsonutils @@ -34,6 +33,7 @@ from nova.i18n import _LI from nova.i18n import _LW from nova.openstack.common import fileutils from nova.openstack.common import log as logging +from nova.openstack.common import processutils from nova import utils from nova.virt import imagecache from nova.virt.libvirt import utils as libvirt_utils diff --git a/nova/virt/libvirt/lvm.py b/nova/virt/libvirt/lvm.py index c159bffbc8..76415d598c 100644 --- a/nova/virt/libvirt/lvm.py +++ b/nova/virt/libvirt/lvm.py @@ -21,7 +21,6 @@ import functools -from oslo.concurrency import processutils from oslo.config import cfg from oslo.utils import units import six @@ -31,6 +30,7 @@ from nova.i18n import _ from nova.i18n import _LE from nova.i18n import _LW from nova.openstack.common import log as logging +from nova.openstack.common import processutils from nova import utils as nova_utils from nova.virt.libvirt import utils diff --git a/nova/virt/libvirt/utils.py b/nova/virt/libvirt/utils.py index 487f45487e..c05de9354e 100644 --- a/nova/virt/libvirt/utils.py +++ b/nova/virt/libvirt/utils.py @@ -24,13 +24,13 @@ import platform import re from lxml import etree -from oslo.concurrency import processutils from oslo.config import cfg from nova.i18n import _ from nova.i18n import _LI from nova.i18n import _LW from nova.openstack.common import log as logging +from nova.openstack.common import processutils from nova import utils from nova.virt import images from nova.virt.libvirt import config as vconfig diff --git a/nova/virt/libvirt/vif.py b/nova/virt/libvirt/vif.py index a4d3d51f47..71404f08ac 100644 --- a/nova/virt/libvirt/vif.py +++ b/nova/virt/libvirt/vif.py @@ -19,7 +19,6 @@ import copy -from oslo.concurrency import processutils from oslo.config import cfg from nova import exception @@ -28,6 +27,7 @@ from nova.i18n import _LE from nova.network import linux_net from nova.network import model as network_model from nova.openstack.common import log as logging +from nova.openstack.common import processutils from nova import utils from nova.virt.libvirt import config as vconfig from nova.virt.libvirt import designer diff --git a/nova/virt/libvirt/volume.py b/nova/virt/libvirt/volume.py index 1ddf4d7a75..8f118ed078 100644 --- a/nova/virt/libvirt/volume.py +++ b/nova/virt/libvirt/volume.py @@ -21,7 +21,6 @@ import os import time import urllib2 -from oslo.concurrency import processutils from oslo.config import cfg from oslo.utils import strutils import six @@ -33,6 +32,7 @@ from nova.i18n import _LE from nova.i18n import _LW from nova.openstack.common import log as logging from nova.openstack.common import loopingcall +from nova.openstack.common import processutils from nova import paths from nova.storage import linuxscsi from nova import utils diff --git a/nova/virt/vmwareapi/imagecache.py b/nova/virt/vmwareapi/imagecache.py index 41b6aeb5d1..946da3aef4 100644 --- a/nova/virt/vmwareapi/imagecache.py +++ b/nova/virt/vmwareapi/imagecache.py @@ -35,12 +35,12 @@ the timestamp folder will be locked and the timestamps will be purged. This will ensure that a image is not deleted during the spawn. """ -from oslo.concurrency import lockutils from oslo.config import cfg from oslo.utils import timeutils from oslo.vmware import exceptions as vexc from nova.i18n import _ +from nova.openstack.common import lockutils from nova.openstack.common import log as logging from nova.virt import imagecache from nova.virt.vmwareapi import ds_util diff --git a/nova/virt/vmwareapi/vmops.py b/nova/virt/vmwareapi/vmops.py index 777e0e1e93..b935a20f3c 100644 --- a/nova/virt/vmwareapi/vmops.py +++ b/nova/virt/vmwareapi/vmops.py @@ -24,7 +24,6 @@ import os import time import decorator -from oslo.concurrency import lockutils from oslo.config import cfg from oslo.utils import excutils from oslo.utils import strutils @@ -41,6 +40,7 @@ from nova import context as nova_context from nova import exception from nova.i18n import _, _LE, _LW from nova import objects +from nova.openstack.common import lockutils from nova.openstack.common import log as logging from nova.openstack.common import uuidutils from nova import utils diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index be34e675ef..04ee9c7327 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -28,7 +28,6 @@ from xml.dom import minidom from xml.parsers import expat from eventlet import greenthread -from oslo.concurrency import processutils from oslo.config import cfg from oslo.utils import excutils from oslo.utils import importutils @@ -47,6 +46,7 @@ from nova import exception from nova.i18n import _, _LE, _LI from nova.network import model as network_model from nova.openstack.common import log as logging +from nova.openstack.common import processutils from nova.openstack.common import versionutils from nova import utils from nova.virt import configdrive diff --git a/nova/volume/encryptors/luks.py b/nova/volume/encryptors/luks.py index 5fe0f74b26..63cdf5e080 100644 --- a/nova/volume/encryptors/luks.py +++ b/nova/volume/encryptors/luks.py @@ -16,9 +16,8 @@ import re -from oslo.concurrency import processutils - from nova.openstack.common import log as logging +from nova.openstack.common import processutils from nova import utils from nova.volume.encryptors import cryptsetup diff --git a/openstack-common.conf b/openstack-common.conf index a52f990b9a..a276846d43 100644 --- a/openstack-common.conf +++ b/openstack-common.conf @@ -13,6 +13,7 @@ module=fixture.moxstubout module=imageutils module=install_venv_common module=local +module=lockutils module=log module=loopingcall module=memorycache @@ -20,6 +21,7 @@ module=middleware.base module=middleware.request_id module=periodic_task module=policy +module=processutils module=report module=report.generators module=report.models diff --git a/requirements.txt b/requirements.txt index db54ec1426..d707941acd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -34,7 +34,6 @@ six>=1.7.0 stevedore>=1.0.0 # Apache-2.0 websockify>=0.6.0,<0.7 wsgiref>=0.1.2 -oslo.concurrency>=0.1.0 # Apache-2.0 oslo.config>=1.4.0 # Apache-2.0 oslo.serialization>=1.0.0 # Apache-2.0 oslo.utils>=1.0.0 # Apache-2.0 diff --git a/run_tests.sh b/run_tests.sh index bd3efddd91..abc10fab15 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -117,7 +117,7 @@ function run_tests { # provided. testrargs="discover ./nova/tests" fi - ${wrapper} python -m oslo.concurrency.lockutils python -m testtools.run $testropts $testrargs + ${wrapper} python -m nova.openstack.common.lockutils python -m testtools.run $testropts $testrargs # Short circuit because all of the testr and coverage stuff # below does not make sense when running testtools.run for @@ -184,7 +184,7 @@ function run_pep8 { } -TESTRTESTS="python -m oslo.concurrency.lockutils python setup.py testr" +TESTRTESTS="python -m nova.openstack.common.lockutils python setup.py testr" if [ $never_venv -eq 0 ] then diff --git a/tools/pretty_tox.sh b/tools/pretty_tox.sh index ddc160b4fe..137f7df3f6 100755 --- a/tools/pretty_tox.sh +++ b/tools/pretty_tox.sh @@ -3,4 +3,4 @@ set -o pipefail TESTRARGS=$1 -python -m oslo.concurrency.lockutils python setup.py testr --slowest --testr-args="--subunit $TESTRARGS" | $(dirname $0)/subunit-trace.py -f +python -m nova.openstack.common.lockutils python setup.py testr --slowest --testr-args="--subunit $TESTRARGS" | $(dirname $0)/subunit-trace.py -f @@ -40,7 +40,7 @@ commands = bash tools/lintstack.sh # tests conflict with coverage. commands = coverage erase - python -m oslo.concurrency.lockutils python setup.py testr --coverage \ + python -m nova.openstack.common.lockutils python setup.py testr --coverage \ --testr-args='{posargs}' coverage combine coverage html --include='nova/*' --omit='nova/openstack/common/*' -d covhtml -i |