diff options
| author | ChangBo Guo(gcb) <eric.guo@easystack.cn> | 2015-12-10 17:31:45 +0800 |
|---|---|---|
| committer | ChangBo Guo(gcb) <eric.guo@easystack.cn> | 2015-12-16 11:31:54 +0800 |
| commit | 2a3d4ac23da5c93cf883d2122ab9cbec769a4d1f (patch) | |
| tree | 6ae4254a7e61a641b46c0b8d833f7e60a22e2348 | |
| parent | 5cd3657bf44ba2b2f4bb465147c147f4deca4186 (diff) | |
| download | oslo-incubator-2a3d4ac23da5c93cf883d2122ab9cbec769a4d1f.tar.gz | |
Follow up for clean up oslo-incubator code
We clean up oslo-incubator in 90ae25e38915cc502d9e9c52d59e8fb668a72ae1,
There are some obsolete things to clean.
* Remove api maintainers file and check script
* Remove sync tool with copy code to other openstack project.
* Remove graduate script
* remove section entry_points in setup.cfg
* Clean test related stuff in tox.ini
* Remove requirements.txt/test-requirements.txt
Change-Id: If79def1607c81a51f3e7e607620fae3995c79358
| -rw-r--r-- | MAINTAINERS | 91 | ||||
| -rw-r--r-- | MANIFEST.in | 6 | ||||
| -rw-r--r-- | README.rst | 19 | ||||
| -rw-r--r-- | TESTING.rst | 88 | ||||
| -rw-r--r-- | etc/.placeholder | 0 | ||||
| -rw-r--r-- | obsolete.txt | 2 | ||||
| -rw-r--r-- | requirements.txt | 22 | ||||
| -rwxr-xr-x | run_tests.sh | 54 | ||||
| -rw-r--r-- | setup.cfg | 18 | ||||
| -rw-r--r-- | test-requirements.txt | 17 | ||||
| -rwxr-xr-x | tools/check_maintainers.py | 77 | ||||
| -rwxr-xr-x | tools/graduate.sh | 91 | ||||
| -rw-r--r-- | tools/requirements.txt | 3 | ||||
| -rw-r--r-- | tox.ini | 41 | ||||
| -rw-r--r-- | update.py | 397 | ||||
| -rwxr-xr-x | update.sh | 14 |
16 files changed, 3 insertions, 937 deletions
diff --git a/MAINTAINERS b/MAINTAINERS deleted file mode 100644 index a3d52ccb..00000000 --- a/MAINTAINERS +++ /dev/null @@ -1,91 +0,0 @@ -= Generalist Code Reviewers = - -Oslo has a cadre of core reviewers who have taken on a generalist role -on the project. They are folks with good taste in Python code, provide -constructive input in their reviews and make time to review any patches -submitted to the project, irrespective of the area which a given patch -targets. - -The current members of oslo-core are listed here: - - https://review.openstack.org/#/admin/groups/106,members - -This the group who can +2 and approve patches in oslo. However, they may -choose to seek feedback from the appropriate specialist API maintainer -before approving a patch if it is in any way controversial or risky. - -= Specialist API Maintainers = - -APIs in oslo-incubator are resting here temporarily until they have been -cleaned up sufficiently so that we can make a commitment to backwards -compatibility and release the API in a properly published library. - -While in oslo-incubator, an API should have one or more specialist -maintainers who have taken on the following responsibilities: - - - Developing a roadmap for the API graduating from the incubator - - - Reviewing changes proposed for that API - a +1 from an API - maintainer should be treated as a +2 by oslo-core members - - - Keeping track of which Integrated projects are using the API and - helping to ensure the API meets the use cases of those projects - - - Ensuring that any incompatible or potentially troublesome changes - are synced from oslo-incubator into the projects using the API, - either by performing the sync themselves are asking that the authors - of the patch perform the sync - -= List of API Maintainers = - -Each API has an entry with the following keys: - - M: Maintainer email address - S: Status, one of the following: - Maintained: Has an active maintainer - Orphan: No current maintainer, feel free to step up! - Graduating: Currently being moved out of the incubator to a standalone project. - Obsolete: Replaced by newer code, or a dead end, or out-dated - F: Wildcard patterns, relative to openstack/common/ - -== cache == - -S: Deprecated -F: cache/ - -== cliutils == - -M: -S: Orphan -F: cliutils.py - -== crypto == - -M: Simo Sorce <simo@redhat.com> -S: Deprecated -F: crypto/ - -== fileutils == - -M: Zhongyue Luo <zhongyue.nah@intel.com> -S: Maintained -F: fileutils.py - -== imageutils == - -M: Zhongyue Luo <zhongyue.nah@intel.com> -S: Maintained -F: imageutils.py - -== memorycache == - -M: -S: Orphan -F: memorycache.py - - -== scheduler == - -M: -S: Orphan -F: scheduler/ diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index c978a52d..00000000 --- a/MANIFEST.in +++ /dev/null @@ -1,6 +0,0 @@ -include AUTHORS -include ChangeLog -exclude .gitignore -exclude .gitreview - -global-exclude *.pyc @@ -16,22 +16,3 @@ library. For more information, see our wiki page: https://wiki.openstack.org/wiki/Oslo - -Running Tests -------------- - -To run tests in virtualenvs (preferred):: - - sudo pip install tox - tox - -To run tests in the current environment:: - - sudo pip install -r requirements.txt - nosetests - -To run tests using MySQL or PostgreSQL as a DB backend do:: - - OS_TEST_DBAPI_ADMIN_CONNECTION=mysql+pymysql://user:password@host/database tox -e py27 - -Note, that your DB user must have permissions to create and drop databases. diff --git a/TESTING.rst b/TESTING.rst deleted file mode 100644 index 4191b1b2..00000000 --- a/TESTING.rst +++ /dev/null @@ -1,88 +0,0 @@ -=========================== -Testing Your OpenStack Code -=========================== ------------- -A Quickstart ------------- - -This is designed to be enough information for you to run your first tests. -Detailed information on testing can be found here: https://wiki.openstack.org/wiki/Testing - -*Install pip*:: - - [apt-get | yum] install python-pip -More information on pip here: http://www.pip-installer.org/en/latest/ - -*Use pip to install tox*:: - - pip install tox - -Run The Tests -------------- - -*Navigate to the project's root directory and execute*:: - - tox -Note: completing this command may take a long time (depends on system resources) -also, you might not see any output until tox is complete. - -Information about tox can be found here: http://testrun.org/tox/latest/ - - -Run The Tests in One Environment --------------------------------- - -Tox will run your entire test suite in the environments specified in the project tox.ini:: - - [tox] - - envlist = <list of available environments> - -To run the test suite in just one of the environments in envlist execute:: - - tox -e <env> -so for example, *run the test suite in py26*:: - - tox -e py26 - -Run One Test ------------- - -To run individual tests with tox: - -if testr is in tox.ini, for example:: - - [testenv] - - includes "python setup.py testr --slowest --testr-args='{posargs}'" - -run individual tests with the following syntax:: - - tox -e <env> -- path.to.module:Class.test -so for example, *run the cpu_limited test in Nova*:: - - tox -e py27 -- nova.tests.test_claims:ClaimTestCase.test_cpu_unlimited - -if nose is in tox.ini, for example:: - - [testenv] - - includes "nosetests {posargs}" - -run individual tests with the following syntax:: - - tox -e <env> -- --tests path.to.module:Class.test -so for example, *run the list test in Glance*:: - - tox -e py27 -- --tests glance.tests.unit.test_auth.py:TestImageRepoProxy.test_list - -Need More Info? ---------------- - -More information about testr: https://wiki.openstack.org/wiki/Testr - -More information about nose: https://nose.readthedocs.org/en/latest/ - - -More information about testing OpenStack code can be found here: -https://wiki.openstack.org/wiki/Testing diff --git a/etc/.placeholder b/etc/.placeholder deleted file mode 100644 index e69de29b..00000000 --- a/etc/.placeholder +++ /dev/null diff --git a/obsolete.txt b/obsolete.txt deleted file mode 100644 index dccae853..00000000 --- a/obsolete.txt +++ /dev/null @@ -1,2 +0,0 @@ -# List of obsolete modules and files as glob patterns (since some -# files in a module may still belong in the incubator). diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index f36f719a..00000000 --- a/requirements.txt +++ /dev/null @@ -1,22 +0,0 @@ -# The order of packages is significant, because pip processes them in the order -# of appearance. Changing the order has an impact on the overall integration -# process, which may cause wedges in the gate later. - -pbr>=1.6 -argparse -debtcollector>=0.3.0 # Apache-2.0 -greenlet>=0.3.2 -Jinja2>=2.8 # BSD License (3 clause) -oslo.concurrency>=2.3.0 # Apache-2.0 -oslo.config>=2.7.0 # Apache-2.0 -oslo.middleware>=3.0.0 # Apache-2.0 -oslo.serialization>=1.10.0 # Apache-2.0 -oslo.utils>=3.2.0 # Apache-2.0 -PrettyTable<0.8,>=0.7 -psutil<2.0.0,>=1.1.1 -pycrypto>=2.6 -python-memcached>=1.56 -requests!=2.9.0,>=2.8.1 -six>=1.9.0 -stevedore>=1.5.0 # Apache-2.0 -oslo.i18n>=1.5.0 # Apache-2.0 diff --git a/run_tests.sh b/run_tests.sh deleted file mode 100755 index dc11b169..00000000 --- a/run_tests.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/bash - -# Current script is a simple wrapper around common script (tools/run_tests_common.sh). -# It passes project specific variables to common script. -# -# Options list (from tools/run_tests_common.sh). -# Use `./run_tests.sh -h` `./run_tests.sh --help` to get help message -# -# -V, --virtual-env Always use virtualenv. Install automatically if not present. -# -N, --no-virtual-env Don't use virtualenv. Run tests in local environment. -# -s, --no-site-packages Isolate the virtualenv from the global Python environment. -# -r, --recreate-db Recreate the test database (deprecated, as this is now the default). -# -n, --no-recreate-db Don't recreate the test database. -# -f, --force Force a clean re-build of the virtual environment. -# Useful when dependencies have been added. -# -u, --update Update the virtual environment with any newer package versions. -# -p, --pep8 Just run PEP8 and HACKING compliance check. -# -P, --no-pep8 Don't run static code checks. -# -c, --coverage Generate coverage report. -# -d, --debug Run tests with testtools instead of testr. -# This allows you to use the debugger. -# -h, --help Print this usage message. -# --hide-elapsed Don't print the elapsed time for each test along with slow test list. -# --virtual-env-path <path> Location of the virtualenv directory. Default: \$(pwd) -# --virtual-env-name <name> Name of the virtualenv directory. Default: .venv -# --tools-path <dir> Location of the tools directory. Default: \$(pwd) -# -# Note: with no options specified, the script will try to run the tests in a -# virtual environment. If no virtualenv is found, the script will ask if -# you would like to create one. If you prefer to run tests NOT in a -# virtual environment, simply pass the -N option. - - -# On Linux, testrepository will inspect /proc/cpuinfo to determine how many -# CPUs are present in the machine, and run one worker per CPU. -# Set workers_count=0 if you want to run one worker per CPU. -# Make our paths available to run_tests_common.sh using `export` statement -# export WORKERS_COUNT=0 - -# It is not possible to run some oslo tests with concurrency > 1 -# or concurrently due to dependencies between tests. See bug #1192207. -export WORKERS_COUNT=1 -# This option includes {PROJECT_NAME}/* directory to coverage report if `-c` or -# `--coverage` is used -export PROJECT_NAME="openstack" -# This option excludes "${PROJECT_NAME}/openstack/common/*" from coverage report -# if equals to 1 -export OMIT_OSLO_FROM_COVERAGE=0 -# Path to the directory where the tests are -export TESTS_DIR="tests/" -export EGG_INFO_FILE="openstack.common.egg-info/entry_points.txt" - -# Run common test script -tools/run_tests_common.sh $* @@ -29,24 +29,6 @@ namespace_packages = setup-hooks = pbr.hooks.setup_hook -[entry_points] -openstack.common.scheduler.filters = - AvailabilityZoneFilter = openstack.common.scheduler.filters.availability_zone_filter:AvailabilityZoneFilter - CapabilitiesFilter = openstack.common.scheduler.filters.capabilities_filter:CapabilitiesFilter - JsonFilter = openstack.common.scheduler.filters.json_filter:JsonFilter - IgnoreAttemptedHostsFilter = openstack.common.scheduler.filters.ignore_attempted_hosts_filter:IgnoreAttemptedHostsFilter - -openstack.common.tests.fakes.weights = - FakeWeigher1 = tests.unit.fakes:FakeWeigher1 - FakeWeigher2 = tests.unit.fakes:FakeWeigher2 - -openstack.common.cache.backends = - memory = openstack.common.cache._backends.memory:MemoryBackend - -oslo.config.opts = - broken_opts_lib = tests.unit.config.broken_opts_lib:list_opts - - [nosetests] # NOTE(jkoelker) To run the test suite under nose install the following # coverage http://pypi.python.org/pypi/coverage diff --git a/test-requirements.txt b/test-requirements.txt deleted file mode 100644 index 3e5ca0e1..00000000 --- a/test-requirements.txt +++ /dev/null @@ -1,17 +0,0 @@ -# The order of packages is significant, because pip processes them in the order -# of appearance. Changing the order has an impact on the overall integration -# process, which may cause wedges in the gate later. - -coverage>=3.6 -discover -hacking<0.11,>=0.10.0 -oslosphinx!=3.4.0,>=2.5.0 # Apache-2.0 -oslotest>=1.10.0 # Apache-2.0 -simplejson>=2.2.0 -sphinx!=1.2.0,!=1.3b1,<1.3,>=1.1.2 -testrepository>=0.0.18 -testscenarios>=0.4 -testtools>=1.4.0 -mox3>=0.7.0 -oslo.log>=1.12.0 # Apache-2.0 -mock>=1.2 diff --git a/tools/check_maintainers.py b/tools/check_maintainers.py deleted file mode 100755 index 08d36261..00000000 --- a/tools/check_maintainers.py +++ /dev/null @@ -1,77 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2014 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import copy -import os -import sys - -# Parse MAINTAINERS file -maintainers = {} -module_template = {'maintainers': [], - 'status': '', - 'files': [], - } -with open('MAINTAINERS', 'r') as maintainers_file: - for line in maintainers_file: - if line.startswith('=='): - module_name = line[3:-4] - maintainers[module_name] = copy.deepcopy(module_template) - elif line.startswith('M:'): - maintainer_name = line[3:] - maintainers[module_name]['maintainers'] = maintainer_name - elif line.startswith('S:'): - status = line[3:] - maintainers[module_name]['status'] = status - elif line.startswith('F:'): - filename = line[3:-1] - maintainers[module_name]['files'].append(filename) - -# Check that all files in the tree are covered in MAINTAINERS -return_value = 0 - - -def find_directory(directory): - for module, values in maintainers.items(): - if (directory + '/') in values['files']: - return - print('Directory %s not found in MAINTAINERS' % directory) - global return_value - return_value = 1 - - -def find_file(filename): - for module, values in maintainers.items(): - if filename in values['files']: - return - print('File %s not found in MAINTAINERS' % filename) - global return_value - return_value = 1 - - -def check_directory(path): - skipped_entries = ['__init__.py', 'deprecated', '__pycache__', '_i18n.py'] - for i in os.listdir(path): - if i.endswith('.pyc') or i in skipped_entries: - continue - if os.path.isdir(os.path.join(path, i)): - find_directory(i) - elif os.path.isfile(os.path.join(path, i)): - find_file(i) - - -check_directory('openstack/common') - -sys.exit(return_value) diff --git a/tools/graduate.sh b/tools/graduate.sh deleted file mode 100755 index 0cd02de6..00000000 --- a/tools/graduate.sh +++ /dev/null @@ -1,91 +0,0 @@ -#!/bin/bash -# -# Use this script to prune a copy of oslo-incubator when graduating -# modules to a brand new library. -# -# To use: -# -# 1. Clone a copy of the oslo-incubator repository to be manipulated. -# 2. Choose the new library name. For "oslo.foo", the argument to the -# script is "foo". -# 3. cd into the copy of oslo-incubator to be changed. -# 4. Run graduate.sh, passing the library name and the names of all -# directories and files to be saved (only code and tests, no project -# configuration): -# -# ../oslo-incubator/tools/graduate.sh foo openstack/common/foo.py tests/unit/test_foo.py ... -# -# 5. Clean up the results a bit by hand to make the tests work -# (update dependencies, etc.). -# - -# Stop if there are any command failures -set -e - -bindir=$(dirname $0) -tmpdir=$(mktemp -d -t oslo-graduate.XXXX) -mkdir -p $tmpdir -logfile=$tmpdir/output.log -echo "Logging to $logfile" - -# Redirect stdout/stderr to tee to write the log file -# (borrowed from verbose mode handling in devstack) -exec 1> >( awk ' - { - cmd ="date +\"%Y-%m-%d %H:%M:%S \"" - cmd | getline now - close("date +\"%Y-%m-%d %H:%M:%S \"") - sub(/^/, now) - print - fflush() - }' | tee "$logfile" ) 2>&1 - -function count_commits { - echo - echo "Have $(git log --oneline | wc -l) commits" -} - -set -x - -# Handle arguments -new_lib="$1" -shift -files_to_keep="$@" - -# FIXME(dhellmann): Make sure they are not running the tool in the -# same copy of the repository where it lives. - -# Filter the repository history down -${bindir}/filter_git_history.sh $files_to_keep - -# Move things around -echo "Moving files into place..." -if [[ -d openstack/common/$new_lib ]]; then - git mv openstack/common/$new_lib oslo_${new_lib} -else - git mv openstack/common oslo_${new_lib} -fi -rmdir openstack -if [[ -d tests ]]; then - git mv tests/* oslo_${new_lib}/tests/ - rmdir tests -fi - -# Fix imports after moving files -echo "Fixing imports..." -if [[ -d oslo_${new_lib} ]]; then - find . -name '*.py' -exec sed -i "s/openstack.common.${new_lib}/oslo_${new_lib}/" {} \; -else - find . -name '*.py' -exec sed -i "s/openstack.common/oslo_${new_lib}/" {} \; -fi - -# Bring in any missing files based on the cookiecutter template -$bindir/apply_cookiecutter.sh $new_lib - -# Commit the work we have done so far. Changes to make -# it work will be applied on top. -git add . -git commit -m "exported from oslo-incubator by graduate.sh" - -echo "The scratch files and logs from the export are in: $tmpdir" -echo "The next step is to make the tests work." diff --git a/tools/requirements.txt b/tools/requirements.txt index 900cdec8..d6dc3f31 100644 --- a/tools/requirements.txt +++ b/tools/requirements.txt @@ -1,8 +1,7 @@ # NOTE(dhellmann): These requirements are just for the tool scripts and # do not need to be synced. +argparse oslo.config -pyyaml jinja2 -oslo.concurrency parawrap @@ -1,6 +1,6 @@ [tox] minversion = 1.6 -envlist = py26,py27,py33,py34,pep8,pylint +envlist = pep8,dashboards,docs skipsdist = True [testenv] @@ -9,12 +9,6 @@ usedevelop = True install_command = pip install -U {opts} {packages} setenv = VIRTUAL_ENV={envdir} PYTHONHASHSEED=0 -deps = -r{toxinidir}/requirements.txt - -r{toxinidir}/test-requirements.txt -commands = - # Use the lockutils wrapper to ensure that external locking works correctly - lockutils-wrapper python setup.py test --slowest --testr-args='{posargs}' - [flake8] show-source = True ignore = E123,H405,H904 @@ -22,46 +16,15 @@ exclude = .venv,.tox,dist,doc,*.egg,.update-venv [testenv:pep8] commands = flake8 {posargs} - python ./tools/check_maintainers.py - -[testenv:pylint] -deps = pylint>=0.26.0 -commands = python ./tools/lint.py ./openstack - python ./tools/lint.py ./tests - -[testenv:cover] -commands = - # due to dependencies between tests (bug 1192207) we use `--concurrency=1` option - # Use the lockutils wrapper to ensure that external locking works correctly - lockutils-wrapper python setup.py test --coverage --testr-args='--concurrency=1 {posargs}' [testenv:docs] commands = python setup.py build_sphinx -[testenv:venv] -commands = {posargs} - -[testenv:relnotes] -deps = -r{toxinidir}/tools/requirements.txt -commands = python ./tools/release_notes.py {posargs} - -[testenv:unreleased] -deps = -r{toxinidir}/tools/requirements.txt -commands = {toxinidir}/tools/list_unreleased_changes.sh {posargs} - -[testenv:pyflakes] -commands = flake8 - [testenv:dashboards] skipdist = True usedevelop = True deps = commands = - git clone https://git.openstack.org/stackforge/gerrit-dash-creator {envdir}/gerrit-dash-creator + git clone https://git.openstack.org/openstack/gerrit-dash-creator {envdir}/gerrit-dash-creator pip install -r {envdir}/gerrit-dash-creator/requirements.txt {toxinidir}/tools/build_dashboards.sh {envdir}/gerrit-dash-creator {toxinidir}/dashboards - -[hacking] -import_exceptions = - openstack.common._i18n - oslo.i18n diff --git a/update.py b/update.py deleted file mode 100644 index 4b04bf77..00000000 --- a/update.py +++ /dev/null @@ -1,397 +0,0 @@ -# Copyright 2012 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -r""" -A simple script to update openstack-common modules which have been copied -into other projects. See: - - https://wiki.openstack.org/wiki/Oslo#Incubation - -The script can be called the following ways: - - $> python update.py ../myproj - $> python update.py --config-file ../myproj/openstack-common.conf - -Where ../myproj is a project directory containing openstack-common.conf which -might look like: - - [DEFAULT] - module = wsgi - module = utils - script = tools/run_cross_tests.sh - base = myproj - -Or: - - $> python update.py ../myproj/myconf.conf - $> python update.py --config-file ../myproj/myconf.conf - -Where ../myproj is a project directory which contains a differently named -configuration file, or: - - $> python update.py --config-file ../myproj/myproj/openstack/common.conf - --dest-dir ../myproj - -Where ../myproj is a project directory, but the configuration file is -stored in a sub-directory, or: - - $> python update.py --modules wsgi,utils --base myproj ../myproj - $> python update.py --modules wsgi,utils --base myproj --dest-dir ../myproj - -Where ../myproj is a project directory, but we explicitly specify -the modules to copy and the base destination module - - $> python update.py --modules wsgi,utils --nodeps --base myproj ../myproj - -Where ../myproj is a project directory, but we explicitly specify -the modules to copy, the base destination module, and do not want to -automatically copy the dependencies of the specified modules - - $> python update.py --script tools/run_cross_tests.sh ../myproj - -Where ../myproj is a project directory, and we explicitly specify -the scripts to copy. - -Obviously, the first way is the easiest! -""" - -from __future__ import print_function - -import fnmatch -import functools -import glob -import os -import os.path -import re -import shutil -import sys - -from oslo_config import cfg - -_OBSOLETE_LIST = None - -opts = [ - cfg.ListOpt('modules', - default=[], - help='The list of modules to copy from oslo-incubator ' - '(deprecated in favor of --module).'), - cfg.MultiStrOpt('module', - default=[], - help='The list of modules to copy from oslo-incubator.'), - cfg.MultiStrOpt( - 'script', - default=[], - help='The list of stand-alone scripts to copy from oslo-incubator.'), - cfg.StrOpt('base', - help='The base module to hold the copy of openstack.common.'), - cfg.StrOpt('dest-dir', - help='Destination project directory.'), - cfg.StrOpt('configfile_or_destdir', - help='A configuration file or destination project directory.', - positional=True), - cfg.BoolOpt('nodeps', - default=False, - help='Enables or disables the use of dependencies for ' - 'configured modules. Default is False, which enables ' - 'dependencies.'), -] - - -def _parse_args(argv): - conf = cfg.ConfigOpts() - conf.register_cli_opts(opts) - conf(argv, usage='Usage: %(prog)s [config-file|dest-dir]') - - if conf.configfile_or_destdir: - def def_config_file(dest_dir): - return os.path.join(dest_dir, 'openstack-common.conf') - - config_file = None - if os.path.isfile(conf.configfile_or_destdir): - config_file = conf.configfile_or_destdir - elif (os.path.isdir(conf.configfile_or_destdir) - and os.path.isfile(def_config_file(conf.configfile_or_destdir))): - config_file = def_config_file(conf.configfile_or_destdir) - - if config_file: - if not (conf.module or conf.modules): - conf(argv + ['--config-file', config_file]) - elif os.path.isdir(conf.configfile_or_destdir): - conf(argv + ['--dest-dir', conf.configfile_or_destdir]) - else: - print('Specifying a config file and a list of modules to ' - 'sync will not work correctly', file=sys.stderr) - sys.exit(1) - - return conf - - -def _mod_to_path(mod): - return os.path.join(*mod.split('.')) - - -def _dest_path(path, base, dest_dir): - return os.path.join(dest_dir, _mod_to_path(base), path) - - -def _replace(path, replacements): - with open(path, "rb+") as f: - lines = f.readlines() - f.seek(0) - f.truncate() - for line in lines: - for pattern, replacement in replacements: - line = re.sub(pattern, replacement, line) - f.write(line) - - -def _make_dirs(path): - if not os.path.isdir(os.path.dirname(path)): - os.makedirs(os.path.dirname(path)) - - -def _check_obsolete(path): - global _OBSOLETE_LIST - if _OBSOLETE_LIST is None: - _OBSOLETE_LIST = [] - with open('obsolete.txt', 'r') as f: - for num, line in enumerate(f): - line = line.strip() - if line.startswith('#') or not line: - continue - try: - pattern, replacement = line.split() - except Exception as e: - print('ERROR: Could not parse obsolete.txt line ' - '%s %r: %s' % (num + 1, line, e)) - else: - _OBSOLETE_LIST.append((pattern, replacement)) - for pattern, replacement in _OBSOLETE_LIST: - if fnmatch.fnmatch(path, pattern): - print('### WARNING: %s is an obsolete module, see %s' % - (path, replacement)) - -OSLO_LIBS = [ - 'cache', - 'concurrency', - 'config', - 'context', - 'db', - 'i18n', - 'log', - 'messaging', - 'middleware', - 'policy', - 'reports', - 'rootwrap', - 'serialization', - 'service', - 'utils', - 'version', - 'vmware', -] - - -def _copy_file(path, dest, base): - _check_obsolete(path) - - _make_dirs(dest) - if not os.path.isdir(os.path.dirname(dest)): - os.makedirs(os.path.dirname(dest)) - - shutil.copy2(path, dest) - - replacements = [] - - replacements.append(('oslo(?!(test|-incubator))', base)) - replacements.append(('OSLO', base.upper())) - - # Restore the imports for modules that are part of the oslo - # namespace package. We can't just do something like 'oslo\..+' - # because there are default configuration settings like - # "oslo.sqlite" that we want to have changed to "nova.sqlite" by - # the above call. - for oslo_module in OSLO_LIBS: - replacements.append((base + r'(.)' + oslo_module, - r'oslo\1' + oslo_module)) - - replacements.append(('^( *)from openstack.common', - r'\1from ' + base + '.openstack.common')) - - replacements.append(('^( *)import openstack.common', - r'\1import ' + base + '.openstack.common')) - - replacements.append(('\'openstack\.common', - '\'' + base + '.openstack.common')) - - replacements.append(('\"openstack\.common', - '\"' + base + '.openstack.common')) - - replacements.append(('=openstack\.common', - '=' + base + '.openstack.common')) - - replacements.append(('possible_topdir, "oslo",$', - 'possible_topdir, "' + base + '",')) - - _replace(dest, replacements) - - -def _copy_pyfile(path, base, dest_dir): - _copy_file(path, _dest_path(path, base, dest_dir), base) - - -def _copy_scripts(scripts, base, dest_dir): - for scr in scripts: - dest = os.path.join(dest_dir, scr) - print("Copying script %s to %s" % (scr, dest)) - _copy_file(scr, dest, base) - - -def _copy_module(mod, base, dest_dir): - print("Copying %s under the %s module in %s" % (mod, base, dest_dir)) - - copy_pyfile = functools.partial(_copy_pyfile, - base=base, dest_dir=dest_dir) - - path = _mod_to_path('openstack.common') - if '.' in mod: - for d in mod.split('.')[:-1]: - path = os.path.join(path, d) - if os.path.isdir(path): - copy_pyfile(os.path.join(path, '__init__.py')) - else: - copy_pyfile(os.path.join(path, '__init__.py')) - - mod_path = _mod_to_path('openstack.common.%s' % mod) - mod_file = '%s.py' % mod_path - if os.path.isfile(mod_file): - copy_pyfile(mod_file) - elif os.path.isdir(mod_path): - dest = os.path.join(dest_dir, _mod_to_path(base), - 'openstack', 'common', mod) - _make_dirs(dest) - sources = filter(lambda x: x[-3:] == '.py', os.listdir(mod_path)) - for s in sources: - copy_pyfile(os.path.join(mod_path, s)) - else: - print("Module not found. Tried: \n\t%s \n\t%s" % (mod_path, mod_file)) - - globs_to_copy = [ - os.path.join('tools', mod, '*.sh'), - os.path.join('tools', mod + '*'), - os.path.join('etc', 'oslo', mod + '*.conf'), - ] - - for matches in [glob.glob(g) for g in globs_to_copy]: - for match in [x for x in matches if not os.path.isdir(x)]: - dest = os.path.join(dest_dir, match.replace('oslo', base)) - print("Copying %s to %s" % (match, dest)) - _copy_file(match, dest, base) - - -def _create_module_init(base, dest_dir, *sub_paths): - """Create module __init__ files.""" - init_path = _dest_path('openstack', base, dest_dir) - - if sub_paths: - init_path = os.path.join(init_path, *sub_paths) - - init_path = os.path.join(init_path, '__init__.py') - - if not os.path.exists(init_path): - _make_dirs(init_path) - open(init_path, 'w').close() - - -def _find_import_modules(srcfile): - oslo_import_pattern = re.compile(r"\s*from\sopenstack\.common" - "(\simport\s|\.)(\w+)($|.+)") - with open(srcfile, 'r') as f: - for line in f: - result = oslo_import_pattern.match(line) - if result: - yield result.group(2) - - -def _build_dependency_tree(): - dep_tree = {} - base_path = os.path.join('openstack', 'common') - for dirpath, _, filenames in os.walk(base_path): - for filename in [x for x in filenames if x.endswith('.py')]: - if dirpath == base_path: - mod_name = filename.split('.')[0] - else: - mod_name = dirpath.split(os.sep)[2] - if mod_name == '__init__': - continue - filepath = os.path.join(dirpath, filename) - dep_list = dep_tree.setdefault(mod_name, []) - dep_list.extend([x for x in _find_import_modules(filepath) - if x != mod_name and x not in dep_list]) - return dep_tree - - -def _dfs_dependency_tree(dep_tree, mod_name, mod_list=None): - mod_list = mod_list or [] - mod_list.append(mod_name) - for mod in dep_tree.get(mod_name, []): - if mod not in mod_list: - mod_list = _dfs_dependency_tree(dep_tree, mod, mod_list) - return mod_list - - -def _complete_module_list(mod_list, nodeps): - if nodeps: - return mod_list - addons = [] - dep_tree = _build_dependency_tree() - for mod in mod_list: - addons.extend([x for x in _dfs_dependency_tree(dep_tree, mod) - if x not in mod_list and x not in addons]) - mod_list.extend(addons) - return mod_list - - -def main(argv): - conf = _parse_args(argv) - - dest_dir = conf.dest_dir - if not dest_dir and conf.config_file: - dest_dir = os.path.dirname(conf.config_file[-1]) - - if not dest_dir or not os.path.isdir(dest_dir): - print("A valid destination dir is required", file=sys.stderr) - sys.exit(1) - - if not conf.module and not conf.modules and not conf.script: - print("A list of modules or scripts to copy is required", - file=sys.stderr) - sys.exit(1) - - if not conf.base: - print("A destination base module is required", file=sys.stderr) - sys.exit(1) - - if conf.module + conf.modules: - _create_module_init(conf.base, dest_dir) - _create_module_init(conf.base, dest_dir, 'common') - - for mod in _complete_module_list(conf.module + conf.modules, conf.nodeps): - _copy_module(mod, conf.base, dest_dir) - - _copy_scripts(conf.script, conf.base, dest_dir) - - -if __name__ == "__main__": - main(sys.argv[1:]) diff --git a/update.sh b/update.sh deleted file mode 100755 index 8ab309cd..00000000 --- a/update.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -VENV=.update-venv - -# -qq gets rid of the deprecation warning, in time we can -# remove --no-site-packages as it's the default now -[ -d $VENV ] || virtualenv -qq --no-site-packages $VENV - -. $VENV/bin/activate - -# need oslo.config for bootstrapping, be quiet for UX reasons -pip -q install oslo.config - -python update.py $* |