Begin adding testing functionality

Take the current run_tests.sh from glance and include
that as well as fill in the rest of the pieces needed
for those that use venv functionality. Start adding
memory tests.

13 files changed, 563 insertions, 16 deletions

diff --git a/openstack-common.conf b/openstack-common.conf
index 073de65..edb170f 100644
--- a/openstack-common.conf
+++ b/openstack-common.conf
@@ -1,7 +1,7 @@
 [DEFAULT]
 
 # The list of modules to copy from oslo-incubator.git
-modules=excutils
+modules=excutils,install_venv_common
 
 # The base module to hold the copy of openstack.common
 base=taskflow
diff --git a/run_tests.sh b/run_tests.sh
new file mode 100755
index 0000000..f96cc10
--- /dev/null
+++ b/run_tests.sh
@@ -0,0 +1,119 @@
+#!/bin/bash
+
+function usage {
+  echo "Usage: $0 [OPTION]..."
+  echo "Run Taskflow's test suite(s)"
+  echo ""
+  echo "  -V, --virtual-env        Always use virtualenv.  Install automatically if not present"
+  echo "  -N, --no-virtual-env     Don't use virtualenv.  Run tests in local environment"
+  echo "  -f, --force              Force a clean re-build of the virtual environment. Useful when dependencies have been added."
+  echo "  -u, --update             Update the virtual environment with any newer package versions"
+  echo "  -p, --pep8               Just run pep8"
+  echo "  -P, --no-pep8            Don't run static code checks"
+  echo "  -h, --help               Print this usage message"
+  echo ""
+  echo "Note: with no options specified, the script will try to run the tests in a virtual environment,"
+  echo "      If no virtualenv is found, the script will ask if you would like to create one.  If you "
+  echo "      prefer to run tests NOT in a virtual environment, simply pass the -N option."
+  exit
+}
+
+function process_option {
+  case "$1" in
+    -h|--help) usage;;
+    -V|--virtual-env) let always_venv=1; let never_venv=0;;
+    -N|--no-virtual-env) let always_venv=0; let never_venv=1;;
+    -p|--pep8) let just_pep8=1;;
+    -P|--no-pep8) let no_pep8=1;;
+    -f|--force) let force=1;;
+    -u|--update) update=1;;
+    -c|--coverage) noseopts="$noseopts --with-coverage --cover-package=taskflow";;
+    -*) noseopts="$noseopts $1";;
+    *) noseargs="$noseargs $1"
+  esac
+}
+
+venv=.venv
+with_venv=tools/with_venv.sh
+always_venv=0
+never_venv=0
+force=0
+noseopts=
+noseargs=
+wrapper=""
+just_pep8=0
+no_pep8=0
+update=0
+
+export NOSE_WITH_OPENSTACK=1
+export NOSE_OPENSTACK_COLOR=1
+export NOSE_OPENSTACK_RED=0.05
+export NOSE_OPENSTACK_YELLOW=0.025
+export NOSE_OPENSTACK_SHOW_ELAPSED=1
+export NOSE_OPENSTACK_STDOUT=1
+
+for arg in "$@"; do
+  process_option $arg
+done
+
+function run_tests {
+  # Cleanup *pyc
+  ${wrapper} find . -type f -name "*.pyc" -delete
+  ${wrapper} $NOSETESTS
+}
+
+function run_pep8 {
+  echo "Running pep8 ..."
+  PEP8_EXCLUDE=".venv,.tox,dist,doc,openstack"
+  PEP8_OPTIONS="--exclude=$PEP8_EXCLUDE --repeat"
+  PEP8_IGNORE="--ignore=E125,E126,E711,E712"
+  PEP8_INCLUDE="."
+
+  ${wrapper} pep8 $PEP8_OPTIONS $PEP8_INCLUDE $PEP8_IGNORE
+}
+
+
+NOSETESTS="nosetests $noseopts $noseargs"
+
+if [ $never_venv -eq 0 ]
+then
+  # Remove the virtual environment if --force used
+  if [ $force -eq 1 ]; then
+    echo "Cleaning virtualenv..."
+    rm -rf ${venv}
+  fi
+  if [ $update -eq 1 ]; then
+    echo "Updating virtualenv..."
+    python tools/install_venv.py
+  fi
+  if [ -e ${venv} ]; then
+    wrapper="${with_venv}"
+  else
+    if [ $always_venv -eq 1 ]; then
+      # Automatically install the virtualenv
+      python tools/install_venv.py
+      wrapper="${with_venv}"
+    else
+      echo -e "No virtual environment found...create one? (Y/n) \c"
+      read use_ve
+      if [ "x$use_ve" = "xY" -o "x$use_ve" = "x" -o "x$use_ve" = "xy" ]; then
+        # Install the virtualenv and run the test suite in it
+        python tools/install_venv.py
+		wrapper=${with_venv}
+      fi
+    fi
+  fi
+fi
+
+if [ $just_pep8 -eq 1 ]; then
+    run_pep8
+    exit
+fi
+
+run_tests || exit
+
+if [ -z "$noseargs" ]; then
+    if [ $no_pep8 -eq 0 ]; then
+        run_pep8
+    fi
+fi
diff --git a/setup.py b/setup.py
index 80723ff..e295657 100644
--- a/setup.py
+++ b/setup.py
@@ -18,7 +18,8 @@ def read_requires(base):
     return requires
 
 
-setuptools.setup(name='taskflow',
+setuptools.setup(
+    name='taskflow',
     version='0.0.1',
     author='OpenStack',
     license='Apache Software License',
@@ -36,4 +37,4 @@ setuptools.setup(name='taskflow',
         'License :: OSI Approved :: Apache Software License',
         'Operating System :: POSIX :: Linux',
         'Programming Language :: Python :: 2.6', ],
-)
\ No newline at end of file
+)
diff --git a/taskflow/backends/memory.py b/taskflow/backends/memory.py
index a0be0ce..13791ee 100644
--- a/taskflow/backends/memory.py
+++ b/taskflow/backends/memory.py
@@ -24,6 +24,7 @@ import threading
 
 from taskflow import catalog
 from taskflow import exceptions as exc
+from taskflow import job
 from taskflow import jobboard
 from taskflow import logbook
 from taskflow import utils
@@ -43,6 +44,14 @@ def check_not_closed(meth):
     return check
 
 
+class MemoryClaimer(job.Claimer):
+    def claim(self, job, owner):
+        # Straight foward check if already claimed.
+        if job.owner is not None:
+            raise exc.UnclaimableJobException()
+        job.owner = owner
+
+
 class MemoryCatalog(catalog.Catalog):
     def __init__(self):
         super(MemoryCatalog, self).__init__()
@@ -50,6 +59,13 @@ class MemoryCatalog(catalog.Catalog):
         self._closed = False
         self._lock = threading.RLock()
 
+    def __contains__(self, job):
+        with self._lock:
+            for (j, b) in self._catalogs:
+                if j == job:
+                    return True
+        return False
+
     def close(self):
         self._closed = True
 
@@ -122,12 +138,11 @@ class MemoryJobBoard(jobboard.JobBoard):
     @check_not_closed
     def post(self, job):
         with self._lock.acquire(read=False):
-            self._active_jobs.append((datetime.utcnow(), job))
+            self._board.append((datetime.utcnow(), job))
         # Let people know a job is here
         self._notify_posted(job)
         self._event.set()
-        # And now that they are notified, wait for
-        # another posting.
+        # And now that they are notified, wait for another posting.
         self._event.clear()
 
     @check_not_closed
@@ -148,9 +163,9 @@ class MemoryJobBoard(jobboard.JobBoard):
                 if j == job:
                     exists = True
                     break
-           if not exists:
-               raise exc.JobNotFound()
-           self._board = [(d, j) for (d, j) in self._board if j != job]
+            if not exists:
+                raise exc.JobNotFound()
+            self._board = [(d, j) for (d, j) in self._board if j != job]
 
     @check_not_closed
     def posted_after(self, date_posted=None):
@@ -163,5 +178,4 @@ class MemoryJobBoard(jobboard.JobBoard):
 
     @check_not_closed
     def await(self, timeout=None):
-        with not self._event.is_set():
-            self._event.wait(timeout)
+        self._event.wait(timeout)
diff --git a/taskflow/job.py b/taskflow/job.py
index a8a0219..31c1b3e 100644
--- a/taskflow/job.py
+++ b/taskflow/job.py
@@ -39,11 +39,9 @@ class Claimer(object):
 
 class Job(object):
     """A job is connection to some set of work to be done by some agent. Basic
-    information is provided about said work to be able to attempt to 
+    information is provided about said work to be able to attempt to
     fullfill said work."""
 
-    __metaclass__ = abc.ABCMeta
-
     def __init__(self, name, context, catalog, claimer):
         self.name = name
         self.context = context
@@ -79,7 +77,7 @@ class Job(object):
         self._claimer.claim(self, owner)
         self.owner = owner
         self._change_state(states.CLAIMED)
- 
+
     def _change_state(self, new_state):
         self.state = new_state
         # TODO(harlowja): update the logbook
diff --git a/taskflow/tests/__init__.py b/taskflow/tests/__init__.py
new file mode 100644
index 0000000..830dd2e
--- /dev/null
+++ b/taskflow/tests/__init__.py
@@ -0,0 +1,17 @@
+# -*- coding: utf-8 -*-
+
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+#    Copyright (C) 2012 Yahoo! Inc. All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
diff --git a/taskflow/tests/unit/__init__.py b/taskflow/tests/unit/__init__.py
new file mode 100644
index 0000000..830dd2e
--- /dev/null
+++ b/taskflow/tests/unit/__init__.py
@@ -0,0 +1,17 @@
+# -*- coding: utf-8 -*-
+
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+#    Copyright (C) 2012 Yahoo! Inc. All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
diff --git a/taskflow/tests/unit/test_memory.py b/taskflow/tests/unit/test_memory.py
new file mode 100644
index 0000000..3ca3d08
--- /dev/null
+++ b/taskflow/tests/unit/test_memory.py
@@ -0,0 +1,69 @@
+# -*- coding: utf-8 -*-
+
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+#    Copyright (C) 2012 Yahoo! Inc. All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from datetime import datetime
+
+import time
+import threading
+import unittest
+
+from taskflow import job
+from taskflow.backends import memory
+
+
+class MemoryBackendTest(unittest.TestCase):
+    def testPostRecvJob(self):
+        job_claimer = memory.MemoryClaimer()
+        book_catalog = memory.MemoryCatalog()
+        j = job.Job("test", {}, book_catalog, job_claimer)
+
+        # Hook up some simulated workers to said job-board.
+        job_board = memory.MemoryJobBoard()
+        receiver_awake = threading.Event()
+        work_items = []
+
+        def post_job():
+            job_board.post(j)
+
+        def work_on_job(j):
+            owner = 'me'
+            j.claim(owner)
+
+        def receive_job():
+            start = datetime.utcnow()
+            receiver_awake.set()
+            new_jobs = []
+            while not new_jobs:
+                job_board.await(0.5)
+                new_jobs = job_board.posted_after(start)
+            work_items.extend(new_jobs)
+            for j in work_items:
+                work_on_job(j)
+
+        poster = threading.Thread(target=post_job)
+        receiver = threading.Thread(target=receive_job)
+        receiver.start()
+        while not receiver_awake.isSet():
+            receiver_awake.wait()
+        poster.start()
+
+        for t in [poster, receiver]:
+            t.join()
+
+        self.assertEquals(1, len(work_items))
+        self.assertEquals(j.owner, 'me')
diff --git a/tools/install_venv.py b/tools/install_venv.py
new file mode 100644
index 0000000..de37891
--- /dev/null
+++ b/tools/install_venv.py
@@ -0,0 +1,76 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2010 United States Government as represented by the
+# Administrator of the National Aeronautics and Space Administration.
+# All Rights Reserved.
+#
+# Copyright 2010 OpenStack LLC.
+# Copyright 2013 IBM Corp.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+Installation script for Taskflows's development virtualenv
+"""
+
+import os
+import subprocess
+import sys
+
+import install_venv_common as install_venv
+
+
+def print_help():
+    help = """
+Taskflow development environment setup is complete.
+
+Taskflow development uses virtualenv to track and manage Python dependencies
+while in development and testing.
+
+To activate the taskflow virtualenv for the extent of your current shell
+session you can run:
+
+$ source .venv/bin/activate
+
+Or, if you prefer, you can run commands in the virtualenv on a case by case
+basis by running:
+
+$ tools/with_venv.sh <your command>
+
+Also, make test will automatically use the virtualenv.
+    """
+    for line in help.splitlines():
+        print(" %s" % (line))
+
+
+def main(argv):
+    root = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
+    venv = os.path.join(root, '.venv')
+    pip_requires = os.path.join(root, 'tools', 'pip-requires')
+    test_requires = os.path.join(root, 'tools', 'test-requires')
+    py_version = "python%s.%s" % (sys.version_info[0], sys.version_info[1])
+    project = 'Taskflow'
+    install = install_venv.InstallVenv(root, venv, pip_requires, test_requires,
+                                       py_version, project)
+    options = install.parse_args(argv)
+    install.check_python_version()
+    install.check_dependencies()
+    install.create_virtualenv(no_site_packages=options.no_site_packages)
+    install.install_dependencies()
+    install.run_command([os.path.join(venv, 'bin/python'),
+                        'setup.py', 'develop'])
+    install.post_process()
+    print_help()
+
+if __name__ == '__main__':
+    main(sys.argv)
diff --git a/tools/install_venv_common.py b/tools/install_venv_common.py
new file mode 100644
index 0000000..914fcf1
--- /dev/null
+++ b/tools/install_venv_common.py
@@ -0,0 +1,221 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2013 OpenStack Foundation
+# Copyright 2013 IBM Corp.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Provides methods needed by installation script for OpenStack development
+virtual environments.
+
+Since this script is used to bootstrap a virtualenv from the system's Python
+environment, it should be kept strictly compatible with Python 2.6.
+
+Synced in from openstack-common
+"""
+
+from __future__ import print_function
+
+import optparse
+import os
+import subprocess
+import sys
+
+
+class InstallVenv(object):
+
+    def __init__(self, root, venv, pip_requires, test_requires, py_version,
+                 project):
+        self.root = root
+        self.venv = venv
+        self.pip_requires = pip_requires
+        self.test_requires = test_requires
+        self.py_version = py_version
+        self.project = project
+
+    def die(self, message, *args):
+        print(message % args, file=sys.stderr)
+        sys.exit(1)
+
+    def check_python_version(self):
+        if sys.version_info < (2, 6):
+            self.die("Need Python Version >= 2.6")
+
+    def run_command_with_code(self, cmd, redirect_output=True,
+                              check_exit_code=True):
+        """Runs a command in an out-of-process shell.
+
+        Returns the output of that command. Working directory is self.root.
+        """
+        if redirect_output:
+            stdout = subprocess.PIPE
+        else:
+            stdout = None
+
+        proc = subprocess.Popen(cmd, cwd=self.root, stdout=stdout)
+        output = proc.communicate()[0]
+        if check_exit_code and proc.returncode != 0:
+            self.die('Command "%s" failed.\n%s', ' '.join(cmd), output)
+        return (output, proc.returncode)
+
+    def run_command(self, cmd, redirect_output=True, check_exit_code=True):
+        return self.run_command_with_code(cmd, redirect_output,
+                                          check_exit_code)[0]
+
+    def get_distro(self):
+        if (os.path.exists('/etc/fedora-release') or
+                os.path.exists('/etc/redhat-release')):
+            return Fedora(self.root, self.venv, self.pip_requires,
+                          self.test_requires, self.py_version, self.project)
+        else:
+            return Distro(self.root, self.venv, self.pip_requires,
+                          self.test_requires, self.py_version, self.project)
+
+    def check_dependencies(self):
+        self.get_distro().install_virtualenv()
+
+    def create_virtualenv(self, no_site_packages=True):
+        """Creates the virtual environment and installs PIP.
+
+        Creates the virtual environment and installs PIP only into the
+        virtual environment.
+        """
+        if not os.path.isdir(self.venv):
+            print('Creating venv...', end=' ')
+            if no_site_packages:
+                self.run_command(['virtualenv', '-q', '--no-site-packages',
+                                 self.venv])
+            else:
+                self.run_command(['virtualenv', '-q', self.venv])
+            print('done.')
+            print('Installing pip in venv...', end=' ')
+            if not self.run_command(['tools/with_venv.sh', 'easy_install',
+                                    'pip>1.0']).strip():
+                self.die("Failed to install pip.")
+            print('done.')
+        else:
+            print("venv already exists...")
+            pass
+
+    def pip_install(self, *args):
+        self.run_command(['tools/with_venv.sh',
+                         'pip', 'install', '--upgrade'] + list(args),
+                         redirect_output=False)
+
+    def install_dependencies(self):
+        print('Installing dependencies with pip (this can take a while)...')
+
+        # First things first, make sure our venv has the latest pip and
+        # distribute.
+        # NOTE: we keep pip at version 1.1 since the most recent version causes
+        # the .venv creation to fail. See:
+        # https://bugs.launchpad.net/nova/+bug/1047120
+        self.pip_install('pip==1.1')
+        self.pip_install('distribute')
+
+        # Install greenlet by hand - just listing it in the requires file does
+        # not
+        # get it installed in the right order
+        self.pip_install('greenlet')
+
+        self.pip_install('-r', self.pip_requires)
+        self.pip_install('-r', self.test_requires)
+
+    def post_process(self):
+        self.get_distro().post_process()
+
+    def parse_args(self, argv):
+        """Parses command-line arguments."""
+        parser = optparse.OptionParser()
+        parser.add_option('-n', '--no-site-packages',
+                          action='store_true',
+                          help="Do not inherit packages from global Python "
+                               "install")
+        return parser.parse_args(argv[1:])[0]
+
+
+class Distro(InstallVenv):
+
+    def check_cmd(self, cmd):
+        return bool(self.run_command(['which', cmd],
+                    check_exit_code=False).strip())
+
+    def install_virtualenv(self):
+        if self.check_cmd('virtualenv'):
+            return
+
+        if self.check_cmd('easy_install'):
+            print('Installing virtualenv via easy_install...', end=' ')
+            if self.run_command(['easy_install', 'virtualenv']):
+                print('Succeeded')
+                return
+            else:
+                print('Failed')
+
+        self.die('ERROR: virtualenv not found.\n\n%s development'
+                 ' requires virtualenv, please install it using your'
+                 ' favorite package management tool' % self.project)
+
+    def post_process(self):
+        """Any distribution-specific post-processing gets done here.
+
+        In particular, this is useful for applying patches to code inside
+        the venv.
+        """
+        pass
+
+
+class Fedora(Distro):
+    """This covers all Fedora-based distributions.
+
+    Includes: Fedora, RHEL, CentOS, Scientific Linux
+    """
+
+    def check_pkg(self, pkg):
+        return self.run_command_with_code(['rpm', '-q', pkg],
+                                          check_exit_code=False)[1] == 0
+
+    def apply_patch(self, originalfile, patchfile):
+        self.run_command(['patch', '-N', originalfile, patchfile],
+                         check_exit_code=False)
+
+    def install_virtualenv(self):
+        if self.check_cmd('virtualenv'):
+            return
+
+        if not self.check_pkg('python-virtualenv'):
+            self.die("Please install 'python-virtualenv'.")
+
+        super(Fedora, self).install_virtualenv()
+
+    def post_process(self):
+        """Workaround for a bug in eventlet.
+
+        This currently affects RHEL6.1, but the fix can safely be
+        applied to all RHEL and Fedora distributions.
+
+        This can be removed when the fix is applied upstream.
+
+        Nova: https://bugs.launchpad.net/nova/+bug/884915
+        Upstream: https://bitbucket.org/which_linden/eventlet/issue/89
+        """
+
+        # Install "patch" program if it's not there
+        if not self.check_pkg('patch'):
+            self.die("Please install 'patch'.")
+
+        # Apply the eventlet patch
+        self.apply_patch(os.path.join(self.venv, 'lib', self.py_version,
+                                      'site-packages',
+                                      'eventlet/green/subprocess.py'),
+                         'contrib/redhat-eventlet.patch')
diff --git a/tools/pip-requires b/tools/pip-requires
index c15e57a..483a09a 100644
--- a/tools/pip-requires
+++ b/tools/pip-requires
@@ -1 +1,3 @@
-oslo.cfg>=1.1.0
+# Packages needed for using this library.
+
+oslo.config>=1.1.0
diff --git a/tools/test-requires b/tools/test-requires
new file mode 100644
index 0000000..8e7679a
--- /dev/null
+++ b/tools/test-requires
@@ -0,0 +1,8 @@
+# Packages needed for testing this library.
+
+nose
+pep8==1.4.5
+nose-exclude
+openstack.nose_plugin>=0.7
+mox
+coverage
diff --git a/tools/with_venv.sh b/tools/with_venv.sh
new file mode 100755
index 0000000..f34cfc2
--- /dev/null
+++ b/tools/with_venv.sh
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+TOOLS=`dirname $0`
+VENV=$TOOLS/../.venv
+source $VENV/bin/activate && $@