Add wsgi_scripts support to PBR

As we attempt to get more of OpenStack API servers to be runnable as a
regular WSGI application, we should provide support in PBR for
building these base scripts just like the console scripts.

This adds a new 'wsgi_scripts' group which builds a base script that
will run under mod_wsgi as expected. It also has a CLI fallback mode,
so that the application can be brought up as a wsgiref simple_server
for quick local testing and development. All wsgiref servers default
to binding to port 8000, but that can be overridden.

To support this, and possible future expansion of the script types,
the group list now iterates over a dictionary of group_name =>
template mappings.

This includes basic testing.

It also includes tests which would run an actual wsgi
environment. These are currently skipped as they can't be reliably run
in the gate for timing reasons on stdout processing.

Change-Id: I334639d7ecaad2703d1ff675880a314cc28e2334

4 files changed, 247 insertions, 2 deletions

diff --git a/pbr/packaging.py b/pbr/packaging.py
index ebdfb09..be5eb74 100644
--- a/pbr/packaging.py
+++ b/pbr/packaging.py
@@ -212,6 +212,36 @@ except ImportError:
 def have_nose():
     return _have_nose
 
+_wsgi_text = """#PBR Generated from %(group)r
+
+from %(module_name)s import %(import_target)s
+
+if __name__ == "__main__":
+    import argparse
+    import socket
+    import wsgiref.simple_server as wss
+
+    my_ip = socket.gethostbyname(socket.gethostname())
+    parser = argparse.ArgumentParser(
+        description=%(import_target)s.__doc__,
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+    parser.add_argument('--port', '-p', type=int, default=8000,
+                        help='TCP port to listen on')
+    args = parser.parse_args()
+    server = wss.make_server('', args.port, %(invoke_target)s())
+
+    print("*" * 80)
+    print("STARTING test server %(module_name)s.%(invoke_target)s")
+    url = "http://%%s:%%d/" %% (my_ip, server.server_port)
+    print("Available at %%s" %% url)
+    print("DANGER! For testing only, do not use in production")
+    print("*" * 80)
+
+    server.serve_forever()
+else:
+    application = %(invoke_target)s()
+
+"""
 
 _script_text = """# PBR Generated from %(group)r
 
@@ -225,16 +255,25 @@ if __name__ == "__main__":
 """
 
 
+# the following allows us to specify different templates per entry
+# point group when generating pbr scripts.
+ENTRY_POINTS_MAP = {
+    'console_scripts': _script_text,
+    'gui_scripts': _script_text,
+    'wsgi_scripts': _wsgi_text
+}
+
+
 def override_get_script_args(
         dist, executable=os.path.normpath(sys.executable), is_wininst=False):
     """Override entrypoints console_script."""
     header = easy_install.get_script_header("", executable, is_wininst)
-    for group in 'console_scripts', 'gui_scripts':
+    for group, template in ENTRY_POINTS_MAP.items():
         for name, ep in dist.get_entry_map(group).items():
             if not ep.attrs or len(ep.attrs) > 2:
                 raise ValueError("Script targets must be of the form "
                                  "'func' or 'Class.class_method'.")
-            script_text = _script_text % dict(
+            script_text = template % dict(
                 group=group,
                 module_name=ep.module_name,
                 import_target=ep.attrs[0],
diff --git a/pbr/tests/test_wsgi.py b/pbr/tests/test_wsgi.py
new file mode 100644
index 0000000..9eded63
--- /dev/null
+++ b/pbr/tests/test_wsgi.py
@@ -0,0 +1,171 @@
+# Copyright (c) 2015 Hewlett-Packard Development Company, L.P. (HP)
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import os
+import re
+import subprocess
+import sys
+import tempfile
+import time
+try:
+    # python 2
+    from urllib2 import urlopen
+except ImportError:
+    # python 3
+    from urllib.request import urlopen
+
+import fixtures
+
+from pbr.tests import base
+
+
+class TestWsgiScripts(base.BaseTestCase):
+
+    cmd_names = ('pbr_test_wsgi', 'pbr_test_wsgi_with_class')
+
+    def test_wsgi_script_install(self):
+        """Test that we install a non-pkg-resources wsgi script."""
+        if os.name == 'nt':
+            self.skipTest('Windows support is passthrough')
+
+        stdout, _, return_code = self.run_setup(
+            'install', '--prefix=%s' % self.temp_dir)
+
+        self.useFixture(
+            fixtures.EnvironmentVariable(
+                'PYTHONPATH', ".:%s/lib/python%s.%s/site-packages" % (
+                    self.temp_dir,
+                    sys.version_info[0],
+                    sys.version_info[1])))
+
+        self._check_wsgi_install_content(stdout)
+
+    def test_wsgi_script_run(self):
+        """Test that we install a runnable wsgi script.
+
+        This test actually attempts to start and interact with the
+        wsgi script in question to demonstrate that it's a working
+        wsgi script using simple server. It's a bit hokey because of
+        process management that has to be done.
+
+        """
+        self.skipTest("Test skipped until we can determine a reliable "
+                      "way to capture subprocess stdout without blocking")
+
+        if os.name == 'nt':
+            self.skipTest('Windows support is passthrough')
+
+        stdout, _, return_code = self.run_setup(
+            'install', '--prefix=%s' % self.temp_dir)
+
+        self.useFixture(
+            fixtures.EnvironmentVariable(
+                'PYTHONPATH', ".:%s/lib/python%s.%s/site-packages" % (
+                    self.temp_dir,
+                    sys.version_info[0],
+                    sys.version_info[1])))
+        # NOTE(sdague): making python unbuffered is critical to
+        # getting output out of the subprocess.
+        self.useFixture(
+            fixtures.EnvironmentVariable(
+                'PYTHONUNBUFFERED', '1'))
+
+        self._check_wsgi_install_content(stdout)
+
+        # Live test run the scripts and see that they respond to wsgi
+        # requests.
+        self._test_wsgi()
+
+    def _test_wsgi(self):
+        for cmd_name in self.cmd_names:
+            cmd = os.path.join(self.temp_dir, 'bin', cmd_name)
+            stdout = tempfile.NamedTemporaryFile()
+            print("Running %s > %s" % (cmd, stdout.name))
+            # NOTE(sdague): ok, this looks a little janky, and it
+            # is. However getting python to not hang with
+            # popen.communicate is beyond me.
+            #
+            # We're opening with a random port (so no conflicts), and
+            # redirecting all stdout and stderr to files. We can then
+            # safely read these files and not deadlock later in the
+            # test. This requires shell expansion.
+            p = subprocess.Popen(
+                "%s -p 0 > %s 2>&1" % (cmd, stdout.name),
+                shell=True,
+                close_fds=True,
+                cwd=self.temp_dir)
+
+            self.addCleanup(p.kill)
+
+            # the sleep is important to force a context switch to the
+            # subprocess
+            time.sleep(0.1)
+
+            stdoutdata = stdout.read()
+            self.assertIn(
+                "STARTING test server pbr_testpackage.wsgi",
+                stdoutdata)
+            self.assertIn(
+                "DANGER! For testing only, do not use in production",
+                stdoutdata)
+
+            m = re.search('(http://[^:]+:\d+)/', stdoutdata)
+            self.assertIsNotNone(m, "Regex failed to match on %s" % stdoutdata)
+
+            f = urlopen(m.group(1))
+            self.assertEqual("Hello World", f.read())
+
+            # the sleep is important to force a context switch to the
+            # subprocess
+            time.sleep(0.1)
+
+            # Kill off the child, it should force a flush of the stdout.
+            p.kill()
+            time.sleep(0.1)
+
+            stdoutdata = stdout.read()
+            # we should have logged an HTTP request, return code 200, that
+            # returned 11 bytes
+            self.assertIn('"GET / HTTP/1.1" 200 11', stdoutdata)
+
+    def _check_wsgi_install_content(self, install_stdout):
+        for cmd_name in self.cmd_names:
+            install_txt = 'Installing %s script to %s' % (cmd_name,
+                                                          self.temp_dir)
+            self.assertIn(install_txt, install_stdout)
+
+            cmd_filename = os.path.join(self.temp_dir, 'bin', cmd_name)
+
+            script_txt = open(cmd_filename, 'r').read()
+            self.assertNotIn('pkg_resources', script_txt)
+
+            main_block = """if __name__ == "__main__":
+    import argparse
+    import socket
+    import wsgiref.simple_server as wss"""
+
+            if cmd_name == 'pbr_test_wsgi':
+                app_name = "main"
+            else:
+                app_name = "WSGI.app"
+
+            starting_block = ("STARTING test server pbr_testpackage.wsgi."
+                              "%s" % app_name)
+
+            else_block = """else:
+    application = %s()""" % app_name
+
+            self.assertIn(main_block, script_txt)
+            self.assertIn(starting_block, script_txt)
+            self.assertIn(else_block, script_txt)
diff --git a/pbr/tests/testpackage/pbr_testpackage/wsgi.py b/pbr/tests/testpackage/pbr_testpackage/wsgi.py
new file mode 100644
index 0000000..7b96e66
--- /dev/null
+++ b/pbr/tests/testpackage/pbr_testpackage/wsgi.py
@@ -0,0 +1,31 @@
+# Copyright (c) 2013 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import print_function
+
+
+def application(env, start_response):
+    start_response('200 OK', [('Content-Type', 'text/html')])
+    return ["Hello World"]
+
+
+def main():
+    return application
+
+
+class WSGI(object):
+
+    @classmethod
+    def app(self):
+        return application
diff --git a/pbr/tests/testpackage/setup.cfg b/pbr/tests/testpackage/setup.cfg
index 0188bd2..7ba209f 100644
--- a/pbr/tests/testpackage/setup.cfg
+++ b/pbr/tests/testpackage/setup.cfg
@@ -38,6 +38,10 @@ console_scripts =
     pbr_test_cmd = pbr_testpackage.cmd:main
     pbr_test_cmd_with_class = pbr_testpackage.cmd:Foo.bar
 
+wsgi_scripts =
+    pbr_test_wsgi = pbr_testpackage.wsgi:main
+    pbr_test_wsgi_with_class = pbr_testpackage.wsgi:WSGI.app
+
 [extension=pbr_testpackage.testext]
 sources = src/testext.c
 optional = True