Merge pull request #1554 from cherrypy/feature/portendv10.0.0

[wip] Use portend for port checks

7 files changed, 75 insertions, 161 deletions

diff --git a/CHANGES.rst b/CHANGES.rst
index 1d7cb6f6..0dfc8523 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -1,3 +1,21 @@
+v10.0.0
+-------
+
+* #1332: CherryPy now uses `portend
+  <https://pypi.org/project/portend>`_ for checking and
+  waiting on ports for startup and teardown checks. The
+  following names are no longer present:
+
+  - cherrypy._cpserver.client_host
+  - cherrypy._cpserver.check_port
+  - cherrypy._cpserver.wait_for_free_port
+  - cherrypy._cpserver.wait_for_occupied_port
+  - cherrypy.process.servers.check_port
+  - cherrypy.process.servers.wait_for_free_port
+  - cherrypy.process.servers.wait_for_occupied_port
+
+  Use this functionality from the portend package directly.
+
 v9.0.0
 -----
 
diff --git a/cherrypy/_cpserver.py b/cherrypy/_cpserver.py
index c2295691..8e66208a 100644
--- a/cherrypy/_cpserver.py
+++ b/cherrypy/_cpserver.py
@@ -7,17 +7,8 @@ from cherrypy.lib.reprconf import attributes
 from cherrypy._cpcompat import text_or_bytes
 from cherrypy.process.servers import ServerAdapter
 
-# export some names from here
-from cherrypy.process.servers import (
-    client_host, check_port, wait_for_free_port, wait_for_occupied_port,
-)
 
-
-__all__ = [
-    'Server',
-    'client_host', 'check_port', 'wait_for_free_port',
-    'wait_for_occupied_port',
-]
+__all__ = ['Server']
 
 
 class Server(ServerAdapter):
diff --git a/cherrypy/process/servers.py b/cherrypy/process/servers.py
index 919d670b..1013f24e 100644
--- a/cherrypy/process/servers.py
+++ b/cherrypy/process/servers.py
@@ -121,6 +121,14 @@ import os
 import sys
 import time
 import warnings
+import contextlib
+
+import portend
+
+
+class Timeouts:
+    occupied = 5
+    free = 1
 
 
 class ServerAdapter(object):
@@ -166,7 +174,7 @@ class ServerAdapter(object):
         if not os.environ.get('LISTEN_PID', None):
             # Start the httpserver in a new thread.
             if isinstance(self.bind_addr, tuple):
-                wait_for_free_port(*self.bind_addr)
+                portend.free(*self.bind_addr, timeout=Timeouts.free)
 
         import threading
         t = threading.Thread(target=self._start_http_thread)
@@ -243,7 +251,8 @@ class ServerAdapter(object):
             # Wait for port to be occupied if not running via socket-activation
             # (for socket-activation the port will be managed by systemd )
             if isinstance(self.bind_addr, tuple):
-                wait_for_occupied_port(*self.bound_addr)
+                with _safe_wait(*self.bound_addr):
+                    portend.occupied(*self.bound_addr, timeout=Timeouts.occupied)
 
     @property
     def bound_addr(self):
@@ -264,7 +273,7 @@ class ServerAdapter(object):
             self.httpserver.stop()
             # Wait for the socket to be truly freed.
             if isinstance(self.bind_addr, tuple):
-                wait_for_free_port(*self.bound_addr)
+                portend.free(*self.bound_addr, timeout=Timeouts.free)
             self.running = False
             self.bus.log('HTTP Server %s shut down' % self.httpserver)
         else:
@@ -385,107 +394,18 @@ class FlupSCGIServer(object):
         self.scgiserver._threadPool.maxSpare = 0
 
 
-def client_host(server_host):
-    """Return the host on which a client can connect to the given listener."""
-    if server_host == '0.0.0.0':
-        # 0.0.0.0 is INADDR_ANY, which should answer on localhost.
-        return '127.0.0.1'
-    if server_host in ('::', '::0', '::0.0.0.0'):
-        # :: is IN6ADDR_ANY, which should answer on localhost.
-        # ::0 and ::0.0.0.0 are non-canonical but common
-        # ways to write IN6ADDR_ANY.
-        return '::1'
-    return server_host
-
-
-def check_port(host, port, timeout=1.0):
-    """Raise an error if the given port is not free on the given host."""
-    if not host:
-        raise ValueError("Host values of '' or None are not allowed.")
-    host = client_host(host)
-    port = int(port)
-
-    import socket
-
-    # AF_INET or AF_INET6 socket
-    # Get the correct address family for our host (allows IPv6 addresses)
+@contextlib.contextmanager
+def _safe_wait(host, port):
+    """
+    On systems where a loopback interface is not available and the
+    server is bound to all interfaces, it's difficult to determine
+    whether the server is in fact occupying the port. In this case,
+    just issue a warning and move on. See issue #1100.
+    """
     try:
-        info = socket.getaddrinfo(host, port, socket.AF_UNSPEC,
-                                  socket.SOCK_STREAM)
-    except socket.gaierror:
-        if ':' in host:
-            info = [(
-                socket.AF_INET6, socket.SOCK_STREAM, 0, '', (host, port, 0, 0)
-            )]
-        else:
-            info = [(socket.AF_INET, socket.SOCK_STREAM, 0, '', (host, port))]
-
-    for res in info:
-        af, socktype, proto, canonname, sa = res
-        s = None
-        try:
-            s = socket.socket(af, socktype, proto)
-            # See http://groups.google.com/group/cherrypy-users/
-            #        browse_frm/thread/bbfe5eb39c904fe0
-            s.settimeout(timeout)
-            s.connect((host, port))
-            s.close()
-        except socket.error:
-            if s:
-                s.close()
-        else:
-            raise IOError('Port %s is in use on %s; perhaps the previous '
-                          'httpserver did not shut down properly.' %
-                          (repr(port), repr(host)))
-
-
-# Feel free to increase these defaults on slow systems:
-free_port_timeout = 0.1
-occupied_port_timeout = 1.0
-
-
-def wait_for_free_port(host, port, timeout=None):
-    """Wait for the specified port to become free (drop requests)."""
-    if not host:
-        raise ValueError("Host values of '' or None are not allowed.")
-    if timeout is None:
-        timeout = free_port_timeout
-
-    for trial in range(50):
-        try:
-            # we are expecting a free port, so reduce the timeout
-            check_port(host, port, timeout=timeout)
-        except IOError:
-            # Give the old server thread time to free the port.
-            time.sleep(timeout)
-        else:
-            return
-
-    raise IOError('Port %r not free on %r' % (port, host))
-
-
-def wait_for_occupied_port(host, port, timeout=None):
-    """Wait for the specified port to become active (receive requests)."""
-    if not host:
-        raise ValueError("Host values of '' or None are not allowed.")
-    if timeout is None:
-        timeout = occupied_port_timeout
-
-    for trial in range(50):
-        try:
-            check_port(host, port, timeout=timeout)
-        except IOError:
-            # port is occupied
-            return
-        else:
-            time.sleep(timeout)
-
-    if host == client_host(host):
-        raise IOError('Port %r not bound on %r' % (port, host))
-
-    # On systems where a loopback interface is not available and the
-    #  server is bound to all interfaces, it's difficult to determine
-    #  whether the server is in fact occupying the port. In this case,
-    # just issue a warning and move on. See issue #1100.
-    msg = 'Unable to verify that the server is bound on %r' % port
-    warnings.warn(msg)
+        yield
+    except portend.Timeout:
+        if host == portend.client_host(host):
+            raise
+        msg = 'Unable to verify that the server is bound on %r' % port
+        warnings.warn(msg)
diff --git a/cherrypy/test/helper.py b/cherrypy/test/helper.py
index 17fe5433..28ec7e36 100644
--- a/cherrypy/test/helper.py
+++ b/cherrypy/test/helper.py
@@ -13,6 +13,7 @@ import warnings
 
 import nose
 import six
+import portend
 
 import cherrypy
 from cherrypy._cpcompat import text_or_bytes, copyitems, HTTPSConnection, ntob
@@ -470,7 +471,7 @@ server.ssl_private_key: r'%s'
 
     def start(self, imports=None):
         """Start cherryd in a subprocess."""
-        cherrypy._cpserver.wait_for_free_port(self.host, self.port)
+        portend.free(self.host, self.port, timeout=1)
 
         args = [
             '-m',
@@ -520,7 +521,7 @@ server.ssl_private_key: r'%s'
         if self.wait:
             self.exit_code = self._proc.wait()
         else:
-            cherrypy._cpserver.wait_for_occupied_port(self.host, self.port)
+            portend.occupied(self.host, self.port, timeout=5)
 
         # Give the engine a wee bit more time to finish STARTING
         if self.daemonize:
diff --git a/cherrypy/test/modwsgi.py b/cherrypy/test/modwsgi.py
index 0177c1c0..cd40ad54 100644
--- a/cherrypy/test/modwsgi.py
+++ b/cherrypy/test/modwsgi.py
@@ -37,7 +37,8 @@ import re
 import sys
 import time
 
-import cherrypy
+import portend
+
 from cherrypy.test import helper, webtest
 
 curdir = os.path.abspath(os.path.dirname(__file__))
@@ -120,7 +121,7 @@ class ModWSGISupervisor(helper.Supervisor):
 
         # Make a request so mod_wsgi starts up our app.
         # If we don't, concurrent initial requests will 404.
-        cherrypy._cpserver.wait_for_occupied_port('127.0.0.1', self.port)
+        portend.occupied('127.0.0.1', self.port, timeout=5)
         webtest.openURL('/ihopetheresnodefault', port=self.port)
         time.sleep(1)
 
diff --git a/cherrypy/test/test_states.py b/cherrypy/test/test_states.py
index ce6830a8..9c000a5a 100644
--- a/cherrypy/test/test_states.py
+++ b/cherrypy/test/test_states.py
@@ -1,11 +1,13 @@
 import os
 import signal
-import socket
 import sys
 import time
 import unittest
 import warnings
 
+import pytest
+import portend
+
 import cherrypy
 import cherrypy.process.servers
 from cherrypy._cpcompat import BadStatusLine, ntob
@@ -110,7 +112,7 @@ class ServerStateTests(helper.CPWebCase):
 
         host = cherrypy.server.socket_host
         port = cherrypy.server.socket_port
-        self.assertRaises(IOError, cherrypy._cpserver.check_port, host, port)
+        portend.occupied(host, port, timeout=0.1)
 
         # The db_connection should be running now
         self.assertEqual(db_connection.running, True)
@@ -288,7 +290,7 @@ class ServerStateTests(helper.CPWebCase):
             time.sleep(2)
             host = cherrypy.server.socket_host
             port = cherrypy.server.socket_port
-            cherrypy._cpserver.wait_for_occupied_port(host, port)
+            portend.occupied(host, port, timeout=5)
 
             self.getPage('/start')
             if not (float(self.body) > start):
@@ -469,7 +471,7 @@ test_case_name: "test_signal_handler_unsubscribe"
 
 class WaitTests(unittest.TestCase):
 
-    def test_wait_for_occupied_port_INADDR_ANY(self):
+    def test_safe_wait_INADDR_ANY(self):
         """
         Wait on INADDR_ANY should not raise IOError
 
@@ -485,43 +487,23 @@ class WaitTests(unittest.TestCase):
         # Simulate the behavior we observe when no loopback interface is
         #  present by: finding a port that's not occupied, then wait on it.
 
-        free_port = self.find_free_port()
+        free_port = portend.find_available_local_port()
 
         servers = cherrypy.process.servers
 
-        def with_shorter_timeouts(func):
-            """
-            A context where occupied_port_timeout is much smaller to speed
-            test runs.
-            """
-            # When we have Python 2.5, simplify using the with_statement.
-            orig_timeout = servers.occupied_port_timeout
-            servers.occupied_port_timeout = .07
-            try:
-                func()
-            finally:
-                servers.occupied_port_timeout = orig_timeout
-
-        def do_waiting():
-            # Wait on the free port that's unbound
-            with warnings.catch_warnings(record=True) as w:
-                servers.wait_for_occupied_port('0.0.0.0', free_port)
-                self.assertEqual(len(w), 1)
-                self.assertTrue(isinstance(w[0], warnings.WarningMessage))
-                self.assertTrue(
-                    'Unable to verify that the server is bound on ' in str(w[0]))
-
-            # The wait should still raise an IO error if INADDR_ANY was
-            #  not supplied.
-            self.assertRaises(IOError, servers.wait_for_occupied_port,
-                              '127.0.0.1', free_port)
-
-        with_shorter_timeouts(do_waiting)
-
-    def find_free_port(self):
-        'Find a free port by binding to port 0 then unbinding.'
-        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-        sock.bind(('', 0))
-        free_port = sock.getsockname()[1]
-        sock.close()
-        return free_port
+        inaddr_any = '0.0.0.0'
+
+        # Wait on the free port that's unbound
+        with warnings.catch_warnings(record=True) as w:
+            with servers._safe_wait(inaddr_any, free_port):
+                portend.occupied(inaddr_any, free_port, timeout=1)
+            self.assertEqual(len(w), 1)
+            self.assertTrue(isinstance(w[0], warnings.WarningMessage))
+            self.assertTrue(
+                'Unable to verify that the server is bound on ' in str(w[0]))
+
+        # The wait should still raise an IO error if INADDR_ANY was
+        #  not supplied.
+        with pytest.raises(IOError):
+            with servers._safe_wait('127.0.0.1', free_port):
+                portend.occupied('127.0.0.1', free_port, timeout=1)
diff --git a/setup.py b/setup.py
index e395cab1..6e148873 100644
--- a/setup.py
+++ b/setup.py
@@ -62,6 +62,7 @@ packages = [
 install_requires = [
     'six',
     'cheroot>=5.0.1',
+    'portend>=1.6.1',
 ]
 
 extras_require = {