Bump up sleep when expecting a timeout

It seemed like some of the tests ment to exercise the proxy's timeout could
would occastionally race and the backend mock would be able to cough up a
chunk before the timeout fired in the proxy - typically resulting in a failed
assertion like " lalala" != 'lalala'.

We paramaterize the timeout value, and bump it up for those cases where we are
expecting to hit the timeout.  Timing shows this change is just as fast for
passing tests.  And if you want to break the node_timeout in
proxy.controller.base you can verify the tests are still just as effective -
if a bit slower to detect the failure path.

Fixes bug #1272509

Change-Id: Iaf91d9d551e94fc317a08e8c0ee02daeed331b60

2 files changed, 38 insertions, 25 deletions

diff --git a/test/unit/__init__.py b/test/unit/__init__.py
index 1e6bc5d6e..b24d5d64d 100644
--- a/test/unit/__init__.py
+++ b/test/unit/__init__.py
@@ -33,6 +33,7 @@ from hashlib import md5
 from eventlet import sleep, Timeout
 import logging.handlers
 from httplib import HTTPException
+from numbers import Number
 
 
 class FakeRing(object):
@@ -450,8 +451,11 @@ def fake_http_connect(*code_iter, **kwargs):
             self.body = body
             self.headers = headers or {}
             self.timestamp = timestamp
-            if kwargs.get('slow') and isinstance(kwargs['slow'], list):
-                kwargs['slow'][0] -= 1
+            if 'slow' in kwargs and isinstance(kwargs['slow'], list):
+                try:
+                    self._next_sleep = kwargs['slow'].pop(0)
+                except IndexError:
+                    self._next_sleep = None
 
         def getresponse(self):
             if kwargs.get('raise_exc'):
@@ -495,31 +499,39 @@ def fake_http_connect(*code_iter, **kwargs):
                     headers['x-container-timestamp'] = '1'
             except StopIteration:
                 pass
-            if self.am_slow():
+            am_slow, value = self.get_slow()
+            if am_slow:
                 headers['content-length'] = '4'
             headers.update(self.headers)
             return headers.items()
 
-        def am_slow(self):
-            if kwargs.get('slow') and isinstance(kwargs['slow'], list):
-                return kwargs['slow'][0] >= 0
-            return bool(kwargs.get('slow'))
+        def get_slow(self):
+            if 'slow' in kwargs and isinstance(kwargs['slow'], list):
+                if self._next_sleep is not None:
+                    return True, self._next_sleep
+                else:
+                    return False, 0.01
+            if kwargs.get('slow') and isinstance(kwargs['slow'], Number):
+                return True, kwargs['slow']
+            return bool(kwargs.get('slow')), 0.1
 
         def read(self, amt=None):
-            if self.am_slow():
+            am_slow, value = self.get_slow()
+            if am_slow:
                 if self.sent < 4:
                     self.sent += 1
-                    sleep(0.1)
+                    sleep(value)
                     return ' '
             rv = self.body[:amt]
             self.body = self.body[amt:]
             return rv
 
         def send(self, amt=None):
-            if self.am_slow():
+            am_slow, value = self.get_slow()
+            if am_slow:
                 if self.received < 4:
                     self.received += 1
-                    sleep(0.1)
+                    sleep(value)
 
         def getheader(self, name, default=None):
             return dict(self.getheaders()).get(name.lower(), default)
diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py
index ed037b8c9..398287d13 100644
--- a/test/unit/proxy/test_server.py
+++ b/test/unit/proxy/test_server.py
@@ -1678,7 +1678,7 @@ class TestObjectController(unittest.TestCase):
                 dev['port'] = 1
             req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'})
             self.app.update_request(req)
-            set_http_connect(200, 200, 200, slow=True)
+            set_http_connect(200, 200, 200, slow=0.1)
             req.sent_size = 0
             resp = req.get_response(self.app)
             got_exc = False
@@ -1688,7 +1688,7 @@ class TestObjectController(unittest.TestCase):
                 got_exc = True
             self.assert_(not got_exc)
             self.app.recoverable_node_timeout = 0.1
-            set_http_connect(200, 200, 200, slow=True)
+            set_http_connect(200, 200, 200, slow=1.0)
             resp = req.get_response(self.app)
             got_exc = False
             try:
@@ -1703,16 +1703,17 @@ class TestObjectController(unittest.TestCase):
             self.app.update_request(req)
 
             self.app.recoverable_node_timeout = 0.1
-            set_http_connect(200, 200, 200, slow=[3])
+            set_http_connect(200, 200, 200, slow=[1.0, 1.0, 1.0])
             resp = req.get_response(self.app)
             got_exc = False
             try:
-                resp.body
+                self.assertEquals('', resp.body)
             except ChunkReadTimeout:
                 got_exc = True
             self.assert_(got_exc)
 
-            set_http_connect(200, 200, 200, body='lalala', slow=[2])
+            set_http_connect(200, 200, 200, body='lalala',
+                             slow=[1.0, 1.0])
             resp = req.get_response(self.app)
             got_exc = False
             try:
@@ -1721,8 +1722,8 @@ class TestObjectController(unittest.TestCase):
                 got_exc = True
             self.assert_(not got_exc)
 
-            set_http_connect(200, 200, 200, body='lalala', slow=[2],
-                             etags=['a', 'a', 'a'])
+            set_http_connect(200, 200, 200, body='lalala',
+                             slow=[1.0, 1.0], etags=['a', 'a', 'a'])
             resp = req.get_response(self.app)
             got_exc = False
             try:
@@ -1731,8 +1732,8 @@ class TestObjectController(unittest.TestCase):
                 got_exc = True
             self.assert_(not got_exc)
 
-            set_http_connect(200, 200, 200, body='lalala', slow=[2],
-                             etags=['a', 'b', 'a'])
+            set_http_connect(200, 200, 200, body='lalala',
+                             slow=[1.0, 1.0], etags=['a', 'b', 'a'])
             resp = req.get_response(self.app)
             got_exc = False
             try:
@@ -1742,8 +1743,8 @@ class TestObjectController(unittest.TestCase):
             self.assert_(not got_exc)
 
             req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'})
-            set_http_connect(200, 200, 200, body='lalala', slow=[2],
-                             etags=['a', 'b', 'b'])
+            set_http_connect(200, 200, 200, body='lalala',
+                             slow=[1.0, 1.0], etags=['a', 'b', 'b'])
             resp = req.get_response(self.app)
             got_exc = False
             try:
@@ -1772,7 +1773,7 @@ class TestObjectController(unittest.TestCase):
                                          'Content-Type': 'text/plain'},
                                 body='    ')
             self.app.update_request(req)
-            set_http_connect(200, 200, 201, 201, 201, slow=True)
+            set_http_connect(200, 200, 201, 201, 201, slow=0.1)
             resp = req.get_response(self.app)
             self.assertEquals(resp.status_int, 201)
             self.app.node_timeout = 0.1
@@ -1782,7 +1783,7 @@ class TestObjectController(unittest.TestCase):
                                          'Content-Type': 'text/plain'},
                                 body='    ')
             self.app.update_request(req)
-            set_http_connect(201, 201, 201, slow=True)
+            set_http_connect(201, 201, 201, slow=1.0)
             resp = req.get_response(self.app)
             self.assertEquals(resp.status_int, 503)
 
@@ -5111,7 +5112,7 @@ class TestContainerController(unittest.TestCase):
         with save_globals():
             req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': 'GET'})
             self.app.node_timeout = 0.1
-            set_http_connect(200, 200, 200, body='abcdef', slow=[2])
+            set_http_connect(200, 200, 200, body='abcdef', slow=[1.0, 1.0])
             resp = req.get_response(self.app)
             got_exc = False
             try: