summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJenkins <jenkins@review.openstack.org>2015-09-26 10:12:02 +0000
committerGerrit Code Review <review@openstack.org>2015-09-26 10:12:02 +0000
commit20ef38112c1b565b698d54916a60550cc8ea491f (patch)
treef0bfcf0ee4cc6cc40dc15425966cf53e91910a58
parentde3f58b4b7d7ca15a412bbec8ec00d2d93dbc752 (diff)
parentccff90e5b604904831e2085b173ea9a66d0441b8 (diff)
downloadoslo-middleware-20ef38112c1b565b698d54916a60550cc8ea491f.tar.gz
Merge "Allow the healthcheck middleware to provide more detailed responses"
Diffstat
-rw-r--r--oslo_middleware/healthcheck/__init__.py161
-rw-r--r--oslo_middleware/tests/test_healthcheck.py30
-rw-r--r--requirements.txt3
3 files changed, 174 insertions, 20 deletions
diff --git a/oslo_middleware/healthcheck/__init__.py b/oslo_middleware/healthcheck/__init__.py
index bdbbd24..9aa2c2e 100644
--- a/oslo_middleware/healthcheck/__init__.py
+++ b/oslo_middleware/healthcheck/__init__.py
@@ -13,6 +13,19 @@
# License for the specific language governing permissions and limitations
# under the License.
+import json
+import socket
+
+try:
+ from collections import OrderedDict # noqa
+except ImportError:
+ # TODO(harlowja): remove this when py2.6 support is dropped...
+ from ordereddict import OrderedDict # noqa
+
+import jinja2
+from oslo_utils import reflection
+from oslo_utils import strutils
+import six
import stevedore
import webob.dec
import webob.exc
@@ -21,6 +34,12 @@ import webob.response
from oslo_middleware import base
+def _expand_template(contents, params):
+ tpl = jinja2.Template(source=contents,
+ undefined=jinja2.StrictUndefined)
+ return tpl.render(**params)
+
+
class Healthcheck(base.ConfigurableMiddleware):
"""Healthcheck middleware used for monitoring.
@@ -28,6 +47,13 @@ class Healthcheck(base.ConfigurableMiddleware):
Or 503 with the reason as the body if one of the backend report
an application issue.
+ This is useful for the following reasons:
+
+ 1. Load balancers can 'ping' this url to determine service availability.
+ 2. Provides an endpoint that is similar to 'mod_status' in apache which
+ can provide details (or no details, depending on if configured) about
+ the activity of the server.
+
Example of paste configuration:
.. code-block:: ini
@@ -71,36 +97,139 @@ class Healthcheck(base.ConfigurableMiddleware):
"""
NAMESPACE = "oslo.middleware.healthcheck"
+ HEALTHY_TO_STATUS_CODES = {
+ True: webob.exc.HTTPOk.code,
+ False: webob.exc.HTTPServiceUnavailable.code,
+ }
+ PLAIN_RESPONSE_TEMPLATE = """
+{% for reason in reasons %}
+{% if reason %}{{reason}}{% endif -%}
+{% endfor %}
+"""
+
+ HTML_RESPONSE_TEMPLATE = """
+<HTML>
+<HEAD><TITLE>Healthcheck Status</TITLE></HEAD>
+<BODY>
+{% if detailed -%}
+{% if hostname -%}
+<H1>Server status for {{hostname|e}}</H1>
+{%- endif %}
+{%- endif %}
+<H2>Result of {{results|length}} checks:</H2>
+<TABLE bgcolor="#ffffff" border="1">
+<TBODY>
+{% for result in results -%}
+{% if result.reason -%}
+<TR>
+{% if detailed -%}
+ <TD>{{result.class|e}}</TD>
+{%- endif %}
+ <TD>{{result.reason|e}}</TD>
+</TR>
+{%- endif %}
+{%- endfor %}
+</TBODY>
+</TABLE>
+</BODY>
+</HTML>
+"""
def __init__(self, application, conf):
super(Healthcheck, self).__init__(application)
self._path = conf.get('path', '/healthcheck')
+ self._show_details = strutils.bool_from_string(conf.get('detailed'))
self._backend_names = []
backends = conf.get('backends')
if backends:
self._backend_names = backends.split(',')
-
self._backends = stevedore.NamedExtensionManager(
self.NAMESPACE, self._backend_names,
name_order=True, invoke_on_load=True,
invoke_args=(conf,))
+ self._accept_to_functor = OrderedDict([
+ # Order here matters...
+ ('text/plain', self._make_text_response),
+ ('text/html', self._make_html_response),
+ ('application/json', self._make_json_response),
+ ])
+ self._accept_order = tuple(six.iterkeys(self._accept_to_functor))
+ # When no accept type matches instead of returning 406 we will
+ # always return text/plain (because sending an error from this
+ # middleware actually can cause issues).
+ self._default_accept = 'text/plain'
+
+ @staticmethod
+ def _pretty_json_dumps(contents):
+ return json.dumps(contents, indent=4, sort_keys=True)
+
+ @staticmethod
+ def _are_results_healthy(results):
+ for result in results:
+ if not result.available:
+ return False
+ return True
+
+ def _make_text_response(self, results, healthy):
+ params = {
+ 'reasons': [result.reason for result in results],
+ 'detailed': self._show_details,
+ }
+ body = _expand_template(self.PLAIN_RESPONSE_TEMPLATE, params)
+ return (body.strip(), 'text/plain')
+
+ def _make_json_response(self, results, healthy):
+ if self._show_details:
+ body = {
+ 'detailed': True,
+ }
+ reasons = []
+ for result in results:
+ reasons.append({
+ 'reason': result.reason,
+ 'class': reflection.get_class_name(result,
+ fully_qualified=False),
+ })
+ body['reasons'] = reasons
+ else:
+ body = {
+ 'reasons': [result.reason for result in results],
+ 'detailed': False,
+ }
+ return (self._pretty_json_dumps(body), 'application/json')
+
+ def _make_html_response(self, results, healthy):
+ try:
+ hostname = socket.gethostname()
+ except socket.error:
+ hostname = None
+ translated_results = []
+ for result in results:
+ translated_results.append({
+ 'reason': result.reason,
+ 'class': reflection.get_class_name(result,
+ fully_qualified=False),
+ })
+ params = {
+ 'healthy': healthy,
+ 'hostname': hostname,
+ 'results': translated_results,
+ 'detailed': self._show_details,
+ }
+ body = _expand_template(self.HTML_RESPONSE_TEMPLATE, params)
+ return (body.strip(), 'text/html')
@webob.dec.wsgify
def process_request(self, req):
if req.path != self._path:
return None
-
- healthy = True
- reasons = []
- for ext in self._backends:
- result = ext.obj.healthcheck()
- healthy &= result.available
- if result.reason:
- reasons.append(result.reason)
-
- return webob.response.Response(
- status=(webob.exc.HTTPOk.code if healthy
- else webob.exc.HTTPServiceUnavailable.code),
- body='\n'.join(reasons),
- content_type="text/plain",
- )
+ results = [ext.obj.healthcheck() for ext in self._backends]
+ healthy = self._are_results_healthy(results)
+ accept_type = req.accept.best_match(self._accept_order)
+ if not accept_type:
+ accept_type = self._default_accept
+ functor = self._accept_to_functor[accept_type]
+ body, content_type = functor(results, healthy)
+ status = self.HEALTHY_TO_STATUS_CODES[healthy]
+ return webob.response.Response(status=status, body=body,
+ content_type=content_type)
diff --git a/oslo_middleware/tests/test_healthcheck.py b/oslo_middleware/tests/test_healthcheck.py
index 13aa5fd..b516d0c 100644
--- a/oslo_middleware/tests/test_healthcheck.py
+++ b/oslo_middleware/tests/test_healthcheck.py
@@ -28,12 +28,17 @@ class HealthcheckTests(test_base.BaseTestCase):
def application(req):
return 'Hello, World!!!'
- def _do_test(self, conf={}, path='/healthcheck',
- expected_code=webob.exc.HTTPOk.code,
- expected_body=b''):
+ def _do_test_request(self, conf={}, path='/healthcheck',
+ accept='text/plain'):
self.app = healthcheck.Healthcheck(self.application, conf)
- req = webob.Request.blank(path)
+ req = webob.Request.blank(path, accept=accept)
res = req.get_response(self.app)
+ return res
+
+ def _do_test(self, conf={}, path='/healthcheck',
+ expected_code=webob.exc.HTTPOk.code,
+ expected_body=b'', accept='text/plain'):
+ res = self._do_test_request(conf=conf, path=path, accept=accept)
self.assertEqual(expected_code, res.status_int)
self.assertEqual(expected_body, res.body)
@@ -69,6 +74,14 @@ class HealthcheckTests(test_base.BaseTestCase):
self._do_test(conf, expected_body=b'OK')
self.assertIn('disable_by_file', self.app._backends.names())
+ def test_disablefile_enabled_html_detailed(self):
+ conf = {'backends': 'disable_by_file',
+ 'disable_by_file_path': '/foobar', 'detailed': True}
+ res = self._do_test_request(conf, accept="text/html")
+ self.assertIn(b'Result of 1 checks:', res.body)
+ self.assertIn(b'<TD>OK</TD>', res.body)
+ self.assertEqual(webob.exc.HTTPOk.code, res.status_int)
+
def test_disablefile_disabled(self):
filename = self.create_tempfiles([('test', 'foobar')])[0]
conf = {'backends': 'disable_by_file',
@@ -78,6 +91,15 @@ class HealthcheckTests(test_base.BaseTestCase):
expected_body=b'DISABLED BY FILE')
self.assertIn('disable_by_file', self.app._backends.names())
+ def test_disablefile_disabled_html_detailed(self):
+ filename = self.create_tempfiles([('test', 'foobar')])[0]
+ conf = {'backends': 'disable_by_file',
+ 'disable_by_file_path': filename, 'detailed': True}
+ res = self._do_test_request(conf, accept="text/html")
+ self.assertIn(b'<TD>DISABLED BY FILE</TD>', res.body)
+ self.assertEqual(webob.exc.HTTPServiceUnavailable.code,
+ res.status_int)
+
def test_two_backends(self):
filename = self.create_tempfiles([('test', 'foobar')])[0]
conf = {'backends': 'disable_by_file,disable_by_file',
diff --git a/requirements.txt b/requirements.txt
index 155d914..3107167 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -4,9 +4,12 @@
pbr>=1.6
Babel>=1.3
+Jinja2>=2.6 # BSD License (3 clause)
+ordereddict
oslo.config>=2.3.0 # Apache-2.0
oslo.context>=0.2.0 # Apache-2.0
oslo.i18n>=1.5.0 # Apache-2.0
+oslo.utils>=2.0.0 # Apache-2.0
six>=1.9.0
stevedore>=1.5.0 # Apache-2.0
WebOb>=1.2.3
View Lorry Controller Status