Enabling self-disabled pollster

Avoid loading pollster if no required environment, and stopped polling if
resources become unavailable

Change-Id: Ib3cba8ab5236e28b05a091619ebcf149767e35d8
Implements: blueprint self-disabled-pollster

3 files changed, 91 insertions, 4 deletions

diff --git a/ceilometer/ipmi/pollsters/__init__.py b/ceilometer/ipmi/pollsters/__init__.py
index e69de29b..9ebbf230 100644
--- a/ceilometer/ipmi/pollsters/__init__.py
+++ b/ceilometer/ipmi/pollsters/__init__.py
@@ -0,0 +1,29 @@
+# Copyright 2014 Intel Corporation.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Pollsters for IPMI and Intel Node Manager
+"""
+
+from oslo_config import cfg
+
+OPTS = [
+    cfg.IntOpt('polling_retry',
+               default=3,
+               help='Tolerance of IPMI/NM polling failures '
+                    'before disable this pollster. '
+                    'Negative indicates retrying forever.')
+]
+
+cfg.CONF.register_opts(OPTS, group='ipmi')
diff --git a/ceilometer/ipmi/pollsters/node.py b/ceilometer/ipmi/pollsters/node.py
index df5f7ea7..bab40a0a 100644
--- a/ceilometer/ipmi/pollsters/node.py
+++ b/ceilometer/ipmi/pollsters/node.py
@@ -19,17 +19,31 @@ from oslo_utils import timeutils
 import six
 
 from ceilometer.agent import plugin_base
+from ceilometer.i18n import _
+from ceilometer.ipmi.platform import exception as nmexcept
 from ceilometer.ipmi.platform import intel_node_manager as node_manager
+from ceilometer.openstack.common import log
 from ceilometer import sample
 
 CONF = cfg.CONF
 CONF.import_opt('host', 'ceilometer.service')
+CONF.import_opt('polling_retry', 'ceilometer.ipmi.pollsters',
+                group='ipmi')
+
+LOG = log.getLogger(__name__)
 
 
 @six.add_metaclass(abc.ABCMeta)
 class _Base(plugin_base.PollsterBase):
-    def __init__(self):
+
+    def setup_environment(self):
+        super(_Base, self).setup_environment()
         self.nodemanager = node_manager.NodeManager()
+        self.polling_failures = 0
+
+        # Do not load this extension if no NM support
+        if not self.nodemanager.nm_support:
+            raise plugin_base.ExtensionLoadError()
 
     @property
     def default_discovery(self):
@@ -40,7 +54,22 @@ class _Base(plugin_base.PollsterBase):
         """Return data sample for IPMI."""
 
     def get_samples(self, manager, cache, resources):
-        stats = self.read_data()
+        # Only one resource for Node Manager pollster
+        try:
+            stats = self.read_data()
+        except nmexcept.IPMIException:
+            self.polling_failures += 1
+            LOG.warning(_('Polling %(name)s faild for %(cnt)s times!')
+                        % ({'name': self.NAME,
+                            'cnt': self.polling_failures}))
+            if (CONF.ipmi.polling_retry >= 0 and
+                    self.polling_failures > CONF.ipmi.polling_retry):
+                LOG.warning(_('Pollster for %s is disabled!') % self.NAME)
+                raise plugin_base.PollsterPermanentError(resources[0])
+            else:
+                return
+
+        self.polling_failures = 0
 
         metadata = {
             'node': CONF.host
diff --git a/ceilometer/ipmi/pollsters/sensor.py b/ceilometer/ipmi/pollsters/sensor.py
index 9dbc0ecb..dbfd95c7 100644
--- a/ceilometer/ipmi/pollsters/sensor.py
+++ b/ceilometer/ipmi/pollsters/sensor.py
@@ -16,12 +16,19 @@ from oslo_config import cfg
 from oslo_utils import timeutils
 
 from ceilometer.agent import plugin_base
+from ceilometer.i18n import _
 from ceilometer.ipmi.notifications import ironic as parser
+from ceilometer.ipmi.platform import exception as ipmiexcept
 from ceilometer.ipmi.platform import ipmi_sensor
+from ceilometer.openstack.common import log
 from ceilometer import sample
 
 CONF = cfg.CONF
 CONF.import_opt('host', 'ceilometer.service')
+CONF.import_opt('polling_retry', 'ceilometer.ipmi.pollsters',
+                group='ipmi')
+
+LOG = log.getLogger(__name__)
 
 
 class InvalidSensorData(ValueError):
@@ -31,8 +38,14 @@ class InvalidSensorData(ValueError):
 class SensorPollster(plugin_base.PollsterBase):
     METRIC = None
 
-    def __init__(self):
+    def setup_environment(self):
+        super(SensorPollster, self).setup_environment()
         self.ipmi = ipmi_sensor.IPMISensor()
+        self.polling_failures = 0
+
+        # Do not load this extension if no IPMI support
+        if not self.ipmi.ipmi_support:
+            raise plugin_base.ExtensionLoadError()
 
     @property
     def default_discovery(self):
@@ -47,7 +60,23 @@ class SensorPollster(plugin_base.PollsterBase):
             return []
 
     def get_samples(self, manager, cache, resources):
-        stats = self.ipmi.read_sensor_any(self.METRIC)
+        # Only one resource for IPMI pollster
+        try:
+            stats = self.ipmi.read_sensor_any(self.METRIC)
+        except ipmiexcept.IPMIException:
+            self.polling_failures += 1
+            LOG.warning(_(
+                'Polling %(mtr)s sensor failed for %(cnt)s times!')
+                % ({'mtr': self.METRIC,
+                    'cnt': self.polling_failures}))
+            if (CONF.ipmi.polling_retry >= 0 and
+                    self.polling_failures > CONF.ipmi.polling_retry):
+                LOG.warning(_('Pollster for %s is disabled!') % self.METRIC)
+                raise plugin_base.PollsterPermanentError(resources[0])
+            else:
+                return
+
+        self.polling_failures = 0
 
         sensor_type_data = self._get_sensor_types(stats, self.METRIC)