sync_power_states handles missing driver info

The ConductorManager._sync_power_states() periodic task will stop
checking power states if getting the power state of a node raises an
exception. Eg, this could happen if a node is not properly configured
(for which node.driver.power.validate() raises an exception).

With this change, sync_power_state catches the exceptions, logs them,
and continues checking other nodes.

Change-Id: I343f722de8f79a50fbaa9fcdd180decdc5a43f95
Closes-Bug: #1262912

2 files changed, 47 insertions, 1 deletions

diff --git a/ironic/conductor/manager.py b/ironic/conductor/manager.py
index 3c5df8bb5..db97dac9c 100644
--- a/ironic/conductor/manager.py
+++ b/ironic/conductor/manager.py
@@ -344,7 +344,18 @@ class ConductorManager(service.PeriodicService):
             try:
                 with task_manager.acquire(context, node_id) as task:
                     node = task.node
-                    power_state = task.driver.power.get_power_state(task, node)
+
+                    try:
+                        power_state = task.driver.power.get_power_state(task,
+                                                                        node)
+                    except Exception as e:
+                        #TODO(rloo): change to IronicException, after
+                        # https://bugs.launchpad.net/ironic/+bug/1267693
+                        LOG.debug(_("During sync_power_state, could not get "
+                            "power state for node %(node)s. Error: %(err)s.") %
+                            {'node': node.uuid, 'err': e})
+                        continue
+
                     if power_state != node['power_state']:
                         # NOTE(deva): don't log a warning the first time we
                         #             sync a node's power state
diff --git a/ironic/tests/conductor/test_manager.py b/ironic/tests/conductor/test_manager.py
index 7d3ad79b4..217dfa28e 100644
--- a/ironic/tests/conductor/test_manager.py
+++ b/ironic/tests/conductor/test_manager.py
@@ -21,6 +21,7 @@
 
 import mock
 from oslo.config import cfg
+from testtools.matchers import HasLength
 
 from ironic.common import driver_factory
 from ironic.common import exception
@@ -145,6 +146,40 @@ class ManagerTestCase(base.DbTestCase):
         self.assertEqual(n1['power_state'], states.POWER_OFF)
         self.assertEqual(n3['power_state'], states.POWER_ON)
 
+    def test__sync_power_state_node_no_power_state(self):
+        self.service.start()
+
+        # create three nodes
+        nodes = []
+        for i in range(0, 3):
+            n = utils.get_test_node(id=i, uuid=ironic_utils.generate_uuid(),
+                    driver='fake', power_state=states.POWER_OFF)
+            self.dbapi.create_node(n)
+            nodes.append(n['uuid'])
+
+        # cannot get power state of node 2; only nodes 1 & 3 have
+        # their power states changed.
+        with mock.patch.object(self.driver.power,
+                               'get_power_state') as get_power_mock:
+            returns = [states.POWER_ON,
+                       exception.InvalidParameterValue("invalid"),
+                       states.POWER_ON]
+
+            def side_effect(*args):
+                result = returns.pop(0)
+                if isinstance(result, Exception):
+                    raise result
+                return result
+
+            get_power_mock.side_effect = side_effect
+            self.service._sync_power_states(self.context)
+            self.assertThat(returns, HasLength(0))
+
+        final = [states.POWER_ON, states.POWER_OFF, states.POWER_ON]
+        for i in range(0, 3):
+            n = self.dbapi.get_node(nodes[i])
+            self.assertEqual(n.power_state, final[i])
+
     def test_get_power_state(self):
         n = utils.get_test_node(driver='fake')
         self.dbapi.create_node(n)