CXMAN-250: Retry 3 times to get fabric IP info

The ECME can sometimes give us incomplete info, so, let's retry
a couple of times and raise a ParseError if they all fail.

2 files changed, 27 insertions, 17 deletions

diff --git a/cxmanage_api/cx_exceptions.py b/cxmanage_api/cx_exceptions.py
index c4b2e0f..8f94955 100644
--- a/cxmanage_api/cx_exceptions.py
+++ b/cxmanage_api/cx_exceptions.py
@@ -401,4 +401,9 @@ class IPDiscoveryError(Exception):
         """String representation of this Exception class."""
         return self.msg
 
+
+class ParseError(Exception):
+    """Raised when there's an error parsing some output"""
+    pass
+
 # End of file: exceptions.py
diff --git a/cxmanage_api/node.py b/cxmanage_api/node.py
index 9e2b42d..9b0c43a 100644
--- a/cxmanage_api/node.py
+++ b/cxmanage_api/node.py
@@ -36,6 +36,7 @@ import os
 import re
 import time
 import tempfile
+import socket
 import subprocess
 
 from pkg_resources import parse_version
@@ -52,7 +53,7 @@ from cxmanage_api.ip_retriever import IPRetriever as IPRETRIEVER
 from cxmanage_api.cx_exceptions import TimeoutError, NoSensorError, \
         SocmanVersionError, FirmwareConfigError, PriorityIncrementError, \
         NoPartitionError, TransferFailure, ImageSizeError, \
-        PartitionInUseError, UbootenvError, EEPROMUpdateError
+        PartitionInUseError, UbootenvError, EEPROMUpdateError, ParseError
 
 
 # pylint: disable=R0902, R0904
@@ -1162,26 +1163,30 @@ communication.
 
         :raises IpmiError: If the IPMI command fails.
         :raises TftpException: If the TFTP transfer fails.
+        :raises ParseError: If we fail to parse IP info
 
         """
-        filename = self._run_fabric_command(
-            function_name='fabric_config_get_ip_info'
-        )
-
-        # Parse addresses from ipinfo file
-        results = {}
-        for line in open(filename):
-            if (line.startswith("Node")):
-                elements = line.split()
-                node_id = int(elements[1].rstrip(":"))
-                node_ip_address = elements[2]
+        for _ in range(3):
+            try:
+                filename = self._run_fabric_command(
+                    function_name='fabric_config_get_ip_info'
+                )
 
-                # Old boards used to return 0.0.0.0 sometimes -- might not be
-                # an issue anymore.
-                if (node_ip_address != "0.0.0.0"):
-                    results[node_id] = node_ip_address
+                results = {}
+                for line in open(filename):
+                    if line.strip():
+                        elements = line.split()
+                        node_id = int(elements[1].rstrip(":"))
+                        ip_address = elements[2]
+                        socket.inet_aton(ip_address)
+                        results[node_id] = ip_address
+                return results
+            except (IndexError, ValueError, socket.error):
+                pass
 
-        return results
+        raise ParseError(
+            "Failed to parse fabric IP info\n%s" % open(filename).read()
+        )
 
     def get_fabric_macaddrs(self):
         """Gets what macaddr information THIS node knows about the Fabric.