Merge citrix into master.

This was a somewhat difficult merge since there was a fair amount of
superficially divergent development on the two branches, especially in the
datapath.

This has been build-tested against XenServer 5.5.0 and XenServer 5.7.0
build 15122.  It has been booted and connected to XenCenter on 5.5.0.

The merge revealed a couple of outstanding bugs, which will be fixed on
citrix and then merged back into master.

1 files changed, 288 insertions, 75 deletions

diff --git a/xenserver/opt_xensource_libexec_interface-reconfigure b/xenserver/opt_xensource_libexec_interface-reconfigure
index 0ae8e70a8..98bcf4c3e 100755
--- a/xenserver/opt_xensource_libexec_interface-reconfigure
+++ b/xenserver/opt_xensource_libexec_interface-reconfigure
@@ -62,8 +62,9 @@ import syslog
 import traceback
 import time
 import re
-import pickle
 import random
+from xml.dom.minidom import getDOMImplementation
+from xml.dom.minidom import parse as parseXML
 
 output_directory = None
 
@@ -276,11 +277,215 @@ def get_netdev_by_mac(mac):
                 maybe = device
     return maybe
 
+#
+# Helper functions for encoding/decoding database attributes to/from XML.
+#
+def str_to_xml(xml, parent, tag, val):
+    e = xml.createElement(tag)
+    parent.appendChild(e)
+    v = xml.createTextNode(val)
+    e.appendChild(v)
+def str_from_xml(n):
+    def getText(nodelist):
+        rc = ""
+        for node in nodelist:
+            if node.nodeType == node.TEXT_NODE:
+                rc = rc + node.data
+        return rc
+    return getText(n.childNodes).strip()
+
+
+def bool_to_xml(xml, parent, tag, val):
+    if val:
+        str_to_xml(xml, parent, tag, "True")
+    else:
+        str_to_xml(xml, parent, tag, "False")
+def bool_from_xml(n):
+    s = str_from_xml(n)
+    if s == "True":
+        return True
+    elif s == "False":
+        return False
+    else:
+        raise Error("Unknown boolean value %s" % s);
+
+def strlist_to_xml(xml, parent, ltag, itag, val):
+    e = xml.createElement(ltag)
+    parent.appendChild(e)
+    for v in val:
+        c = xml.createElement(itag)
+        e.appendChild(c)
+        cv = xml.createTextNode(v)
+        c.appendChild(cv)
+def strlist_from_xml(n, ltag, itag):
+    ret = []
+    for n in n.childNodes:
+        if n.nodeName == itag:
+            ret.append(str_from_xml(n))
+    return ret
+
+def otherconfig_to_xml(xml, parent, val, attrs):
+    otherconfig = xml.createElement("other_config")
+    parent.appendChild(otherconfig)
+    for n,v in val.items():
+        if not n in attrs:
+            raise Error("Unknown other-config attribute: %s" % n)
+        str_to_xml(xml, otherconfig, n, v)
+def otherconfig_from_xml(n, attrs):
+    ret = {}
+    for n in n.childNodes:
+        if n.nodeName in attrs:
+            ret[n.nodeName] = str_from_xml(n)
+    return ret
+
+#
+# Definitions of the database objects (and their attributes) used by interface-reconfigure.
+#
+# Each object is defined by a dictionary mapping an attribute name in
+# the xapi database to a tuple containing two items:
+#  - a function which takes this attribute and encodes it as XML.
+#  - a function which takes XML and decocdes it into a value.
+#
+# other-config attributes are specified as a simple array of strings
+
+PIF_XML_TAG = "pif"
+VLAN_XML_TAG = "vlan"
+BOND_XML_TAG = "bond"
+NETWORK_XML_TAG = "network"
+
+ETHTOOL_OTHERCONFIG_ATTRS = ['ethtool-%s' % x for x in 'autoneg', 'speed', 'duplex', 'rx', 'tx', 'sg', 'tso', 'ufo', 'gso' ]
+
+PIF_ATTRS = { 'uuid': (str_to_xml,str_from_xml),
+              'management': (bool_to_xml,bool_from_xml),
+              'network': (str_to_xml,str_from_xml),
+              'device': (str_to_xml,str_from_xml),
+              'bond_master_of': (lambda x, p, t, v: strlist_to_xml(x, p, 'bond_master_of', 'slave', v),
+                                 lambda n: strlist_from_xml(n, 'bond_master_of', 'slave')),
+              'bond_slave_of': (str_to_xml,str_from_xml),
+              'VLAN': (str_to_xml,str_from_xml),
+              'VLAN_master_of': (str_to_xml,str_from_xml),
+              'VLAN_slave_of': (lambda x, p, t, v: strlist_to_xml(x, p, 'VLAN_slave_of', 'master', v),
+                                lambda n: strlist_from_xml(n, 'VLAN_slave_Of', 'master')),
+              'ip_configuration_mode': (str_to_xml,str_from_xml),
+              'IP': (str_to_xml,str_from_xml),
+              'netmask': (str_to_xml,str_from_xml),
+              'gateway': (str_to_xml,str_from_xml),
+              'DNS': (str_to_xml,str_from_xml),
+              'MAC': (str_to_xml,str_from_xml),
+              'other_config': (lambda x, p, t, v: otherconfig_to_xml(x, p, v, PIF_OTHERCONFIG_ATTRS),
+                               lambda n: otherconfig_from_xml(n, PIF_OTHERCONFIG_ATTRS)),
+            }
+
+PIF_OTHERCONFIG_ATTRS = [ 'domain', 'peerdns', 'defaultroute', 'mtu', 'static-routes' ] + \
+                        [ 'bond-%s' % x for x in 'mode', 'miimon', 'downdelay', 'updelay', 'use_carrier' ] + \
+                        ETHTOOL_OTHERCONFIG_ATTRS
+
+VLAN_ATTRS = { 'uuid': (str_to_xml,str_from_xml),
+               'tagged_PIF': (str_to_xml,str_from_xml),
+               'untagged_PIF': (str_to_xml,str_from_xml),
+             }
+    
+BOND_ATTRS = { 'uuid': (str_to_xml,str_from_xml),
+               'master': (str_to_xml,str_from_xml),
+               'slaves': (lambda x, p, t, v: strlist_to_xml(x, p, 'slaves', 'slave', v),
+                          lambda n: strlist_from_xml(n, 'slaves', 'slave')),
+             }
+
+NETWORK_ATTRS = { 'uuid': (str_to_xml,str_from_xml),
+                  'bridge': (str_to_xml,str_from_xml),
+                  'PIFs': (lambda x, p, t, v: strlist_to_xml(x, p, 'PIFs', 'PIF', v),
+                           lambda n: strlist_from_xml(n, 'PIFs', 'PIF')),
+                  'other_config': (lambda x, p, t, v: otherconfig_to_xml(x, p, v, NETWORK_OTHERCONFIG_ATTRS),
+                                   lambda n: otherconfig_from_xml(n, NETWORK_OTHERCONFIG_ATTRS)),
+                }
+
+NETWORK_OTHERCONFIG_ATTRS = [ 'mtu', 'static-routes' ] + ETHTOOL_OTHERCONFIG_ATTRS
+
 class DatabaseCache(object):
+    def __read_xensource_inventory(self):
+        filename = "/etc/xensource-inventory"
+        f = open(filename, "r")
+        lines = [x.strip("\n") for x in f.readlines()]
+        f.close()
+
+        defs = [ (l[:l.find("=")], l[(l.find("=") + 1):]) for l in lines ]
+        defs = [ (a, b.strip("'")) for (a,b) in defs ]
+
+        return dict(defs)
+    def __pif_on_host(self,pif):
+        return self.__pifs.has_key(pif)
+
+    def __get_pif_records_from_xapi(self, session, host):
+        self.__pifs = {}
+        for (p,rec) in session.xenapi.PIF.get_all_records().items():
+            if rec['host'] != host:
+                continue
+            self.__pifs[p] = {}
+            for f in PIF_ATTRS:
+                self.__pifs[p][f] = rec[f]
+            self.__pifs[p]['other_config'] = {}
+            for f in PIF_OTHERCONFIG_ATTRS:
+                if not rec['other_config'].has_key(f): continue
+                self.__pifs[p]['other_config'][f] = rec['other_config'][f]
+
+    def __get_vlan_records_from_xapi(self, session):
+        self.__vlans = {}
+        for v in session.xenapi.VLAN.get_all():
+            rec = session.xenapi.VLAN.get_record(v)
+            if not self.__pif_on_host(rec['untagged_PIF']):
+                continue
+            self.__vlans[v] = {}
+            for f in VLAN_ATTRS:
+                self.__vlans[v][f] = rec[f]
+
+    def __get_bond_records_from_xapi(self, session):
+        self.__bonds = {}
+        for b in session.xenapi.Bond.get_all():
+            rec = session.xenapi.Bond.get_record(b)
+            if not self.__pif_on_host(rec['master']):
+                continue
+            self.__bonds[b] = {}
+            for f in BOND_ATTRS:
+                self.__bonds[b][f] = rec[f]
+
+    def __get_network_records_from_xapi(self, session):
+        self.__networks = {}
+        for n in session.xenapi.network.get_all():
+            rec = session.xenapi.network.get_record(n)
+            self.__networks[n] = {}
+            for f in NETWORK_ATTRS:
+                self.__networks[n][f] = rec[f]
+            self.__networks[n]['other_config'] = {}
+            for f in NETWORK_OTHERCONFIG_ATTRS:
+                if not rec['other_config'].has_key(f): continue
+                self.__networks[n]['other_config'][f] = rec['other_config'][f]
+
+    def __to_xml(self, xml, parent, key, ref, rec, attrs):
+        """Encode a database object as XML"""
+        e = xml.createElement(key)
+        parent.appendChild(e)
+        if ref:
+            e.setAttribute('ref', ref)
+
+        for n,v in rec.items():
+            if attrs.has_key(n):
+                h,_ = attrs[n]
+                h(xml, e, n, v)
+            else:
+                raise Error("Unknown attribute %s" % n)
+    def __from_xml(self, e, attrs):
+        """Decode a database object from XML"""
+        ref = e.attributes['ref'].value
+        rec = {}
+        for n in e.childNodes:
+            if n.nodeName in attrs:
+                _,h = attrs[n.nodeName]
+                rec[n.nodeName] = h(n)
+        return (ref,rec)
+    
     def __init__(self, session_ref=None, cache_file=None):
         if session_ref and cache_file:
             raise Error("can't specify session reference and cache file")
-
         if cache_file == None:
             session = XenAPI.xapi_local()
 
@@ -291,32 +496,70 @@ class DatabaseCache(object):
                 session._session = session_ref
 
             try:
-                self.__vlans = session.xenapi.VLAN.get_all_records()
-                self.__bonds = session.xenapi.Bond.get_all_records()
-                self.__pifs = session.xenapi.PIF.get_all_records()
-                self.__networks = session.xenapi.network.get_all_records()
+                
+                inventory = self.__read_xensource_inventory()
+                assert(inventory.has_key('INSTALLATION_UUID'))
+                log("host uuid is %s" % inventory['INSTALLATION_UUID'])
+                
+                host = session.xenapi.host.get_by_uuid(inventory['INSTALLATION_UUID'])
+                
+                self.__get_pif_records_from_xapi(session, host)
+
+                self.__get_vlan_records_from_xapi(session)
+                self.__get_bond_records_from_xapi(session)
+                self.__get_network_records_from_xapi(session)
             finally:
                 if not session_ref:
                     session.xenapi.session.logout()
         else:
             log("Loading xapi database cache from %s" % cache_file)
-            f = open(cache_file, 'r')
-            members = pickle.load(f)
-            self.extras = pickle.load(f)
-            f.close()
 
-            self.__vlans = members['vlans']
-            self.__bonds = members['bonds']
-            self.__pifs = members['pifs']
-            self.__networks = members['networks']
+            xml = parseXML(cache_file)
+
+            self.__pifs = {}
+            self.__bonds = {}
+            self.__vlans = {}
+            self.__networks = {}
 
-    def save(self, cache_file, extras):
+            assert(len(xml.childNodes) == 1)
+            toplevel = xml.childNodes[0]
+            
+            assert(toplevel.nodeName == "xenserver-network-configuration")
+            
+            for n in toplevel.childNodes:
+                if n.nodeName == "#text":
+                    pass
+                elif n.nodeName == PIF_XML_TAG:
+                    (ref,rec) = self.__from_xml(n, PIF_ATTRS)
+                    self.__pifs[ref] = rec
+                elif n.nodeName == BOND_XML_TAG:
+                    (ref,rec) = self.__from_xml(n, BOND_ATTRS)
+                    self.__bonds[ref] = rec
+                elif n.nodeName == VLAN_XML_TAG:
+                    (ref,rec) = self.__from_xml(n, VLAN_ATTRS)
+                    self.__vlans[ref] = rec
+                elif n.nodeName == NETWORK_XML_TAG:
+                    (ref,rec) = self.__from_xml(n, NETWORK_ATTRS)
+                    self.__networks[ref] = rec
+                else:
+                    raise Error("Unknown XML element %s" % n.nodeName)
+
+    def save(self, cache_file):
+
+        xml = getDOMImplementation().createDocument(
+            None, "xenserver-network-configuration", None)
+        for (ref,rec) in self.__pifs.items():
+            self.__to_xml(xml, xml.documentElement, PIF_XML_TAG, ref, rec, PIF_ATTRS)
+        for (ref,rec) in self.__bonds.items():
+            self.__to_xml(xml, xml.documentElement, BOND_XML_TAG, ref, rec, BOND_ATTRS)
+        for (ref,rec) in self.__vlans.items():
+            self.__to_xml(xml, xml.documentElement, VLAN_XML_TAG, ref, rec, VLAN_ATTRS)
+        for (ref,rec) in self.__networks.items():
+            self.__to_xml(xml, xml.documentElement, NETWORK_XML_TAG, ref, rec,
+                          NETWORK_ATTRS)
+            
         f = open(cache_file, 'w')
-        pickle.dump({'vlans': self.__vlans,
-                     'bonds': self.__bonds,
-                     'pifs': self.__pifs,
-                     'networks': self.__networks}, f)
-        pickle.dump(extras, f)
+        f.write(xml.toprettyxml())
         f.close()
 
     def get_pif_by_uuid(self, uuid):
@@ -330,33 +573,12 @@ class DatabaseCache(object):
 
         return pifs[0]
 
-    def get_pifs_by_record(self, record):
-        """record is partial pif record.
-        Get the pif(s) whose record matches.
-        """
-        def match(pifrec):
-            for key in record:
-                if record[key] != pifrec[key]:
-                    return False
-            return True
-            
+    def get_pifs_by_device(self, device):
         return map(lambda (ref,rec): ref,
-                   filter(lambda (ref,rec): match(rec),
+                   filter(lambda (ref,rec): rec['device'] == device,
                           self.__pifs.items()))
 
-    def get_pif_by_record(self, record):
-        """record is partial pif record.
-        Get the pif whose record matches.
-        """
-        pifs = self.get_pifs_by_record(record)
-        if len(pifs) == 0:
-            raise Error("No matching PIF \"%s\"" % str(record))
-        elif len(pifs) > 1:
-            raise Error("Multiple matching PIFs \"%s\"" % str(record))
-
-        return pifs[0]
-
-    def get_pif_by_bridge(self, host, bridge):
+    def get_pif_by_bridge(self, bridge):
         networks = map(lambda (ref,rec): ref,
                        filter(lambda (ref,rec): rec['bridge'] == bridge,
                               self.__networks.items()))
@@ -368,32 +590,29 @@ class DatabaseCache(object):
             nwrec = self.get_network_record(network)
             for pif in nwrec['PIFs']:
                 pifrec = self.get_pif_record(pif)
-                if pifrec['host'] != host:
-                    continue
                 if answer:
-                    raise Error("Multiple PIFs on %s for network %s" % (host, bridge))
+                    raise Error("Multiple PIFs on host for network %s" % (bridge))
                 answer = pif
         if not answer:
-            raise Error("No PIF on %s for network %s" % (host, bridge))
+            raise Error("No PIF on host for network %s" % (bridge))
         return answer
 
     def get_pif_record(self, pif):
         if self.__pifs.has_key(pif):
             return self.__pifs[pif]
-        raise Error("Unknown PIF \"%s\"" % pif)
+        raise Error("Unknown PIF \"%s\" (get_pif_record)" % pif)
     def get_all_pifs(self):
         return self.__pifs
     def pif_exists(self, pif):
         return self.__pifs.has_key(pif)
     
-    def get_management_pif(self, host):
+    def get_management_pif(self):
         """ Returns the management pif on host
         """
         all = self.get_all_pifs()
         for pif in all: 
             pifrec = self.get_pif_record(pif)
-            if pifrec['management'] and pifrec['host'] == host :
-                return pif
+            if pifrec['management']: return pif
         return None
 
     def get_network_record(self, network):
@@ -445,6 +664,7 @@ For a VLAN PIF, the datapath name is the bridge name for the PIF's VLAN slave.
 use it.)
 """
 
+
     pifrec = db.get_pif_record(pif)
 
     if pifrec['VLAN'] == '-1':
@@ -467,7 +687,6 @@ For a VLAN PIF, this is the VLAN slave's physical device PIF.
 For a bond master PIF, these are the bond slave PIFs.
 For a non-VLAN, non-bond master PIF, the PIF is its own physical device PIF.
 """
-
     pifrec = db.get_pif_record(pif)
 
     if pifrec['VLAN'] != '-1':
@@ -488,13 +707,16 @@ For a non-VLAN, non-bond master PIF, the physical device is the PIF itself.
 
 def log_pif_action(action, pif):
     pifrec = db.get_pif_record(pif)
-    pifrec['action'] = action
-    pifrec['interface-name'] = interface_name(pif)
+    rec = {}
+    rec['uuid'] = pifrec['uuid']
+    rec['ip_configuration_mode'] = pifrec['ip_configuration_mode']
+    rec['action'] = action
+    rec['interface-name'] = interface_name(pif)
     if action == "rewrite":
-        pifrec['message'] = "Rewrite PIF %(uuid)s configuration" % pifrec
+        rec['message'] = "Rewrite PIF %(uuid)s configuration" % rec
     else:
-        pifrec['message'] = "Bring %(action)s PIF %(uuid)s" % pifrec
-    log("%(message)s: %(interface-name)s configured as %(ip_configuration_mode)s" % pifrec)
+        rec['message'] = "Bring %(action)s PIF %(uuid)s" % rec
+    log("%(message)s: %(interface-name)s configured as %(ip_configuration_mode)s" % rec)
 
 def get_bond_masters_of_pif(pif):
     """Returns a list of PIFs which are bond masters of this PIF"""
@@ -520,7 +742,6 @@ def get_bond_slaves_of_pif(pif):
     """Returns a list of PIFs which make up the given bonded pif."""
     
     pifrec = db.get_pif_record(pif)
-    host = pifrec['host']
 
     bmo = pifrec['bond_master_of']
     if len(bmo) > 1:
@@ -661,10 +882,8 @@ This is because when we are called to bring up an interface with a bond master,
 we should bring down that master."""
 
     pifrec = db.get_pif_record(pif)
-    host = pifrec['host']
 
-    pifs_on_host = [ __pif for __pif in db.get_all_pifs() if
-                     db.get_pif_record(__pif)['host'] == host and 
+    pifs = [ __pif for __pif in db.get_all_pifs() if
                      (not  __pif in get_bond_masters_of_pif(pif)) ]
 
     peerdns_pif = None
@@ -673,7 +892,7 @@ we should bring down that master."""
     # loop through all the pifs on this host looking for one with
     #   other-config:peerdns = true, and one with
     #   other-config:default-route=true
-    for __pif in pifs_on_host:
+    for __pif in pifs:
         __pifrec = db.get_pif_record(__pif)
         __oc = __pifrec['other_config']
         if __oc.has_key('peerdns') and __oc['peerdns'] == 'true':
@@ -751,7 +970,6 @@ def configure_local_port(pif):
     datapath = datapath_name(pif)
     ipdev = ipdev_name(pif)
 
-    host = pifrec['host']
     nw = pifrec['network']
     nwrec = db.get_network_record(nw)
 
@@ -984,8 +1202,7 @@ def action_up(pif):
     # - The networks corresponding to any VLANs attached to the
     #   datapath's PIF.
     network_uuids = []
-    for nwpif in db.get_pifs_by_record({'device': pifrec['device'],
-                                        'host': pifrec['host']}):
+    for nwpif in db.get_pifs_by_device({'device': pifrec['device']}):
         net = db.get_pif_record(nwpif)['network']
         network_uuids += [db.get_network_record(net)['uuid']]
 
@@ -1244,9 +1461,8 @@ def main(argv=None):
                 action_force_rewrite(force_interface, force_rewrite_config)
             else:
                 db = DatabaseCache(cache_file=dbcache_file)
-                host = db.extras['host']
-                pif = db.get_pif_by_bridge(host, force_interface)
-                management_pif = db.get_management_pif(host)
+                pif = db.get_pif_by_bridge(force_interface)
+                management_pif = db.get_management_pif()
 
                 if action == "up":
                     action_up(pif)
@@ -1270,8 +1486,7 @@ def main(argv=None):
                 # pif is not going to be the management pif.
                 # Search DB cache for pif on same host with management=true
                 pifrec = db.get_pif_record(pif)
-                host = pifrec['host']
-                management_pif = db.get_management_pif(host)
+                management_pif = db.get_management_pif()
 
             log_pif_action(action, pif)
 
@@ -1289,7 +1504,7 @@ def main(argv=None):
 
             # Save cache.
             pifrec = db.get_pif_record(pif)
-            db.save(dbcache_file, {'host': pifrec['host']})
+            db.save(dbcache_file)
         
     except Usage, err:
         print >>sys.stderr, err.msg
@@ -1455,7 +1670,6 @@ def configure_network(pif, f):
     """
     
     pifrec = db.get_pif_record(pif)
-    host = pifrec['host']
     nw = pifrec['network']
     nwrec = db.get_network_record(nw)
     oc = None
@@ -1510,8 +1724,7 @@ def configure_network(pif, f):
     # This is because when we are called to bring up an interface with a bond master, it is implicit that
     # we should bring down that master.
     pifs_on_host = [ __pif for __pif in db.get_all_pifs() if
-                     db.get_pif_record(__pif)['host'] == host and 
-                     (not  __pif in get_bond_masters_of_pif(pif)) ]
+                     not __pif in get_bond_masters_of_pif(pif) ]
     other_pifs_on_host = [ __pif for __pif in pifs_on_host if __pif != pif ]
 
     peerdns_pif = None