macOS: rework list_ports to support unicode product descriptors.

This commit makes some changes to try to improve the behavior of
serial.tools.list_ports on macOS and to reduce the amount of magic
numbers in this code. These changes include a better adherence to iokit
function type declarations as taken from the iokit header files, as
well as switching all iokit calls to use UTF-8 encoding rather than
mac_roman.

The main functional change of this commit is that the devicename is
now retrieved through IORegistryEntryName, which avoids some weird
USB descriptor mangling that happens somewhere deep within the BSD
serial device subsystem in XNU. In particular, on serial devices, all
USB product descriptor characters that are not in the basic
alphanumeric set are replaced with underscores, which is silly as USB
descriptors are unicode strings. This mangling ONLY happens to the
product descriptor string, so the vendor and serial number strings do
not need to be handled differently than before.

1 files changed, 55 insertions, 22 deletions

diff --git a/serial/tools/list_ports_osx.py b/serial/tools/list_ports_osx.py
index f46a820..34a7f5a 100644
--- a/serial/tools/list_ports_osx.py
+++ b/serial/tools/list_ports_osx.py
@@ -35,25 +35,40 @@ kIOMasterPortDefault = ctypes.c_void_p.in_dll(iokit, "kIOMasterPortDefault")
 kCFAllocatorDefault = ctypes.c_void_p.in_dll(cf, "kCFAllocatorDefault")
 
 kCFStringEncodingMacRoman = 0
+kCFStringEncodingUTF8 = 0x08000100
+
+# defined in `IOKit/usb/USBSpec.h`
+kUSBVendorString = 'USB Vendor Name'
+kUSBSerialNumberString = 'USB Serial Number'
+
+# `io_name_t` defined as `typedef char io_name_t[128];`
+# in `device/device_types.h`
+io_name_size = 128
+
+# defined in `mach/kern_return.h`
+KERN_SUCCESS = 0
+# kern_return_t defined as `typedef int kern_return_t;` in `mach/i386/kern_return.h`
+kern_return_t = ctypes.c_int
 
 iokit.IOServiceMatching.restype = ctypes.c_void_p
 
 iokit.IOServiceGetMatchingServices.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p]
-iokit.IOServiceGetMatchingServices.restype = ctypes.c_void_p
+iokit.IOServiceGetMatchingServices.restype = kern_return_t
 
 iokit.IORegistryEntryGetParentEntry.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p]
+iokit.IOServiceGetMatchingServices.restype = kern_return_t
 
 iokit.IORegistryEntryCreateCFProperty.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_uint32]
 iokit.IORegistryEntryCreateCFProperty.restype = ctypes.c_void_p
 
 iokit.IORegistryEntryGetPath.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p]
-iokit.IORegistryEntryGetPath.restype = ctypes.c_void_p
+iokit.IORegistryEntryGetPath.restype = kern_return_t
 
 iokit.IORegistryEntryGetName.argtypes = [ctypes.c_void_p, ctypes.c_void_p]
-iokit.IORegistryEntryGetName.restype = ctypes.c_void_p
+iokit.IORegistryEntryGetName.restype = kern_return_t
 
 iokit.IOObjectGetClass.argtypes = [ctypes.c_void_p, ctypes.c_void_p]
-iokit.IOObjectGetClass.restype = ctypes.c_void_p
+iokit.IOObjectGetClass.restype = kern_return_t
 
 iokit.IOObjectRelease.argtypes = [ctypes.c_void_p]
 
@@ -64,6 +79,9 @@ cf.CFStringCreateWithCString.restype = ctypes.c_void_p
 cf.CFStringGetCStringPtr.argtypes = [ctypes.c_void_p, ctypes.c_uint32]
 cf.CFStringGetCStringPtr.restype = ctypes.c_char_p
 
+cf.CFStringGetCString.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_long, ctypes.c_uint32]
+cf.CFStringGetCString.restype = ctypes.c_bool
+
 cf.CFNumberGetValue.argtypes = [ctypes.c_void_p, ctypes.c_uint32, ctypes.c_void_p]
 cf.CFNumberGetValue.restype = ctypes.c_void_p
 
@@ -88,8 +106,8 @@ def get_string_property(device_type, property):
     """
     key = cf.CFStringCreateWithCString(
             kCFAllocatorDefault,
-            property.encode("mac_roman"),
-            kCFStringEncodingMacRoman)
+            property.encode("utf-8"),
+            kCFStringEncodingUTF8)
 
     CFContainer = iokit.IORegistryEntryCreateCFProperty(
             device_type,
@@ -101,7 +119,12 @@ def get_string_property(device_type, property):
     if CFContainer:
         output = cf.CFStringGetCStringPtr(CFContainer, 0)
         if output is not None:
-            output = output.decode('mac_roman')
+            output = output.decode('utf-8')
+        else:
+            buffer = ctypes.create_string_buffer(io_name_size);
+            success = cf.CFStringGetCString(CFContainer, ctypes.byref(buffer), io_name_size, kCFStringEncodingUTF8)
+            if success:
+                output = buffer.value.decode('utf-8')
         cf.CFRelease(CFContainer)
     return output
 
@@ -118,8 +141,8 @@ def get_int_property(device_type, property, cf_number_type):
     """
     key = cf.CFStringCreateWithCString(
             kCFAllocatorDefault,
-            property.encode("mac_roman"),
-            kCFStringEncodingMacRoman)
+            property.encode("utf-8"),
+            kCFStringEncodingUTF8)
 
     CFContainer = iokit.IORegistryEntryCreateCFProperty(
             device_type,
@@ -137,12 +160,19 @@ def get_int_property(device_type, property, cf_number_type):
         return number.value
     return None
 
-
 def IORegistryEntryGetName(device):
-    pathname = ctypes.create_string_buffer(100)  # TODO: Is this ok?
-    iokit.IOObjectGetClass(device, ctypes.byref(pathname))
-    return pathname.value
-
+    devicename = ctypes.create_string_buffer(io_name_size);
+    res = iokit.IORegistryEntryGetName(device, ctypes.byref(devicename))
+    if res != KERN_SUCCESS:
+        return None
+    # this works in python2 but may not be valid. Also I don't know if
+    # this encoding is guaranteed. It may be dependent on system locale.
+    return devicename.value.decode('utf-8')
+
+def IOObjectGetClass(device):
+    classname = ctypes.create_string_buffer(io_name_size)
+    iokit.IOObjectGetClass(device, ctypes.byref(classname))
+    return classname.value
 
 def GetParentDeviceByType(device, parent_type):
     """ Find the first parent of a device that implements the parent_type
@@ -150,15 +180,15 @@ def GetParentDeviceByType(device, parent_type):
         @return Pointer to the parent type, or None if it was not found.
     """
     # First, try to walk up the IOService tree to find a parent of this device that is a IOUSBDevice.
-    parent_type = parent_type.encode('mac_roman')
-    while IORegistryEntryGetName(device) != parent_type:
+    parent_type = parent_type.encode('utf-8')
+    while IOObjectGetClass(device) != parent_type:
         parent = ctypes.c_void_p()
         response = iokit.IORegistryEntryGetParentEntry(
                 device,
-                "IOService".encode("mac_roman"),
+                "IOService".encode("utf-8"),
                 ctypes.byref(parent))
         # If we weren't able to find a parent for the device, we're done.
-        if response != 0:
+        if response != KERN_SUCCESS:
             return None
         device = parent
     return device
@@ -172,7 +202,7 @@ def GetIOServicesByType(service_type):
 
     iokit.IOServiceGetMatchingServices(
             kIOMasterPortDefault,
-            iokit.IOServiceMatching(service_type.encode('mac_roman')),
+            iokit.IOServiceMatching(service_type.encode('utf-8')),
             ctypes.byref(serial_port_iterator))
 
     services = []
@@ -246,9 +276,12 @@ def comports(include_links=False):
                 # fetch some useful informations from properties
                 info.vid = get_int_property(usb_device, "idVendor", kCFNumberSInt16Type)
                 info.pid = get_int_property(usb_device, "idProduct", kCFNumberSInt16Type)
-                info.serial_number = get_string_property(usb_device, "USB Serial Number")
-                info.product = get_string_property(usb_device, "USB Product Name") or 'n/a'
-                info.manufacturer = get_string_property(usb_device, "USB Vendor Name")
+                info.serial_number = get_string_property(usb_device, kUSBSerialNumberString)
+                # We know this is a usb device, so the
+                # IORegistryEntryName should always be aliased to the
+                # usb product name string descriptor.
+                info.product = IORegistryEntryGetName(usb_device) or 'n/a'
+                info.manufacturer = get_string_property(usb_device, kUSBVendorString)
                 locationID = get_int_property(usb_device, "locationID", kCFNumberSInt32Type)
                 info.location = location_to_string(locationID)
                 info.interface = search_for_locationID_in_interfaces(serial_interfaces, locationID)